From 1cd9aebc48cceced04eb93503183ca8828a240e4 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 19 Jun 2026 14:05:12 +0000 Subject: [PATCH 01/30] KOKKOS: port surf_collide adiabatic to Kokkos Add SurfCollideAdiabaticKokkos, the GPU-capable port of the adiabatic surface collision model (isotropic scattering conserving particle speed). Follows the established surf_collide_diffuse_kokkos pattern: device-callable collide_kokkos, Kokkos RNG pool with SPARTA_KOKKOS_EXACT support, DualView counters, and the surf-react KKCopy dispatch. Wire the new sc_type id 5 into update_kokkos (model selection, the 3D/2D/ boundary collision dispatch ladders, post_collide, and backup/restore), and register the files in the KOKKOS Install.sh. Verified: builds with the Kokkos serial backend and runs examples/surf_collide/in.circle.adiabatic with -sf kk, producing surface- collision statistics consistent with the CPU path. Co-Authored-By: Claude Opus 4.8 Co-Authored-By: Stan Moore Claude-Session: https://claude.ai/code/session_01Q6nQVuxEiTUqSzDoDppCUM --- src/KOKKOS/Install.sh | 2 + src/KOKKOS/surf_collide_adiabatic_kokkos.cpp | 271 +++++++++++++++++++ src/KOKKOS/surf_collide_adiabatic_kokkos.h | 258 ++++++++++++++++++ src/KOKKOS/update_kokkos.cpp | 44 ++- src/KOKKOS/update_kokkos.h | 2 + 5 files changed, 568 insertions(+), 9 deletions(-) create mode 100644 src/KOKKOS/surf_collide_adiabatic_kokkos.cpp create mode 100644 src/KOKKOS/surf_collide_adiabatic_kokkos.h diff --git a/src/KOKKOS/Install.sh b/src/KOKKOS/Install.sh index 58fc4ce43..ca6169ff8 100644 --- a/src/KOKKOS/Install.sh +++ b/src/KOKKOS/Install.sh @@ -117,6 +117,8 @@ action react_bird_kokkos.cpp action react_bird_kokkos.h action react_tce_kokkos.cpp action react_tce_kokkos.h +action surf_collide_adiabatic_kokkos.cpp +action surf_collide_adiabatic_kokkos.h action surf_collide_diffuse_kokkos.cpp action surf_collide_diffuse_kokkos.h action surf_collide_piston_kokkos.cpp diff --git a/src/KOKKOS/surf_collide_adiabatic_kokkos.cpp b/src/KOKKOS/surf_collide_adiabatic_kokkos.cpp new file mode 100644 index 000000000..da274f9cc --- /dev/null +++ b/src/KOKKOS/surf_collide_adiabatic_kokkos.cpp @@ -0,0 +1,271 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#include "math.h" +#include "stdlib.h" +#include "string.h" +#include "surf_collide_adiabatic_kokkos.h" +#include "surf_kokkos.h" +#include "input.h" +#include "variable.h" +#include "particle.h" +#include "domain.h" +#include "update.h" +#include "modify.h" +#include "comm.h" +#include "random_mars.h" +#include "random_knuth.h" +#include "math_const.h" +#include "math_extra.h" +#include "error.h" +#include "particle_kokkos.h" +#include "sparta_masks.h" +#include "collide.h" + +using namespace SPARTA_NS; +using namespace MathConst; + +#define VAL_1(X) X +#define VAL_2(X) VAL_1(X), VAL_1(X) + +/* ---------------------------------------------------------------------- */ + +SurfCollideAdiabaticKokkos::SurfCollideAdiabaticKokkos(SPARTA *sparta, int narg, char **arg) : + SurfCollideAdiabatic(sparta, narg, arg), + fix_ambi_kk_copy(sparta), + fix_vibmode_kk_copy(sparta), + sr_kk_global_copy{VAL_2(KKCopy(sparta))}, + sr_kk_prob_copy{VAL_2(KKCopy(sparta))}, + rand_pool(12345 + comm->me +#ifdef SPARTA_KOKKOS_EXACT + , sparta +#endif + ) +{ + kokkosable = 1; + + random_backup = NULL; + +#ifdef SPARTA_KOKKOS_EXACT + rand_pool.init(random); +#endif + + // use 1D view for scalars to reduce GPU memory operations + + d_scalars = t_int_2("surf_collide_adiabatic:scalars"); + d_nsingle = Kokkos::subview(d_scalars,0); + d_nreact_one = Kokkos::subview(d_scalars,1); + + h_scalars = t_host_int_2("surf_collide_adiabatic:scalars_mirror"); + h_nsingle = Kokkos::subview(h_scalars,0); + h_nreact_one = Kokkos::subview(h_scalars,1); +} + +SurfCollideAdiabaticKokkos::SurfCollideAdiabaticKokkos(SPARTA *sparta) : + SurfCollideAdiabatic(sparta), + fix_ambi_kk_copy(sparta), + fix_vibmode_kk_copy(sparta), + sr_kk_global_copy{VAL_2(KKCopy(sparta))}, + sr_kk_prob_copy{VAL_2(KKCopy(sparta))}, + rand_pool(12345 // seed doesn't matter since it will just be copied over +#ifdef SPARTA_KOKKOS_EXACT + , sparta +#endif + ) +{ + copy = 1; +} + +/* ---------------------------------------------------------------------- */ + +SurfCollideAdiabaticKokkos::~SurfCollideAdiabaticKokkos() +{ + if (uncopy) { + fix_ambi_kk_copy.uncopy(); + fix_vibmode_kk_copy.uncopy(); + + for (int i = 0; i < KOKKOS_MAX_SURF_REACT_PER_TYPE; i++) { + sr_kk_global_copy[i].uncopy(); + sr_kk_prob_copy[i].uncopy(); + } + } + + if (copy) return; + +#ifdef SPARTA_KOKKOS_EXACT + rand_pool.destroy(); + if (random_backup) + delete random_backup; +#endif +} + +/* ---------------------------------------------------------------------- */ + +void SurfCollideAdiabaticKokkos::init() +{ + SurfCollideAdiabatic::init(); + + ambi_flag = vibmode_flag = 0; + if (modify->n_update_custom) { + for (int ifix = 0; ifix < modify->nfix; ifix++) { + if (strcmp(modify->fix[ifix]->style,"ambipolar") == 0) { + ambi_flag = 1; + FixAmbipolar *afix = (FixAmbipolar *) modify->fix[ifix]; + if (!afix->kokkos_flag) + error->all(FLERR,"Must use fix ambipolar/kk when Kokkos is enabled"); + afix_kk = (FixAmbipolarKokkos*)afix; + } else if (strcmp(modify->fix[ifix]->style,"vibmode") == 0) { + vibmode_flag = 1; + FixVibmode *vfix = (FixVibmode *) modify->fix[ifix]; + if (!vfix->kokkos_flag) + error->all(FLERR,"Must use fix vibmode/kk when Kokkos is enabled"); + vfix_kk = (FixVibmodeKokkos*)vfix; + } + } + } +} + +/* ---------------------------------------------------------------------- */ + +void SurfCollideAdiabaticKokkos::pre_collide() +{ + if (ambi_flag) { + afix_kk->pre_update_custom_kokkos(); + fix_ambi_kk_copy.copy(afix_kk); + } + + if (vibmode_flag) { + vfix_kk->pre_update_custom_kokkos(); + fix_vibmode_kk_copy.copy(vfix_kk); + } + + if (surf->nsr > KOKKOS_MAX_TOT_SURF_REACT) + error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); + + if (surf->nsr > 0) { + int nglob,nprob; + nglob = nprob = 0; + for (int n = 0; n < surf->nsr; n++) { + if (!surf->sr[n]->kokkosable) + error->all(FLERR,"Must use Kokkos-enabled surface reaction method with Kokkos"); + if (strcmp(surf->sr[n]->style,"global") == 0) { + sr_kk_global_copy[nglob].copy((SurfReactGlobalKokkos*)(surf->sr[n])); + sr_kk_global_copy[nglob].obj.pre_react(); + sr_type_list[n] = 0; + sr_map[n] = nglob; + nglob++; + } else if (strcmp(surf->sr[n]->style,"prob") == 0) { + sr_kk_prob_copy[nprob].copy((SurfReactProbKokkos*)(surf->sr[n])); + sr_kk_prob_copy[nprob].obj.pre_react(); + sr_type_list[n] = 1; + sr_map[n] = nprob; + nprob++; + } else { + error->all(FLERR,"Unknown Kokkos surface reaction method"); + } + } + + if (nglob > KOKKOS_MAX_SURF_REACT_PER_TYPE || nprob > KOKKOS_MAX_SURF_REACT_PER_TYPE) + error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); + } + + if (random == NULL) { + // initialize RNG + + random = new RanKnuth(update->ranmaster->uniform()); + double seed = update->ranmaster->uniform(); + random->reset(seed,comm->me,100); + +#ifdef SPARTA_KOKKOS_EXACT + rand_pool.init(random); +#endif + } + + ParticleKokkos* particle_kk = (ParticleKokkos*) particle; + particle_kk->sync(Device,PARTICLE_MASK|SPECIES_MASK); + d_particles = particle_kk->k_particles.view_device(); + d_species = particle_kk->k_species.view_device(); + + Kokkos::deep_copy(d_scalars,0); +} + +/* ---------------------------------------------------------------------- */ + +void SurfCollideAdiabaticKokkos::post_collide() +{ + ParticleKokkos* particle_kk = (ParticleKokkos*) particle; + if (ambi_flag || vibmode_flag) particle_kk->modify(Device,CUSTOM_MASK); + + Kokkos::deep_copy(h_scalars,d_scalars); + + int m = surf->find_collide(id); + auto sc = surf->sc[m]; // can't modify the copy directly, use the original + sc->nsingle += h_nsingle(); + surf->nreact_one += h_nreact_one(); + + d_particles = {}; +} + +/* ---------------------------------------------------------------------- */ + +void SurfCollideAdiabaticKokkos::backup() +{ + ParticleKokkos* particle_kk = (ParticleKokkos*) particle; + d_particles = particle_kk->k_particles.view_device(); + + if (surf->nsr > 0) { + int nglob,nprob; + nglob = nprob = 0; + for (int n = 0; n < surf->nsr; n++) { + if (strcmp(surf->sr[n]->style,"global") == 0) { + sr_kk_global_copy[nglob].obj.backup(); + nglob++; + } else if (strcmp(surf->sr[n]->style,"prob") == 0) { + sr_kk_prob_copy[nprob].obj.backup(); + nprob++; + } + } + } + +#ifdef SPARTA_KOKKOS_EXACT + if (!random_backup) + random_backup = new RanKnuth(12345 + comm->me); + memcpy(random_backup,random,sizeof(RanKnuth)); +#endif +} + +/* ---------------------------------------------------------------------- */ + +void SurfCollideAdiabaticKokkos::restore() +{ + if (surf->nsr > 0) { + int nglob,nprob; + nglob = nprob = 0; + for (int n = 0; n < surf->nsr; n++) { + if (strcmp(surf->sr[n]->style,"global") == 0) { + sr_kk_global_copy[nglob].obj.restore(); + nglob++; + } else if (strcmp(surf->sr[n]->style,"prob") == 0) { + sr_kk_prob_copy[nprob].obj.restore(); + nprob++; + } + } + } + + Kokkos::deep_copy(d_scalars,0); + +#ifdef SPARTA_KOKKOS_EXACT + memcpy(random,random_backup,sizeof(RanKnuth)); +#endif +} diff --git a/src/KOKKOS/surf_collide_adiabatic_kokkos.h b/src/KOKKOS/surf_collide_adiabatic_kokkos.h new file mode 100644 index 000000000..bdfb1b1af --- /dev/null +++ b/src/KOKKOS/surf_collide_adiabatic_kokkos.h @@ -0,0 +1,258 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#ifdef SURF_COLLIDE_CLASS + +SurfCollideStyle(adiabatic/kk,SurfCollideAdiabaticKokkos) + +#else + +#ifndef SPARTA_SURF_COLLIDE_ADIABATIC_KOKKOS_H +#define SPARTA_SURF_COLLIDE_ADIABATIC_KOKKOS_H + +#include "surf_collide_adiabatic.h" +#include "kokkos_type.h" +#include "math_extra_kokkos.h" +#include "Kokkos_Random.hpp" +#include "rand_pool_wrap.h" +#include "kokkos_copy.h" +#include "fix_ambipolar_kokkos.h" +#include "fix_vibmode_kokkos.h" +#include "surf_react_global_kokkos.h" +#include "surf_react_prob_kokkos.h" + +namespace SPARTA_NS { + +class SurfCollideAdiabaticKokkos : public SurfCollideAdiabatic { + public: + + enum{PKEEP,PINSERT,PDONE,PDISCARD,PENTRY,PEXIT,PSURF}; // several files + + SurfCollideAdiabaticKokkos(class SPARTA *, int, char **); + SurfCollideAdiabaticKokkos(class SPARTA *); + ~SurfCollideAdiabaticKokkos(); + void init(); + void pre_collide(); + void post_collide(); + void backup(); + void restore(); + + private: + +#ifndef SPARTA_KOKKOS_EXACT + Kokkos::Random_XorShift64_Pool rand_pool; + typedef typename Kokkos::Random_XorShift64_Pool::generator_type rand_type; +#else + RandPoolWrap rand_pool; + typedef RandWrap rand_type; +#endif + + RanKnuth* random_backup; + + typedef Kokkos::DualView tdual_int_2; + typedef tdual_int_2::t_dev t_int_2; + typedef tdual_int_2::t_host t_host_int_2; + t_int_2 d_scalars; + t_host_int_2 h_scalars; + + DAT::t_int_scalar d_nsingle; + DAT::t_int_scalar d_nreact_one; + + HAT::t_int_scalar h_nsingle; + HAT::t_int_scalar h_nreact_one; + + t_particle_1d d_particles; + t_species_1d d_species; + + int ambi_flag,vibmode_flag; + FixAmbipolarKokkos* afix_kk; + FixVibmodeKokkos* vfix_kk; + KKCopy fix_ambi_kk_copy; + KKCopy fix_vibmode_kk_copy; + + int sr_type_list[KOKKOS_MAX_TOT_SURF_REACT]; + int sr_map[KOKKOS_MAX_TOT_SURF_REACT]; + KKCopy sr_kk_global_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; + KKCopy sr_kk_prob_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; + + public: + + /* ---------------------------------------------------------------------- + particle collision with surface with optional chemistry + ip = particle with current x = collision pt, current v = incident v + isurf = index of surface element + norm = surface normal unit vector + isr = index of reaction model if >= 0, -1 for no chemistry + ip = set to NULL if destroyed by chemistry + return jp = new particle if created by chemistry + return reaction = index of reaction (1 to N) that took place, 0 = no reaction + resets particle(s) to post-collision outward velocity + + note that the adiabatic condition (i.e. no energy transfer of flow to + surf) only applies to particle collisions. Chemistry (e.g. particle + adsorptions) can lead to energy transfer in both directions, so the + velocities reset by SurfReact are kept as-is. + ------------------------------------------------------------------------- */ + + template + KOKKOS_INLINE_FUNCTION + Particle::OnePart* collide_kokkos(Particle::OnePart *&ip, double &, + int isurf, const double *norm, int isr, int &reaction, + const DAT::t_int_scalar &d_retry, const DAT::t_int_scalar &d_nlocal) const + { + if (ATOMIC_REDUCTION == 0) + d_nsingle()++; + else + Kokkos::atomic_inc(&d_nsingle()); + + // if surface chemistry defined, attempt reaction + // reaction = 1 to N for which reaction took place, 0 for none + // velreset = 1 if reaction reset post-collision velocity, else 0 + + Particle::OnePart iorig; + Particle::OnePart *jp = NULL; + reaction = 0; + int velreset = 0; + + if (REACT) { + if (ambi_flag || vibmode_flag) memcpy(&iorig,ip,sizeof(Particle::OnePart)); + + int sr_type = sr_type_list[isr]; + int m = sr_map[isr]; + + if (sr_type == 0) { + reaction = sr_kk_global_copy[m].obj. + react_kokkos(ip,isurf,norm,jp,velreset,d_retry,d_nlocal); + } else if (sr_type == 1) { + reaction = sr_kk_prob_copy[m].obj. + react_kokkos(ip,isurf,norm,jp,velreset,d_retry,d_nlocal); + } + + if (reaction) { + if (ATOMIC_REDUCTION == 0) + d_nreact_one()++; + else + Kokkos::atomic_inc(&d_nreact_one()); + } + } + + // isotropic scattering conserving velocity magnitude (kinetic energy) + // of each particle + // only if SurfReact did not already reset velocities + // cannot trigger fixes that require temperature of particle here + // because temperature of wall is not known + + if (ip) { + if (!velreset) scatter_isotropic(ip,norm); + } + if (REACT && jp) { + if (!velreset) scatter_isotropic(jp,norm); + } + + // call any fixes with a surf_react() method + // they may reset j to -1, e.g. fix ambipolar + // in which case newly created j is deleted + + if (REACT && reaction && ambi_flag) { + int i = -1; + if (ip) i = ip - d_particles.data(); + int j = -1; + if (jp) j = jp - d_particles.data(); + int j_orig = j; + fix_ambi_kk_copy.obj.surf_react_kokkos(&iorig,i,j); + if (jp && j < 0) { + d_particles[j_orig].flag = PDISCARD; + jp = NULL; + } + } + + return jp; + }; + + private: + + /* ---------------------------------------------------------------------- + particle collision with adiabatic surface + p = particle with current x = collision pt, current v = incident v + norm = surface normal unit vector + particle is scattered isotropically while conserving its velocity + magnitude (i.e. no energy transfer to surf) + ------------------------------------------------------------------------- */ + + KOKKOS_INLINE_FUNCTION + void scatter_isotropic(Particle::OnePart *p, const double *norm) const + { + rand_type rand_gen = rand_pool.get_state(); + + double *v = p->v; + double dot = MathExtraKokkos::dot3(v,norm); + + // tangent1/2 = surface tangential unit vectors + + double tangent1[3],tangent2[3]; + tangent1[0] = v[0] - dot*norm[0]; + tangent1[1] = v[1] - dot*norm[1]; + tangent1[2] = v[2] - dot*norm[2]; + + // if mag(tangent1) == 0, normal collision: choose a random tangent vector + + if (MathExtraKokkos::lensq3(tangent1) == 0.0) { + tangent2[0] = rand_gen.drand(); + tangent2[1] = rand_gen.drand(); + tangent2[2] = rand_gen.drand(); + MathExtraKokkos::cross3(norm,tangent2,tangent1); + } + + MathExtraKokkos::norm3(tangent1); + MathExtraKokkos::cross3(norm,tangent1,tangent2); + + // isotropic scattering + // vmag = magnitude of incident particle velocity vector + // vperp = velocity component perpendicular to surface along norm + // vtan1/2 = 2 remaining velocity components tangential to surface + + double vmag = MathExtraKokkos::len3(v); + + double theta = MathConst::MY_2PI * rand_gen.drand(); + double f_phi = rand_gen.drand(); + double sqrt_f_phi = sqrt(f_phi); + + double vperp = vmag * sqrt(1.0 - f_phi); + double vtan1 = vmag * sqrt_f_phi * sin(theta); + double vtan2 = vmag * sqrt_f_phi * cos(theta); + + v[0] = vperp*norm[0] + vtan1*tangent1[0] + vtan2*tangent2[0]; + v[1] = vperp*norm[1] + vtan1*tangent1[1] + vtan2*tangent2[1]; + v[2] = vperp*norm[2] + vtan1*tangent1[2] + vtan2*tangent2[2]; + + // p->erot and p->evib stay identical + + rand_pool.free_state(rand_gen); + } +}; + +} + +#endif +#endif + +/* ERROR/WARNING messages: + +E: Illegal ... command + +Self-explanatory. Check the input script syntax and compare to the +documentation for the command. You can use -echo screen as a +command-line option when running SPARTA to see the offending line. + +*/ diff --git a/src/KOKKOS/update_kokkos.cpp b/src/KOKKOS/update_kokkos.cpp index 5f82f5a0f..80e4dd136 100644 --- a/src/KOKKOS/update_kokkos.cpp +++ b/src/KOKKOS/update_kokkos.cpp @@ -80,6 +80,7 @@ UpdateKokkos::UpdateKokkos(SPARTA *sparta) : Update(sparta), sc_kk_vanish_copy{VAL_2(KKCopy(sparta))}, sc_kk_piston_copy{VAL_2(KKCopy(sparta))}, sc_kk_transparent_copy{VAL_2(KKCopy(sparta))}, + sc_kk_adiabatic_copy{VAL_2(KKCopy(sparta))}, blist_active_copy{VAL_2(KKCopy(sparta))}, slist_active_copy{VAL_2(KKCopy(sparta))}, tmp_compute_boundary_kk(sparta), @@ -145,6 +146,7 @@ UpdateKokkos::~UpdateKokkos() sc_kk_vanish_copy[i].uncopy(); sc_kk_piston_copy[i].uncopy(); sc_kk_transparent_copy[i].uncopy(); + sc_kk_adiabatic_copy[i].uncopy(); } for (int i=0; i void UpdateKokkos::move() error->all(FLERR,"Kokkos currently supports two instances of each surface collide method"); if (surf->nsc > 0) { - int nspec,ndiff,nvan,npist,ntrans; - nspec = ndiff = nvan = npist = ntrans = 0; + int nspec,ndiff,nvan,npist,ntrans,nadia; + nspec = ndiff = nvan = npist = ntrans = nadia = 0; for (int n = 0; n < surf->nsc; n++) { if (!surf->sc[n]->kokkosable) error->all(FLERR,"Must use Kokkos-enabled surface collide method with Kokkos"); @@ -566,13 +568,19 @@ template < int DIM, int SURF, int REACT, int OPT > void UpdateKokkos::move() sc_type_list[n] = 4; sc_map[n] = ntrans; ntrans++; + } else if (strcmp(surf->sc[n]->style,"adiabatic") == 0) { + sc_kk_adiabatic_copy[nadia].copy((SurfCollideAdiabaticKokkos*)(surf->sc[n])); + sc_kk_adiabatic_copy[nadia].obj.pre_collide(); + sc_type_list[n] = 5; + sc_map[n] = nadia; + nadia++; } else { error->all(FLERR,"Unknown Kokkos surface collide method"); } } if (nspec > KOKKOS_MAX_SURF_COLL_PER_TYPE || ndiff > KOKKOS_MAX_SURF_COLL_PER_TYPE || nvan > KOKKOS_MAX_SURF_COLL_PER_TYPE || npist > KOKKOS_MAX_SURF_COLL_PER_TYPE || - ntrans > KOKKOS_MAX_SURF_COLL_PER_TYPE) + ntrans > KOKKOS_MAX_SURF_COLL_PER_TYPE || nadia > KOKKOS_MAX_SURF_COLL_PER_TYPE) error->all(FLERR,"Kokkos currently supports two instances of each surface collide method"); } @@ -713,8 +721,8 @@ template < int DIM, int SURF, int REACT, int OPT > void UpdateKokkos::move() } if (surf->nsc > 0) { - int nspec,ndiff,nvan,npist,ntrans; - nspec = ndiff = nvan = npist = ntrans = 0; + int nspec,ndiff,nvan,npist,ntrans,nadia; + nspec = ndiff = nvan = npist = ntrans = nadia = 0; for (int n = 0; n < surf->nsc; n++) { if (strcmp(surf->sc[n]->style,"specular") == 0) { sc_kk_specular_copy[nspec].obj.post_collide(); @@ -731,6 +739,9 @@ template < int DIM, int SURF, int REACT, int OPT > void UpdateKokkos::move() } else if (strcmp(surf->sc[n]->style,"transparent") == 0) { sc_kk_transparent_copy[ntrans].obj.post_collide(); ntrans++; + } else if (strcmp(surf->sc[n]->style,"adiabatic") == 0) { + sc_kk_adiabatic_copy[nadia].obj.post_collide(); + nadia++; } } } @@ -1382,6 +1393,9 @@ void UpdateKokkos::operator()(TagUpdateMove } else if (sc_type == 4) { jpart = sc_kk_transparent_copy[m].obj. collide_kokkos(ipart,dtremain,minsurf,tri->norm,tri->isr,reaction,d_retry,d_nlocal); + } else if (sc_type == 5) { + jpart = sc_kk_adiabatic_copy[m].obj. + collide_kokkos(ipart,dtremain,minsurf,tri->norm,tri->isr,reaction,d_retry,d_nlocal); } } @@ -1401,6 +1415,9 @@ void UpdateKokkos::operator()(TagUpdateMove } else if (sc_type == 4) { jpart = sc_kk_transparent_copy[m].obj. collide_kokkos(ipart,dtremain,minsurf,line->norm,line->isr,reaction,d_retry,d_nlocal); + } else if (sc_type == 5) { + jpart = sc_kk_adiabatic_copy[m].obj. + collide_kokkos(ipart,dtremain,minsurf,line->norm,line->isr,reaction,d_retry,d_nlocal); } } @@ -1648,6 +1665,9 @@ void UpdateKokkos::operator()(TagUpdateMove else if (sc_type == 4) jpart = sc_kk_transparent_copy[m].obj. collide_kokkos(ipart,dtremain,-(outface+1),domain_kk_copy.obj.norm[outface],domain_kk_copy.obj.surf_react[outface],reaction,d_retry,d_nlocal); + else if (sc_type == 5) + jpart = sc_kk_adiabatic_copy[m].obj. + collide_kokkos(ipart,dtremain,-(outface+1),domain_kk_copy.obj.norm[outface],domain_kk_copy.obj.surf_react[outface],reaction,d_retry,d_nlocal); if (ipart) { double *x = ipart->x; @@ -2005,8 +2025,8 @@ void UpdateKokkos::backup() Kokkos::deep_copy(d_particles_backup,d_particles); if (surf->nsc > 0) { - int nspec,ndiff,npist; - nspec = ndiff = npist = 0; + int nspec,ndiff,npist,nadia; + nspec = ndiff = npist = nadia = 0; for (int n = 0; n < surf->nsc; n++) { if (strcmp(surf->sc[n]->style,"specular") == 0) { sc_kk_specular_copy[nspec].obj.backup(); @@ -2017,6 +2037,9 @@ void UpdateKokkos::backup() } else if (strcmp(surf->sc[n]->style,"piston") == 0) { sc_kk_piston_copy[npist].obj.backup(); npist++; + } else if (strcmp(surf->sc[n]->style,"adiabatic") == 0) { + sc_kk_adiabatic_copy[nadia].obj.backup(); + nadia++; } } } @@ -2031,8 +2054,8 @@ void UpdateKokkos::restore() d_particles = particle_kk->k_particles.view_device(); if (surf->nsc > 0) { - int nspec,ndiff,npist; - nspec = ndiff = npist = 0; + int nspec,ndiff,npist,nadia; + nspec = ndiff = npist = nadia = 0; for (int n = 0; n < surf->nsc; n++) { if (strcmp(surf->sc[n]->style,"specular") == 0) { sc_kk_specular_copy[nspec].obj.restore(); @@ -2043,6 +2066,9 @@ void UpdateKokkos::restore() } else if (strcmp(surf->sc[n]->style,"piston") == 0) { sc_kk_piston_copy[npist].obj.restore(); npist++; + } else if (strcmp(surf->sc[n]->style,"adiabatic") == 0) { + sc_kk_adiabatic_copy[nadia].obj.restore(); + nadia++; } } } diff --git a/src/KOKKOS/update_kokkos.h b/src/KOKKOS/update_kokkos.h index 7ac60fa88..02da81cc8 100644 --- a/src/KOKKOS/update_kokkos.h +++ b/src/KOKKOS/update_kokkos.h @@ -26,6 +26,7 @@ #include "surf_collide_vanish_kokkos.h" #include "surf_collide_piston_kokkos.h" #include "surf_collide_transparent_kokkos.h" +#include "surf_collide_adiabatic_kokkos.h" #include "compute_boundary_kokkos.h" #include "compute_surf_kokkos.h" @@ -136,6 +137,7 @@ class UpdateKokkos : public Update { KKCopy sc_kk_vanish_copy[KOKKOS_MAX_SURF_COLL_PER_TYPE]; KKCopy sc_kk_piston_copy[KOKKOS_MAX_SURF_COLL_PER_TYPE]; KKCopy sc_kk_transparent_copy[KOKKOS_MAX_SURF_COLL_PER_TYPE]; + KKCopy sc_kk_adiabatic_copy[KOKKOS_MAX_SURF_COLL_PER_TYPE]; //KKCopy blist_active_copy[KOKKOS_MAX_GLIST]; KKCopy slist_active_copy[KOKKOS_MAX_SLIST]; From 2f0b4e5c5245cc72c8c074887f85d9f51ed35c84 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 19 Jun 2026 17:53:36 +0000 Subject: [PATCH 02/30] KOKKOS: port surf_collide impulsive to Kokkos Add SurfCollideImpulsiveKokkos, the GPU-capable port of the impulsive surface collision model (Rettner-style rejection-sampled reflection with optional soft-sphere energy exchange, step/double-cosine angular variants, and internal-energy redistribution). Follows the surf_collide_diffuse_kokkos pattern: device-callable collide_kokkos, the impulsive() reflection kernel as KOKKOS_INLINE_FUNCTION (v_f_avg demoted to a local so the method is const/device-safe), Kokkos RNG pool with SPARTA_KOKKOS_EXACT support, DualView counters, dynamic() for VARSURF/CUSTOM Tsurf, and surf-react dispatch. Give the base SurfCollideImpulsive a Kokkos empty constructor, make its members protected, and guard its destructor with copy so KKCopy shallow copies don't double-free the RNG. Wire sc_type id 6 through update_kokkos (selection, the 3D/2D/boundary dispatch ladders, post_collide, backup/restore) and register the files in the KOKKOS Install.sh. Verified: exact-match gate (Serial + SPARTA_KOKKOS_EXACT, 1 thread) is bit-for-bit identical CPU vs -sf kk on examples/surf_collide/in.circle.impulsive and in.beam.impulsive; OpenMP 4-thread run is clean and statistically consistent. Co-Authored-By: Claude Opus 4.8 Co-Authored-By: Stan Moore Claude-Session: https://claude.ai/code/session_01Q6nQVuxEiTUqSzDoDppCUM --- src/KOKKOS/Install.sh | 2 + src/KOKKOS/surf_collide_impulsive_kokkos.cpp | 358 ++++++++++++++++++ src/KOKKOS/surf_collide_impulsive_kokkos.h | 376 +++++++++++++++++++ src/KOKKOS/update_kokkos.cpp | 45 ++- src/KOKKOS/update_kokkos.h | 2 + src/surf_collide_impulsive.cpp | 2 + src/surf_collide_impulsive.h | 3 +- 7 files changed, 778 insertions(+), 10 deletions(-) create mode 100644 src/KOKKOS/surf_collide_impulsive_kokkos.cpp create mode 100644 src/KOKKOS/surf_collide_impulsive_kokkos.h diff --git a/src/KOKKOS/Install.sh b/src/KOKKOS/Install.sh index ca6169ff8..69d95a48d 100644 --- a/src/KOKKOS/Install.sh +++ b/src/KOKKOS/Install.sh @@ -121,6 +121,8 @@ action surf_collide_adiabatic_kokkos.cpp action surf_collide_adiabatic_kokkos.h action surf_collide_diffuse_kokkos.cpp action surf_collide_diffuse_kokkos.h +action surf_collide_impulsive_kokkos.cpp +action surf_collide_impulsive_kokkos.h action surf_collide_piston_kokkos.cpp action surf_collide_piston_kokkos.h action surf_collide_specular_kokkos.cpp diff --git a/src/KOKKOS/surf_collide_impulsive_kokkos.cpp b/src/KOKKOS/surf_collide_impulsive_kokkos.cpp new file mode 100644 index 000000000..e8e357e76 --- /dev/null +++ b/src/KOKKOS/surf_collide_impulsive_kokkos.cpp @@ -0,0 +1,358 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#include "math.h" +#include "stdlib.h" +#include "string.h" +#include "surf_collide_impulsive_kokkos.h" +#include "surf_kokkos.h" +#include "input.h" +#include "variable.h" +#include "particle.h" +#include "domain.h" +#include "update.h" +#include "modify.h" +#include "comm.h" +#include "random_mars.h" +#include "random_knuth.h" +#include "math_const.h" +#include "math_extra.h" +#include "memory.h" +#include "error.h" +#include "particle_kokkos.h" +#include "sparta_masks.h" +#include "collide.h" + +using namespace SPARTA_NS; +using namespace MathConst; + +enum{INT,DOUBLE}; // several files +enum{NUMERIC,CUSTOM,VARIABLE,VAREQUAL,VARSURF}; // surf_collide classes + +#define VAL_1(X) X +#define VAL_2(X) VAL_1(X), VAL_1(X) + +/* ---------------------------------------------------------------------- */ + +SurfCollideImpulsiveKokkos::SurfCollideImpulsiveKokkos(SPARTA *sparta, int narg, char **arg) : + SurfCollideImpulsive(sparta, narg, arg), + fix_ambi_kk_copy(sparta), + fix_vibmode_kk_copy(sparta), + sr_kk_global_copy{VAL_2(KKCopy(sparta))}, + sr_kk_prob_copy{VAL_2(KKCopy(sparta))}, + rand_pool(12345 + comm->me +#ifdef SPARTA_KOKKOS_EXACT + , sparta +#endif + ) +{ + kokkosable = 1; + + random_backup = NULL; + +#ifdef SPARTA_KOKKOS_EXACT + rand_pool.init(random); +#endif + + // use 1D view for scalars to reduce GPU memory operations + + d_scalars = t_int_2("surf_collide_impulsive:scalars"); + d_nsingle = Kokkos::subview(d_scalars,0); + d_nreact_one = Kokkos::subview(d_scalars,1); + + h_scalars = t_host_int_2("surf_collide_impulsive:scalars_mirror"); + h_nsingle = Kokkos::subview(h_scalars,0); + h_nreact_one = Kokkos::subview(h_scalars,1); +} + +SurfCollideImpulsiveKokkos::SurfCollideImpulsiveKokkos(SPARTA *sparta) : + SurfCollideImpulsive(sparta), + fix_ambi_kk_copy(sparta), + fix_vibmode_kk_copy(sparta), + sr_kk_global_copy{VAL_2(KKCopy(sparta))}, + sr_kk_prob_copy{VAL_2(KKCopy(sparta))}, + rand_pool(12345 // seed doesn't matter since it will just be copied over +#ifdef SPARTA_KOKKOS_EXACT + , sparta +#endif + ) +{ + copy = 1; +} + +/* ---------------------------------------------------------------------- */ + +SurfCollideImpulsiveKokkos::~SurfCollideImpulsiveKokkos() +{ + if (uncopy) { + fix_ambi_kk_copy.uncopy(); + fix_vibmode_kk_copy.uncopy(); + + for (int i = 0; i < KOKKOS_MAX_SURF_REACT_PER_TYPE; i++) { + sr_kk_global_copy[i].uncopy(); + sr_kk_prob_copy[i].uncopy(); + } + } + + if (copy) return; + +#ifdef SPARTA_KOKKOS_EXACT + rand_pool.destroy(); + if (random_backup) + delete random_backup; +#endif +} + +/* ---------------------------------------------------------------------- */ + +void SurfCollideImpulsiveKokkos::init() +{ + SurfCollideImpulsive::init(); + + ambi_flag = vibmode_flag = 0; + if (modify->n_update_custom) { + for (int ifix = 0; ifix < modify->nfix; ifix++) { + if (strcmp(modify->fix[ifix]->style,"ambipolar") == 0) { + ambi_flag = 1; + FixAmbipolar *afix = (FixAmbipolar *) modify->fix[ifix]; + if (!afix->kokkos_flag) + error->all(FLERR,"Must use fix ambipolar/kk when Kokkos is enabled"); + afix_kk = (FixAmbipolarKokkos*)afix; + } else if (strcmp(modify->fix[ifix]->style,"vibmode") == 0) { + vibmode_flag = 1; + FixVibmode *vfix = (FixVibmode *) modify->fix[ifix]; + if (!vfix->kokkos_flag) + error->all(FLERR,"Must use fix vibmode/kk when Kokkos is enabled"); + vfix_kk = (FixVibmodeKokkos*)vfix; + } + } + } +} + +/* ---------------------------------------------------------------------- + recalculate Tsurf values which are dynamic + called by Update::setup() and Update::run() +---------------------------------------------------------------------- */ + +void SurfCollideImpulsiveKokkos::dynamic() +{ + // VAREQUAL mode + // equal-style variable sets single tsurf value for all surfs + + if (tmode == VAREQUAL) { + + // only evaluate variable if timestep is multiple of tfreq + + if (update->ntimestep % tfreq) return; + tsurf = input->variable->compute_equal(tindex_var); + if (tsurf <= 0.0) error->all(FLERR,"Surf_collide tsurf <= 0.0"); + + // VARSURF mode + // surf-style variable sets new tsurf values for all surfs + // particle/surf collisions access t_persurf for local+ghost values + + } else if (tmode == VARSURF) { + + // only evaluate variable if timestep is multiple of tfreq + + int spreadflag = 0; + if (update->ntimestep % tfreq == 0) { + if (n_owned != surf->nown) { + memory->destroy(t_owned); + n_owned = surf->nown; + memory->create(t_owned,n_owned,"surfcollide:t_owned"); + } + + input->variable->compute_surf(tindex_var,t_owned,1,0); + spreadflag = 1; + } + + // spread t_owned values to t_localghost values via spread_own2local() + // if just re-computed variable OR surfs are + // distributed and load balance/adaptation took place on previous step + + if (spreadflag || + (surf->distributed && surf->localghost_changed_step == update->ntimestep-1)) { + if (n_localghost != surf->nlocal + surf->nghost) { + memory->destroy(t_localghost); + n_localghost = surf->nlocal + surf->nghost; + memory->create(t_localghost,n_localghost,"surfcollide:t_localghost"); + } + + surf->spread_own2local(1,DOUBLE,t_owned,t_localghost); + t_persurf = t_localghost; + + auto h_t_persurf = HAT::t_float_1d(t_persurf,n_localghost); + d_t_persurf = Kokkos::create_mirror_view_and_copy(SPADeviceType(),h_t_persurf); + } + + // CUSTOM mode + // ensure access to custom per-surf vec for tsurf values for all surfs + // particle/surf collisions access t_persurf for local+ghost values + + } else if (tmode == CUSTOM) { + SurfKokkos* surf_kk = (SurfKokkos*) surf; + auto h_edvec_local = surf_kk->k_edvec_local.view_host(); + + // spread owned values to local+ghost values via spread_custom() + // estatus == 1 means owned values already spread to local+ghost values + // if estatus == 0: owned values are new OR + // surfs are distributed and load balance/adaptation took place + + if (surf->estatus[tindex_custom] == 0) surf->spread_custom(tindex_custom); + + h_edvec_local[tindex_custom].k_view.sync_device(); + d_t_persurf = h_edvec_local[tindex_custom].k_view.view_device(); + } +} + +/* ---------------------------------------------------------------------- */ + +void SurfCollideImpulsiveKokkos::pre_collide() +{ + if (ambi_flag) { + afix_kk->pre_update_custom_kokkos(); + fix_ambi_kk_copy.copy(afix_kk); + } + + if (vibmode_flag) { + vfix_kk->pre_update_custom_kokkos(); + fix_vibmode_kk_copy.copy(vfix_kk); + } + + if (surf->nsr > KOKKOS_MAX_TOT_SURF_REACT) + error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); + + if (surf->nsr > 0) { + int nglob,nprob; + nglob = nprob = 0; + for (int n = 0; n < surf->nsr; n++) { + if (!surf->sr[n]->kokkosable) + error->all(FLERR,"Must use Kokkos-enabled surface reaction method with Kokkos"); + if (strcmp(surf->sr[n]->style,"global") == 0) { + sr_kk_global_copy[nglob].copy((SurfReactGlobalKokkos*)(surf->sr[n])); + sr_kk_global_copy[nglob].obj.pre_react(); + sr_type_list[n] = 0; + sr_map[n] = nglob; + nglob++; + } else if (strcmp(surf->sr[n]->style,"prob") == 0) { + sr_kk_prob_copy[nprob].copy((SurfReactProbKokkos*)(surf->sr[n])); + sr_kk_prob_copy[nprob].obj.pre_react(); + sr_type_list[n] = 1; + sr_map[n] = nprob; + nprob++; + } else { + error->all(FLERR,"Unknown Kokkos surface reaction method"); + } + } + + if (nglob > KOKKOS_MAX_SURF_REACT_PER_TYPE || nprob > KOKKOS_MAX_SURF_REACT_PER_TYPE) + error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); + } + + if (random == NULL) { + // initialize RNG + + random = new RanKnuth(update->ranmaster->uniform()); + double seed = update->ranmaster->uniform(); + random->reset(seed,comm->me,100); + +#ifdef SPARTA_KOKKOS_EXACT + rand_pool.init(random); +#endif + } + + ParticleKokkos* particle_kk = (ParticleKokkos*) particle; + particle_kk->sync(Device,PARTICLE_MASK|SPECIES_MASK); + d_particles = particle_kk->k_particles.view_device(); + d_species = particle_kk->k_species.view_device(); + boltz = update->boltz; + + rotstyle = NONE; + if (Pointers::collide) rotstyle = Pointers::collide->rotstyle; + vibstyle = NONE; + if (Pointers::collide) vibstyle = Pointers::collide->vibstyle; + + Kokkos::deep_copy(d_scalars,0); +} + +/* ---------------------------------------------------------------------- */ + +void SurfCollideImpulsiveKokkos::post_collide() +{ + ParticleKokkos* particle_kk = (ParticleKokkos*) particle; + if (ambi_flag || vibmode_flag) particle_kk->modify(Device,CUSTOM_MASK); + + Kokkos::deep_copy(h_scalars,d_scalars); + + int m = surf->find_collide(id); + auto sc = surf->sc[m]; // can't modify the copy directly, use the original + sc->nsingle += h_nsingle(); + surf->nreact_one += h_nreact_one(); + + d_particles = {}; +} + +/* ---------------------------------------------------------------------- */ + +void SurfCollideImpulsiveKokkos::backup() +{ + ParticleKokkos* particle_kk = (ParticleKokkos*) particle; + d_particles = particle_kk->k_particles.view_device(); + + if (surf->nsr > 0) { + int nglob,nprob; + nglob = nprob = 0; + for (int n = 0; n < surf->nsr; n++) { + if (strcmp(surf->sr[n]->style,"global") == 0) { + sr_kk_global_copy[nglob].obj.backup(); + nglob++; + } else if (strcmp(surf->sr[n]->style,"prob") == 0) { + sr_kk_prob_copy[nprob].obj.backup(); + nprob++; + } + } + } + +#ifdef SPARTA_KOKKOS_EXACT + if (!random_backup) + random_backup = new RanKnuth(12345 + comm->me); + memcpy(random_backup,random,sizeof(RanKnuth)); +#endif +} + +/* ---------------------------------------------------------------------- */ + +void SurfCollideImpulsiveKokkos::restore() +{ + if (surf->nsr > 0) { + int nglob,nprob; + nglob = nprob = 0; + for (int n = 0; n < surf->nsr; n++) { + if (strcmp(surf->sr[n]->style,"global") == 0) { + sr_kk_global_copy[nglob].obj.restore(); + nglob++; + } else if (strcmp(surf->sr[n]->style,"prob") == 0) { + sr_kk_prob_copy[nprob].obj.restore(); + nprob++; + } + } + } + + Kokkos::deep_copy(d_scalars,0); + +#ifdef SPARTA_KOKKOS_EXACT + memcpy(random,random_backup,sizeof(RanKnuth)); +#endif +} diff --git a/src/KOKKOS/surf_collide_impulsive_kokkos.h b/src/KOKKOS/surf_collide_impulsive_kokkos.h new file mode 100644 index 000000000..9f909e3fc --- /dev/null +++ b/src/KOKKOS/surf_collide_impulsive_kokkos.h @@ -0,0 +1,376 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#ifdef SURF_COLLIDE_CLASS + +SurfCollideStyle(impulsive/kk,SurfCollideImpulsiveKokkos) + +#else + +#ifndef SPARTA_SURF_COLLIDE_IMPULSIVE_KOKKOS_H +#define SPARTA_SURF_COLLIDE_IMPULSIVE_KOKKOS_H + +#include "surf_collide_impulsive.h" +#include "kokkos_type.h" +#include "math_extra_kokkos.h" +#include "Kokkos_Random.hpp" +#include "rand_pool_wrap.h" +#include "kokkos_copy.h" +#include "fix_ambipolar_kokkos.h" +#include "fix_vibmode_kokkos.h" +#include "surf_react_global_kokkos.h" +#include "surf_react_prob_kokkos.h" + +namespace SPARTA_NS { + +class SurfCollideImpulsiveKokkos : public SurfCollideImpulsive { + public: + + enum{NONE,DISCRETE,SMOOTH}; // several files + enum{PKEEP,PINSERT,PDONE,PDISCARD,PENTRY,PEXIT,PSURF}; // several files + + SurfCollideImpulsiveKokkos(class SPARTA *, int, char **); + SurfCollideImpulsiveKokkos(class SPARTA *); + ~SurfCollideImpulsiveKokkos(); + void init(); + void dynamic(); + void pre_collide(); + void post_collide(); + void backup(); + void restore(); + + private: + double boltz; + int rotstyle, vibstyle; + +#ifndef SPARTA_KOKKOS_EXACT + Kokkos::Random_XorShift64_Pool rand_pool; + typedef typename Kokkos::Random_XorShift64_Pool::generator_type rand_type; +#else + RandPoolWrap rand_pool; + typedef RandWrap rand_type; +#endif + + RanKnuth* random_backup; + + DAT::t_float_1d d_t_persurf; + + typedef Kokkos::DualView tdual_int_2; + typedef tdual_int_2::t_dev t_int_2; + typedef tdual_int_2::t_host t_host_int_2; + t_int_2 d_scalars; + t_host_int_2 h_scalars; + + DAT::t_int_scalar d_nsingle; + DAT::t_int_scalar d_nreact_one; + + HAT::t_int_scalar h_nsingle; + HAT::t_int_scalar h_nreact_one; + + t_particle_1d d_particles; + t_species_1d d_species; + + int ambi_flag,vibmode_flag; + FixAmbipolarKokkos* afix_kk; + FixVibmodeKokkos* vfix_kk; + KKCopy fix_ambi_kk_copy; + KKCopy fix_vibmode_kk_copy; + + int sr_type_list[KOKKOS_MAX_TOT_SURF_REACT]; + int sr_map[KOKKOS_MAX_TOT_SURF_REACT]; + KKCopy sr_kk_global_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; + KKCopy sr_kk_prob_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; + + public: + + /* ---------------------------------------------------------------------- + particle collision with surface with optional chemistry + ip = particle with current x = collision pt, current v = incident v + isurf = index of surface element + norm = surface normal unit vector + isr = index of reaction model if >= 0, -1 for no chemistry + ip = set to NULL if destroyed by chemistry + return jp = new particle if created by chemistry + return reaction = index of reaction (1 to N) that took place, 0 = no reaction + resets particle(s) to post-collision outward velocity + ------------------------------------------------------------------------- */ + + template + KOKKOS_INLINE_FUNCTION + Particle::OnePart* collide_kokkos(Particle::OnePart *&ip, double &, + int isurf, const double *norm, int isr, int &reaction, + const DAT::t_int_scalar &d_retry, const DAT::t_int_scalar &d_nlocal) const + { + if (ATOMIC_REDUCTION == 0) + d_nsingle()++; + else + Kokkos::atomic_inc(&d_nsingle()); + + // if surface chemistry defined, attempt reaction + // reaction = 1 to N for which reaction took place, 0 for none + // velreset = 1 if reaction reset post-collision velocity, else 0 + + Particle::OnePart iorig; + Particle::OnePart *jp = NULL; + reaction = 0; + int velreset = 0; + + if (REACT) { + if (ambi_flag || vibmode_flag) memcpy(&iorig,ip,sizeof(Particle::OnePart)); + + int sr_type = sr_type_list[isr]; + int m = sr_map[isr]; + + if (sr_type == 0) { + reaction = sr_kk_global_copy[m].obj. + react_kokkos(ip,isurf,norm,jp,velreset,d_retry,d_nlocal); + } else if (sr_type == 1) { + reaction = sr_kk_prob_copy[m].obj. + react_kokkos(ip,isurf,norm,jp,velreset,d_retry,d_nlocal); + } + + if (reaction) { + if (ATOMIC_REDUCTION == 0) + d_nreact_one()++; + else + Kokkos::atomic_inc(&d_nreact_one()); + } + } + + // set temperature of isurf if VARSURF or CUSTOM + + double tsurf_local = tsurf; + if (persurf_temperature) { + tsurf_local = d_t_persurf[isurf]; + if (tsurf_local <= 0.0) Kokkos::abort("Surf_collide tsurf <= 0.0"); + } + + // impulsive reflection for each particle + // only if SurfReact did not already reset velocities + // also both particles need to trigger any fixes + // to update per-particle properties which depend on + // temperature of the particle, e.g. fix vibmode and fix ambipolar + + if (ip) { + if (!velreset) impulsive(ip,norm,tsurf_local); + int i = ip - d_particles.data(); + if (ambi_flag) + fix_ambi_kk_copy.obj.update_custom_kokkos(i,tsurf_local,tsurf_local,tsurf_local,vstream); + if (vibmode_flag) + fix_vibmode_kk_copy.obj.update_custom_kokkos(i,tsurf_local,tsurf_local,tsurf_local,vstream); + } + if (REACT && jp) { + if (!velreset) impulsive(jp,norm,tsurf_local); + int j = jp - d_particles.data(); + if (ambi_flag) + fix_ambi_kk_copy.obj.update_custom_kokkos(j,tsurf_local,tsurf_local,tsurf_local,vstream); + if (vibmode_flag) + fix_vibmode_kk_copy.obj.update_custom_kokkos(j,tsurf_local,tsurf_local,tsurf_local,vstream); + } + + // call any fixes with a surf_react() method + // they may reset j to -1, e.g. fix ambipolar + // in which case newly created j is deleted + + if (REACT && reaction && ambi_flag) { + int i = -1; + if (ip) i = ip - d_particles.data(); + int j = -1; + if (jp) j = jp - d_particles.data(); + int j_orig = j; + fix_ambi_kk_copy.obj.surf_react_kokkos(&iorig,i,j); + if (jp && j < 0) { + d_particles[j_orig].flag = PDISCARD; + jp = NULL; + } + } + + return jp; + }; + + private: + + /* ---------------------------------------------------------------------- + impulsive reflection + ------------------------------------------------------------------------- */ + + KOKKOS_INLINE_FUNCTION + void impulsive(Particle::OnePart *p, const double *norm, const double twall) const + { + rand_type rand_gen = rand_pool.get_state(); + + double tangent1[3],tangent2[3]; + int ispecies = p->ispecies; + + double vperp, vtan1, vtan2; + double mass = d_species[ispecies].mass; + + double *v = p->v; + double dot = MathExtraKokkos::dot3(v,norm); + + tangent1[0] = v[0] - dot*norm[0]; + tangent1[1] = v[1] - dot*norm[1]; + tangent1[2] = v[2] - dot*norm[2]; + + if (MathExtraKokkos::lensq3(tangent1) == 0.0) { + tangent2[0] = rand_gen.drand(); + tangent2[1] = rand_gen.drand(); + tangent2[2] = rand_gen.drand(); + MathExtraKokkos::cross3(norm,tangent2,tangent1); + } + + MathExtraKokkos::norm3(tangent1); + MathExtraKokkos::cross3(norm,tangent1,tangent2); + + // compute final polar (theta) and azimuthal (phi) angles + + double tan1 = MathExtraKokkos::dot3(v,tangent1); + double tan2 = MathExtraKokkos::dot3(v,tangent2); + + double v_i_mag_sq = MathExtraKokkos::lensq3(v); + double E_i = 0.5 * mass * v_i_mag_sq; + double theta_i = acos(-dot/sqrt(v_i_mag_sq)); + double phi_i = atan2(tan2,tan1); + double phi_peak = MathConst::MY_2PI - phi_i; + + double theta_f, phi_f; + double P = 0.0; + + // theta_f calculation + + while (rand_gen.drand() > P) { + theta_f = MathConst::MY_PI2 * rand_gen.drand(); + P = pow(cos( theta_f - theta_peak ),cos_theta_pow) * sin(theta_f); + if (double_flag) { + if (theta_f > theta_peak) + P = pow(cos( theta_f - theta_peak ),cos_theta_pow_2) * sin(theta_f); + } + + if (step_flag) { + double func_step = 0.0; + double tan_theta = tan(theta_f); + double cotangent = 1.0/tan_theta; + if (cotangent > step_size) func_step = 1 - step_size*tan_theta; + P *= func_step; + } + } + + // phi_f calculations + + P = 0.0; + while (rand_gen.drand() > P) { + phi_f = phi_peak + MathConst::MY_PI * (2*rand_gen.drand() - 1); + P = pow(cos( 0.5*(phi_f - phi_peak) ),cos_phi_pow); + } + + if (phi_f > MathConst::MY_PI) phi_f -= MathConst::MY_2PI; + else if (phi_f < -MathConst::MY_PI) phi_f += MathConst::MY_2PI; + + double v_f_avg = 0.0; + if (softsphere_flag) { + double mu = d_species[ispecies].molwt/eff_mass; + double cos_khi = cos(MathConst::MY_PI - theta_i - theta_f); + double sin_khi_sq = 1 - cos_khi*cos_khi; + double dE, E_f_avg; + + dE = 2*mu/((mu+1)*(mu+1)) * + (1 + mu*sin_khi_sq + eng_ratio*(mu+1)/(2*mu) - + cos_khi*sqrt(1 - mu*mu*sin_khi_sq - eng_ratio*(mu + 1))); + E_f_avg = E_i * (1 - dE); + v_f_avg = var_alpha_sq * sqrt(mass/(2*E_f_avg)) * + (2*E_f_avg/(mass*var_alpha_sq) - 1); + } else { + v_f_avg = u0_a*twall + u0_b; + } + + double v_f_max = 0.5 * (v_f_avg + sqrt(v_f_avg*v_f_avg + 6*var_alpha_sq)); + double f_max = v_f_max*v_f_max*v_f_max * + exp(-(v_f_max - v_f_avg) * (v_f_max - v_f_avg)/(var_alpha_sq)); + + double v_f_mag; + P = 0.0; + while (rand_gen.drand() > P) { + v_f_mag = v_f_max + 3 * var_alpha * ( 2 * rand_gen.drand() - 1 ); + P = v_f_mag*v_f_mag*v_f_mag/(f_max) * + exp(-(v_f_mag - v_f_avg)*(v_f_mag - v_f_avg)/(var_alpha_sq)); + } + + vperp = v_f_mag * cos(theta_f); + vtan1 = v_f_mag * sin(theta_f) * cos(phi_f); + vtan2 = v_f_mag * sin(theta_f) * sin(phi_f); + + v[0] = vperp*norm[0] + vtan1*tangent1[0] + vtan2*tangent2[0]; + v[1] = vperp*norm[1] + vtan1*tangent1[1] + vtan2*tangent2[1]; + v[2] = vperp*norm[2] + vtan1*tangent1[2] + vtan2*tangent2[2]; + + if (intenergy_flag) { + double E_f = 0.5 * mass * v_f_mag * v_f_mag; + double extra_energy = E_i - E_f; + + // rotational component + + if (rotstyle == NONE || d_species[ispecies].rotdof < 2) p->erot = 0.0; + else p->erot += rot_frac*extra_energy; + + // vibrational component + + int vibdof = d_species[ispecies].vibdof; + + if (vibstyle == NONE || vibdof < 2) { + p->evib = 0.0; + } else { + double *vibtemp = d_species[ispecies].vibtemp; + double evib_val = p->evib + vib_frac*extra_energy; + + if (vibstyle == SMOOTH) p->evib = evib_val; + if (vibstyle == DISCRETE && vibdof==2) { + int ivib = evib_val / (boltz*vibtemp[0]); + p->evib = ivib * boltz * vibtemp[0]; + } else { + int nvibmode = d_species[ispecies].nvibmode; + int *vibdegen = d_species[ispecies].vibdegen; + double tot_temp=0.0; + double evib_sum = 0.0; + + for (int imode=0; imodeevib = evib_sum; + } + } + } + + rand_pool.free_state(rand_gen); + } +}; + +} + +#endif +#endif + +/* ERROR/WARNING messages: + +E: Illegal ... command + +Self-explanatory. Check the input script syntax and compare to the +documentation for the command. You can use -echo screen as a +command-line option when running SPARTA to see the offending line. + +*/ diff --git a/src/KOKKOS/update_kokkos.cpp b/src/KOKKOS/update_kokkos.cpp index 80e4dd136..aade901b5 100644 --- a/src/KOKKOS/update_kokkos.cpp +++ b/src/KOKKOS/update_kokkos.cpp @@ -81,6 +81,7 @@ UpdateKokkos::UpdateKokkos(SPARTA *sparta) : Update(sparta), sc_kk_piston_copy{VAL_2(KKCopy(sparta))}, sc_kk_transparent_copy{VAL_2(KKCopy(sparta))}, sc_kk_adiabatic_copy{VAL_2(KKCopy(sparta))}, + sc_kk_impulsive_copy{VAL_2(KKCopy(sparta))}, blist_active_copy{VAL_2(KKCopy(sparta))}, slist_active_copy{VAL_2(KKCopy(sparta))}, tmp_compute_boundary_kk(sparta), @@ -147,6 +148,7 @@ UpdateKokkos::~UpdateKokkos() sc_kk_piston_copy[i].uncopy(); sc_kk_transparent_copy[i].uncopy(); sc_kk_adiabatic_copy[i].uncopy(); + sc_kk_impulsive_copy[i].uncopy(); } for (int i=0; i void UpdateKokkos::move() error->all(FLERR,"Kokkos currently supports two instances of each surface collide method"); if (surf->nsc > 0) { - int nspec,ndiff,nvan,npist,ntrans,nadia; - nspec = ndiff = nvan = npist = ntrans = nadia = 0; + int nspec,ndiff,nvan,npist,ntrans,nadia,nimpul; + nspec = ndiff = nvan = npist = ntrans = nadia = nimpul = 0; for (int n = 0; n < surf->nsc; n++) { if (!surf->sc[n]->kokkosable) error->all(FLERR,"Must use Kokkos-enabled surface collide method with Kokkos"); @@ -574,13 +576,20 @@ template < int DIM, int SURF, int REACT, int OPT > void UpdateKokkos::move() sc_type_list[n] = 5; sc_map[n] = nadia; nadia++; + } else if (strcmp(surf->sc[n]->style,"impulsive") == 0) { + sc_kk_impulsive_copy[nimpul].copy((SurfCollideImpulsiveKokkos*)(surf->sc[n])); + sc_kk_impulsive_copy[nimpul].obj.pre_collide(); + sc_type_list[n] = 6; + sc_map[n] = nimpul; + nimpul++; } else { error->all(FLERR,"Unknown Kokkos surface collide method"); } } if (nspec > KOKKOS_MAX_SURF_COLL_PER_TYPE || ndiff > KOKKOS_MAX_SURF_COLL_PER_TYPE || nvan > KOKKOS_MAX_SURF_COLL_PER_TYPE || npist > KOKKOS_MAX_SURF_COLL_PER_TYPE || - ntrans > KOKKOS_MAX_SURF_COLL_PER_TYPE || nadia > KOKKOS_MAX_SURF_COLL_PER_TYPE) + ntrans > KOKKOS_MAX_SURF_COLL_PER_TYPE || nadia > KOKKOS_MAX_SURF_COLL_PER_TYPE || + nimpul > KOKKOS_MAX_SURF_COLL_PER_TYPE) error->all(FLERR,"Kokkos currently supports two instances of each surface collide method"); } @@ -721,8 +730,8 @@ template < int DIM, int SURF, int REACT, int OPT > void UpdateKokkos::move() } if (surf->nsc > 0) { - int nspec,ndiff,nvan,npist,ntrans,nadia; - nspec = ndiff = nvan = npist = ntrans = nadia = 0; + int nspec,ndiff,nvan,npist,ntrans,nadia,nimpul; + nspec = ndiff = nvan = npist = ntrans = nadia = nimpul = 0; for (int n = 0; n < surf->nsc; n++) { if (strcmp(surf->sc[n]->style,"specular") == 0) { sc_kk_specular_copy[nspec].obj.post_collide(); @@ -742,6 +751,9 @@ template < int DIM, int SURF, int REACT, int OPT > void UpdateKokkos::move() } else if (strcmp(surf->sc[n]->style,"adiabatic") == 0) { sc_kk_adiabatic_copy[nadia].obj.post_collide(); nadia++; + } else if (strcmp(surf->sc[n]->style,"impulsive") == 0) { + sc_kk_impulsive_copy[nimpul].obj.post_collide(); + nimpul++; } } } @@ -1396,6 +1408,9 @@ void UpdateKokkos::operator()(TagUpdateMove } else if (sc_type == 5) { jpart = sc_kk_adiabatic_copy[m].obj. collide_kokkos(ipart,dtremain,minsurf,tri->norm,tri->isr,reaction,d_retry,d_nlocal); + } else if (sc_type == 6) { + jpart = sc_kk_impulsive_copy[m].obj. + collide_kokkos(ipart,dtremain,minsurf,tri->norm,tri->isr,reaction,d_retry,d_nlocal); } } @@ -1418,6 +1433,9 @@ void UpdateKokkos::operator()(TagUpdateMove } else if (sc_type == 5) { jpart = sc_kk_adiabatic_copy[m].obj. collide_kokkos(ipart,dtremain,minsurf,line->norm,line->isr,reaction,d_retry,d_nlocal); + } else if (sc_type == 6) { + jpart = sc_kk_impulsive_copy[m].obj. + collide_kokkos(ipart,dtremain,minsurf,line->norm,line->isr,reaction,d_retry,d_nlocal); } } @@ -1668,6 +1686,9 @@ void UpdateKokkos::operator()(TagUpdateMove else if (sc_type == 5) jpart = sc_kk_adiabatic_copy[m].obj. collide_kokkos(ipart,dtremain,-(outface+1),domain_kk_copy.obj.norm[outface],domain_kk_copy.obj.surf_react[outface],reaction,d_retry,d_nlocal); + else if (sc_type == 6) + jpart = sc_kk_impulsive_copy[m].obj. + collide_kokkos(ipart,dtremain,-(outface+1),domain_kk_copy.obj.norm[outface],domain_kk_copy.obj.surf_react[outface],reaction,d_retry,d_nlocal); if (ipart) { double *x = ipart->x; @@ -2025,8 +2046,8 @@ void UpdateKokkos::backup() Kokkos::deep_copy(d_particles_backup,d_particles); if (surf->nsc > 0) { - int nspec,ndiff,npist,nadia; - nspec = ndiff = npist = nadia = 0; + int nspec,ndiff,npist,nadia,nimpul; + nspec = ndiff = npist = nadia = nimpul = 0; for (int n = 0; n < surf->nsc; n++) { if (strcmp(surf->sc[n]->style,"specular") == 0) { sc_kk_specular_copy[nspec].obj.backup(); @@ -2040,6 +2061,9 @@ void UpdateKokkos::backup() } else if (strcmp(surf->sc[n]->style,"adiabatic") == 0) { sc_kk_adiabatic_copy[nadia].obj.backup(); nadia++; + } else if (strcmp(surf->sc[n]->style,"impulsive") == 0) { + sc_kk_impulsive_copy[nimpul].obj.backup(); + nimpul++; } } } @@ -2054,8 +2078,8 @@ void UpdateKokkos::restore() d_particles = particle_kk->k_particles.view_device(); if (surf->nsc > 0) { - int nspec,ndiff,npist,nadia; - nspec = ndiff = npist = nadia = 0; + int nspec,ndiff,npist,nadia,nimpul; + nspec = ndiff = npist = nadia = nimpul = 0; for (int n = 0; n < surf->nsc; n++) { if (strcmp(surf->sc[n]->style,"specular") == 0) { sc_kk_specular_copy[nspec].obj.restore(); @@ -2069,6 +2093,9 @@ void UpdateKokkos::restore() } else if (strcmp(surf->sc[n]->style,"adiabatic") == 0) { sc_kk_adiabatic_copy[nadia].obj.restore(); nadia++; + } else if (strcmp(surf->sc[n]->style,"impulsive") == 0) { + sc_kk_impulsive_copy[nimpul].obj.restore(); + nimpul++; } } } diff --git a/src/KOKKOS/update_kokkos.h b/src/KOKKOS/update_kokkos.h index 02da81cc8..abe3e6b31 100644 --- a/src/KOKKOS/update_kokkos.h +++ b/src/KOKKOS/update_kokkos.h @@ -27,6 +27,7 @@ #include "surf_collide_piston_kokkos.h" #include "surf_collide_transparent_kokkos.h" #include "surf_collide_adiabatic_kokkos.h" +#include "surf_collide_impulsive_kokkos.h" #include "compute_boundary_kokkos.h" #include "compute_surf_kokkos.h" @@ -138,6 +139,7 @@ class UpdateKokkos : public Update { KKCopy sc_kk_piston_copy[KOKKOS_MAX_SURF_COLL_PER_TYPE]; KKCopy sc_kk_transparent_copy[KOKKOS_MAX_SURF_COLL_PER_TYPE]; KKCopy sc_kk_adiabatic_copy[KOKKOS_MAX_SURF_COLL_PER_TYPE]; + KKCopy sc_kk_impulsive_copy[KOKKOS_MAX_SURF_COLL_PER_TYPE]; //KKCopy blist_active_copy[KOKKOS_MAX_GLIST]; KKCopy slist_active_copy[KOKKOS_MAX_SLIST]; diff --git a/src/surf_collide_impulsive.cpp b/src/surf_collide_impulsive.cpp index 314735245..263df8a92 100644 --- a/src/surf_collide_impulsive.cpp +++ b/src/surf_collide_impulsive.cpp @@ -141,6 +141,8 @@ SurfCollideImpulsive::SurfCollideImpulsive(SPARTA *sparta, int narg, char **arg) SurfCollideImpulsive::~SurfCollideImpulsive() { + if (copy) return; + delete random; } diff --git a/src/surf_collide_impulsive.h b/src/surf_collide_impulsive.h index aa5c1eda2..0ee87c53c 100644 --- a/src/surf_collide_impulsive.h +++ b/src/surf_collide_impulsive.h @@ -28,6 +28,7 @@ namespace SPARTA_NS { class SurfCollideImpulsive : public SurfCollide { public: SurfCollideImpulsive(class SPARTA *, int, char **); + SurfCollideImpulsive(class SPARTA *sparta) : SurfCollide(sparta) {} // needed for Kokkos ~SurfCollideImpulsive(); void init(); Particle::OnePart *collide(Particle::OnePart *&, double &, @@ -35,7 +36,7 @@ class SurfCollideImpulsive : public SurfCollide { void wrapper(Particle::OnePart *, double *, int *, double*); void flags_and_coeffs(int *, double *); - private: + protected: double eng_ratio,eff_mass; // energy ratio and effective mass // of the surface for soft-sphere model double u0_a, u0_b; // u0 values for the direct case From 49696f12acb9c29a6e61e30d4b7194bad7df7420 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 19 Jun 2026 17:59:11 +0000 Subject: [PATCH 03/30] KOKKOS: port surf_collide td to Kokkos Add SurfCollideTDKokkos, the GPU-capable port of the td (thermal desorption) surface collision model, including the barrier/initenergy/bond options. Follows the surf_collide_diffuse_kokkos pattern: device-callable collide_kokkos, the td() reflection kernel plus device erot()/evib() helpers as KOKKOS_INLINE_FUNCTION, Kokkos RNG pool with SPARTA_KOKKOS_EXACT support, DualView counters, dynamic() for VARSURF/CUSTOM Tsurf, and surf-react dispatch. Give the base SurfCollideTD a Kokkos empty constructor, make its members protected, and guard its destructor with copy. Wire sc_type id 7 through update_kokkos (selection, the 3D/2D/boundary dispatch ladders, post_collide, backup/restore) and register the files in the KOKKOS Install.sh. Verified: exact-match gate (Serial + SPARTA_KOKKOS_EXACT, 1 thread) is bit-for-bit identical CPU vs -sf kk on examples/surf_collide/in.circle.td and in.beam.td; OpenMP 4-thread run is clean and statistically consistent. Co-Authored-By: Claude Opus 4.8 Co-Authored-By: Stan Moore Claude-Session: https://claude.ai/code/session_01Q6nQVuxEiTUqSzDoDppCUM --- src/KOKKOS/Install.sh | 2 + src/KOKKOS/surf_collide_td_kokkos.cpp | 358 +++++++++++++++++++++++++ src/KOKKOS/surf_collide_td_kokkos.h | 363 ++++++++++++++++++++++++++ src/KOKKOS/update_kokkos.cpp | 44 +++- src/KOKKOS/update_kokkos.h | 2 + src/surf_collide_td.cpp | 2 + src/surf_collide_td.h | 3 +- 7 files changed, 764 insertions(+), 10 deletions(-) create mode 100644 src/KOKKOS/surf_collide_td_kokkos.cpp create mode 100644 src/KOKKOS/surf_collide_td_kokkos.h diff --git a/src/KOKKOS/Install.sh b/src/KOKKOS/Install.sh index 69d95a48d..451ac3a55 100644 --- a/src/KOKKOS/Install.sh +++ b/src/KOKKOS/Install.sh @@ -127,6 +127,8 @@ action surf_collide_piston_kokkos.cpp action surf_collide_piston_kokkos.h action surf_collide_specular_kokkos.cpp action surf_collide_specular_kokkos.h +action surf_collide_td_kokkos.cpp +action surf_collide_td_kokkos.h action surf_collide_transparent_kokkos.cpp action surf_collide_transparent_kokkos.h action surf_collide_vanish_kokkos.cpp diff --git a/src/KOKKOS/surf_collide_td_kokkos.cpp b/src/KOKKOS/surf_collide_td_kokkos.cpp new file mode 100644 index 000000000..7a49d7443 --- /dev/null +++ b/src/KOKKOS/surf_collide_td_kokkos.cpp @@ -0,0 +1,358 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#include "math.h" +#include "stdlib.h" +#include "string.h" +#include "surf_collide_td_kokkos.h" +#include "surf_kokkos.h" +#include "input.h" +#include "variable.h" +#include "particle.h" +#include "domain.h" +#include "update.h" +#include "modify.h" +#include "comm.h" +#include "random_mars.h" +#include "random_knuth.h" +#include "math_const.h" +#include "math_extra.h" +#include "memory.h" +#include "error.h" +#include "particle_kokkos.h" +#include "sparta_masks.h" +#include "collide.h" + +using namespace SPARTA_NS; +using namespace MathConst; + +enum{INT,DOUBLE}; // several files +enum{NUMERIC,CUSTOM,VARIABLE,VAREQUAL,VARSURF}; // surf_collide classes + +#define VAL_1(X) X +#define VAL_2(X) VAL_1(X), VAL_1(X) + +/* ---------------------------------------------------------------------- */ + +SurfCollideTDKokkos::SurfCollideTDKokkos(SPARTA *sparta, int narg, char **arg) : + SurfCollideTD(sparta, narg, arg), + fix_ambi_kk_copy(sparta), + fix_vibmode_kk_copy(sparta), + sr_kk_global_copy{VAL_2(KKCopy(sparta))}, + sr_kk_prob_copy{VAL_2(KKCopy(sparta))}, + rand_pool(12345 + comm->me +#ifdef SPARTA_KOKKOS_EXACT + , sparta +#endif + ) +{ + kokkosable = 1; + + random_backup = NULL; + +#ifdef SPARTA_KOKKOS_EXACT + rand_pool.init(random); +#endif + + // use 1D view for scalars to reduce GPU memory operations + + d_scalars = t_int_2("surf_collide_td:scalars"); + d_nsingle = Kokkos::subview(d_scalars,0); + d_nreact_one = Kokkos::subview(d_scalars,1); + + h_scalars = t_host_int_2("surf_collide_td:scalars_mirror"); + h_nsingle = Kokkos::subview(h_scalars,0); + h_nreact_one = Kokkos::subview(h_scalars,1); +} + +SurfCollideTDKokkos::SurfCollideTDKokkos(SPARTA *sparta) : + SurfCollideTD(sparta), + fix_ambi_kk_copy(sparta), + fix_vibmode_kk_copy(sparta), + sr_kk_global_copy{VAL_2(KKCopy(sparta))}, + sr_kk_prob_copy{VAL_2(KKCopy(sparta))}, + rand_pool(12345 // seed doesn't matter since it will just be copied over +#ifdef SPARTA_KOKKOS_EXACT + , sparta +#endif + ) +{ + copy = 1; +} + +/* ---------------------------------------------------------------------- */ + +SurfCollideTDKokkos::~SurfCollideTDKokkos() +{ + if (uncopy) { + fix_ambi_kk_copy.uncopy(); + fix_vibmode_kk_copy.uncopy(); + + for (int i = 0; i < KOKKOS_MAX_SURF_REACT_PER_TYPE; i++) { + sr_kk_global_copy[i].uncopy(); + sr_kk_prob_copy[i].uncopy(); + } + } + + if (copy) return; + +#ifdef SPARTA_KOKKOS_EXACT + rand_pool.destroy(); + if (random_backup) + delete random_backup; +#endif +} + +/* ---------------------------------------------------------------------- */ + +void SurfCollideTDKokkos::init() +{ + SurfCollideTD::init(); + + ambi_flag = vibmode_flag = 0; + if (modify->n_update_custom) { + for (int ifix = 0; ifix < modify->nfix; ifix++) { + if (strcmp(modify->fix[ifix]->style,"ambipolar") == 0) { + ambi_flag = 1; + FixAmbipolar *afix = (FixAmbipolar *) modify->fix[ifix]; + if (!afix->kokkos_flag) + error->all(FLERR,"Must use fix ambipolar/kk when Kokkos is enabled"); + afix_kk = (FixAmbipolarKokkos*)afix; + } else if (strcmp(modify->fix[ifix]->style,"vibmode") == 0) { + vibmode_flag = 1; + FixVibmode *vfix = (FixVibmode *) modify->fix[ifix]; + if (!vfix->kokkos_flag) + error->all(FLERR,"Must use fix vibmode/kk when Kokkos is enabled"); + vfix_kk = (FixVibmodeKokkos*)vfix; + } + } + } +} + +/* ---------------------------------------------------------------------- + recalculate Tsurf values which are dynamic + called by Update::setup() and Update::run() +---------------------------------------------------------------------- */ + +void SurfCollideTDKokkos::dynamic() +{ + // VAREQUAL mode + // equal-style variable sets single tsurf value for all surfs + + if (tmode == VAREQUAL) { + + // only evaluate variable if timestep is multiple of tfreq + + if (update->ntimestep % tfreq) return; + tsurf = input->variable->compute_equal(tindex_var); + if (tsurf <= 0.0) error->all(FLERR,"Surf_collide tsurf <= 0.0"); + + // VARSURF mode + // surf-style variable sets new tsurf values for all surfs + // particle/surf collisions access t_persurf for local+ghost values + + } else if (tmode == VARSURF) { + + // only evaluate variable if timestep is multiple of tfreq + + int spreadflag = 0; + if (update->ntimestep % tfreq == 0) { + if (n_owned != surf->nown) { + memory->destroy(t_owned); + n_owned = surf->nown; + memory->create(t_owned,n_owned,"surfcollide:t_owned"); + } + + input->variable->compute_surf(tindex_var,t_owned,1,0); + spreadflag = 1; + } + + // spread t_owned values to t_localghost values via spread_own2local() + // if just re-computed variable OR surfs are + // distributed and load balance/adaptation took place on previous step + + if (spreadflag || + (surf->distributed && surf->localghost_changed_step == update->ntimestep-1)) { + if (n_localghost != surf->nlocal + surf->nghost) { + memory->destroy(t_localghost); + n_localghost = surf->nlocal + surf->nghost; + memory->create(t_localghost,n_localghost,"surfcollide:t_localghost"); + } + + surf->spread_own2local(1,DOUBLE,t_owned,t_localghost); + t_persurf = t_localghost; + + auto h_t_persurf = HAT::t_float_1d(t_persurf,n_localghost); + d_t_persurf = Kokkos::create_mirror_view_and_copy(SPADeviceType(),h_t_persurf); + } + + // CUSTOM mode + // ensure access to custom per-surf vec for tsurf values for all surfs + // particle/surf collisions access t_persurf for local+ghost values + + } else if (tmode == CUSTOM) { + SurfKokkos* surf_kk = (SurfKokkos*) surf; + auto h_edvec_local = surf_kk->k_edvec_local.view_host(); + + // spread owned values to local+ghost values via spread_custom() + // estatus == 1 means owned values already spread to local+ghost values + // if estatus == 0: owned values are new OR + // surfs are distributed and load balance/adaptation took place + + if (surf->estatus[tindex_custom] == 0) surf->spread_custom(tindex_custom); + + h_edvec_local[tindex_custom].k_view.sync_device(); + d_t_persurf = h_edvec_local[tindex_custom].k_view.view_device(); + } +} + +/* ---------------------------------------------------------------------- */ + +void SurfCollideTDKokkos::pre_collide() +{ + if (ambi_flag) { + afix_kk->pre_update_custom_kokkos(); + fix_ambi_kk_copy.copy(afix_kk); + } + + if (vibmode_flag) { + vfix_kk->pre_update_custom_kokkos(); + fix_vibmode_kk_copy.copy(vfix_kk); + } + + if (surf->nsr > KOKKOS_MAX_TOT_SURF_REACT) + error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); + + if (surf->nsr > 0) { + int nglob,nprob; + nglob = nprob = 0; + for (int n = 0; n < surf->nsr; n++) { + if (!surf->sr[n]->kokkosable) + error->all(FLERR,"Must use Kokkos-enabled surface reaction method with Kokkos"); + if (strcmp(surf->sr[n]->style,"global") == 0) { + sr_kk_global_copy[nglob].copy((SurfReactGlobalKokkos*)(surf->sr[n])); + sr_kk_global_copy[nglob].obj.pre_react(); + sr_type_list[n] = 0; + sr_map[n] = nglob; + nglob++; + } else if (strcmp(surf->sr[n]->style,"prob") == 0) { + sr_kk_prob_copy[nprob].copy((SurfReactProbKokkos*)(surf->sr[n])); + sr_kk_prob_copy[nprob].obj.pre_react(); + sr_type_list[n] = 1; + sr_map[n] = nprob; + nprob++; + } else { + error->all(FLERR,"Unknown Kokkos surface reaction method"); + } + } + + if (nglob > KOKKOS_MAX_SURF_REACT_PER_TYPE || nprob > KOKKOS_MAX_SURF_REACT_PER_TYPE) + error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); + } + + if (random == NULL) { + // initialize RNG + + random = new RanKnuth(update->ranmaster->uniform()); + double seed = update->ranmaster->uniform(); + random->reset(seed,comm->me,100); + +#ifdef SPARTA_KOKKOS_EXACT + rand_pool.init(random); +#endif + } + + ParticleKokkos* particle_kk = (ParticleKokkos*) particle; + particle_kk->sync(Device,PARTICLE_MASK|SPECIES_MASK); + d_particles = particle_kk->k_particles.view_device(); + d_species = particle_kk->k_species.view_device(); + boltz = update->boltz; + + rotstyle = NONE; + if (Pointers::collide) rotstyle = Pointers::collide->rotstyle; + vibstyle = NONE; + if (Pointers::collide) vibstyle = Pointers::collide->vibstyle; + + Kokkos::deep_copy(d_scalars,0); +} + +/* ---------------------------------------------------------------------- */ + +void SurfCollideTDKokkos::post_collide() +{ + ParticleKokkos* particle_kk = (ParticleKokkos*) particle; + if (ambi_flag || vibmode_flag) particle_kk->modify(Device,CUSTOM_MASK); + + Kokkos::deep_copy(h_scalars,d_scalars); + + int m = surf->find_collide(id); + auto sc = surf->sc[m]; // can't modify the copy directly, use the original + sc->nsingle += h_nsingle(); + surf->nreact_one += h_nreact_one(); + + d_particles = {}; +} + +/* ---------------------------------------------------------------------- */ + +void SurfCollideTDKokkos::backup() +{ + ParticleKokkos* particle_kk = (ParticleKokkos*) particle; + d_particles = particle_kk->k_particles.view_device(); + + if (surf->nsr > 0) { + int nglob,nprob; + nglob = nprob = 0; + for (int n = 0; n < surf->nsr; n++) { + if (strcmp(surf->sr[n]->style,"global") == 0) { + sr_kk_global_copy[nglob].obj.backup(); + nglob++; + } else if (strcmp(surf->sr[n]->style,"prob") == 0) { + sr_kk_prob_copy[nprob].obj.backup(); + nprob++; + } + } + } + +#ifdef SPARTA_KOKKOS_EXACT + if (!random_backup) + random_backup = new RanKnuth(12345 + comm->me); + memcpy(random_backup,random,sizeof(RanKnuth)); +#endif +} + +/* ---------------------------------------------------------------------- */ + +void SurfCollideTDKokkos::restore() +{ + if (surf->nsr > 0) { + int nglob,nprob; + nglob = nprob = 0; + for (int n = 0; n < surf->nsr; n++) { + if (strcmp(surf->sr[n]->style,"global") == 0) { + sr_kk_global_copy[nglob].obj.restore(); + nglob++; + } else if (strcmp(surf->sr[n]->style,"prob") == 0) { + sr_kk_prob_copy[nprob].obj.restore(); + nprob++; + } + } + } + + Kokkos::deep_copy(d_scalars,0); + +#ifdef SPARTA_KOKKOS_EXACT + memcpy(random,random_backup,sizeof(RanKnuth)); +#endif +} diff --git a/src/KOKKOS/surf_collide_td_kokkos.h b/src/KOKKOS/surf_collide_td_kokkos.h new file mode 100644 index 000000000..5df3b9102 --- /dev/null +++ b/src/KOKKOS/surf_collide_td_kokkos.h @@ -0,0 +1,363 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#ifdef SURF_COLLIDE_CLASS + +SurfCollideStyle(td/kk,SurfCollideTDKokkos) + +#else + +#ifndef SPARTA_SURF_COLLIDE_TD_KOKKOS_H +#define SPARTA_SURF_COLLIDE_TD_KOKKOS_H + +#include "surf_collide_td.h" +#include "kokkos_type.h" +#include "math_extra_kokkos.h" +#include "Kokkos_Random.hpp" +#include "rand_pool_wrap.h" +#include "kokkos_copy.h" +#include "fix_ambipolar_kokkos.h" +#include "fix_vibmode_kokkos.h" +#include "surf_react_global_kokkos.h" +#include "surf_react_prob_kokkos.h" + +namespace SPARTA_NS { + +class SurfCollideTDKokkos : public SurfCollideTD { + public: + + enum{NONE,DISCRETE,SMOOTH}; // several files + enum{PKEEP,PINSERT,PDONE,PDISCARD,PENTRY,PEXIT,PSURF}; // several files + + SurfCollideTDKokkos(class SPARTA *, int, char **); + SurfCollideTDKokkos(class SPARTA *); + ~SurfCollideTDKokkos(); + void init(); + void dynamic(); + void pre_collide(); + void post_collide(); + void backup(); + void restore(); + + private: + double boltz; + int rotstyle, vibstyle; + +#ifndef SPARTA_KOKKOS_EXACT + Kokkos::Random_XorShift64_Pool rand_pool; + typedef typename Kokkos::Random_XorShift64_Pool::generator_type rand_type; +#else + RandPoolWrap rand_pool; + typedef RandWrap rand_type; +#endif + + RanKnuth* random_backup; + + DAT::t_float_1d d_t_persurf; + + typedef Kokkos::DualView tdual_int_2; + typedef tdual_int_2::t_dev t_int_2; + typedef tdual_int_2::t_host t_host_int_2; + t_int_2 d_scalars; + t_host_int_2 h_scalars; + + DAT::t_int_scalar d_nsingle; + DAT::t_int_scalar d_nreact_one; + + HAT::t_int_scalar h_nsingle; + HAT::t_int_scalar h_nreact_one; + + t_particle_1d d_particles; + t_species_1d d_species; + + int ambi_flag,vibmode_flag; + FixAmbipolarKokkos* afix_kk; + FixVibmodeKokkos* vfix_kk; + KKCopy fix_ambi_kk_copy; + KKCopy fix_vibmode_kk_copy; + + int sr_type_list[KOKKOS_MAX_TOT_SURF_REACT]; + int sr_map[KOKKOS_MAX_TOT_SURF_REACT]; + KKCopy sr_kk_global_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; + KKCopy sr_kk_prob_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; + + public: + + /* ---------------------------------------------------------------------- + particle collision with surface with optional chemistry + ip = particle with current x = collision pt, current v = incident v + isurf = index of surface element + norm = surface normal unit vector + isr = index of reaction model if >= 0, -1 for no chemistry + ip = set to NULL if destroyed by chemistry + return jp = new particle if created by chemistry + return reaction = index of reaction (1 to N) that took place, 0 = no reaction + resets particle(s) to post-collision outward velocity + ------------------------------------------------------------------------- */ + + template + KOKKOS_INLINE_FUNCTION + Particle::OnePart* collide_kokkos(Particle::OnePart *&ip, double &, + int isurf, const double *norm, int isr, int &reaction, + const DAT::t_int_scalar &d_retry, const DAT::t_int_scalar &d_nlocal) const + { + if (ATOMIC_REDUCTION == 0) + d_nsingle()++; + else + Kokkos::atomic_inc(&d_nsingle()); + + // if surface chemistry defined, attempt reaction + // reaction = 1 to N for which reaction took place, 0 for none + // velreset = 1 if reaction reset post-collision velocity, else 0 + + Particle::OnePart iorig; + Particle::OnePart *jp = NULL; + reaction = 0; + int velreset = 0; + + if (REACT) { + if (ambi_flag || vibmode_flag) memcpy(&iorig,ip,sizeof(Particle::OnePart)); + + int sr_type = sr_type_list[isr]; + int m = sr_map[isr]; + + if (sr_type == 0) { + reaction = sr_kk_global_copy[m].obj. + react_kokkos(ip,isurf,norm,jp,velreset,d_retry,d_nlocal); + } else if (sr_type == 1) { + reaction = sr_kk_prob_copy[m].obj. + react_kokkos(ip,isurf,norm,jp,velreset,d_retry,d_nlocal); + } + + if (reaction) { + if (ATOMIC_REDUCTION == 0) + d_nreact_one()++; + else + Kokkos::atomic_inc(&d_nreact_one()); + } + } + + // set temperature of isurf if VARSURF or CUSTOM + + double tsurf_local = tsurf; + if (persurf_temperature) { + tsurf_local = d_t_persurf[isurf]; + if (tsurf_local <= 0.0) Kokkos::abort("Surf_collide tsurf <= 0.0"); + } + + // TD reflection for each particle + // only if SurfReact did not already reset velocities + // also both particles need to trigger any fixes + // to update per-particle properties which depend on + // temperature of the particle, e.g. fix vibmode and fix ambipolar + + if (ip) { + if (!velreset) td(ip,norm,tsurf_local); + int i = ip - d_particles.data(); + if (ambi_flag) + fix_ambi_kk_copy.obj.update_custom_kokkos(i,tsurf_local,tsurf_local,tsurf_local,vstream); + if (vibmode_flag) + fix_vibmode_kk_copy.obj.update_custom_kokkos(i,tsurf_local,tsurf_local,tsurf_local,vstream); + } + if (REACT && jp) { + if (!velreset) td(jp,norm,tsurf_local); + int j = jp - d_particles.data(); + if (ambi_flag) + fix_ambi_kk_copy.obj.update_custom_kokkos(j,tsurf_local,tsurf_local,tsurf_local,vstream); + if (vibmode_flag) + fix_vibmode_kk_copy.obj.update_custom_kokkos(j,tsurf_local,tsurf_local,tsurf_local,vstream); + } + + // call any fixes with a surf_react() method + // they may reset j to -1, e.g. fix ambipolar + // in which case newly created j is deleted + + if (REACT && reaction && ambi_flag) { + int i = -1; + if (ip) i = ip - d_particles.data(); + int j = -1; + if (jp) j = jp - d_particles.data(); + int j_orig = j; + fix_ambi_kk_copy.obj.surf_react_kokkos(&iorig,i,j); + if (jp && j < 0) { + d_particles[j_orig].flag = PDISCARD; + jp = NULL; + } + } + + return jp; + }; + + private: + + /* ---------------------------------------------------------------------- + TD (thermal desorption) reflection + ------------------------------------------------------------------------- */ + + KOKKOS_INLINE_FUNCTION + void td(Particle::OnePart *p, const double *norm, const double twall) const + { + rand_type rand_gen = rand_pool.get_state(); + + double tangent1[3],tangent2[3]; + int ispecies = p->ispecies; + + double *v = p->v; + double dot = MathExtraKokkos::dot3(v,norm); + + tangent1[0] = v[0] - dot*norm[0]; + tangent1[1] = v[1] - dot*norm[1]; + tangent1[2] = v[2] - dot*norm[2]; + + if (MathExtraKokkos::lensq3(tangent1) == 0.0) { + tangent2[0] = rand_gen.drand(); + tangent2[1] = rand_gen.drand(); + tangent2[2] = rand_gen.drand(); + MathExtraKokkos::cross3(norm,tangent2,tangent1); + } + + MathExtraKokkos::norm3(tangent1); + MathExtraKokkos::cross3(norm,tangent1,tangent2); + + double mass = d_species[ispecies].mass; + double E_i = 0.5 * mass * MathExtraKokkos::lensq3(v); + + double E_t = boltz * twall; + if (bond_flag) E_t += boltz*bond_trans; + if (initen_flag) E_t += E_i*initen_trans; + + double E_n = E_t; + if (barrier_flag) E_n += boltz*barrier_val; + + double vrm_n = sqrt(2.0*E_n / mass); + double vrm_t = sqrt(2.0*E_t / mass); + double vperp = vrm_n * sqrt(-log(rand_gen.drand())); + + double theta = MathConst::MY_2PI * rand_gen.drand(); + double vtangent = vrm_t * sqrt(-log(rand_gen.drand())); + double vtan1 = vtangent * sin(theta); + double vtan2 = vtangent * cos(theta); + + v[0] = vperp*norm[0] + vtan1*tangent1[0] + vtan2*tangent2[0]; + v[1] = vperp*norm[1] + vtan1*tangent1[1] + vtan2*tangent2[1]; + v[2] = vperp*norm[2] + vtan1*tangent1[2] + vtan2*tangent2[2]; + + double twall_rot = twall; + double twall_vib = twall; + + if (bond_flag) { + twall_rot += bond_rot; + twall_vib += bond_vib; + } + + if (initen_flag) { + twall_rot += E_i*initen_rot/boltz; + twall_vib += E_i*initen_vib/boltz; + } + + p->erot = erot(ispecies,twall_rot,rand_gen,boltz); + p->evib = evib(ispecies,twall_vib,rand_gen,boltz); + + rand_pool.free_state(rand_gen); + } + + /* ---------------------------------------------------------------------- + generate random rotational energy for a particle + only a function of species index and species properties + ------------------------------------------------------------------------- */ + + KOKKOS_INLINE_FUNCTION + double erot(int isp, double temp_thermal, rand_type &rand_gen, double boltz) const + { + double eng,a,erm,b; + + if (rotstyle == NONE) return 0.0; + if (d_species[isp].rotdof < 2) return 0.0; + + if (rotstyle == DISCRETE && d_species[isp].rotdof == 2) { + int irot = -log(rand_gen.drand()) * temp_thermal / + d_species[isp].rottemp[0]; + eng = irot * boltz * d_species[isp].rottemp[0]; + } else if (rotstyle == SMOOTH && d_species[isp].rotdof == 2) { + eng = -log(rand_gen.drand()) * boltz * temp_thermal; + } else { + a = 0.5*d_species[isp].rotdof-1.0; + while (1) { + // energy cut-off at 10 kT + erm = 10.0*rand_gen.drand(); + b = pow(erm/a,a) * exp(a-erm); + if (b > rand_gen.drand()) break; + } + eng = erm * boltz * temp_thermal; + } + + return eng; + } + + /* ---------------------------------------------------------------------- + generate random vibrational energy for a particle + only a function of species index and species properties + index_vibmode = index of extra per-particle vibrational mode storage + -1 if not defined for this model + ------------------------------------------------------------------------- */ + + KOKKOS_INLINE_FUNCTION + double evib(int isp, double temp_thermal, rand_type &rand_gen, double boltz) const + { + double eng,a,erm,b; + + if (vibstyle == NONE || d_species[isp].vibdof < 2) return 0.0; + + // for DISCRETE, only need set evib for vibdof = 2 + // mode levels and evib will be set by FixVibmode::update_custom() + + eng = 0.0; + + if (vibstyle == DISCRETE && d_species[isp].vibdof == 2) { + int ivib = -log(rand_gen.drand()) * temp_thermal / + d_species[isp].vibtemp[0]; + eng = ivib * boltz * d_species[isp].vibtemp[0]; + } else if (vibstyle == SMOOTH || d_species[isp].vibdof >= 2) { + if (d_species[isp].vibdof == 2) + eng = -log(rand_gen.drand()) * boltz * temp_thermal; + else if (d_species[isp].vibdof > 2) { + a = 0.5*d_species[isp].vibdof-1.; + while (1) { + // energy cut-off at 10 kT + erm = 10.0*rand_gen.drand(); + b = pow(erm/a,a) * exp(a-erm); + if (b > rand_gen.drand()) break; + } + eng = erm * boltz * temp_thermal; + } + } + + return eng; + } +}; + +} + +#endif +#endif + +/* ERROR/WARNING messages: + +E: Illegal ... command + +Self-explanatory. Check the input script syntax and compare to the +documentation for the command. You can use -echo screen as a +command-line option when running SPARTA to see the offending line. + +*/ diff --git a/src/KOKKOS/update_kokkos.cpp b/src/KOKKOS/update_kokkos.cpp index aade901b5..bc9bb0ab4 100644 --- a/src/KOKKOS/update_kokkos.cpp +++ b/src/KOKKOS/update_kokkos.cpp @@ -82,6 +82,7 @@ UpdateKokkos::UpdateKokkos(SPARTA *sparta) : Update(sparta), sc_kk_transparent_copy{VAL_2(KKCopy(sparta))}, sc_kk_adiabatic_copy{VAL_2(KKCopy(sparta))}, sc_kk_impulsive_copy{VAL_2(KKCopy(sparta))}, + sc_kk_td_copy{VAL_2(KKCopy(sparta))}, blist_active_copy{VAL_2(KKCopy(sparta))}, slist_active_copy{VAL_2(KKCopy(sparta))}, tmp_compute_boundary_kk(sparta), @@ -149,6 +150,7 @@ UpdateKokkos::~UpdateKokkos() sc_kk_transparent_copy[i].uncopy(); sc_kk_adiabatic_copy[i].uncopy(); sc_kk_impulsive_copy[i].uncopy(); + sc_kk_td_copy[i].uncopy(); } for (int i=0; i void UpdateKokkos::move() error->all(FLERR,"Kokkos currently supports two instances of each surface collide method"); if (surf->nsc > 0) { - int nspec,ndiff,nvan,npist,ntrans,nadia,nimpul; - nspec = ndiff = nvan = npist = ntrans = nadia = nimpul = 0; + int nspec,ndiff,nvan,npist,ntrans,nadia,nimpul,ntd; + nspec = ndiff = nvan = npist = ntrans = nadia = nimpul = ntd = 0; for (int n = 0; n < surf->nsc; n++) { if (!surf->sc[n]->kokkosable) error->all(FLERR,"Must use Kokkos-enabled surface collide method with Kokkos"); @@ -582,6 +584,12 @@ template < int DIM, int SURF, int REACT, int OPT > void UpdateKokkos::move() sc_type_list[n] = 6; sc_map[n] = nimpul; nimpul++; + } else if (strcmp(surf->sc[n]->style,"td") == 0) { + sc_kk_td_copy[ntd].copy((SurfCollideTDKokkos*)(surf->sc[n])); + sc_kk_td_copy[ntd].obj.pre_collide(); + sc_type_list[n] = 7; + sc_map[n] = ntd; + ntd++; } else { error->all(FLERR,"Unknown Kokkos surface collide method"); } @@ -589,7 +597,7 @@ template < int DIM, int SURF, int REACT, int OPT > void UpdateKokkos::move() if (nspec > KOKKOS_MAX_SURF_COLL_PER_TYPE || ndiff > KOKKOS_MAX_SURF_COLL_PER_TYPE || nvan > KOKKOS_MAX_SURF_COLL_PER_TYPE || npist > KOKKOS_MAX_SURF_COLL_PER_TYPE || ntrans > KOKKOS_MAX_SURF_COLL_PER_TYPE || nadia > KOKKOS_MAX_SURF_COLL_PER_TYPE || - nimpul > KOKKOS_MAX_SURF_COLL_PER_TYPE) + nimpul > KOKKOS_MAX_SURF_COLL_PER_TYPE || ntd > KOKKOS_MAX_SURF_COLL_PER_TYPE) error->all(FLERR,"Kokkos currently supports two instances of each surface collide method"); } @@ -730,8 +738,8 @@ template < int DIM, int SURF, int REACT, int OPT > void UpdateKokkos::move() } if (surf->nsc > 0) { - int nspec,ndiff,nvan,npist,ntrans,nadia,nimpul; - nspec = ndiff = nvan = npist = ntrans = nadia = nimpul = 0; + int nspec,ndiff,nvan,npist,ntrans,nadia,nimpul,ntd; + nspec = ndiff = nvan = npist = ntrans = nadia = nimpul = ntd = 0; for (int n = 0; n < surf->nsc; n++) { if (strcmp(surf->sc[n]->style,"specular") == 0) { sc_kk_specular_copy[nspec].obj.post_collide(); @@ -754,6 +762,9 @@ template < int DIM, int SURF, int REACT, int OPT > void UpdateKokkos::move() } else if (strcmp(surf->sc[n]->style,"impulsive") == 0) { sc_kk_impulsive_copy[nimpul].obj.post_collide(); nimpul++; + } else if (strcmp(surf->sc[n]->style,"td") == 0) { + sc_kk_td_copy[ntd].obj.post_collide(); + ntd++; } } } @@ -1411,6 +1422,9 @@ void UpdateKokkos::operator()(TagUpdateMove } else if (sc_type == 6) { jpart = sc_kk_impulsive_copy[m].obj. collide_kokkos(ipart,dtremain,minsurf,tri->norm,tri->isr,reaction,d_retry,d_nlocal); + } else if (sc_type == 7) { + jpart = sc_kk_td_copy[m].obj. + collide_kokkos(ipart,dtremain,minsurf,tri->norm,tri->isr,reaction,d_retry,d_nlocal); } } @@ -1436,6 +1450,9 @@ void UpdateKokkos::operator()(TagUpdateMove } else if (sc_type == 6) { jpart = sc_kk_impulsive_copy[m].obj. collide_kokkos(ipart,dtremain,minsurf,line->norm,line->isr,reaction,d_retry,d_nlocal); + } else if (sc_type == 7) { + jpart = sc_kk_td_copy[m].obj. + collide_kokkos(ipart,dtremain,minsurf,line->norm,line->isr,reaction,d_retry,d_nlocal); } } @@ -1689,6 +1706,9 @@ void UpdateKokkos::operator()(TagUpdateMove else if (sc_type == 6) jpart = sc_kk_impulsive_copy[m].obj. collide_kokkos(ipart,dtremain,-(outface+1),domain_kk_copy.obj.norm[outface],domain_kk_copy.obj.surf_react[outface],reaction,d_retry,d_nlocal); + else if (sc_type == 7) + jpart = sc_kk_td_copy[m].obj. + collide_kokkos(ipart,dtremain,-(outface+1),domain_kk_copy.obj.norm[outface],domain_kk_copy.obj.surf_react[outface],reaction,d_retry,d_nlocal); if (ipart) { double *x = ipart->x; @@ -2046,8 +2066,8 @@ void UpdateKokkos::backup() Kokkos::deep_copy(d_particles_backup,d_particles); if (surf->nsc > 0) { - int nspec,ndiff,npist,nadia,nimpul; - nspec = ndiff = npist = nadia = nimpul = 0; + int nspec,ndiff,npist,nadia,nimpul,ntd; + nspec = ndiff = npist = nadia = nimpul = ntd = 0; for (int n = 0; n < surf->nsc; n++) { if (strcmp(surf->sc[n]->style,"specular") == 0) { sc_kk_specular_copy[nspec].obj.backup(); @@ -2064,6 +2084,9 @@ void UpdateKokkos::backup() } else if (strcmp(surf->sc[n]->style,"impulsive") == 0) { sc_kk_impulsive_copy[nimpul].obj.backup(); nimpul++; + } else if (strcmp(surf->sc[n]->style,"td") == 0) { + sc_kk_td_copy[ntd].obj.backup(); + ntd++; } } } @@ -2078,8 +2101,8 @@ void UpdateKokkos::restore() d_particles = particle_kk->k_particles.view_device(); if (surf->nsc > 0) { - int nspec,ndiff,npist,nadia,nimpul; - nspec = ndiff = npist = nadia = nimpul = 0; + int nspec,ndiff,npist,nadia,nimpul,ntd; + nspec = ndiff = npist = nadia = nimpul = ntd = 0; for (int n = 0; n < surf->nsc; n++) { if (strcmp(surf->sc[n]->style,"specular") == 0) { sc_kk_specular_copy[nspec].obj.restore(); @@ -2096,6 +2119,9 @@ void UpdateKokkos::restore() } else if (strcmp(surf->sc[n]->style,"impulsive") == 0) { sc_kk_impulsive_copy[nimpul].obj.restore(); nimpul++; + } else if (strcmp(surf->sc[n]->style,"td") == 0) { + sc_kk_td_copy[ntd].obj.restore(); + ntd++; } } } diff --git a/src/KOKKOS/update_kokkos.h b/src/KOKKOS/update_kokkos.h index abe3e6b31..afd121369 100644 --- a/src/KOKKOS/update_kokkos.h +++ b/src/KOKKOS/update_kokkos.h @@ -28,6 +28,7 @@ #include "surf_collide_transparent_kokkos.h" #include "surf_collide_adiabatic_kokkos.h" #include "surf_collide_impulsive_kokkos.h" +#include "surf_collide_td_kokkos.h" #include "compute_boundary_kokkos.h" #include "compute_surf_kokkos.h" @@ -140,6 +141,7 @@ class UpdateKokkos : public Update { KKCopy sc_kk_transparent_copy[KOKKOS_MAX_SURF_COLL_PER_TYPE]; KKCopy sc_kk_adiabatic_copy[KOKKOS_MAX_SURF_COLL_PER_TYPE]; KKCopy sc_kk_impulsive_copy[KOKKOS_MAX_SURF_COLL_PER_TYPE]; + KKCopy sc_kk_td_copy[KOKKOS_MAX_SURF_COLL_PER_TYPE]; //KKCopy blist_active_copy[KOKKOS_MAX_GLIST]; KKCopy slist_active_copy[KOKKOS_MAX_SLIST]; diff --git a/src/surf_collide_td.cpp b/src/surf_collide_td.cpp index 1da0b2a48..a291b5d21 100644 --- a/src/surf_collide_td.cpp +++ b/src/surf_collide_td.cpp @@ -102,6 +102,8 @@ SurfCollideTD::SurfCollideTD(SPARTA *sparta, int narg, char **arg) : SurfCollideTD::~SurfCollideTD() { + if (copy) return; + delete random; } diff --git a/src/surf_collide_td.h b/src/surf_collide_td.h index 3b6b516f5..cea5ac30e 100644 --- a/src/surf_collide_td.h +++ b/src/surf_collide_td.h @@ -28,6 +28,7 @@ namespace SPARTA_NS { class SurfCollideTD : public SurfCollide { public: SurfCollideTD(class SPARTA *, int, char **); + SurfCollideTD(class SPARTA *sparta) : SurfCollide(sparta) {} // needed for Kokkos ~SurfCollideTD(); void init(); Particle::OnePart *collide(Particle::OnePart *&, double &, @@ -35,7 +36,7 @@ class SurfCollideTD : public SurfCollide { void wrapper(Particle::OnePart *, double *, int *, double*); void flags_and_coeffs(int *, double *); - private: + protected: double barrier_val; double initen_trans, initen_rot, initen_vib; double bond_trans, bond_rot, bond_vib; From 3d66c62a6d0fb27c0240d1257029eb25238a81a2 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 19 Jun 2026 18:04:57 +0000 Subject: [PATCH 04/30] KOKKOS: port surf_collide cll to Kokkos Add SurfCollideCLLKokkos, the GPU-capable port of the cll (Cercignani-Lampis- Lord) surface collision model, including the partial (eccentricity) and translate/rotate options and the rotational/vibrational accommodation. Follows the surf_collide_diffuse_kokkos pattern: device-callable collide_kokkos, the cll() reflection kernel as KOKKOS_INLINE_FUNCTION, Kokkos RNG pool with SPARTA_KOKKOS_EXACT support, DualView counters, dynamic() for VARSURF/CUSTOM Tsurf, and surf-react dispatch. The translate path's random->gaussian() maps to rand_gen.normal(), which is RanKnuth::gaussian() under SPARTA_KOKKOS_EXACT (via RandWrap::normal). Give the base SurfCollideCLL a Kokkos empty constructor, make its members protected, and guard its destructor with copy. Wire sc_type id 8 through update_kokkos (selection, the 3D/2D/boundary dispatch ladders, post_collide, backup/restore) and register the files in the KOKKOS Install.sh. Verified: exact-match gate (Serial + SPARTA_KOKKOS_EXACT, 1 thread) is bit-for-bit identical CPU vs -sf kk on examples/surf_collide/in.circle.cll and in.beam.cll; OpenMP 4-thread run is clean and statistically consistent. Note: those inputs use default cll, so the translate/gaussian branch is not exercised by the gate (it matches by construction in exact mode). Co-Authored-By: Claude Opus 4.8 Co-Authored-By: Stan Moore Claude-Session: https://claude.ai/code/session_01Q6nQVuxEiTUqSzDoDppCUM --- src/KOKKOS/Install.sh | 2 + src/KOKKOS/surf_collide_cll_kokkos.cpp | 358 +++++++++++++++++++++ src/KOKKOS/surf_collide_cll_kokkos.h | 413 +++++++++++++++++++++++++ src/KOKKOS/update_kokkos.cpp | 45 ++- src/KOKKOS/update_kokkos.h | 2 + src/surf_collide_cll.cpp | 2 + src/surf_collide_cll.h | 3 +- 7 files changed, 815 insertions(+), 10 deletions(-) create mode 100644 src/KOKKOS/surf_collide_cll_kokkos.cpp create mode 100644 src/KOKKOS/surf_collide_cll_kokkos.h diff --git a/src/KOKKOS/Install.sh b/src/KOKKOS/Install.sh index 451ac3a55..51e33f826 100644 --- a/src/KOKKOS/Install.sh +++ b/src/KOKKOS/Install.sh @@ -119,6 +119,8 @@ action react_tce_kokkos.cpp action react_tce_kokkos.h action surf_collide_adiabatic_kokkos.cpp action surf_collide_adiabatic_kokkos.h +action surf_collide_cll_kokkos.cpp +action surf_collide_cll_kokkos.h action surf_collide_diffuse_kokkos.cpp action surf_collide_diffuse_kokkos.h action surf_collide_impulsive_kokkos.cpp diff --git a/src/KOKKOS/surf_collide_cll_kokkos.cpp b/src/KOKKOS/surf_collide_cll_kokkos.cpp new file mode 100644 index 000000000..262867b4e --- /dev/null +++ b/src/KOKKOS/surf_collide_cll_kokkos.cpp @@ -0,0 +1,358 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#include "math.h" +#include "stdlib.h" +#include "string.h" +#include "surf_collide_cll_kokkos.h" +#include "surf_kokkos.h" +#include "input.h" +#include "variable.h" +#include "particle.h" +#include "domain.h" +#include "update.h" +#include "modify.h" +#include "comm.h" +#include "random_mars.h" +#include "random_knuth.h" +#include "math_const.h" +#include "math_extra.h" +#include "memory.h" +#include "error.h" +#include "particle_kokkos.h" +#include "sparta_masks.h" +#include "collide.h" + +using namespace SPARTA_NS; +using namespace MathConst; + +enum{INT,DOUBLE}; // several files +enum{NUMERIC,CUSTOM,VARIABLE,VAREQUAL,VARSURF}; // surf_collide classes + +#define VAL_1(X) X +#define VAL_2(X) VAL_1(X), VAL_1(X) + +/* ---------------------------------------------------------------------- */ + +SurfCollideCLLKokkos::SurfCollideCLLKokkos(SPARTA *sparta, int narg, char **arg) : + SurfCollideCLL(sparta, narg, arg), + fix_ambi_kk_copy(sparta), + fix_vibmode_kk_copy(sparta), + sr_kk_global_copy{VAL_2(KKCopy(sparta))}, + sr_kk_prob_copy{VAL_2(KKCopy(sparta))}, + rand_pool(12345 + comm->me +#ifdef SPARTA_KOKKOS_EXACT + , sparta +#endif + ) +{ + kokkosable = 1; + + random_backup = NULL; + +#ifdef SPARTA_KOKKOS_EXACT + rand_pool.init(random); +#endif + + // use 1D view for scalars to reduce GPU memory operations + + d_scalars = t_int_2("surf_collide_cll:scalars"); + d_nsingle = Kokkos::subview(d_scalars,0); + d_nreact_one = Kokkos::subview(d_scalars,1); + + h_scalars = t_host_int_2("surf_collide_cll:scalars_mirror"); + h_nsingle = Kokkos::subview(h_scalars,0); + h_nreact_one = Kokkos::subview(h_scalars,1); +} + +SurfCollideCLLKokkos::SurfCollideCLLKokkos(SPARTA *sparta) : + SurfCollideCLL(sparta), + fix_ambi_kk_copy(sparta), + fix_vibmode_kk_copy(sparta), + sr_kk_global_copy{VAL_2(KKCopy(sparta))}, + sr_kk_prob_copy{VAL_2(KKCopy(sparta))}, + rand_pool(12345 // seed doesn't matter since it will just be copied over +#ifdef SPARTA_KOKKOS_EXACT + , sparta +#endif + ) +{ + copy = 1; +} + +/* ---------------------------------------------------------------------- */ + +SurfCollideCLLKokkos::~SurfCollideCLLKokkos() +{ + if (uncopy) { + fix_ambi_kk_copy.uncopy(); + fix_vibmode_kk_copy.uncopy(); + + for (int i = 0; i < KOKKOS_MAX_SURF_REACT_PER_TYPE; i++) { + sr_kk_global_copy[i].uncopy(); + sr_kk_prob_copy[i].uncopy(); + } + } + + if (copy) return; + +#ifdef SPARTA_KOKKOS_EXACT + rand_pool.destroy(); + if (random_backup) + delete random_backup; +#endif +} + +/* ---------------------------------------------------------------------- */ + +void SurfCollideCLLKokkos::init() +{ + SurfCollideCLL::init(); + + ambi_flag = vibmode_flag = 0; + if (modify->n_update_custom) { + for (int ifix = 0; ifix < modify->nfix; ifix++) { + if (strcmp(modify->fix[ifix]->style,"ambipolar") == 0) { + ambi_flag = 1; + FixAmbipolar *afix = (FixAmbipolar *) modify->fix[ifix]; + if (!afix->kokkos_flag) + error->all(FLERR,"Must use fix ambipolar/kk when Kokkos is enabled"); + afix_kk = (FixAmbipolarKokkos*)afix; + } else if (strcmp(modify->fix[ifix]->style,"vibmode") == 0) { + vibmode_flag = 1; + FixVibmode *vfix = (FixVibmode *) modify->fix[ifix]; + if (!vfix->kokkos_flag) + error->all(FLERR,"Must use fix vibmode/kk when Kokkos is enabled"); + vfix_kk = (FixVibmodeKokkos*)vfix; + } + } + } +} + +/* ---------------------------------------------------------------------- + recalculate Tsurf values which are dynamic + called by Update::setup() and Update::run() +---------------------------------------------------------------------- */ + +void SurfCollideCLLKokkos::dynamic() +{ + // VAREQUAL mode + // equal-style variable sets single tsurf value for all surfs + + if (tmode == VAREQUAL) { + + // only evaluate variable if timestep is multiple of tfreq + + if (update->ntimestep % tfreq) return; + tsurf = input->variable->compute_equal(tindex_var); + if (tsurf <= 0.0) error->all(FLERR,"Surf_collide tsurf <= 0.0"); + + // VARSURF mode + // surf-style variable sets new tsurf values for all surfs + // particle/surf collisions access t_persurf for local+ghost values + + } else if (tmode == VARSURF) { + + // only evaluate variable if timestep is multiple of tfreq + + int spreadflag = 0; + if (update->ntimestep % tfreq == 0) { + if (n_owned != surf->nown) { + memory->destroy(t_owned); + n_owned = surf->nown; + memory->create(t_owned,n_owned,"surfcollide:t_owned"); + } + + input->variable->compute_surf(tindex_var,t_owned,1,0); + spreadflag = 1; + } + + // spread t_owned values to t_localghost values via spread_own2local() + // if just re-computed variable OR surfs are + // distributed and load balance/adaptation took place on previous step + + if (spreadflag || + (surf->distributed && surf->localghost_changed_step == update->ntimestep-1)) { + if (n_localghost != surf->nlocal + surf->nghost) { + memory->destroy(t_localghost); + n_localghost = surf->nlocal + surf->nghost; + memory->create(t_localghost,n_localghost,"surfcollide:t_localghost"); + } + + surf->spread_own2local(1,DOUBLE,t_owned,t_localghost); + t_persurf = t_localghost; + + auto h_t_persurf = HAT::t_float_1d(t_persurf,n_localghost); + d_t_persurf = Kokkos::create_mirror_view_and_copy(SPADeviceType(),h_t_persurf); + } + + // CUSTOM mode + // ensure access to custom per-surf vec for tsurf values for all surfs + // particle/surf collisions access t_persurf for local+ghost values + + } else if (tmode == CUSTOM) { + SurfKokkos* surf_kk = (SurfKokkos*) surf; + auto h_edvec_local = surf_kk->k_edvec_local.view_host(); + + // spread owned values to local+ghost values via spread_custom() + // estatus == 1 means owned values already spread to local+ghost values + // if estatus == 0: owned values are new OR + // surfs are distributed and load balance/adaptation took place + + if (surf->estatus[tindex_custom] == 0) surf->spread_custom(tindex_custom); + + h_edvec_local[tindex_custom].k_view.sync_device(); + d_t_persurf = h_edvec_local[tindex_custom].k_view.view_device(); + } +} + +/* ---------------------------------------------------------------------- */ + +void SurfCollideCLLKokkos::pre_collide() +{ + if (ambi_flag) { + afix_kk->pre_update_custom_kokkos(); + fix_ambi_kk_copy.copy(afix_kk); + } + + if (vibmode_flag) { + vfix_kk->pre_update_custom_kokkos(); + fix_vibmode_kk_copy.copy(vfix_kk); + } + + if (surf->nsr > KOKKOS_MAX_TOT_SURF_REACT) + error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); + + if (surf->nsr > 0) { + int nglob,nprob; + nglob = nprob = 0; + for (int n = 0; n < surf->nsr; n++) { + if (!surf->sr[n]->kokkosable) + error->all(FLERR,"Must use Kokkos-enabled surface reaction method with Kokkos"); + if (strcmp(surf->sr[n]->style,"global") == 0) { + sr_kk_global_copy[nglob].copy((SurfReactGlobalKokkos*)(surf->sr[n])); + sr_kk_global_copy[nglob].obj.pre_react(); + sr_type_list[n] = 0; + sr_map[n] = nglob; + nglob++; + } else if (strcmp(surf->sr[n]->style,"prob") == 0) { + sr_kk_prob_copy[nprob].copy((SurfReactProbKokkos*)(surf->sr[n])); + sr_kk_prob_copy[nprob].obj.pre_react(); + sr_type_list[n] = 1; + sr_map[n] = nprob; + nprob++; + } else { + error->all(FLERR,"Unknown Kokkos surface reaction method"); + } + } + + if (nglob > KOKKOS_MAX_SURF_REACT_PER_TYPE || nprob > KOKKOS_MAX_SURF_REACT_PER_TYPE) + error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); + } + + if (random == NULL) { + // initialize RNG + + random = new RanKnuth(update->ranmaster->uniform()); + double seed = update->ranmaster->uniform(); + random->reset(seed,comm->me,100); + +#ifdef SPARTA_KOKKOS_EXACT + rand_pool.init(random); +#endif + } + + ParticleKokkos* particle_kk = (ParticleKokkos*) particle; + particle_kk->sync(Device,PARTICLE_MASK|SPECIES_MASK); + d_particles = particle_kk->k_particles.view_device(); + d_species = particle_kk->k_species.view_device(); + boltz = update->boltz; + + rotstyle = NONE; + if (Pointers::collide) rotstyle = Pointers::collide->rotstyle; + vibstyle = NONE; + if (Pointers::collide) vibstyle = Pointers::collide->vibstyle; + + Kokkos::deep_copy(d_scalars,0); +} + +/* ---------------------------------------------------------------------- */ + +void SurfCollideCLLKokkos::post_collide() +{ + ParticleKokkos* particle_kk = (ParticleKokkos*) particle; + if (ambi_flag || vibmode_flag) particle_kk->modify(Device,CUSTOM_MASK); + + Kokkos::deep_copy(h_scalars,d_scalars); + + int m = surf->find_collide(id); + auto sc = surf->sc[m]; // can't modify the copy directly, use the original + sc->nsingle += h_nsingle(); + surf->nreact_one += h_nreact_one(); + + d_particles = {}; +} + +/* ---------------------------------------------------------------------- */ + +void SurfCollideCLLKokkos::backup() +{ + ParticleKokkos* particle_kk = (ParticleKokkos*) particle; + d_particles = particle_kk->k_particles.view_device(); + + if (surf->nsr > 0) { + int nglob,nprob; + nglob = nprob = 0; + for (int n = 0; n < surf->nsr; n++) { + if (strcmp(surf->sr[n]->style,"global") == 0) { + sr_kk_global_copy[nglob].obj.backup(); + nglob++; + } else if (strcmp(surf->sr[n]->style,"prob") == 0) { + sr_kk_prob_copy[nprob].obj.backup(); + nprob++; + } + } + } + +#ifdef SPARTA_KOKKOS_EXACT + if (!random_backup) + random_backup = new RanKnuth(12345 + comm->me); + memcpy(random_backup,random,sizeof(RanKnuth)); +#endif +} + +/* ---------------------------------------------------------------------- */ + +void SurfCollideCLLKokkos::restore() +{ + if (surf->nsr > 0) { + int nglob,nprob; + nglob = nprob = 0; + for (int n = 0; n < surf->nsr; n++) { + if (strcmp(surf->sr[n]->style,"global") == 0) { + sr_kk_global_copy[nglob].obj.restore(); + nglob++; + } else if (strcmp(surf->sr[n]->style,"prob") == 0) { + sr_kk_prob_copy[nprob].obj.restore(); + nprob++; + } + } + } + + Kokkos::deep_copy(d_scalars,0); + +#ifdef SPARTA_KOKKOS_EXACT + memcpy(random,random_backup,sizeof(RanKnuth)); +#endif +} diff --git a/src/KOKKOS/surf_collide_cll_kokkos.h b/src/KOKKOS/surf_collide_cll_kokkos.h new file mode 100644 index 000000000..dd0ec9be5 --- /dev/null +++ b/src/KOKKOS/surf_collide_cll_kokkos.h @@ -0,0 +1,413 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#ifdef SURF_COLLIDE_CLASS + +SurfCollideStyle(cll/kk,SurfCollideCLLKokkos) + +#else + +#ifndef SPARTA_SURF_COLLIDE_CLL_KOKKOS_H +#define SPARTA_SURF_COLLIDE_CLL_KOKKOS_H + +#include "surf_collide_cll.h" +#include "kokkos_type.h" +#include "math_extra_kokkos.h" +#include "Kokkos_Random.hpp" +#include "rand_pool_wrap.h" +#include "kokkos_copy.h" +#include "fix_ambipolar_kokkos.h" +#include "fix_vibmode_kokkos.h" +#include "surf_react_global_kokkos.h" +#include "surf_react_prob_kokkos.h" + +namespace SPARTA_NS { + +class SurfCollideCLLKokkos : public SurfCollideCLL { + public: + + enum{NONE,DISCRETE,SMOOTH}; // several files + enum{PKEEP,PINSERT,PDONE,PDISCARD,PENTRY,PEXIT,PSURF}; // several files + + SurfCollideCLLKokkos(class SPARTA *, int, char **); + SurfCollideCLLKokkos(class SPARTA *); + ~SurfCollideCLLKokkos(); + void init(); + void dynamic(); + void pre_collide(); + void post_collide(); + void backup(); + void restore(); + + private: + double boltz; + int rotstyle, vibstyle; + +#ifndef SPARTA_KOKKOS_EXACT + Kokkos::Random_XorShift64_Pool rand_pool; + typedef typename Kokkos::Random_XorShift64_Pool::generator_type rand_type; +#else + RandPoolWrap rand_pool; + typedef RandWrap rand_type; +#endif + + RanKnuth* random_backup; + + DAT::t_float_1d d_t_persurf; + + typedef Kokkos::DualView tdual_int_2; + typedef tdual_int_2::t_dev t_int_2; + typedef tdual_int_2::t_host t_host_int_2; + t_int_2 d_scalars; + t_host_int_2 h_scalars; + + DAT::t_int_scalar d_nsingle; + DAT::t_int_scalar d_nreact_one; + + HAT::t_int_scalar h_nsingle; + HAT::t_int_scalar h_nreact_one; + + t_particle_1d d_particles; + t_species_1d d_species; + + int ambi_flag,vibmode_flag; + FixAmbipolarKokkos* afix_kk; + FixVibmodeKokkos* vfix_kk; + KKCopy fix_ambi_kk_copy; + KKCopy fix_vibmode_kk_copy; + + int sr_type_list[KOKKOS_MAX_TOT_SURF_REACT]; + int sr_map[KOKKOS_MAX_TOT_SURF_REACT]; + KKCopy sr_kk_global_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; + KKCopy sr_kk_prob_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; + + public: + + /* ---------------------------------------------------------------------- + particle collision with surface with optional chemistry + ip = particle with current x = collision pt, current v = incident v + isurf = index of surface element + norm = surface normal unit vector + isr = index of reaction model if >= 0, -1 for no chemistry + ip = set to NULL if destroyed by chemistry + return jp = new particle if created by chemistry + return reaction = index of reaction (1 to N) that took place, 0 = no reaction + resets particle(s) to post-collision outward velocity + ------------------------------------------------------------------------- */ + + template + KOKKOS_INLINE_FUNCTION + Particle::OnePart* collide_kokkos(Particle::OnePart *&ip, double &, + int isurf, const double *norm, int isr, int &reaction, + const DAT::t_int_scalar &d_retry, const DAT::t_int_scalar &d_nlocal) const + { + if (ATOMIC_REDUCTION == 0) + d_nsingle()++; + else + Kokkos::atomic_inc(&d_nsingle()); + + // if surface chemistry defined, attempt reaction + // reaction = 1 to N for which reaction took place, 0 for none + // velreset = 1 if reaction reset post-collision velocity, else 0 + + Particle::OnePart iorig; + Particle::OnePart *jp = NULL; + reaction = 0; + int velreset = 0; + + if (REACT) { + if (ambi_flag || vibmode_flag) memcpy(&iorig,ip,sizeof(Particle::OnePart)); + + int sr_type = sr_type_list[isr]; + int m = sr_map[isr]; + + if (sr_type == 0) { + reaction = sr_kk_global_copy[m].obj. + react_kokkos(ip,isurf,norm,jp,velreset,d_retry,d_nlocal); + } else if (sr_type == 1) { + reaction = sr_kk_prob_copy[m].obj. + react_kokkos(ip,isurf,norm,jp,velreset,d_retry,d_nlocal); + } + + if (reaction) { + if (ATOMIC_REDUCTION == 0) + d_nreact_one()++; + else + Kokkos::atomic_inc(&d_nreact_one()); + } + } + + // set temperature of isurf if VARSURF or CUSTOM + + double tsurf_local = tsurf; + if (persurf_temperature) { + tsurf_local = d_t_persurf[isurf]; + if (tsurf_local <= 0.0) Kokkos::abort("Surf_collide tsurf <= 0.0"); + } + + // CLL reflection for each particle + // only if SurfReact did not already reset velocities + // also both particles need to trigger any fixes + // to update per-particle properties which depend on + // temperature of the particle, e.g. fix vibmode and fix ambipolar + + if (ip) { + if (!velreset) cll(ip,norm,tsurf_local); + int i = ip - d_particles.data(); + if (ambi_flag) + fix_ambi_kk_copy.obj.update_custom_kokkos(i,tsurf_local,tsurf_local,tsurf_local,vstream); + if (vibmode_flag) + fix_vibmode_kk_copy.obj.update_custom_kokkos(i,tsurf_local,tsurf_local,tsurf_local,vstream); + } + if (REACT && jp) { + if (!velreset) cll(jp,norm,tsurf_local); + int j = jp - d_particles.data(); + if (ambi_flag) + fix_ambi_kk_copy.obj.update_custom_kokkos(j,tsurf_local,tsurf_local,tsurf_local,vstream); + if (vibmode_flag) + fix_vibmode_kk_copy.obj.update_custom_kokkos(j,tsurf_local,tsurf_local,tsurf_local,vstream); + } + + // call any fixes with a surf_react() method + // they may reset j to -1, e.g. fix ambipolar + // in which case newly created j is deleted + + if (REACT && reaction && ambi_flag) { + int i = -1; + if (ip) i = ip - d_particles.data(); + int j = -1; + if (jp) j = jp - d_particles.data(); + int j_orig = j; + fix_ambi_kk_copy.obj.surf_react_kokkos(&iorig,i,j); + if (jp && j < 0) { + d_particles[j_orig].flag = PDISCARD; + jp = NULL; + } + } + + return jp; + }; + + private: + + /* ---------------------------------------------------------------------- + cll reflection + ------------------------------------------------------------------------- */ + + KOKKOS_INLINE_FUNCTION + void cll(Particle::OnePart *p, const double *norm, const double twall) const + { + rand_type rand_gen = rand_pool.get_state(); + + double tangent1[3],tangent2[3]; + int ispecies = p->ispecies; + double beta_un,normalized_distbn_fn; + + double *v = p->v; + double dot = MathExtraKokkos::dot3(v,norm); + double vrm, vperp, vtan1, vtan2; + + tangent1[0] = v[0] - dot*norm[0]; + tangent1[1] = v[1] - dot*norm[1]; + tangent1[2] = v[2] - dot*norm[2]; + + if (MathExtraKokkos::lensq3(tangent1) == 0.0) { + tangent2[0] = rand_gen.drand(); + tangent2[1] = rand_gen.drand(); + tangent2[2] = rand_gen.drand(); + MathExtraKokkos::cross3(norm,tangent2,tangent1); + } + + MathExtraKokkos::norm3(tangent1); + MathExtraKokkos::cross3(norm,tangent1,tangent2); + + double tan1 = MathExtraKokkos::dot3(v,tangent1); + + vrm = sqrt(2.0*boltz * twall / d_species[ispecies].mass); + + // CLL model normal velocity + + double r_1 = sqrt(-acc_n*log(rand_gen.drand())); + double theta_1 = MathConst::MY_2PI * rand_gen.drand(); + double dot_norm = dot/vrm * sqrt(1-acc_n); + vperp = vrm * sqrt(r_1*r_1 + dot_norm*dot_norm + 2*r_1*dot_norm*cos(theta_1)); + + // CLL model tangential velocities + + double r_2 = sqrt(-acc_t*log(rand_gen.drand())); + double theta_2 = MathConst::MY_2PI * rand_gen.drand(); + double vtangent = tan1/vrm * sqrt(1-acc_t); + vtan1 = vrm * (vtangent + r_2*cos(theta_2)); + vtan2 = vrm * r_2 * sin(theta_2); + + // partial keyword + // incomplete energy accommodation with partial/fully diffuse scattering + // adjust the final angle of the particle while keeping + // the velocity magnitude or speed according to CLL scattering + + if (pflag) { + double tan2 = MathExtraKokkos::dot3(v,tangent2); + double phi_i, psi_i, theta_f, phi_f, psi_f, cos_beta; + + psi_i = acos(dot*dot/MathExtraKokkos::lensq3(v)); + phi_i = atan2(tan2,tan1); + + double v_mag = sqrt(vperp*vperp + vtan1*vtan1 + vtan2*vtan2); + + double P = 0; + while (rand_gen.drand() > P) { + phi_f = MathConst::MY_2PI*rand_gen.drand(); + psi_f = acos(1-rand_gen.drand()); + cos_beta = cos(psi_i)*cos(psi_f) + + sin(psi_i)*sin(psi_f)*cos(phi_i - phi_f); + P = (1-eccen)/(1-eccen*cos_beta); + } + + theta_f = acos(sqrt(cos(psi_f))); + + vperp = v_mag * cos(theta_f); + vtan1 = v_mag * sin(theta_f) * cos(phi_f); + vtan2 = v_mag * sin(theta_f) * sin(phi_f); + } + + // add in translation or rotation vector if specified + // only keep portion of vector tangential to surface element + + if (trflag) { + double vxdelta,vydelta,vzdelta; + if (tflag) { + vxdelta = vx; vydelta = vy; vzdelta = vz; + double dot = vxdelta*norm[0] + vydelta*norm[1] + vzdelta*norm[2]; + + if (fabs(dot) > 0.001) { + dot /= vrm; + do { + do { + beta_un = (6.0*rand_gen.normal() - 3.0); + } while (beta_un + dot < 0.0); + normalized_distbn_fn = 2.0 * (beta_un + dot) / + (dot + sqrt(dot*dot + 2.0)) * + exp(0.5 + (0.5*dot)*(dot-sqrt(dot*dot + 2.0)) - beta_un*beta_un); + } while (normalized_distbn_fn < rand_gen.drand()); + vperp = beta_un*vrm; + } + + } else { + double *x = p->x; + vxdelta = wy*(x[2]-pz) - wz*(x[1]-py); + vydelta = wz*(x[0]-px) - wx*(x[2]-pz); + vzdelta = wx*(x[1]-py) - wy*(x[0]-px); + double dot = vxdelta*norm[0] + vydelta*norm[1] + vzdelta*norm[2]; + vxdelta -= dot*norm[0]; + vydelta -= dot*norm[1]; + vzdelta -= dot*norm[2]; + } + + v[0] = vperp*norm[0] + vtan1*tangent1[0] + vtan2*tangent2[0] + vxdelta; + v[1] = vperp*norm[1] + vtan1*tangent1[1] + vtan2*tangent2[1] + vydelta; + v[2] = vperp*norm[2] + vtan1*tangent1[2] + vtan2*tangent2[2] + vzdelta; + + // no translation or rotation + + } else { + v[0] = vperp*norm[0] + vtan1*tangent1[0] + vtan2*tangent2[0]; + v[1] = vperp*norm[1] + vtan1*tangent1[1] + vtan2*tangent2[1]; + v[2] = vperp*norm[2] + vtan1*tangent1[2] + vtan2*tangent2[2]; + } + + // rotational component + + if (rotstyle == NONE || d_species[ispecies].rotdof < 2) p->erot = 0.0; + + else { + double erot_mag = sqrt(p->erot*(1-acc_rot)/(boltz*twall)); + + double r_rot,cos_theta_rot,A_rot,X_rot; + if (d_species[ispecies].rotdof == 2) { + r_rot = sqrt(-acc_rot*log(rand_gen.drand())); + cos_theta_rot = cos(MathConst::MY_2PI*rand_gen.drand()); + } + else if (d_species[ispecies].rotdof > 2) { + A_rot = 0; + while (A_rot < rand_gen.drand()) { + X_rot = 4*rand_gen.drand(); + A_rot = 2.71828182845904523536028747*X_rot*X_rot*exp(-X_rot*X_rot); + } + r_rot = sqrt(acc_rot)*X_rot; + cos_theta_rot = 2*rand_gen.drand() - 1; + } + + p->erot = boltz * twall * + (r_rot*r_rot + erot_mag*erot_mag + 2*r_rot*erot_mag*cos_theta_rot); + } + + // vibrational component + + int vibdof = d_species[ispecies].vibdof; + double r_vib, cos_theta_vib, A_vib, X_vib, evib_mag, evib_val; + + if (vibstyle == NONE || vibdof < 2) + p->evib = 0.0; + + else if (vibstyle == DISCRETE && vibdof == 2) { + double evib_star = + -log(1 - rand_gen.drand() * + (1 - exp(-boltz*d_species[ispecies].vibtemp[0]))); + evib_val = p->evib + evib_star; + evib_mag = sqrt(evib_val*(1-acc_vib)/(boltz*twall)); + r_vib = sqrt(-acc_vib*log(rand_gen.drand())); + cos_theta_vib = cos(MathConst::MY_2PI*rand_gen.drand()); + evib_val = boltz * twall * + (r_vib*r_vib + evib_mag*evib_mag + 2*r_vib*evib_mag*cos_theta_vib); + int ivib = evib_val / (boltz*d_species[ispecies].vibtemp[0]); + p->evib = ivib * boltz * d_species[ispecies].vibtemp[0]; + } + + else if (vibstyle == SMOOTH || vibdof >= 2) { + evib_mag = sqrt(p->evib*(1-acc_vib)/(boltz*twall)); + if (vibdof == 2) { + r_vib = sqrt(-acc_vib*log(rand_gen.drand())); + cos_theta_vib = cos(MathConst::MY_2PI*rand_gen.drand()); + } else if (vibdof > 2) { + A_vib = 0; + while (A_vib < rand_gen.drand()) { + X_vib = 4*rand_gen.drand(); + A_vib = 2.71828182845904523536028747*X_vib*X_vib*exp(-X_vib*X_vib); + } + r_vib = sqrt(acc_vib)*X_vib; + cos_theta_vib = 2*rand_gen.drand() - 1; + } + + p->evib = boltz * twall * + (r_vib*r_vib + evib_mag*evib_mag + 2*r_vib*evib_mag*cos_theta_vib); + } + + rand_pool.free_state(rand_gen); + } +}; + +} + +#endif +#endif + +/* ERROR/WARNING messages: + +E: Illegal ... command + +Self-explanatory. Check the input script syntax and compare to the +documentation for the command. You can use -echo screen as a +command-line option when running SPARTA to see the offending line. + +*/ diff --git a/src/KOKKOS/update_kokkos.cpp b/src/KOKKOS/update_kokkos.cpp index bc9bb0ab4..0d45f9489 100644 --- a/src/KOKKOS/update_kokkos.cpp +++ b/src/KOKKOS/update_kokkos.cpp @@ -83,6 +83,7 @@ UpdateKokkos::UpdateKokkos(SPARTA *sparta) : Update(sparta), sc_kk_adiabatic_copy{VAL_2(KKCopy(sparta))}, sc_kk_impulsive_copy{VAL_2(KKCopy(sparta))}, sc_kk_td_copy{VAL_2(KKCopy(sparta))}, + sc_kk_cll_copy{VAL_2(KKCopy(sparta))}, blist_active_copy{VAL_2(KKCopy(sparta))}, slist_active_copy{VAL_2(KKCopy(sparta))}, tmp_compute_boundary_kk(sparta), @@ -151,6 +152,7 @@ UpdateKokkos::~UpdateKokkos() sc_kk_adiabatic_copy[i].uncopy(); sc_kk_impulsive_copy[i].uncopy(); sc_kk_td_copy[i].uncopy(); + sc_kk_cll_copy[i].uncopy(); } for (int i=0; i void UpdateKokkos::move() error->all(FLERR,"Kokkos currently supports two instances of each surface collide method"); if (surf->nsc > 0) { - int nspec,ndiff,nvan,npist,ntrans,nadia,nimpul,ntd; - nspec = ndiff = nvan = npist = ntrans = nadia = nimpul = ntd = 0; + int nspec,ndiff,nvan,npist,ntrans,nadia,nimpul,ntd,ncll; + nspec = ndiff = nvan = npist = ntrans = nadia = nimpul = ntd = ncll = 0; for (int n = 0; n < surf->nsc; n++) { if (!surf->sc[n]->kokkosable) error->all(FLERR,"Must use Kokkos-enabled surface collide method with Kokkos"); @@ -590,6 +592,12 @@ template < int DIM, int SURF, int REACT, int OPT > void UpdateKokkos::move() sc_type_list[n] = 7; sc_map[n] = ntd; ntd++; + } else if (strcmp(surf->sc[n]->style,"cll") == 0) { + sc_kk_cll_copy[ncll].copy((SurfCollideCLLKokkos*)(surf->sc[n])); + sc_kk_cll_copy[ncll].obj.pre_collide(); + sc_type_list[n] = 8; + sc_map[n] = ncll; + ncll++; } else { error->all(FLERR,"Unknown Kokkos surface collide method"); } @@ -597,7 +605,8 @@ template < int DIM, int SURF, int REACT, int OPT > void UpdateKokkos::move() if (nspec > KOKKOS_MAX_SURF_COLL_PER_TYPE || ndiff > KOKKOS_MAX_SURF_COLL_PER_TYPE || nvan > KOKKOS_MAX_SURF_COLL_PER_TYPE || npist > KOKKOS_MAX_SURF_COLL_PER_TYPE || ntrans > KOKKOS_MAX_SURF_COLL_PER_TYPE || nadia > KOKKOS_MAX_SURF_COLL_PER_TYPE || - nimpul > KOKKOS_MAX_SURF_COLL_PER_TYPE || ntd > KOKKOS_MAX_SURF_COLL_PER_TYPE) + nimpul > KOKKOS_MAX_SURF_COLL_PER_TYPE || ntd > KOKKOS_MAX_SURF_COLL_PER_TYPE || + ncll > KOKKOS_MAX_SURF_COLL_PER_TYPE) error->all(FLERR,"Kokkos currently supports two instances of each surface collide method"); } @@ -738,8 +747,8 @@ template < int DIM, int SURF, int REACT, int OPT > void UpdateKokkos::move() } if (surf->nsc > 0) { - int nspec,ndiff,nvan,npist,ntrans,nadia,nimpul,ntd; - nspec = ndiff = nvan = npist = ntrans = nadia = nimpul = ntd = 0; + int nspec,ndiff,nvan,npist,ntrans,nadia,nimpul,ntd,ncll; + nspec = ndiff = nvan = npist = ntrans = nadia = nimpul = ntd = ncll = 0; for (int n = 0; n < surf->nsc; n++) { if (strcmp(surf->sc[n]->style,"specular") == 0) { sc_kk_specular_copy[nspec].obj.post_collide(); @@ -765,6 +774,9 @@ template < int DIM, int SURF, int REACT, int OPT > void UpdateKokkos::move() } else if (strcmp(surf->sc[n]->style,"td") == 0) { sc_kk_td_copy[ntd].obj.post_collide(); ntd++; + } else if (strcmp(surf->sc[n]->style,"cll") == 0) { + sc_kk_cll_copy[ncll].obj.post_collide(); + ncll++; } } } @@ -1425,6 +1437,9 @@ void UpdateKokkos::operator()(TagUpdateMove } else if (sc_type == 7) { jpart = sc_kk_td_copy[m].obj. collide_kokkos(ipart,dtremain,minsurf,tri->norm,tri->isr,reaction,d_retry,d_nlocal); + } else if (sc_type == 8) { + jpart = sc_kk_cll_copy[m].obj. + collide_kokkos(ipart,dtremain,minsurf,tri->norm,tri->isr,reaction,d_retry,d_nlocal); } } @@ -1453,6 +1468,9 @@ void UpdateKokkos::operator()(TagUpdateMove } else if (sc_type == 7) { jpart = sc_kk_td_copy[m].obj. collide_kokkos(ipart,dtremain,minsurf,line->norm,line->isr,reaction,d_retry,d_nlocal); + } else if (sc_type == 8) { + jpart = sc_kk_cll_copy[m].obj. + collide_kokkos(ipart,dtremain,minsurf,line->norm,line->isr,reaction,d_retry,d_nlocal); } } @@ -1709,6 +1727,9 @@ void UpdateKokkos::operator()(TagUpdateMove else if (sc_type == 7) jpart = sc_kk_td_copy[m].obj. collide_kokkos(ipart,dtremain,-(outface+1),domain_kk_copy.obj.norm[outface],domain_kk_copy.obj.surf_react[outface],reaction,d_retry,d_nlocal); + else if (sc_type == 8) + jpart = sc_kk_cll_copy[m].obj. + collide_kokkos(ipart,dtremain,-(outface+1),domain_kk_copy.obj.norm[outface],domain_kk_copy.obj.surf_react[outface],reaction,d_retry,d_nlocal); if (ipart) { double *x = ipart->x; @@ -2066,8 +2087,8 @@ void UpdateKokkos::backup() Kokkos::deep_copy(d_particles_backup,d_particles); if (surf->nsc > 0) { - int nspec,ndiff,npist,nadia,nimpul,ntd; - nspec = ndiff = npist = nadia = nimpul = ntd = 0; + int nspec,ndiff,npist,nadia,nimpul,ntd,ncll; + nspec = ndiff = npist = nadia = nimpul = ntd = ncll = 0; for (int n = 0; n < surf->nsc; n++) { if (strcmp(surf->sc[n]->style,"specular") == 0) { sc_kk_specular_copy[nspec].obj.backup(); @@ -2087,6 +2108,9 @@ void UpdateKokkos::backup() } else if (strcmp(surf->sc[n]->style,"td") == 0) { sc_kk_td_copy[ntd].obj.backup(); ntd++; + } else if (strcmp(surf->sc[n]->style,"cll") == 0) { + sc_kk_cll_copy[ncll].obj.backup(); + ncll++; } } } @@ -2101,8 +2125,8 @@ void UpdateKokkos::restore() d_particles = particle_kk->k_particles.view_device(); if (surf->nsc > 0) { - int nspec,ndiff,npist,nadia,nimpul,ntd; - nspec = ndiff = npist = nadia = nimpul = ntd = 0; + int nspec,ndiff,npist,nadia,nimpul,ntd,ncll; + nspec = ndiff = npist = nadia = nimpul = ntd = ncll = 0; for (int n = 0; n < surf->nsc; n++) { if (strcmp(surf->sc[n]->style,"specular") == 0) { sc_kk_specular_copy[nspec].obj.restore(); @@ -2122,6 +2146,9 @@ void UpdateKokkos::restore() } else if (strcmp(surf->sc[n]->style,"td") == 0) { sc_kk_td_copy[ntd].obj.restore(); ntd++; + } else if (strcmp(surf->sc[n]->style,"cll") == 0) { + sc_kk_cll_copy[ncll].obj.restore(); + ncll++; } } } diff --git a/src/KOKKOS/update_kokkos.h b/src/KOKKOS/update_kokkos.h index afd121369..9136a81d2 100644 --- a/src/KOKKOS/update_kokkos.h +++ b/src/KOKKOS/update_kokkos.h @@ -29,6 +29,7 @@ #include "surf_collide_adiabatic_kokkos.h" #include "surf_collide_impulsive_kokkos.h" #include "surf_collide_td_kokkos.h" +#include "surf_collide_cll_kokkos.h" #include "compute_boundary_kokkos.h" #include "compute_surf_kokkos.h" @@ -142,6 +143,7 @@ class UpdateKokkos : public Update { KKCopy sc_kk_adiabatic_copy[KOKKOS_MAX_SURF_COLL_PER_TYPE]; KKCopy sc_kk_impulsive_copy[KOKKOS_MAX_SURF_COLL_PER_TYPE]; KKCopy sc_kk_td_copy[KOKKOS_MAX_SURF_COLL_PER_TYPE]; + KKCopy sc_kk_cll_copy[KOKKOS_MAX_SURF_COLL_PER_TYPE]; //KKCopy blist_active_copy[KOKKOS_MAX_GLIST]; KKCopy slist_active_copy[KOKKOS_MAX_SLIST]; diff --git a/src/surf_collide_cll.cpp b/src/surf_collide_cll.cpp index 2751c3fd3..836789bb3 100644 --- a/src/surf_collide_cll.cpp +++ b/src/surf_collide_cll.cpp @@ -136,6 +136,8 @@ SurfCollideCLL::SurfCollideCLL(SPARTA *sparta, int narg, char **arg) : SurfCollideCLL::~SurfCollideCLL() { + if (copy) return; + delete random; } diff --git a/src/surf_collide_cll.h b/src/surf_collide_cll.h index 9bdfb200b..b80bcdb4d 100644 --- a/src/surf_collide_cll.h +++ b/src/surf_collide_cll.h @@ -29,6 +29,7 @@ namespace SPARTA_NS { class SurfCollideCLL : public SurfCollide { public: SurfCollideCLL(class SPARTA *, int, char **); + SurfCollideCLL(class SPARTA *sparta) : SurfCollide(sparta) {} // needed for Kokkos ~SurfCollideCLL(); void init(); Particle::OnePart *collide(Particle::OnePart *&, double &, @@ -36,7 +37,7 @@ class SurfCollideCLL : public SurfCollide { void wrapper(Particle::OnePart *, double *, int *, double*); void flags_and_coeffs(int *, double *); - private: + protected: double acc_n,acc_t,acc_rot,acc_vib; // surface accomodation coeffs double vx,vy,vz; // translational velocity of surface double wx,wy,wz; // angular velocity of surface From 14b03ffae09374be0a5fad1b656e27de722cc45e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 19 Jun 2026 18:46:58 +0000 Subject: [PATCH 05/30] KOKKOS: add device wrapper_kokkos to diffuse/specular surf collide Prerequisite for the surf_react adsorb Kokkos port (GS chemistry re-scatters a product particle via cmodels[...]->wrapper()). Add a device-callable KOKKOS_INLINE_FUNCTION wrapper_kokkos(p,norm,flags,coeffs) to SurfCollideDiffuseKokkos and SurfCollideSpecularKokkos mirroring the host wrapper(): diffuse applies coeffs[0]=tsurf, coeffs[1]=acc then reflects; specular reflects. To keep the object const in the functor copy, the device diffuse() now takes acc as a parameter (collide_kokkos passes the member, so that path is unchanged) and wrapper_kokkos passes the reaction's coeffs. These wrappers are not yet called (SurfReactAdsorbKokkos lands in later slices); this commit only adds the infrastructure. Verified no regression: diffuse and specular remain bit-for-bit identical CPU vs -sf kk under Serial+EXACT on examples/surf_collide/in.circle.{diffuse,specular}. Co-Authored-By: Claude Opus 4.8 Co-Authored-By: Stan Moore Claude-Session: https://claude.ai/code/session_01Q6nQVuxEiTUqSzDoDppCUM --- src/KOKKOS/surf_collide_diffuse_kokkos.h | 27 ++++++++++++++++++++--- src/KOKKOS/surf_collide_specular_kokkos.h | 13 +++++++++++ 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/src/KOKKOS/surf_collide_diffuse_kokkos.h b/src/KOKKOS/surf_collide_diffuse_kokkos.h index 25826164a..321ccdbb1 100644 --- a/src/KOKKOS/surf_collide_diffuse_kokkos.h +++ b/src/KOKKOS/surf_collide_diffuse_kokkos.h @@ -165,7 +165,7 @@ class SurfCollideDiffuseKokkos : public SurfCollideDiffuse { // temperature of the particle, e.g. fix vibmode and fix ambipolar if (ip) { - if (!velreset) diffuse(ip,norm,tsurf_local); + if (!velreset) diffuse(ip,norm,tsurf_local,acc); int i = ip - d_particles.data(); if (ambi_flag) fix_ambi_kk_copy.obj.update_custom_kokkos(i,tsurf_local,tsurf_local,tsurf_local,vstream); @@ -173,7 +173,7 @@ class SurfCollideDiffuseKokkos : public SurfCollideDiffuse { fix_vibmode_kk_copy.obj.update_custom_kokkos(i,tsurf_local,tsurf_local,tsurf_local,vstream); } if (REACT && jp) { - if (!velreset) diffuse(jp,norm,tsurf_local); + if (!velreset) diffuse(jp,norm,tsurf_local,acc); int j = jp - d_particles.data(); if (ambi_flag) fix_ambi_kk_copy.obj.update_custom_kokkos(j,tsurf_local,tsurf_local,tsurf_local,vstream); @@ -201,10 +201,31 @@ class SurfCollideDiffuseKokkos : public SurfCollideDiffuse { return jp; }; + public: + + /* ---------------------------------------------------------------------- + wrapper on diffuse() to perform a collision for a single particle + called on-device by SurfReactAdsorbKokkos GS chemistry + flags, coeffs can be NULL; matches SurfCollideDiffuse::wrapper + ------------------------------------------------------------------------- */ + + KOKKOS_INLINE_FUNCTION + void wrapper_kokkos(Particle::OnePart *p, const double *norm, + int *, double *coeffs) const + { + double twall = tsurf; + double acc_local = acc; + if (coeffs) { + twall = coeffs[0]; + acc_local = coeffs[1]; + } + diffuse(p,norm,twall,acc_local); + } + private: KOKKOS_INLINE_FUNCTION - void diffuse(Particle::OnePart *p, const double *norm, const double twall) const + void diffuse(Particle::OnePart *p, const double *norm, const double twall, const double acc) const { // specular reflection // reflect incident v around norm diff --git a/src/KOKKOS/surf_collide_specular_kokkos.h b/src/KOKKOS/surf_collide_specular_kokkos.h index d225b150c..d36c0eb70 100644 --- a/src/KOKKOS/surf_collide_specular_kokkos.h +++ b/src/KOKKOS/surf_collide_specular_kokkos.h @@ -202,6 +202,19 @@ class SurfCollideSpecularKokkos : public SurfCollideSpecular { return jp; }; + + /* ---------------------------------------------------------------------- + wrapper on specular reflection to perform a collision for a single particle + called on-device by SurfReactAdsorbKokkos GS chemistry + flags, coeffs can be NULL; matches SurfCollideSpecular::wrapper + ------------------------------------------------------------------------- */ + + KOKKOS_INLINE_FUNCTION + void wrapper_kokkos(Particle::OnePart *p, const double *norm, + int *, double *) const + { + MathExtraKokkos::reflect3(p->v,norm); + } }; } From 20ee4712fd9edcb32c5a95ae69dc606cb8d7c6b2 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 19 Jun 2026 19:05:45 +0000 Subject: [PATCH 06/30] KOKKOS: port fix temp/global/rescale to Kokkos Add FixTempGlobalRescaleKokkos (style temp/global/rescale/kk), a device port of fix temp/global/rescale. end_of_step() runs two device passes over all particles: a parallel_reduce to accumulate t = sum mass*(v.v), and a parallel_for to rescale velocities by vscale = sqrt(t_target/t_current). The global reduction / MPI_Allreduce / scale-factor math is unchanged from the host version, and particle data is synced to Device and marked modified after rescaling. Verified on a 2d circle flow (in.circle.diffuse + fix temp/global/rescale): CPU vs -sf kk are bit-for-bit identical under Serial+EXACT (the Serial reduce accumulates in index order, matching the host serial sum), the fix demonstrably changes the run vs the no-fix baseline, and a 4-thread OpenMP run is clean and statistically consistent (np within ~0.4% of serial). Co-Authored-By: Claude Opus 4.8 Co-Authored-By: Stan Moore Claude-Session: https://claude.ai/code/session_01Q6nQVuxEiTUqSzDoDppCUM --- src/KOKKOS/fix_temp_global_rescale_kokkos.cpp | 106 ++++++++++++++++++ src/KOKKOS/fix_temp_global_rescale_kokkos.h | 64 +++++++++++ 2 files changed, 170 insertions(+) create mode 100644 src/KOKKOS/fix_temp_global_rescale_kokkos.cpp create mode 100644 src/KOKKOS/fix_temp_global_rescale_kokkos.h diff --git a/src/KOKKOS/fix_temp_global_rescale_kokkos.cpp b/src/KOKKOS/fix_temp_global_rescale_kokkos.cpp new file mode 100644 index 000000000..2cc470e35 --- /dev/null +++ b/src/KOKKOS/fix_temp_global_rescale_kokkos.cpp @@ -0,0 +1,106 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#include "fix_temp_global_rescale_kokkos.h" +#include "update.h" +#include "particle_kokkos.h" +#include "sparta_masks.h" + +using namespace SPARTA_NS; + +/* ---------------------------------------------------------------------- */ + +FixTempGlobalRescaleKokkos::FixTempGlobalRescaleKokkos(SPARTA *sparta, int narg, char **arg) : + FixTempGlobalRescale(sparta, narg, arg) +{ + kokkos_flag = 1; + execution_space = Device; + datamask_read = EMPTY_MASK; + datamask_modify = EMPTY_MASK; +} + +/* ---------------------------------------------------------------------- */ + +void FixTempGlobalRescaleKokkos::end_of_step() +{ + if (update->ntimestep % nevery) return; + + // set current t_target + + double delta = update->ntimestep - update->beginstep; + if (delta != 0.0) delta /= update->endstep - update->beginstep; + double t_target = tstart + delta * (tstop-tstart); + + // t_current = global temperature + // just return if no particles or t_current = 0.0 + + ParticleKokkos* particle_kk = (ParticleKokkos*) particle; + particle_kk->sync(Device,PARTICLE_MASK|SPECIES_MASK); + d_particles = particle_kk->k_particles.view_device(); + d_species = particle_kk->k_species.view_device(); + + int nlocal = particle->nlocal; + + // 1st pass: t = sum over my particles of mass*(v.v) + + double t = 0.0; + + copymode = 1; + Kokkos::parallel_reduce(Kokkos::RangePolicy(0,nlocal),*this,t); + copymode = 0; + + double t_current; + MPI_Allreduce(&t,&t_current,1,MPI_DOUBLE,MPI_SUM,world); + + bigint n = particle->nlocal; + MPI_Allreduce(&n,&particle->nglobal,1,MPI_SPARTA_BIGINT,MPI_SUM,world); + if (particle->nglobal == 0 || t_current == 0.0) return; + + double tscale = update->mvv2e / (3.0 * particle->nglobal * update->boltz); + t_current *= tscale; + + // rescale all particle velocities + + t_target = t_current - fraction*(t_current-t_target); + vscale = sqrt(t_target/t_current); + + // 2nd pass: rescale velocities of all my particles + + copymode = 1; + Kokkos::parallel_for(Kokkos::RangePolicy(0,nlocal),*this); + copymode = 0; + + particle_kk->modify(Device,PARTICLE_MASK); +} + +/* ---------------------------------------------------------------------- */ + +KOKKOS_INLINE_FUNCTION +void FixTempGlobalRescaleKokkos::operator()(TagFixTempGlobalRescale_reduce, + const int &i, double &t) const { + const double *v = d_particles[i].v; + t += (v[0]*v[0] + v[1]*v[1] + v[2]*v[2]) * + d_species[d_particles[i].ispecies].mass; +} + +/* ---------------------------------------------------------------------- */ + +KOKKOS_INLINE_FUNCTION +void FixTempGlobalRescaleKokkos::operator()(TagFixTempGlobalRescale_scale, + const int &i) const { + double *v = d_particles[i].v; + v[0] *= vscale; + v[1] *= vscale; + v[2] *= vscale; +} diff --git a/src/KOKKOS/fix_temp_global_rescale_kokkos.h b/src/KOKKOS/fix_temp_global_rescale_kokkos.h new file mode 100644 index 000000000..3dfa260cb --- /dev/null +++ b/src/KOKKOS/fix_temp_global_rescale_kokkos.h @@ -0,0 +1,64 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#ifdef FIX_CLASS + +FixStyle(temp/global/rescale/kk,FixTempGlobalRescaleKokkos) + +#else + +#ifndef SPARTA_FIX_TEMP_GLOBAL_RESCALE_KOKKOS_H +#define SPARTA_FIX_TEMP_GLOBAL_RESCALE_KOKKOS_H + +#include "fix_temp_global_rescale.h" +#include "kokkos_type.h" + +namespace SPARTA_NS { + +struct TagFixTempGlobalRescale_reduce{}; +struct TagFixTempGlobalRescale_scale{}; + +class FixTempGlobalRescaleKokkos : public FixTempGlobalRescale { + public: + FixTempGlobalRescaleKokkos(class SPARTA *, int, char **); + virtual ~FixTempGlobalRescaleKokkos() {} + void end_of_step() override; + + KOKKOS_INLINE_FUNCTION + void operator()(TagFixTempGlobalRescale_reduce, const int&, double&) const; + + KOKKOS_INLINE_FUNCTION + void operator()(TagFixTempGlobalRescale_scale, const int&) const; + + private: + double vscale; + + t_particle_1d d_particles; + t_species_1d d_species; +}; + +} + +#endif +#endif + +/* ERROR/WARNING messages: + +E: Illegal ... command + +Self-explanatory. Check the input script syntax and compare to the +documentation for the command. You can use -echo screen as a +command-line option when running SPARTA to see the offending line. + +*/ From f947b477fc82a18272928cfa61555196ec56a174 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 20 Jun 2026 17:05:24 +0000 Subject: [PATCH 07/30] KOKKOS: enable flux-driven implicit-surface ablation end-to-end Make the standard ablation pipeline (compute isurf/grid -> fix ave/grid -> fix ablate) run under -sf kk. Previously update_kokkos errored with "Kokkos doesn't yet support compute isurf/grid" and fix ave/grid errored on grid/surf (PERGRIDSURF) inputs, so only random-decrement ablation worked on device. - ComputeISurfGridKokkos: new Kokkos compute that tallies per-surf flux/force on-device in surf_tally_kk() (mirrors ComputeSurfKokkos, with the isurf/grid keyword subset). tallyinfo() syncs+compresses the tally to the host arrays the host fix ave/grid PERGRIDSURF path consumes; post-processing to per-grid (grid->collate_array_implicit) stays on the host. Base ComputeISurfGrid gets the Kokkos copy ctor, virtual init_normflux/grow_tally, and the copy/copymode destructor guard. - update_kokkos: partition the active surf-tally computes into compute surf (slist_active_copy) and compute isurf/grid (slist_active_isurf_copy) typed copy arrays; the move kernel's surface-collision loop now invokes surf_tally_kk() on both. nsurf_tally still counts the total (so iorig is saved correctly); nslist_surf + nslist_isurf == nsurf_tally. - fix_ave_grid_kokkos: PERGRIDSURF runs on the host (the device per-surf tally is brought to the host by the compute's tallyinfo(), then the host base class collates to per-grid). Skip Kokkos allocation in the ctor and delegate init/setup/end_of_step/grow_percell to FixAveGrid; the grow_percell delegation fixes a shutdown invalid-free when fix balance changed the grid (host arrays were being reallocated with Kokkos memory). Verified bit-for-bit CPU vs -sf kk under Serial+EXACT on examples/ablation/ in.ablation.2d (full compute isurf/grid -> fix ave/grid -> fix ablate -> fix balance chain): Np, Nscoll, Nscheck and f_ablate (surface state 716805 -> 714047 over 500 steps) all identical. Also verified no regression in the compute surf tally path (adjust_temp/in.circle.constant, compute surf etot) and the no-tally path (surf_collide/in.circle.diffuse). Co-Authored-By: Claude Opus 4.8 Co-Authored-By: Stan Moore Claude-Session: https://claude.ai/code/session_01Q6nQVuxEiTUqSzDoDppCUM --- src/KOKKOS/compute_isurf_grid_kokkos.cpp | 199 ++++++++++++++++ src/KOKKOS/compute_isurf_grid_kokkos.h | 287 +++++++++++++++++++++++ src/KOKKOS/fix_ave_grid_kokkos.cpp | 36 ++- src/KOKKOS/update_kokkos.cpp | 75 ++++-- src/KOKKOS/update_kokkos.h | 10 + src/compute_isurf_grid.cpp | 2 + src/compute_isurf_grid.h | 5 +- 7 files changed, 589 insertions(+), 25 deletions(-) create mode 100644 src/KOKKOS/compute_isurf_grid_kokkos.cpp create mode 100644 src/KOKKOS/compute_isurf_grid_kokkos.h diff --git a/src/KOKKOS/compute_isurf_grid_kokkos.cpp b/src/KOKKOS/compute_isurf_grid_kokkos.cpp new file mode 100644 index 000000000..a325b5aa7 --- /dev/null +++ b/src/KOKKOS/compute_isurf_grid_kokkos.cpp @@ -0,0 +1,199 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#include "string.h" +#include "compute_isurf_grid_kokkos.h" +#include "particle_kokkos.h" +#include "mixture.h" +#include "surf_kokkos.h" +#include "grid.h" +#include "update.h" +#include "memory_kokkos.h" +#include "error.h" +#include "sparta_masks.h" +#include "kokkos.h" + +using namespace SPARTA_NS; + +/* ---------------------------------------------------------------------- */ + +ComputeISurfGridKokkos::ComputeISurfGridKokkos(SPARTA *sparta, int narg, char **arg) : + ComputeISurfGrid(sparta, narg, arg) +{ + kokkos_flag = 1; + + // hash is allocated/used only on the host; not needed for device tally + + d_which = DAT::t_int_1d("isurf/grid:which",nvalue); +} + +ComputeISurfGridKokkos::ComputeISurfGridKokkos(SPARTA *sparta) : + ComputeISurfGrid(sparta) +{ + copy = 1; + uncopy = 0; +} + +/* ---------------------------------------------------------------------- */ + +ComputeISurfGridKokkos::~ComputeISurfGridKokkos() +{ + if (copy) return; + + memoryKK->destroy_kokkos(k_tally2surf,tally2surf); + memoryKK->destroy_kokkos(k_array_surf_tally,array_surf_tally); +} + +/* ---------------------------------------------------------------------- */ + +void ComputeISurfGridKokkos::init() +{ + ComputeISurfGrid::init(); + + auto h_which = Kokkos::create_mirror_view(d_which); + for (int n=0; nnlocal + surf->nghost; + + d_normflux = DAT::t_float_1d("isurf/grid:normflux",nsurf); + auto h_normflux = Kokkos::create_mirror_view(d_normflux); + for (int n=0; ngrow_kokkos(k_tally2surf,tally2surf,nsurf,"isurf/grid:tally2surf"); + d_tally2surf = k_tally2surf.view_device(); + d_surf2tally = DAT::t_int_1d("isurf/grid:surf2tally",nsurf); + Kokkos::deep_copy(d_surf2tally,-1); + + memoryKK->grow_kokkos(k_array_surf_tally,array_surf_tally,nsurf,ntotal,"isurf/grid:array_surf_tally"); + d_array_surf_tally = k_array_surf_tally.view_device(); +} + +/* ---------------------------------------------------------------------- */ + +void ComputeISurfGridKokkos::clear() +{ + // reset all set surf2tally values to -1 + // called by Update at beginning of timesteps surf tallying is done + + Kokkos::deep_copy(d_array_surf_tally,0); + Kokkos::deep_copy(d_surf2tally,-1); + + ntally = 0; + combined = 0; +} + +/* ---------------------------------------------------------------------- */ + +void ComputeISurfGridKokkos::pre_surf_tally() +{ + mvv2e = update->mvv2e; + + ParticleKokkos* particle_kk = (ParticleKokkos*) particle; + particle_kk->sync(Device,SPECIES_MASK); + d_species = particle_kk->k_species.view_device(); + d_s2g = particle_kk->k_species2group.view_device(); + + SurfKokkos* surf_kk = (SurfKokkos*) surf; + surf_kk->sync(Device,ALL_MASK); + d_lines = surf_kk->k_lines.view_device(); + d_tris = surf_kk->k_tris.view_device(); + + need_dup = sparta->kokkos->need_dup(); + if (need_dup) + dup_array_surf_tally = Kokkos::Experimental::create_scatter_view(d_array_surf_tally); + else + ndup_array_surf_tally = Kokkos::Experimental::create_scatter_view(d_array_surf_tally); +} + +/* ---------------------------------------------------------------------- */ + +void ComputeISurfGridKokkos::post_surf_tally() +{ + if (need_dup) { + Kokkos::Experimental::contribute(d_array_surf_tally, dup_array_surf_tally); + dup_array_surf_tally = {}; // free duplicated memory + } + + k_tally2surf.modify_device(); + k_array_surf_tally.modify_device(); +} + +/* ---------------------------------------------------------------------- + sync device tallies to host and compress to dense list (ntally tallies) + matches ComputeSurfKokkos::tallyinfo(); + host fix ave/grid (PERGRIDSURF) consumes array_surf_tally + tally2surf +------------------------------------------------------------------------- */ + +int ComputeISurfGridKokkos::tallyinfo(surfint *&ptr) +{ + k_tally2surf.sync_host(); + ptr = tally2surf; + + k_array_surf_tally.sync_host(); + auto h_surf2tally = Kokkos::create_mirror_view(d_surf2tally); + Kokkos::deep_copy(h_surf2tally,d_surf2tally); + + // compress array_surf_tally + + int nsurf = surf->nlocal + surf->nghost; + int istart = 0; + int iend = nsurf-1; + + while (1) { + while (h_surf2tally[istart] != -1 && istart < nsurf-2) istart++; + while (h_surf2tally[iend] == -1 && iend > 0) iend--; + if (istart >= iend) { + ntally = istart; + break; + } + for (int k = 0; k < ntotal; k++) { + array_surf_tally[istart][k] = array_surf_tally[iend][k]; + } + h_surf2tally[istart] = h_surf2tally[iend]; + h_surf2tally[iend] = -1; + tally2surf[istart] = tally2surf[iend]; + } + + return ntally; +} + +/* ---------------------------------------------------------------------- */ + +void ComputeISurfGridKokkos::grow_tally() +{ + // Cannot realloc inside a Kokkos parallel region, so size as nsurf + + int nsurf = surf->nlocal + surf->nghost; + + memoryKK->grow_kokkos(k_tally2surf,tally2surf,nsurf,"isurf/grid:tally2surf"); + d_tally2surf = k_tally2surf.view_device(); + + memoryKK->grow_kokkos(k_array_surf_tally,array_surf_tally,nsurf,ntotal,"isurf/grid:array_surf_tally"); + d_array_surf_tally = k_array_surf_tally.view_device(); +} diff --git a/src/KOKKOS/compute_isurf_grid_kokkos.h b/src/KOKKOS/compute_isurf_grid_kokkos.h new file mode 100644 index 000000000..35679bb22 --- /dev/null +++ b/src/KOKKOS/compute_isurf_grid_kokkos.h @@ -0,0 +1,287 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#ifdef COMPUTE_CLASS + +ComputeStyle(isurf/grid/kk,ComputeISurfGridKokkos) + +#else + +#ifndef SPARTA_COMPUTE_ISURF_GRID_KOKKOS_H +#define SPARTA_COMPUTE_ISURF_GRID_KOKKOS_H + +#include "compute_isurf_grid.h" +#include "kokkos_type.h" +#include "math_extra_kokkos.h" + +namespace SPARTA_NS { + +class ComputeISurfGridKokkos : public ComputeISurfGrid { + public: + ComputeISurfGridKokkos(class SPARTA *, int, char **); + ComputeISurfGridKokkos(class SPARTA *); + ~ComputeISurfGridKokkos(); + void init(); + void init_normflux(); + void clear(); + int tallyinfo(surfint *&); + void pre_surf_tally(); + void post_surf_tally(); + + enum{NUM,NUMWT,MFLUX,FX,FY,FZ,PRESS,XPRESS,YPRESS,ZPRESS, + XSHEAR,YSHEAR,ZSHEAR,KE,EROT,EVIB,ETOT}; + +/* ---------------------------------------------------------------------- + tally values for a single particle in icell + colliding with surface element isurf, performing reaction (1 to N) + iorig = particle ip before collision + ip,jp = particles after collision + ip = NULL means no particles after collision + jp = NULL means one particle after collision + jp != NULL means two particles after collision + this method mirrors ComputeISurfGrid::surf_tally(), tallying per-surf; + post-processing (collate to per-grid) is done on the host +------------------------------------------------------------------------- */ + +template +KOKKOS_INLINE_FUNCTION +void surf_tally_kk(double /*dtremain*/, int isurf, int /*icell*/, int /*reaction*/, + Particle::OnePart *iorig, + Particle::OnePart *ip, Particle::OnePart *jp) const +{ + // skip if species not in mixture group + + int origspecies = iorig->ispecies; + int igroup = d_s2g(imix,origspecies); + if (igroup < 0) return; + + // itally = tally index of isurf (thread-safe; compressed later on host) + + surfint surfID; + if (dim == 2) surfID = d_lines[isurf].id; + else surfID = d_tris[isurf].id; + + int itally = isurf; + d_tally2surf(itally) = surfID; + d_surf2tally(isurf) = isurf; + + double fluxscale = d_normflux(isurf); + + double vsqpre,ivsqpost,jvsqpost; + double ierot,jerot,ievib,jevib,iother,jother,otherpre,etot; + double pdelta[3],pnorm[3],ptang[3],pdelta_force[3]; + + double *norm; + if (dim == 2) norm = d_lines(isurf).norm; + else norm = d_tris(isurf).norm; + + double weight = 1.0; + if (weightflag) weight = iorig->weight; + double origmass = d_species[origspecies].mass * weight; + double imass = 0.0, jmass = 0.0; + if (ip) imass = d_species(ip->ispecies).mass * weight; + if (jp) jmass = d_species(jp->ispecies).mass * weight; + + double *vorig = iorig->v; + + auto v_array_surf_tally = ScatterViewHelper::value,decltype(dup_array_surf_tally),decltype(ndup_array_surf_tally)>::get(dup_array_surf_tally,ndup_array_surf_tally); + auto a_array_surf_tally = v_array_surf_tally.template access::value>(); + + int k = igroup*nvalue; + int fflag = 0; + int nflag = 0; + int tflag = 0; + + for (int m = 0; m < nvalue; m++) { + switch (d_which(m)) { + case NUM: + a_array_surf_tally(itally,k++) += 1.0; + break; + case NUMWT: + a_array_surf_tally(itally,k++) += weight; + break; + case MFLUX: + a_array_surf_tally(itally,k) += origmass * fluxscale; + if (ip) a_array_surf_tally(itally,k) -= imass * fluxscale; + if (jp) a_array_surf_tally(itally,k) -= jmass * fluxscale; + k++; + break; + case FX: + if (!fflag) { + fflag = 1; + MathExtraKokkos::scale3(-origmass,vorig,pdelta_force); + if (ip) MathExtraKokkos::axpy3(imass,ip->v,pdelta_force); + if (jp) MathExtraKokkos::axpy3(jmass,jp->v,pdelta_force); + } + a_array_surf_tally(itally,k++) -= pdelta_force[0] * nfactor_inverse; + break; + case FY: + if (!fflag) { + fflag = 1; + MathExtraKokkos::scale3(-origmass,vorig,pdelta_force); + if (ip) MathExtraKokkos::axpy3(imass,ip->v,pdelta_force); + if (jp) MathExtraKokkos::axpy3(jmass,jp->v,pdelta_force); + } + a_array_surf_tally(itally,k++) -= pdelta_force[1] * nfactor_inverse; + break; + case FZ: + if (!fflag) { + fflag = 1; + MathExtraKokkos::scale3(-origmass,vorig,pdelta_force); + if (ip) MathExtraKokkos::axpy3(imass,ip->v,pdelta_force); + if (jp) MathExtraKokkos::axpy3(jmass,jp->v,pdelta_force); + } + a_array_surf_tally(itally,k++) -= pdelta_force[2] * nfactor_inverse; + break; + case PRESS: + MathExtraKokkos::scale3(-origmass,vorig,pdelta); + if (ip) MathExtraKokkos::axpy3(imass,ip->v,pdelta); + if (jp) MathExtraKokkos::axpy3(jmass,jp->v,pdelta); + a_array_surf_tally(itally,k++) += MathExtraKokkos::dot3(pdelta,norm) * fluxscale; + break; + case XPRESS: + if (!nflag) { + nflag = 1; + MathExtraKokkos::scale3(-origmass,vorig,pdelta); + if (ip) MathExtraKokkos::axpy3(imass,ip->v,pdelta); + if (jp) MathExtraKokkos::axpy3(jmass,jp->v,pdelta); + MathExtraKokkos::scale3(MathExtraKokkos::dot3(pdelta,norm),norm,pnorm); + } + a_array_surf_tally(itally,k++) -= pnorm[0] * fluxscale; + break; + case YPRESS: + if (!nflag) { + nflag = 1; + MathExtraKokkos::scale3(-origmass,vorig,pdelta); + if (ip) MathExtraKokkos::axpy3(imass,ip->v,pdelta); + if (jp) MathExtraKokkos::axpy3(jmass,jp->v,pdelta); + MathExtraKokkos::scale3(MathExtraKokkos::dot3(pdelta,norm),norm,pnorm); + } + a_array_surf_tally(itally,k++) -= pnorm[1] * fluxscale; + break; + case ZPRESS: + if (!nflag) { + nflag = 1; + MathExtraKokkos::scale3(-origmass,vorig,pdelta); + if (ip) MathExtraKokkos::axpy3(imass,ip->v,pdelta); + if (jp) MathExtraKokkos::axpy3(jmass,jp->v,pdelta); + MathExtraKokkos::scale3(MathExtraKokkos::dot3(pdelta,norm),norm,pnorm); + } + a_array_surf_tally(itally,k++) -= pnorm[2] * fluxscale; + break; + case XSHEAR: + if (!tflag) { + tflag = 1; + MathExtraKokkos::scale3(-origmass,vorig,pdelta); + if (ip) MathExtraKokkos::axpy3(imass,ip->v,pdelta); + if (jp) MathExtraKokkos::axpy3(jmass,jp->v,pdelta); + MathExtraKokkos::scale3(MathExtraKokkos::dot3(pdelta,norm),norm,pnorm); + MathExtraKokkos::sub3(pdelta,pnorm,ptang); + } + a_array_surf_tally(itally,k++) -= ptang[0] * fluxscale; + break; + case YSHEAR: + if (!tflag) { + tflag = 1; + MathExtraKokkos::scale3(-origmass,vorig,pdelta); + if (ip) MathExtraKokkos::axpy3(imass,ip->v,pdelta); + if (jp) MathExtraKokkos::axpy3(jmass,jp->v,pdelta); + MathExtraKokkos::scale3(MathExtraKokkos::dot3(pdelta,norm),norm,pnorm); + MathExtraKokkos::sub3(pdelta,pnorm,ptang); + } + a_array_surf_tally(itally,k++) -= ptang[1] * fluxscale; + break; + case ZSHEAR: + if (!tflag) { + tflag = 1; + MathExtraKokkos::scale3(-origmass,vorig,pdelta); + if (ip) MathExtraKokkos::axpy3(imass,ip->v,pdelta); + if (jp) MathExtraKokkos::axpy3(jmass,jp->v,pdelta); + MathExtraKokkos::scale3(MathExtraKokkos::dot3(pdelta,norm),norm,pnorm); + MathExtraKokkos::sub3(pdelta,pnorm,ptang); + } + a_array_surf_tally(itally,k++) -= ptang[2] * fluxscale; + break; + case KE: + vsqpre = origmass * MathExtraKokkos::lensq3(vorig); + if (ip) ivsqpost = imass * MathExtraKokkos::lensq3(ip->v); + else ivsqpost = 0.0; + if (jp) jvsqpost = jmass * MathExtraKokkos::lensq3(jp->v); + else jvsqpost = 0.0; + a_array_surf_tally(itally,k++) -= 0.5*mvv2e * (ivsqpost + jvsqpost - vsqpre) * fluxscale; + break; + case EROT: + if (ip) ierot = ip->erot; + else ierot = 0.0; + if (jp) jerot = jp->erot; + else jerot = 0.0; + a_array_surf_tally(itally,k++) -= weight * (ierot + jerot - iorig->erot) * fluxscale; + break; + case EVIB: + if (ip) ievib = ip->evib; + else ievib = 0.0; + if (jp) jevib = jp->evib; + else jevib = 0.0; + a_array_surf_tally(itally,k++) -= weight * (ievib + jevib - iorig->evib) * fluxscale; + break; + case ETOT: + vsqpre = origmass * MathExtraKokkos::lensq3(vorig); + otherpre = iorig->erot + iorig->evib; + if (ip) { + ivsqpost = imass * MathExtraKokkos::lensq3(ip->v); + iother = ip->erot + ip->evib; + } else ivsqpost = iother = 0.0; + if (jp) { + jvsqpost = jmass * MathExtraKokkos::lensq3(jp->v); + jother = jp->erot + jp->evib; + } else jvsqpost = jother = 0.0; + etot = 0.5*mvv2e*(ivsqpost + jvsqpost - vsqpre) + + weight * (iother + jother - otherpre); + a_array_surf_tally(itally,k++) -= etot * fluxscale; + break; + } + } +} + + private: + double mvv2e; + + DAT::t_int_1d d_which; + + DAT::tdual_float_2d_lr k_array_surf_tally; + DAT::t_float_2d_lr d_array_surf_tally; // tally values for local surfs + + int need_dup; + Kokkos::Experimental::ScatterView dup_array_surf_tally; + Kokkos::Experimental::ScatterView ndup_array_surf_tally; + + DAT::t_surfint_1d d_tally2surf; // tally2surf[I] = surf ID of Ith tally + DAT::tdual_surfint_1d k_tally2surf; + DAT::t_int_1d d_surf2tally; + + DAT::t_float_1d d_normflux; // normalization factor for each surf element + + t_species_1d d_species; + DAT::t_int_2d d_s2g; + + t_line_1d d_lines; + t_tri_1d d_tris; + + void grow_tally(); +}; + +} + +#endif +#endif diff --git a/src/KOKKOS/fix_ave_grid_kokkos.cpp b/src/KOKKOS/fix_ave_grid_kokkos.cpp index 431e07609..320b12a8f 100644 --- a/src/KOKKOS/fix_ave_grid_kokkos.cpp +++ b/src/KOKKOS/fix_ave_grid_kokkos.cpp @@ -51,8 +51,13 @@ FixAveGridKokkos::FixAveGridKokkos(SPARTA *sparta, int narg, char **arg) : datamask_read = EMPTY_MASK; datamask_modify = EMPTY_MASK; - if (flavor == PERGRIDSURF) - error->all(FLERR,"Cannot yet use Kokkos with fix ave/grid for grid/surf inputs"); + // PERGRIDSURF (grid/surf inputs, e.g. compute isurf/grid) runs on the host: + // the per-surf tally is produced on-device by the Kokkos compute and brought + // to the host by its tallyinfo(), then collated to per-grid by the host base + // class. Skip all Kokkos-specific allocation and leave the host base ctor's + // allocations intact; the overridden methods below delegate to FixAveGrid. + + if (flavor == PERGRIDSURF) return; nglocal = maxgrid = grid->nlocal; @@ -109,6 +114,8 @@ FixAveGridKokkos::~FixAveGridKokkos() { if (copymode) return; + if (flavor == PERGRIDSURF) return; + if (nvalues == 1) memoryKK->destroy_kokkos(k_vector_grid,vector_grid); else memoryKK->destroy_kokkos(k_array_grid,array_grid); memoryKK->destroy_kokkos(k_tally,tally); @@ -120,6 +127,13 @@ FixAveGridKokkos::~FixAveGridKokkos() void FixAveGridKokkos::init() { + // PERGRIDSURF path runs entirely on the host + + if (flavor == PERGRIDSURF) { + FixAveGrid::init(); + return; + } + // set indices and check validity of all computes,fixes,variables,custom attributes for (int m = 0; m < nvalues; m++) { @@ -161,6 +175,15 @@ void FixAveGridKokkos::end_of_step() int j,n; //int *itmp; + // PERGRIDSURF path runs entirely on the host: the Kokkos compute's device + // surf tally is brought to the host by its tallyinfo(), then the host base + // class collates per-surf tallies to per-grid output + + if (flavor == PERGRIDSURF) { + FixAveGrid::end_of_step(); + return; + } + // skip if not step which requires doing something bigint ntimestep = update->ntimestep; @@ -492,6 +515,15 @@ void FixAveGridKokkos::operator()(TagFixAveGrid_Norm_array_grid, const int &i) c void FixAveGridKokkos::grow_percell(int nnew) { + // PERGRIDSURF keeps its per-cell arrays in host memory (managed by the host + // base class); reallocating them with Kokkos memory here would make the host + // base destructor free a Kokkos-allocated pointer + + if (flavor == PERGRIDSURF) { + FixAveGrid::grow_percell(nnew); + return; + } + if (nglocal+nnew < maxgrid) return; maxgrid += DELTAGRID; int n = maxgrid; diff --git a/src/KOKKOS/update_kokkos.cpp b/src/KOKKOS/update_kokkos.cpp index 0d45f9489..45c2d3d1c 100644 --- a/src/KOKKOS/update_kokkos.cpp +++ b/src/KOKKOS/update_kokkos.cpp @@ -86,9 +86,13 @@ UpdateKokkos::UpdateKokkos(SPARTA *sparta) : Update(sparta), sc_kk_cll_copy{VAL_2(KKCopy(sparta))}, blist_active_copy{VAL_2(KKCopy(sparta))}, slist_active_copy{VAL_2(KKCopy(sparta))}, + slist_active_isurf_copy{VAL_2(KKCopy(sparta))}, tmp_compute_boundary_kk(sparta), - tmp_compute_surf_kk(sparta) + tmp_compute_surf_kk(sparta), + tmp_compute_isurf_grid_kk(sparta) { + nslist_surf = nslist_isurf = 0; + // use 1D view for scalars to reduce GPU memory operations @@ -142,6 +146,7 @@ UpdateKokkos::~UpdateKokkos() tmp_compute_boundary_kk.uncopy = 1; tmp_compute_surf_kk.uncopy = 1; + tmp_compute_isurf_grid_kk.uncopy = 1; for (int i=0; i void UpdateKokkos::move() if (nsurf_tally) { for (int m = 0; m < nsurf_tally; m++) { - ComputeSurfKokkos* compute_surf_kk = (ComputeSurfKokkos*)(slist_active[m]); - compute_surf_kk->post_surf_tally(); + if (strcmp(slist_active[m]->style,"isurf/grid") == 0) { + ComputeISurfGridKokkos* compute_isurf_kk = + (ComputeISurfGridKokkos*)(slist_active[m]); + compute_isurf_kk->post_surf_tally(); + } else { + ComputeSurfKokkos* compute_surf_kk = (ComputeSurfKokkos*)(slist_active[m]); + compute_surf_kk->post_surf_tally(); + } } } @@ -1482,10 +1494,14 @@ void UpdateKokkos::operator()(TagUpdateMove jpart->weight = particle_i.weight; } - if (nsurf_tally) - for (m = 0; m < nsurf_tally; m++) + if (nsurf_tally) { + for (m = 0; m < nslist_surf; m++) slist_active_copy[m].obj. surf_tally_kk(dtremain,minsurf,icell,reaction,&iorig,ipart,jpart); + for (m = 0; m < nslist_isurf; m++) + slist_active_isurf_copy[m].obj. + surf_tally_kk(dtremain,minsurf,icell,reaction,&iorig,ipart,jpart); + } // stuck_iterate = consecutive iterations particle is immobile @@ -2049,28 +2065,45 @@ void UpdateKokkos::tally_set(bigint ntimestep) } } - if (nsurf_tally > KOKKOS_MAX_SLIST) - error->all(FLERR,"Kokkos currently only supports two instances of compute surface"); + // partition surf tally computes into "compute surf" (slist_active_copy) and + // "compute isurf/grid" (slist_active_isurf_copy); both tally on-device via + // surf_tally_kk(), invoked from the move kernel's surface collision loop + + nslist_surf = nslist_isurf = 0; if (nsurf_tally) { for (i = 0; i < nsurf_tally; i++) { - if (strcmp(slist_active[i]->style,"isurf/grid") == 0) - error->all(FLERR,"Kokkos doesn't yet support compute isurf/grid"); - ComputeSurfKokkos* compute_surf_kk = dynamic_cast(slist_active[i]); - if (!compute_surf_kk) - error->all(FLERR,"Kokkos does not (yet) support compute surf/collision/tally or compute surf/reaction/tally"); - compute_surf_kk->pre_surf_tally(); - slist_active_copy[i].copy(compute_surf_kk); + if (strcmp(slist_active[i]->style,"isurf/grid") == 0) { + ComputeISurfGridKokkos* compute_isurf_kk = + dynamic_cast(slist_active[i]); + if (!compute_isurf_kk) + error->all(FLERR,"Must use Kokkos-enabled compute isurf/grid with Kokkos"); + if (nslist_isurf >= KOKKOS_MAX_SLIST) + error->all(FLERR,"Kokkos currently only supports two instances of compute isurf/grid"); + compute_isurf_kk->pre_surf_tally(); + slist_active_isurf_copy[nslist_isurf].copy(compute_isurf_kk); + nslist_isurf++; + } else { + ComputeSurfKokkos* compute_surf_kk = + dynamic_cast(slist_active[i]); + if (!compute_surf_kk) + error->all(FLERR,"Kokkos does not (yet) support compute surf/collision/tally or compute surf/reaction/tally"); + if (nslist_surf >= KOKKOS_MAX_SLIST) + error->all(FLERR,"Kokkos currently only supports two instances of compute surface"); + compute_surf_kk->pre_surf_tally(); + slist_active_copy[nslist_surf].copy(compute_surf_kk); + nslist_surf++; + } } - } else { - for (int i = 0; i < KOKKOS_MAX_SLIST; i++) { + } - // use temporary to avoid the copy getting stale leading to an issue - // with view reference counting + // fill unused slots of each typed copy list with the temporary + // to avoid the copy getting stale leading to an issue with view ref counting - slist_active_copy[i].copy(&tmp_compute_surf_kk); - } - } + for (i = nslist_surf; i < KOKKOS_MAX_SLIST; i++) + slist_active_copy[i].copy(&tmp_compute_surf_kk); + for (i = nslist_isurf; i < KOKKOS_MAX_SLIST; i++) + slist_active_isurf_copy[i].copy(&tmp_compute_isurf_grid_kk); if (ngas_tally) error->all(FLERR,"Kokkos does not (yet) support tallying gas/gas collisions or reactions"); diff --git a/src/KOKKOS/update_kokkos.h b/src/KOKKOS/update_kokkos.h index 9136a81d2..01920afac 100644 --- a/src/KOKKOS/update_kokkos.h +++ b/src/KOKKOS/update_kokkos.h @@ -32,6 +32,7 @@ #include "surf_collide_cll_kokkos.h" #include "compute_boundary_kokkos.h" #include "compute_surf_kokkos.h" +#include "compute_isurf_grid_kokkos.h" namespace SPARTA_NS { @@ -147,10 +148,19 @@ class UpdateKokkos : public Update { //KKCopy blist_active_copy[KOKKOS_MAX_GLIST]; KKCopy slist_active_copy[KOKKOS_MAX_SLIST]; + KKCopy slist_active_isurf_copy[KOKKOS_MAX_SLIST]; KKCopy blist_active_copy[KOKKOS_MAX_BLIST]; + // partition of slist_active (set in tally_set): + // nslist_surf = # of compute surf style tallies (slist_active_copy) + // nslist_isurf = # of compute isurf/grid tallies (slist_active_isurf_copy) + // nslist_surf + nslist_isurf == nsurf_tally + + int nslist_surf,nslist_isurf; + ComputeBoundaryKokkos tmp_compute_boundary_kk; ComputeSurfKokkos tmp_compute_surf_kk; + ComputeISurfGridKokkos tmp_compute_isurf_grid_kk; typedef Kokkos::DualView tdual_int_14; typedef tdual_int_14::t_dev t_int_14; diff --git a/src/compute_isurf_grid.cpp b/src/compute_isurf_grid.cpp index fec71e56e..11c4e758d 100644 --- a/src/compute_isurf_grid.cpp +++ b/src/compute_isurf_grid.cpp @@ -101,6 +101,8 @@ ComputeISurfGrid::ComputeISurfGrid(SPARTA *sparta, int narg, char **arg) : ComputeISurfGrid::~ComputeISurfGrid() { + if (copy || copymode) return; + delete [] which; memory->destroy(array_surf_tally); memory->destroy(tally2surf); diff --git a/src/compute_isurf_grid.h b/src/compute_isurf_grid.h index 817de99fe..844123d32 100644 --- a/src/compute_isurf_grid.h +++ b/src/compute_isurf_grid.h @@ -31,6 +31,7 @@ namespace SPARTA_NS { class ComputeISurfGrid : public Compute { public: ComputeISurfGrid(class SPARTA *, int, char **); + ComputeISurfGrid(class SPARTA* sparta) : Compute(sparta) {} // needed for Kokkos ~ComputeISurfGrid(); virtual void init(); void compute_per_grid(); @@ -73,8 +74,8 @@ class ComputeISurfGrid : public Compute { double weight; // particle weight, based on initial cell double *normflux; // normalization factor for each surf element - void init_normflux(); - void grow_tally(); + virtual void init_normflux(); + virtual void grow_tally(); }; } From b8a7b3161b0bcb7547a0f9c9c87af03ee70cc179 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 20 Jun 2026 17:47:59 +0000 Subject: [PATCH 08/30] KOKKOS: port surf_react adsorb GS chemistry (face mode) to Kokkos Add SurfReactAdsorbKokkos (style adsorb/kk), a device port of the gas-surface (GS) chemistry in surf_react adsorb for box-face (face) mode. - SurfReactAdsorbKokkos: flattens the GS reaction tables (type/style/k_react, kisliuk + energy coeffs, per-reactant/product state/part/stoich/ad-index, product species) and the per-face state (total_state, area, weight, species_state, atomic species_delta) into device views. react_kokkos() mirrors SurfReactAdsorb::react() for face/GS: per-reaction probability (including kisliuk and surf-coverage S_theta), scatter probability, and the reaction executions that need no post-reaction collision model (AA/EXCHANGE/RECOMBINATION/LH3/CD, plus DISSOCIATION particle creation). tally_update() pulls device counts + per-face deltas to the host and reuses the host MPI state-sync (update_state_face). Unsupported features (ps chemistry, surf-element mode, and the DA/LH1/ER/CI types or any cmodel post-reaction scatter) error clearly at init rather than silently mis-running. - surf_collide cll: dispatch sr_type==2 (adsorb) in collide_kokkos, with the setup/backup/restore wiring; guard the react block with isr >= 0 so box-face collisions on non-reacting faces are safe. - update_kokkos: select a REACT=1 move variant when reactions exist without explicit surfs (move), so box-face/boundary reactions are invoked on device (previously REACT was only enabled when surf->exist). - surf_react_adsorb base: add the Kokkos copy ctor, make init/tally_update virtual, members protected, and guard the destructor for copies. Verified bit-for-bit CPU vs -sf kk under Serial+EXACT on a face-mode GS run (beam onto a zlo box face, cll collide + adsorb, sample-GS_1.surf O(g)->O(s)): identical stats and 28194 surface reactions. GS_2 (cmodel reaction types) errors cleanly under -sf kk. No regression: surf_collide/in.circle.{diffuse, cll} and adjust_temp/in.circle.constant (compute surf) remain bit-for-bit. Co-Authored-By: Claude Opus 4.8 Co-Authored-By: Stan Moore Claude-Session: https://claude.ai/code/session_01Q6nQVuxEiTUqSzDoDppCUM --- src/KOKKOS/surf_collide_cll_kokkos.cpp | 30 +- src/KOKKOS/surf_collide_cll_kokkos.h | 7 +- src/KOKKOS/surf_react_adsorb_kokkos.cpp | 359 ++++++++++++++++++++++++ src/KOKKOS/surf_react_adsorb_kokkos.h | 328 ++++++++++++++++++++++ src/KOKKOS/update_kokkos.cpp | 12 +- src/surf_react_adsorb.cpp | 2 + src/surf_react_adsorb.h | 7 +- 7 files changed, 731 insertions(+), 14 deletions(-) create mode 100644 src/KOKKOS/surf_react_adsorb_kokkos.cpp create mode 100644 src/KOKKOS/surf_react_adsorb_kokkos.h diff --git a/src/KOKKOS/surf_collide_cll_kokkos.cpp b/src/KOKKOS/surf_collide_cll_kokkos.cpp index 262867b4e..2cf188db6 100644 --- a/src/KOKKOS/surf_collide_cll_kokkos.cpp +++ b/src/KOKKOS/surf_collide_cll_kokkos.cpp @@ -51,6 +51,7 @@ SurfCollideCLLKokkos::SurfCollideCLLKokkos(SPARTA *sparta, int narg, char **arg) fix_vibmode_kk_copy(sparta), sr_kk_global_copy{VAL_2(KKCopy(sparta))}, sr_kk_prob_copy{VAL_2(KKCopy(sparta))}, + sr_kk_adsorb_copy{VAL_2(KKCopy(sparta))}, rand_pool(12345 + comm->me #ifdef SPARTA_KOKKOS_EXACT , sparta @@ -82,6 +83,7 @@ SurfCollideCLLKokkos::SurfCollideCLLKokkos(SPARTA *sparta) : fix_vibmode_kk_copy(sparta), sr_kk_global_copy{VAL_2(KKCopy(sparta))}, sr_kk_prob_copy{VAL_2(KKCopy(sparta))}, + sr_kk_adsorb_copy{VAL_2(KKCopy(sparta))}, rand_pool(12345 // seed doesn't matter since it will just be copied over #ifdef SPARTA_KOKKOS_EXACT , sparta @@ -102,6 +104,7 @@ SurfCollideCLLKokkos::~SurfCollideCLLKokkos() for (int i = 0; i < KOKKOS_MAX_SURF_REACT_PER_TYPE; i++) { sr_kk_global_copy[i].uncopy(); sr_kk_prob_copy[i].uncopy(); + sr_kk_adsorb_copy[i].uncopy(); } } @@ -235,8 +238,8 @@ void SurfCollideCLLKokkos::pre_collide() error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); if (surf->nsr > 0) { - int nglob,nprob; - nglob = nprob = 0; + int nglob,nprob,nadsorb; + nglob = nprob = nadsorb = 0; for (int n = 0; n < surf->nsr; n++) { if (!surf->sr[n]->kokkosable) error->all(FLERR,"Must use Kokkos-enabled surface reaction method with Kokkos"); @@ -252,12 +255,19 @@ void SurfCollideCLLKokkos::pre_collide() sr_type_list[n] = 1; sr_map[n] = nprob; nprob++; + } else if (strcmp(surf->sr[n]->style,"adsorb") == 0) { + sr_kk_adsorb_copy[nadsorb].copy((SurfReactAdsorbKokkos*)(surf->sr[n])); + sr_kk_adsorb_copy[nadsorb].obj.pre_react(); + sr_type_list[n] = 2; + sr_map[n] = nadsorb; + nadsorb++; } else { error->all(FLERR,"Unknown Kokkos surface reaction method"); } } - if (nglob > KOKKOS_MAX_SURF_REACT_PER_TYPE || nprob > KOKKOS_MAX_SURF_REACT_PER_TYPE) + if (nglob > KOKKOS_MAX_SURF_REACT_PER_TYPE || nprob > KOKKOS_MAX_SURF_REACT_PER_TYPE || + nadsorb > KOKKOS_MAX_SURF_REACT_PER_TYPE) error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); } @@ -312,8 +322,8 @@ void SurfCollideCLLKokkos::backup() d_particles = particle_kk->k_particles.view_device(); if (surf->nsr > 0) { - int nglob,nprob; - nglob = nprob = 0; + int nglob,nprob,nadsorb; + nglob = nprob = nadsorb = 0; for (int n = 0; n < surf->nsr; n++) { if (strcmp(surf->sr[n]->style,"global") == 0) { sr_kk_global_copy[nglob].obj.backup(); @@ -321,6 +331,9 @@ void SurfCollideCLLKokkos::backup() } else if (strcmp(surf->sr[n]->style,"prob") == 0) { sr_kk_prob_copy[nprob].obj.backup(); nprob++; + } else if (strcmp(surf->sr[n]->style,"adsorb") == 0) { + sr_kk_adsorb_copy[nadsorb].obj.backup(); + nadsorb++; } } } @@ -337,8 +350,8 @@ void SurfCollideCLLKokkos::backup() void SurfCollideCLLKokkos::restore() { if (surf->nsr > 0) { - int nglob,nprob; - nglob = nprob = 0; + int nglob,nprob,nadsorb; + nglob = nprob = nadsorb = 0; for (int n = 0; n < surf->nsr; n++) { if (strcmp(surf->sr[n]->style,"global") == 0) { sr_kk_global_copy[nglob].obj.restore(); @@ -346,6 +359,9 @@ void SurfCollideCLLKokkos::restore() } else if (strcmp(surf->sr[n]->style,"prob") == 0) { sr_kk_prob_copy[nprob].obj.restore(); nprob++; + } else if (strcmp(surf->sr[n]->style,"adsorb") == 0) { + sr_kk_adsorb_copy[nadsorb].obj.restore(); + nadsorb++; } } } diff --git a/src/KOKKOS/surf_collide_cll_kokkos.h b/src/KOKKOS/surf_collide_cll_kokkos.h index dd0ec9be5..fd39787c9 100644 --- a/src/KOKKOS/surf_collide_cll_kokkos.h +++ b/src/KOKKOS/surf_collide_cll_kokkos.h @@ -31,6 +31,7 @@ SurfCollideStyle(cll/kk,SurfCollideCLLKokkos) #include "fix_vibmode_kokkos.h" #include "surf_react_global_kokkos.h" #include "surf_react_prob_kokkos.h" +#include "surf_react_adsorb_kokkos.h" namespace SPARTA_NS { @@ -91,6 +92,7 @@ class SurfCollideCLLKokkos : public SurfCollideCLL { int sr_map[KOKKOS_MAX_TOT_SURF_REACT]; KKCopy sr_kk_global_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; KKCopy sr_kk_prob_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; + KKCopy sr_kk_adsorb_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; public: @@ -126,7 +128,7 @@ class SurfCollideCLLKokkos : public SurfCollideCLL { reaction = 0; int velreset = 0; - if (REACT) { + if (REACT && isr >= 0) { if (ambi_flag || vibmode_flag) memcpy(&iorig,ip,sizeof(Particle::OnePart)); int sr_type = sr_type_list[isr]; @@ -138,6 +140,9 @@ class SurfCollideCLLKokkos : public SurfCollideCLL { } else if (sr_type == 1) { reaction = sr_kk_prob_copy[m].obj. react_kokkos(ip,isurf,norm,jp,velreset,d_retry,d_nlocal); + } else if (sr_type == 2) { + reaction = sr_kk_adsorb_copy[m].obj. + react_kokkos(ip,isurf,norm,jp,velreset,d_retry,d_nlocal); } if (reaction) { diff --git a/src/KOKKOS/surf_react_adsorb_kokkos.cpp b/src/KOKKOS/surf_react_adsorb_kokkos.cpp new file mode 100644 index 000000000..44fab7b8e --- /dev/null +++ b/src/KOKKOS/surf_react_adsorb_kokkos.cpp @@ -0,0 +1,359 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#include "math.h" +#include "string.h" +#include "surf_react_adsorb_kokkos.h" +#include "input.h" +#include "update.h" +#include "comm.h" +#include "domain.h" +#include "particle.h" +#include "error.h" +#include "particle_kokkos.h" +#include "sparta_masks.h" + +using namespace SPARTA_NS; + +/* ---------------------------------------------------------------------- */ + +SurfReactAdsorbKokkos::SurfReactAdsorbKokkos(SPARTA *sparta, int narg, char **arg) : + SurfReactAdsorb(sparta,narg,arg), + rand_pool(12345 + comm->me +#ifdef SPARTA_KOKKOS_EXACT + , sparta +#endif + ) +{ + kokkosable = 1; + + d_scalars = DAT::t_int_1d("surf_react_adsorb:scalars",nlist_gs+1); + d_nsingle = Kokkos::subview(d_scalars,0); + d_tally_single = Kokkos::subview(d_scalars,std::make_pair(1,nlist_gs+1)); + + h_scalars = HAT::t_int_1d("surf_react_adsorb:scalars_mirror",nlist_gs+1); + h_nsingle = Kokkos::subview(h_scalars,0); + h_tally_single = Kokkos::subview(h_scalars,std::make_pair(1,nlist_gs+1)); + + random_backup = NULL; +} + +SurfReactAdsorbKokkos::SurfReactAdsorbKokkos(SPARTA *sparta) : + SurfReactAdsorb(sparta), + rand_pool(12345 +#ifdef SPARTA_KOKKOS_EXACT + , sparta +#endif + ) +{ + copy = 1; +} + +/* ---------------------------------------------------------------------- */ + +SurfReactAdsorbKokkos::~SurfReactAdsorbKokkos() +{ + if (copy) return; + +#ifdef SPARTA_KOKKOS_EXACT + rand_pool.destroy(); + if (random_backup) delete random_backup; +#endif +} + +/* ---------------------------------------------------------------------- */ + +void SurfReactAdsorbKokkos::init() +{ + SurfReactAdsorb::init(); + + // Kokkos GS adsorb currently supports a restricted feature set; + // error clearly at init rather than silently producing wrong results + + if (!gsflag) + error->all(FLERR,"Kokkos surf_react adsorb requires gas-surface (gs) chemistry"); + if (psflag) + error->all(FLERR,"Kokkos surf_react adsorb does not yet support on-surface (ps) chemistry"); + if (mode != SRA_KK::FACE) + error->all(FLERR,"Kokkos surf_react adsorb only supports the box-face (face) option"); + + for (int i = 0; i < nlist_gs; i++) { + OneReaction_GS *r = &rlist_gs[i]; + if (!r->active) continue; + if (r->cmodel_ip != SRA_KK::NOMODEL || r->cmodel_jp != SRA_KK::NOMODEL) + error->all(FLERR,"Kokkos surf_react adsorb does not yet support reactions " + "with a post-reaction surface collision model"); + if (r->type == SRA_KK::DA || r->type == SRA_KK::LH1 || + r->type == SRA_KK::ER || r->type == SRA_KK::CI) + error->all(FLERR,"Kokkos surf_react adsorb does not yet support DA/LH1/ER/CI " + "reaction types (they require a post-reaction collision model)"); + } + + Kokkos::deep_copy(d_scalars,0); + + init_reactions_gs_kokkos(); + +#ifdef SPARTA_KOKKOS_EXACT + rand_pool.init(random); +#endif +} + +/* ---------------------------------------------------------------------- */ + +void SurfReactAdsorbKokkos::init_reactions_gs_kokkos() +{ + int nspecies = particle->nspecies; + + // per-species reaction lists + + int nmax = 0; + d_reactions_n = DAT::t_int_1d("surf_react_adsorb:reactions_n",nspecies); + auto h_reactions_n = Kokkos::create_mirror_view(d_reactions_n); + for (int i = 0; i < nspecies; i++) { + int n = reactions_gs[i].n; + h_reactions_n(i) = n; + nmax = MAX(nmax,n); + } + if (nmax > SRA_KK_MAXPERSPECIES) + error->all(FLERR,"Too many Kokkos surf_react adsorb reactions per species"); + + d_list = DAT::t_int_2d("surf_react_adsorb:list",nspecies,MAX(nmax,1)); + auto h_list = Kokkos::create_mirror_view(d_list); + for (int i = 0; i < nspecies; i++) + for (int j = 0; j < reactions_gs[i].n; j++) + h_list(i,j) = reactions_gs[i].list[j]; + + // flattened per-reaction tables + + int nr = MAX(nlist_gs,1); + d_type = DAT::t_int_1d("sra:type",nr); + d_style = DAT::t_int_1d("sra:style",nr); + d_kreact = DAT::t_float_1d("sra:kreact",nr); + d_kisliuk_flag = DAT::t_int_1d("sra:kflag",nr); + d_kisliuk = DAT::t_float_2d("sra:kisliuk",nr,3); + d_energy_flag = DAT::t_int_1d("sra:eflag",nr); + d_energy = DAT::t_float_2d("sra:energy",nr,2); + d_coeff = DAT::t_float_2d("sra:coeff",nr,SRA_KK_MAXCOEFF); + d_nreactant = DAT::t_int_1d("sra:nreactant",nr); + d_nproduct = DAT::t_int_1d("sra:nproduct",nr); + d_nprod_g = DAT::t_int_1d("sra:nprod_g",nr); + d_nprod_g_tot = DAT::t_int_1d("sra:nprod_g_tot",nr); + d_cmodel_ip = DAT::t_int_1d("sra:cmodel_ip",nr); + d_cmodel_jp = DAT::t_int_1d("sra:cmodel_jp",nr); + d_rstate = DAT::t_int_2d("sra:rstate",nr,SRA_KK_MAXREACTANT); + d_rpart = DAT::t_int_2d("sra:rpart",nr,SRA_KK_MAXREACTANT); + d_rstoich = DAT::t_int_2d("sra:rstoich",nr,SRA_KK_MAXREACTANT); + d_rad = DAT::t_int_2d("sra:rad",nr,SRA_KK_MAXREACTANT); + d_pstate = DAT::t_int_2d("sra:pstate",nr,SRA_KK_MAXPRODUCT); + d_ppart = DAT::t_int_2d("sra:ppart",nr,SRA_KK_MAXPRODUCT); + d_pstoich = DAT::t_int_2d("sra:pstoich",nr,SRA_KK_MAXPRODUCT); + d_pad = DAT::t_int_2d("sra:pad",nr,SRA_KK_MAXPRODUCT); + d_products = DAT::t_int_2d("sra:products",nr,SRA_KK_MAXPRODUCT); + + auto h_type = Kokkos::create_mirror_view(d_type); + auto h_style = Kokkos::create_mirror_view(d_style); + auto h_kreact = Kokkos::create_mirror_view(d_kreact); + auto h_kflag = Kokkos::create_mirror_view(d_kisliuk_flag); + auto h_kisliuk = Kokkos::create_mirror_view(d_kisliuk); + auto h_eflag = Kokkos::create_mirror_view(d_energy_flag); + auto h_energy = Kokkos::create_mirror_view(d_energy); + auto h_coeff = Kokkos::create_mirror_view(d_coeff); + auto h_nreactant = Kokkos::create_mirror_view(d_nreactant); + auto h_nproduct = Kokkos::create_mirror_view(d_nproduct); + auto h_nprod_g = Kokkos::create_mirror_view(d_nprod_g); + auto h_nprod_g_tot = Kokkos::create_mirror_view(d_nprod_g_tot); + auto h_cmodel_ip = Kokkos::create_mirror_view(d_cmodel_ip); + auto h_cmodel_jp = Kokkos::create_mirror_view(d_cmodel_jp); + auto h_rstate = Kokkos::create_mirror_view(d_rstate); + auto h_rpart = Kokkos::create_mirror_view(d_rpart); + auto h_rstoich = Kokkos::create_mirror_view(d_rstoich); + auto h_rad = Kokkos::create_mirror_view(d_rad); + auto h_pstate = Kokkos::create_mirror_view(d_pstate); + auto h_ppart = Kokkos::create_mirror_view(d_ppart); + auto h_pstoich = Kokkos::create_mirror_view(d_pstoich); + auto h_pad = Kokkos::create_mirror_view(d_pad); + auto h_products = Kokkos::create_mirror_view(d_products); + + for (int i = 0; i < nlist_gs; i++) { + OneReaction_GS *r = &rlist_gs[i]; + h_type(i) = r->type; + h_style(i) = r->style; + h_kreact(i) = r->k_react; + h_kflag(i) = r->kisliuk_flag; + for (int k = 0; k < 3; k++) h_kisliuk(i,k) = r->kisliuk_coeff[k]; + h_eflag(i) = r->energy_flag; + for (int k = 0; k < 2; k++) h_energy(i,k) = r->energy_coeff[k]; + for (int k = 0; k < SRA_KK_MAXCOEFF; k++) + h_coeff(i,k) = (k < r->ncoeff) ? r->coeff[k] : 0.0; + h_nreactant(i) = r->nreactant; + h_nproduct(i) = r->nproduct; + h_nprod_g(i) = r->nprod_g; + h_nprod_g_tot(i) = r->nprod_g_tot; + h_cmodel_ip(i) = r->cmodel_ip; + h_cmodel_jp(i) = r->cmodel_jp; + for (int k = 0; k < r->nreactant && k < SRA_KK_MAXREACTANT; k++) { + h_rstate(i,k) = r->state_reactants[k][0]; + h_rpart(i,k) = r->part_reactants[k]; + h_rstoich(i,k) = r->stoich_reactants[k]; + h_rad(i,k) = r->reactants_ad_index[k]; + } + for (int k = 0; k < r->nproduct && k < SRA_KK_MAXPRODUCT; k++) { + h_pstate(i,k) = r->state_products[k][0]; + h_ppart(i,k) = r->part_products[k]; + h_pstoich(i,k) = r->stoich_products[k]; + h_pad(i,k) = r->products_ad_index[k]; + h_products(i,k) = r->products[k]; + } + } + + Kokkos::deep_copy(d_reactions_n,h_reactions_n); + Kokkos::deep_copy(d_list,h_list); + Kokkos::deep_copy(d_type,h_type); + Kokkos::deep_copy(d_style,h_style); + Kokkos::deep_copy(d_kreact,h_kreact); + Kokkos::deep_copy(d_kisliuk_flag,h_kflag); + Kokkos::deep_copy(d_kisliuk,h_kisliuk); + Kokkos::deep_copy(d_energy_flag,h_eflag); + Kokkos::deep_copy(d_energy,h_energy); + Kokkos::deep_copy(d_coeff,h_coeff); + Kokkos::deep_copy(d_nreactant,h_nreactant); + Kokkos::deep_copy(d_nproduct,h_nproduct); + Kokkos::deep_copy(d_nprod_g,h_nprod_g); + Kokkos::deep_copy(d_nprod_g_tot,h_nprod_g_tot); + Kokkos::deep_copy(d_cmodel_ip,h_cmodel_ip); + Kokkos::deep_copy(d_cmodel_jp,h_cmodel_jp); + Kokkos::deep_copy(d_rstate,h_rstate); + Kokkos::deep_copy(d_rpart,h_rpart); + Kokkos::deep_copy(d_rstoich,h_rstoich); + Kokkos::deep_copy(d_rad,h_rad); + Kokkos::deep_copy(d_pstate,h_pstate); + Kokkos::deep_copy(d_ppart,h_ppart); + Kokkos::deep_copy(d_pstoich,h_pstoich); + Kokkos::deep_copy(d_pad,h_pad); + Kokkos::deep_copy(d_products,h_products); + + // per-face state device storage (SRA_KK::FACE mode) + + d_total_state = DAT::t_int_1d("sra:total_state",nface); + d_area = DAT::t_float_1d("sra:area",nface); + d_weight = DAT::t_float_1d("sra:weight",nface); + d_species_state = DAT::t_int_2d("sra:species_state",nface,nspecies_surf); + + k_species_delta = DAT::tdual_int_2d("sra:species_delta",nface,nspecies_surf); + d_species_delta = k_species_delta.view_device(); + Kokkos::deep_copy(d_species_delta,0); +} + +/* ---------------------------------------------------------------------- + sync per-face state host->device and refresh particle/scalar views + called each step from the surf collide pre_collide +------------------------------------------------------------------------- */ + +void SurfReactAdsorbKokkos::pre_react() +{ + ParticleKokkos* particle_kk = (ParticleKokkos*) particle; + particle_kk->sync(Device,PARTICLE_MASK|SPECIES_MASK); + d_particles = particle_kk->k_particles.view_device(); + d_species = particle_kk->k_species.view_device(); + + fnum_ = update->fnum; + + // copy current per-face state (changes only at sync) host->device + + auto h_total = Kokkos::create_mirror_view(d_total_state); + auto h_area = Kokkos::create_mirror_view(d_area); + auto h_weight = Kokkos::create_mirror_view(d_weight); + auto h_sstate = Kokkos::create_mirror_view(d_species_state); + for (int i = 0; i < nface; i++) { + h_total(i) = total_state[i]; + h_area(i) = area[i]; + h_weight(i) = weight[i]; + for (int j = 0; j < nspecies_surf; j++) + h_sstate(i,j) = species_state[i][j]; + } + Kokkos::deep_copy(d_total_state,h_total); + Kokkos::deep_copy(d_area,h_area); + Kokkos::deep_copy(d_weight,h_weight); + Kokkos::deep_copy(d_species_state,h_sstate); +} + +/* ---------------------------------------------------------------------- */ + +void SurfReactAdsorbKokkos::tally_reset() +{ + SurfReact::tally_reset(); + Kokkos::deep_copy(d_scalars,0); +} + +/* ---------------------------------------------------------------------- + bring device tallies + per-face deltas to host, then run the host + state-sync logic (MPI reduce + per-face state update), then re-zero +------------------------------------------------------------------------- */ + +void SurfReactAdsorbKokkos::tally_update() +{ + // device -> host: reaction counts + + Kokkos::deep_copy(h_scalars,d_scalars); + nsingle = h_nsingle(); + for (int i = 0; i < nlist_gs; i++) tally_single[i] = h_tally_single[i]; + + // device -> host: per-face perspecies deltas accumulated since last sync + + k_species_delta.modify_device(); + k_species_delta.sync_host(); + auto h_delta = k_species_delta.view_host(); + for (int i = 0; i < nface; i++) + for (int j = 0; j < nspecies_surf; j++) + species_delta[i][j] = h_delta(i,j); + + // host logic: accumulate tallies and (every nsync) MPI-sync per-face state; + // update_state_face() also re-zeros host species_delta + + SurfReactAdsorb::tally_update(); + + // mirror re-zeroed host deltas back to device (only changed on a sync step) + + if (update->ntimestep % nsync == 0) { + for (int i = 0; i < nface; i++) + for (int j = 0; j < nspecies_surf; j++) + h_delta(i,j) = species_delta[i][j]; + k_species_delta.modify_host(); + k_species_delta.sync_device(); + Kokkos::deep_copy(d_scalars,0); + } +} + +/* ---------------------------------------------------------------------- */ + +void SurfReactAdsorbKokkos::backup() +{ + ParticleKokkos* particle_kk = (ParticleKokkos*) particle; + d_particles = particle_kk->k_particles.view_device(); + +#ifdef SPARTA_KOKKOS_EXACT + if (!random_backup) + random_backup = new RanKnuth(12345 + comm->me); + memcpy(random_backup,random,sizeof(RanKnuth)); +#endif +} + +/* ---------------------------------------------------------------------- */ + +void SurfReactAdsorbKokkos::restore() +{ +#ifdef SPARTA_KOKKOS_EXACT + memcpy(random,random_backup,sizeof(RanKnuth)); +#endif +} diff --git a/src/KOKKOS/surf_react_adsorb_kokkos.h b/src/KOKKOS/surf_react_adsorb_kokkos.h new file mode 100644 index 000000000..544e7a267 --- /dev/null +++ b/src/KOKKOS/surf_react_adsorb_kokkos.h @@ -0,0 +1,328 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#ifdef SURF_REACT_CLASS + +SurfReactStyle(adsorb/kk,SurfReactAdsorbKokkos) + +#else + +#ifndef SPARTA_SURF_REACT_ADSORB_KOKKOS_H +#define SPARTA_SURF_REACT_ADSORB_KOKKOS_H + +#include "surf_react_adsorb.h" +#include "kokkos_type.h" +#include "rand_pool_wrap.h" +#include "Kokkos_Random.hpp" +#include "particle_kokkos.h" + +namespace SPARTA_NS { + +// must match enums in surf_react_adsorb.cpp + +namespace SRA_KK { + enum{DISSOCIATION,EXCHANGE,RECOMBINATION,AA,DA,LH1,LH3,CD,ER,CI}; + enum{FACE,SURF}; + enum{NOMODEL,SPECULAR,DIFFUSE,ADIABATIC,CLL,TD,IMPULSIVE,MAXMODELS}; + enum{SIMPLE,ARRHENIUS}; +} + +#define SRA_KK_MAXREACTANT 5 +#define SRA_KK_MAXPRODUCT 5 +#define SRA_KK_MAXCOEFF 4 +#define SRA_KK_MAXPERSPECIES 16 // max GS reactions a single species can be in + +class SurfReactAdsorbKokkos : public SurfReactAdsorb { + public: + SurfReactAdsorbKokkos(class SPARTA *, int, char **); + SurfReactAdsorbKokkos(class SPARTA *); + ~SurfReactAdsorbKokkos(); + void init(); + void tally_reset(); + void tally_update(); + + void pre_react(); + void backup(); + void restore(); + + private: + // flattened GS reaction tables (indexed by reaction j in 0..nlist_gs) + + DAT::t_int_1d d_reactions_n; // # of GS reactions for each species + DAT::t_int_2d d_list; // per-species list of reaction indices + + DAT::t_int_1d d_type; // reaction type (DISSOCIATION,...) + DAT::t_int_1d d_style; // SIMPLE or ARRHENIUS + DAT::t_float_1d d_kreact; // precomputed rate coefficient + DAT::t_int_1d d_kisliuk_flag; + DAT::t_float_2d d_kisliuk; // [j][3] + DAT::t_int_1d d_energy_flag; + DAT::t_float_2d d_energy; // [j][2] + DAT::t_float_2d d_coeff; // [j][MAXCOEFF] + DAT::t_int_1d d_nreactant,d_nproduct; + DAT::t_int_1d d_nprod_g,d_nprod_g_tot; + DAT::t_int_1d d_cmodel_ip,d_cmodel_jp; + + DAT::t_int_2d d_rstate,d_rpart,d_rstoich,d_rad; // reactant slots [j][MAXREACTANT] + DAT::t_int_2d d_pstate,d_ppart,d_pstoich,d_pad; // product slots [j][MAXPRODUCT] + DAT::t_int_2d d_products; // product species indices + + // per-face state (FACE mode); small (nface <= 6) + + DAT::t_int_1d d_total_state; // [nface] + DAT::t_float_1d d_area,d_weight; // [nface] + DAT::t_int_2d d_species_state; // [nface][nspecies_surf] + DAT::t_int_2d d_species_delta; // [nface][nspecies_surf] (atomic) + + DAT::tdual_int_2d k_species_delta; + + double fnum_; // update->fnum, set in pre_react + + void init_reactions_gs_kokkos(); + +#ifndef SPARTA_KOKKOS_EXACT + Kokkos::Random_XorShift64_Pool rand_pool; + typedef typename Kokkos::Random_XorShift64_Pool::generator_type rand_type; +#else + RandPoolWrap rand_pool; + typedef RandWrap rand_type; +#endif + + RanKnuth* random_backup; + + DAT::t_int_1d d_scalars; + HAT::t_int_1d h_scalars; + DAT::t_int_scalar d_nsingle; + DAT::t_int_1d d_tally_single; + HAT::t_int_scalar h_nsingle; + HAT::t_int_1d h_tally_single; + + t_particle_1d d_particles; + t_species_1d d_species; + + public: + + /* ---------------------------------------------------------------------- + select GS surface reaction to perform for particle IP on box face + mirrors SurfReactAdsorb::react() for mode == FACE, gsflag == 1 + return reaction 1 to N, 0 = no reaction + only reaction types/cmodels validated at init are reachable here + ------------------------------------------------------------------------- */ + + template + KOKKOS_INLINE_FUNCTION + int react_kokkos(Particle::OnePart *&ip, int isurf, const double *norm, + Particle::OnePart *&jp, int &velreset, + const DAT::t_int_scalar &d_retry, + const DAT::t_int_scalar &d_nlocal) const + { + // convert face index from negative value to 0..5 inclusive + + int iface = -(isurf+1); + + int n = d_reactions_n[ip->ispecies]; + if (n == 0) return 0; + + double fnum = fnum_; + long int maxstick = ceil(max_cover*d_area[iface] / (fnum*d_weight[iface])); + double factor = fnum * d_weight[iface] / d_area[iface]; + double ms_inv = factor / max_cover; + + double prob_value[SRA_KK_MAXPERSPECIES]; + double sum_prob = 0.0; + double scatter_prob = 0.0, correction = 1.0; + int coeff_val = 1; + + rand_type rand_gen = rand_pool.get_state(); + + for (int i = 0; i < n; i++) { + int j = d_list(ip->ispecies,i); + + if (d_style(j) == SRA_KK::ARRHENIUS) coeff_val = 3; + + double surf_cover,S_theta,K_ads; + + switch (d_type(j)) { + case SRA_KK::DISSOCIATION: + case SRA_KK::EXCHANGE: + case SRA_KK::RECOMBINATION: + prob_value[i] = d_kreact(j); + break; + + case SRA_KK::AA: + case SRA_KK::DA: + case SRA_KK::LH1: + case SRA_KK::LH3: + case SRA_KK::CD: + surf_cover = d_total_state[iface] * ms_inv; + S_theta = 0.0; + if (d_kisliuk_flag(j)) { + K_ads = d_kisliuk(j,0) * pow(twall,d_kisliuk(j,1)) * + exp(-d_kisliuk(j,2)/twall); + if (surf_cover < 1) + S_theta = pow((1 - surf_cover) / + (1 - surf_cover + K_ads*surf_cover),d_coeff(j,coeff_val)); + } else { + S_theta = pow((1-surf_cover),d_coeff(j,coeff_val)); + } + prob_value[i] = d_kreact(j)*S_theta; + break; + + case SRA_KK::ER: + { + double dot = 2.0; + if (d_nreactant(j) == 1) + prob_value[i] = 2.0 * d_kreact(j) * + (maxstick - d_total_state[iface]) * ms_inv / fabs(dot); + else + prob_value[i] = 2.0 * d_kreact(j) / fabs(dot); + break; + } + + case SRA_KK::CI: + prob_value[i] = d_kreact(j); + if (d_energy_flag(j)) { + double *v = ip->v; + double dot = v[0]*norm[0]+v[1]*norm[1]+v[2]*norm[2]; + double vmag_sq = v[0]*v[0]+v[1]*v[1]+v[2]*v[2]; + double E_i = 0.5 * d_species[ip->ispecies].mass * vmag_sq; + double cos_theta = fabs(dot) / sqrt(vmag_sq); + prob_value[i] *= pow(E_i,d_energy(j,0)) * pow(cos_theta,d_energy(j,1)); + } + break; + } + + for (int k = 1; k < d_nreactant(j); k++) { + if (d_rstate(j,k) == 's') { + if (d_rpart(j,k) == 0) + prob_value[i] *= stoich_pow_kk(d_total_state[iface],d_rstoich(j,k)) * + pow(ms_inv,d_rstoich(j,k)); + else + prob_value[i] *= stoich_pow_kk(d_species_state(iface,d_rad(j,k)), + d_rstoich(j,k)) * + pow(ms_inv,d_rstoich(j,k)); + } + } + + sum_prob += prob_value[i]; + } + + if (sum_prob > 1.0) correction = 1.0/sum_prob; + else scatter_prob = 1.0 - sum_prob; + + double react_prob = scatter_prob; + double random_prob = rand_gen.drand(); + + if (react_prob > random_prob) { + rand_pool.free_state(rand_gen); + return 0; + } + + for (int i = 0; i < n; i++) { + int j = d_list(ip->ispecies,i); + react_prob += prob_value[i] * correction; + if (react_prob <= random_prob) continue; + + // reaction j fires + + if (ATOMIC_REDUCTION == 0) { + d_nsingle()++; + d_tally_single(j)++; + } else { + Kokkos::atomic_inc(&d_nsingle()); + Kokkos::atomic_inc(&d_tally_single(j)); + } + + // update per-face perspecies deltas for participating surf reactants/products + + auto a_species_delta = d_species_delta; + for (int k = 0; k < d_nreactant(j); k++) + if (d_rpart(j,k) == 1 && d_rstate(j,k) == 's') + Kokkos::atomic_add(&a_species_delta(iface,d_rad(j,k)),-d_rstoich(j,k)); + for (int k = 0; k < d_nproduct(j); k++) + if (d_ppart(j,k) == 1 && d_pstate(j,k) == 's') + Kokkos::atomic_add(&a_species_delta(iface,d_pad(j,k)),d_pstoich(j,k)); + + // post-reaction particle handling + // only types validated at init (no cmodel post-scatter) are reachable + + switch (d_type(j)) { + + case SRA_KK::DISSOCIATION: + { + double x[3],v[3]; + ip->ispecies = d_products(j,0); + int id = MAXSMALLINT*rand_gen.drand(); + memcpy(x,ip->x,3*sizeof(double)); + memcpy(v,ip->v,3*sizeof(double)); + int jp_species; + if (d_pstoich(j,0) == 2) jp_species = d_products(j,0); + else jp_species = d_products(j,1); + + int index; + if (ATOMIC_REDUCTION == 0) { index = d_nlocal(); d_nlocal()++; } + else index = Kokkos::atomic_fetch_add(&d_nlocal(),1); + + int reallocflag = ParticleKokkos::add_particle_kokkos(d_particles,index, + id,jp_species,ip->icell,x,v,0.0,0.0); + if (reallocflag) { + d_retry() = 1; + rand_pool.free_state(rand_gen); + return 0; + } + jp = &d_particles[index]; + rand_pool.free_state(rand_gen); + return (j + 1); + } + + case SRA_KK::EXCHANGE: + ip->ispecies = d_products(j,0); + rand_pool.free_state(rand_gen); + return (j + 1); + + case SRA_KK::RECOMBINATION: + case SRA_KK::AA: + case SRA_KK::LH3: + case SRA_KK::CD: + ip = NULL; + rand_pool.free_state(rand_gen); + return (j + 1); + } + } + + rand_pool.free_state(rand_gen); + return 0; + } + + KOKKOS_INLINE_FUNCTION + double stoich_pow_kk(int base, int p) const + { + const double THIRD = 1.0/3.0; + switch (p) { + case 0: return 1.0; + case 1: return (base >= p) ? double(base) : 0.0; + case 2: return (base >= p) ? 0.5*base*(base-1) : 0.0; + case 3: return (base >= p) ? 0.5*THIRD*base*(base-1)*(base-2) : 0.0; + case 4: return (base >= p) ? 0.125*THIRD*base*(base-1)*(base-2)*(base-3) : 0.0; + case 5: return (base >= p) ? 0.025*THIRD*base*(base-1)*(base-2)*(base-3)*(base-4) : 0.0; + case 6: return (base >= p) ? 0.0125*THIRD*THIRD*base*(base-1)*(base-2)*(base-3)*(base-4)*(base-5) : 0.0; + } + return 0.0; + } +}; + +} + +#endif +#endif diff --git a/src/KOKKOS/update_kokkos.cpp b/src/KOKKOS/update_kokkos.cpp index 45c2d3d1c..364f3e2a5 100644 --- a/src/KOKKOS/update_kokkos.cpp +++ b/src/KOKKOS/update_kokkos.cpp @@ -195,12 +195,16 @@ void UpdateKokkos::init() // choose the appropriate move method + // REACT=1 is also needed without explicit surfs when box-face/boundary + // reactions are defined (e.g. surf_react adsorb in face mode) + if (domain->dimension == 3) { if (surf->exist) { if (surf->nsr) moveptr = &UpdateKokkos::move<3,1,1,0>; else moveptr = &UpdateKokkos::move<3,1,0,0>; } else { - if (optmove_flag) moveptr = &UpdateKokkos::move<3,0,0,1>; + if (surf->nsr) moveptr = &UpdateKokkos::move<3,0,1,0>; + else if (optmove_flag) moveptr = &UpdateKokkos::move<3,0,0,1>; else moveptr = &UpdateKokkos::move<3,0,0,0>; } } else if (domain->axisymmetric) { @@ -208,7 +212,8 @@ void UpdateKokkos::init() if (surf->nsr) moveptr = &UpdateKokkos::move<1,1,1,0>; else moveptr = &UpdateKokkos::move<1,1,0,0>; } else { - if (optmove_flag) moveptr = &UpdateKokkos::move<1,0,0,1>; + if (surf->nsr) moveptr = &UpdateKokkos::move<1,0,1,0>; + else if (optmove_flag) moveptr = &UpdateKokkos::move<1,0,0,1>; else moveptr = &UpdateKokkos::move<1,0,0,0>; } } else if (domain->dimension == 2) { @@ -216,7 +221,8 @@ void UpdateKokkos::init() if (surf->nsr) moveptr = &UpdateKokkos::move<2,1,1,0>; else moveptr = &UpdateKokkos::move<2,1,0,0>; } else { - if (optmove_flag) moveptr = &UpdateKokkos::move<2,0,0,1>; + if (surf->nsr) moveptr = &UpdateKokkos::move<2,0,1,0>; + else if (optmove_flag) moveptr = &UpdateKokkos::move<2,0,0,1>; else moveptr = &UpdateKokkos::move<2,0,0,0>; } } diff --git a/src/surf_react_adsorb.cpp b/src/surf_react_adsorb.cpp index c099edc5b..ccaa76fca 100644 --- a/src/surf_react_adsorb.cpp +++ b/src/surf_react_adsorb.cpp @@ -212,6 +212,8 @@ SurfReactAdsorb::SurfReactAdsorb(SPARTA *sparta, int narg, char **arg) : SurfReactAdsorb::~SurfReactAdsorb() { + if (copy) return; + delete random; // surface species diff --git a/src/surf_react_adsorb.h b/src/surf_react_adsorb.h index 611d45598..7ef0bfd03 100644 --- a/src/surf_react_adsorb.h +++ b/src/surf_react_adsorb.h @@ -28,8 +28,9 @@ namespace SPARTA_NS { class SurfReactAdsorb : public SurfReact { public: SurfReactAdsorb(class SPARTA *, int, char **); + SurfReactAdsorb(class SPARTA *sparta) : SurfReact(sparta) {} // needed for Kokkos ~SurfReactAdsorb(); - void init(); + virtual void init(); int react(Particle::OnePart *&, int, double *, Particle::OnePart *&, int &); char *reactionID(int); @@ -37,10 +38,10 @@ class SurfReactAdsorb : public SurfReact { int match_reactant(char *, int); int match_product(char *, int); - void tally_update(); + virtual void tally_update(); void grid_changed(); - private: + protected: int me,nprocs; int distributed; From c1305e0885265b5219e9afe9d144798ff53443d8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 20 Jun 2026 21:46:21 +0000 Subject: [PATCH 09/30] KOKKOS: adsorb GS face mode - add DA/LH1/ER/CI types + specular cmodel Extend SurfReactAdsorbKokkos::react_kokkos() to the remaining GS reaction types (DA/LH1/ER/CI) and post-reaction collision-model (cmodel) scatter for the NOMODEL and SPECULAR cases (specular mirrors SurfCollideSpecular::wrapper, a velocity reflect with no RNG, so it is exactly bit-for-bit). - react_kokkos: DA (gas-product handling + optional second-particle creation), LH1/ER (set product species + cmodel scatter), CI (set product species + optional second-particle creation + cmodel scatter), mirroring the host SurfReactAdsorb::react() control flow and RNG draw order (particle-id draws use the surf-react RNG, matching the host). Add apply_cmodel() (SPECULAR reflect via MathExtraKokkos::reflect3) and create_particle() helpers. - init guard relaxed: DA/LH1/ER/CI are now supported; only RNG-based cmodels (diffuse/cll/td/adiabatic/impulsive) still error clearly at init. Verified bit-for-bit CPU vs -sf kk (Serial+EXACT) on the beam/face GS_2 set with cmodels switched to specular: identical stats and per-reaction tallies (AA 21055, LH1 2, LH3 6475, ER 662, CI 10; total 28204). Co-Authored-By: Claude Opus 4.8 Co-Authored-By: Stan Moore Claude-Session: https://claude.ai/code/session_01Q6nQVuxEiTUqSzDoDppCUM --- src/KOKKOS/surf_react_adsorb_kokkos.cpp | 14 ++-- src/KOKKOS/surf_react_adsorb_kokkos.h | 103 +++++++++++++++++++++++- 2 files changed, 108 insertions(+), 9 deletions(-) diff --git a/src/KOKKOS/surf_react_adsorb_kokkos.cpp b/src/KOKKOS/surf_react_adsorb_kokkos.cpp index 44fab7b8e..9e3f5357f 100644 --- a/src/KOKKOS/surf_react_adsorb_kokkos.cpp +++ b/src/KOKKOS/surf_react_adsorb_kokkos.cpp @@ -91,13 +91,13 @@ void SurfReactAdsorbKokkos::init() for (int i = 0; i < nlist_gs; i++) { OneReaction_GS *r = &rlist_gs[i]; if (!r->active) continue; - if (r->cmodel_ip != SRA_KK::NOMODEL || r->cmodel_jp != SRA_KK::NOMODEL) - error->all(FLERR,"Kokkos surf_react adsorb does not yet support reactions " - "with a post-reaction surface collision model"); - if (r->type == SRA_KK::DA || r->type == SRA_KK::LH1 || - r->type == SRA_KK::ER || r->type == SRA_KK::CI) - error->all(FLERR,"Kokkos surf_react adsorb does not yet support DA/LH1/ER/CI " - "reaction types (they require a post-reaction collision model)"); + // post-reaction collision model (cmodel) scatter on device currently + // supports NOMODEL and SPECULAR (no RNG); RNG-based cmodels deferred + + if ((r->cmodel_ip != SRA_KK::NOMODEL && r->cmodel_ip != SRA_KK::SPECULAR) || + (r->cmodel_jp != SRA_KK::NOMODEL && r->cmodel_jp != SRA_KK::SPECULAR)) + error->all(FLERR,"Kokkos surf_react adsorb does not yet support reactions with " + "a diffuse/cll/td/adiabatic/impulsive post-reaction collision model"); } Kokkos::deep_copy(d_scalars,0); diff --git a/src/KOKKOS/surf_react_adsorb_kokkos.h b/src/KOKKOS/surf_react_adsorb_kokkos.h index 544e7a267..af3ba1384 100644 --- a/src/KOKKOS/surf_react_adsorb_kokkos.h +++ b/src/KOKKOS/surf_react_adsorb_kokkos.h @@ -23,6 +23,7 @@ SurfReactStyle(adsorb/kk,SurfReactAdsorbKokkos) #include "surf_react_adsorb.h" #include "kokkos_type.h" +#include "math_extra_kokkos.h" #include "rand_pool_wrap.h" #include "Kokkos_Random.hpp" #include "particle_kokkos.h" @@ -254,8 +255,9 @@ class SurfReactAdsorbKokkos : public SurfReactAdsorb { if (d_ppart(j,k) == 1 && d_pstate(j,k) == 's') Kokkos::atomic_add(&a_species_delta(iface,d_pad(j,k)),d_pstoich(j,k)); - // post-reaction particle handling - // only types validated at init (no cmodel post-scatter) are reachable + // post-reaction particle handling, mirrors SurfReactAdsorb::react() + // cmodel post-reaction scatter currently supports NOMODEL and SPECULAR + // (validated at init); RNG-based cmodels (diffuse/cll/td/...) deferred switch (d_type(j)) { @@ -298,6 +300,63 @@ class SurfReactAdsorbKokkos : public SurfReactAdsorb { ip = NULL; rand_pool.free_state(rand_gen); return (j + 1); + + case SRA_KK::DA: + { + if (d_nprod_g(j) == 0) ip = NULL; + else { + int nn = 1; + for (int pj = 1; pj < d_nproduct(j); pj++) { + if (d_pstate(j,pj) == 'g') { + if (nn == 1) { + nn++; + ip->ispecies = d_products(j,pj); + apply_cmodel(ip,norm,d_cmodel_ip(j)); + if (d_pstoich(j,pj) == 2) { + jp = create_particle(ip,d_products(j,pj),rand_gen,d_nlocal,d_retry); + if (!jp) { rand_pool.free_state(rand_gen); return 0; } + apply_cmodel(jp,norm,d_cmodel_ip(j)); + } + } else { + jp = create_particle(ip,d_products(j,pj),rand_gen,d_nlocal,d_retry); + if (!jp) { rand_pool.free_state(rand_gen); return 0; } + apply_cmodel(jp,norm,d_cmodel_jp(j)); + } + } + } + } + if (d_cmodel_ip(j) != SRA_KK::NOMODEL) velreset = 1; + rand_pool.free_state(rand_gen); + return (j + 1); + } + + case SRA_KK::LH1: + case SRA_KK::ER: + ip->ispecies = d_products(j,0); + apply_cmodel(ip,norm,d_cmodel_ip(j)); + if (d_cmodel_ip(j) != SRA_KK::NOMODEL) velreset = 1; + rand_pool.free_state(rand_gen); + return (j + 1); + + case SRA_KK::CI: + { + ip->ispecies = d_products(j,0); + apply_cmodel(ip,norm,d_cmodel_ip(j)); + if (d_nprod_g_tot(j) == 2) { + if (d_pstoich(j,0) == 2) { + jp = create_particle(ip,d_products(j,0),rand_gen,d_nlocal,d_retry); + if (!jp) { rand_pool.free_state(rand_gen); return 0; } + apply_cmodel(jp,norm,d_cmodel_ip(j)); + } else { + jp = create_particle(ip,d_products(j,1),rand_gen,d_nlocal,d_retry); + if (!jp) { rand_pool.free_state(rand_gen); return 0; } + apply_cmodel(jp,norm,d_cmodel_jp(j)); + } + } + if (d_cmodel_ip(j) != SRA_KK::NOMODEL) velreset = 1; + rand_pool.free_state(rand_gen); + return (j + 1); + } } } @@ -305,6 +364,46 @@ class SurfReactAdsorbKokkos : public SurfReactAdsorb { return 0; } + /* ---------------------------------------------------------------------- + apply a post-reaction collision model (cmodel) scatter to particle p + SPECULAR mirrors SurfCollideSpecular::wrapper() (reflect, no RNG) + NOMODEL is a no-op; RNG-based cmodels are rejected at init + ------------------------------------------------------------------------- */ + + KOKKOS_INLINE_FUNCTION + void apply_cmodel(Particle::OnePart *p, const double *norm, int cmodel) const + { + if (cmodel == SRA_KK::SPECULAR) + MathExtraKokkos::reflect3(p->v,norm); + } + + /* ---------------------------------------------------------------------- + create a new particle (copy of ip's x,v) of species sp, mirrors the + add_particle path in SurfReactAdsorb::react(); returns ptr or NULL on + realloc (caller retries the whole move) + ------------------------------------------------------------------------- */ + + KOKKOS_INLINE_FUNCTION + Particle::OnePart *create_particle(Particle::OnePart *ip, int sp, + rand_type &rand_gen, + const DAT::t_int_scalar &d_nlocal, + const DAT::t_int_scalar &d_retry) const + { + double x[3],v[3]; + memcpy(x,ip->x,3*sizeof(double)); + memcpy(v,ip->v,3*sizeof(double)); + int id = MAXSMALLINT*rand_gen.drand(); + + int index = Kokkos::atomic_fetch_add(&d_nlocal(),1); + int reallocflag = ParticleKokkos::add_particle_kokkos(d_particles,index, + id,sp,ip->icell,x,v,0.0,0.0); + if (reallocflag) { + d_retry() = 1; + return NULL; + } + return &d_particles[index]; + } + KOKKOS_INLINE_FUNCTION double stoich_pow_kk(int base, int p) const { From c0789454ac4129f21e5d3e8c3ec019deef01c8ad Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 20 Jun 2026 23:51:11 +0000 Subject: [PATCH 10/30] KOKKOS: adsorb GS face mode - bit-exact cmodel scatter (diffuse/cll/td) Add device post-reaction collision-model (cmodel) scatter so the real gas-surface examples (GS_2: cll + td cmodels, DA/LH1/ER/CI types) run under -sf kk and match the host bit-for-bit. RNG approach (per guidance): each cmodel scatter draws from the Kokkos RNG wrapper (RandPoolWrap) initialized from that cmodel's own RanKnuth (cmodels[idx]->random, reached via a new SurfCollide::kokkos_random() accessor). In EXACT serial RandPoolWrap::init points thread 0 at the same RanKnuth the host uses, so the replicated scatter draws in the host order and matches exactly. The Kokkos scatter device functions for diffuse/cll/td (incl. rotational/vibrational energy accommodation, mirroring the surf-collide Kokkos kernels) are replicated inline in SurfReactAdsorbKokkos to avoid the cll<->adsorb include cycle; specular is an inline reflect (no RNG). - SurfReactAdsorbKokkos: scatter_cmodel() dispatch + diffuse_scatter/cll_scatter/ td_scatter/erot_kk/evib_kk; flatten per-reaction cmodel coeffs/flags (ip+jp) to device; one RandPoolWrap per cmodel type built in init_cmodels_kokkos(); capture boltz + collide rot/vib styles + per-cmodel RNG in pre_react(). - surf_collide: add kokkos_random() accessor (base returns NULL; cll/td/diffuse/ adiabatic/impulsive return their RanKnuth). - init guard now only rejects adiabatic/impulsive cmodels. Verified bit-for-bit CPU vs -sf kk (Serial+EXACT) on examples/surf_react_adsorb/ in.beam.face.gs (GS_2): identical stats and per-reaction tallies (total 28160; AA 20986, LH1 2, LH3 6529, ER 638, CI 5). Co-Authored-By: Claude Opus 4.8 Co-Authored-By: Stan Moore Claude-Session: https://claude.ai/code/session_01Q6nQVuxEiTUqSzDoDppCUM --- src/KOKKOS/surf_react_adsorb_kokkos.cpp | 96 ++++++- src/KOKKOS/surf_react_adsorb_kokkos.h | 342 +++++++++++++++++++++++- src/surf_collide.h | 1 + src/surf_collide_adiabatic.h | 2 + src/surf_collide_cll.h | 2 + src/surf_collide_diffuse.h | 2 + src/surf_collide_impulsive.h | 2 + src/surf_collide_td.h | 2 + 8 files changed, 435 insertions(+), 14 deletions(-) diff --git a/src/KOKKOS/surf_react_adsorb_kokkos.cpp b/src/KOKKOS/surf_react_adsorb_kokkos.cpp index 9e3f5357f..b10085c1e 100644 --- a/src/KOKKOS/surf_react_adsorb_kokkos.cpp +++ b/src/KOKKOS/surf_react_adsorb_kokkos.cpp @@ -17,6 +17,9 @@ #include "surf_react_adsorb_kokkos.h" #include "input.h" #include "update.h" +#include "collide.h" +#include "surf_collide.h" +#include "random_knuth.h" #include "comm.h" #include "domain.h" #include "particle.h" @@ -26,6 +29,25 @@ using namespace SPARTA_NS; +// cmodel coeff/flag counts (must match SurfReactAdsorb::readfile_gs) + +static void cmodel_sizes(int model, int &nc, int &nf) +{ + nc = nf = 0; + switch (model) { + case SRA_KK::SPECULAR: nf = 1; break; + case SRA_KK::DIFFUSE: nc = 2; break; + case SRA_KK::CLL: nc = 5; nf = 1; break; + case SRA_KK::TD: nc = 8; nf = 3; break; + case SRA_KK::IMPULSIVE: nc = 11; nf = 4; break; + } +} + +static bool cmodel_unsupported(int m) +{ + return (m == SRA_KK::ADIABATIC || m == SRA_KK::IMPULSIVE); +} + /* ---------------------------------------------------------------------- */ SurfReactAdsorbKokkos::SurfReactAdsorbKokkos(SPARTA *sparta, int narg, char **arg) : @@ -47,6 +69,8 @@ SurfReactAdsorbKokkos::SurfReactAdsorbKokkos(SPARTA *sparta, int narg, char **ar h_tally_single = Kokkos::subview(h_scalars,std::make_pair(1,nlist_gs+1)); random_backup = NULL; + + for (int i = 0; i < SRA_KK_MAXMODELS; i++) cmodel_pool[i] = NULL; } SurfReactAdsorbKokkos::SurfReactAdsorbKokkos(SPARTA *sparta) : @@ -69,6 +93,8 @@ SurfReactAdsorbKokkos::~SurfReactAdsorbKokkos() #ifdef SPARTA_KOKKOS_EXACT rand_pool.destroy(); if (random_backup) delete random_backup; + for (int i = 0; i < SRA_KK_MAXMODELS; i++) + if (cmodel_pool[i]) { cmodel_pool[i]->destroy(); delete cmodel_pool[i]; } #endif } @@ -91,24 +117,71 @@ void SurfReactAdsorbKokkos::init() for (int i = 0; i < nlist_gs; i++) { OneReaction_GS *r = &rlist_gs[i]; if (!r->active) continue; - // post-reaction collision model (cmodel) scatter on device currently - // supports NOMODEL and SPECULAR (no RNG); RNG-based cmodels deferred + // post-reaction collision model (cmodel) scatter on device supports + // NOMODEL/SPECULAR/DIFFUSE/CLL/TD; adiabatic/impulsive deferred - if ((r->cmodel_ip != SRA_KK::NOMODEL && r->cmodel_ip != SRA_KK::SPECULAR) || - (r->cmodel_jp != SRA_KK::NOMODEL && r->cmodel_jp != SRA_KK::SPECULAR)) + if (cmodel_unsupported(r->cmodel_ip) || cmodel_unsupported(r->cmodel_jp)) error->all(FLERR,"Kokkos surf_react adsorb does not yet support reactions with " - "a diffuse/cll/td/adiabatic/impulsive post-reaction collision model"); + "an adiabatic or impulsive post-reaction collision model"); } Kokkos::deep_copy(d_scalars,0); init_reactions_gs_kokkos(); + init_cmodels_kokkos(); #ifdef SPARTA_KOKKOS_EXACT rand_pool.init(random); #endif } +/* ---------------------------------------------------------------------- + flatten per-reaction cmodel coeffs/flags and build one RNG pool per + cmodel type, each wrapping that cmodel's RanKnuth so the device scatter + matches the host SurfCollide::wrapper bit-for-bit (EXACT serial) +------------------------------------------------------------------------- */ + +void SurfReactAdsorbKokkos::init_cmodels_kokkos() +{ + int nr = MAX(nlist_gs,1); + d_cmip_coeffs = DAT::t_float_2d("sra:cmip_coeffs",nr,SRA_KK_MAXCMCOEFF); + d_cmjp_coeffs = DAT::t_float_2d("sra:cmjp_coeffs",nr,SRA_KK_MAXCMCOEFF); + d_cmip_flags = DAT::t_int_2d("sra:cmip_flags",nr,SRA_KK_MAXCMFLAG); + d_cmjp_flags = DAT::t_int_2d("sra:cmjp_flags",nr,SRA_KK_MAXCMFLAG); + + auto h_cmip_coeffs = Kokkos::create_mirror_view(d_cmip_coeffs); + auto h_cmjp_coeffs = Kokkos::create_mirror_view(d_cmjp_coeffs); + auto h_cmip_flags = Kokkos::create_mirror_view(d_cmip_flags); + auto h_cmjp_flags = Kokkos::create_mirror_view(d_cmjp_flags); + + for (int i = 0; i < nlist_gs; i++) { + OneReaction_GS *r = &rlist_gs[i]; + int nc,nf; + cmodel_sizes(r->cmodel_ip,nc,nf); + for (int k = 0; k < nc; k++) h_cmip_coeffs(i,k) = r->cmodel_ip_coeffs[k]; + for (int k = 0; k < nf; k++) h_cmip_flags(i,k) = r->cmodel_ip_flags[k]; + cmodel_sizes(r->cmodel_jp,nc,nf); + for (int k = 0; k < nc; k++) h_cmjp_coeffs(i,k) = r->cmodel_jp_coeffs[k]; + for (int k = 0; k < nf; k++) h_cmjp_flags(i,k) = r->cmodel_jp_flags[k]; + } + + Kokkos::deep_copy(d_cmip_coeffs,h_cmip_coeffs); + Kokkos::deep_copy(d_cmjp_coeffs,h_cmjp_coeffs); + Kokkos::deep_copy(d_cmip_flags,h_cmip_flags); + Kokkos::deep_copy(d_cmjp_flags,h_cmjp_flags); + +#ifdef SPARTA_KOKKOS_EXACT + for (int idx = 0; idx < SRA_KK_MAXMODELS; idx++) { + if (cmodel_pool[idx]) continue; + if (!cmodels[idx]) continue; + RanKnuth *cmrand = cmodels[idx]->kokkos_random(); + if (!cmrand) continue; // e.g. specular has no RNG + cmodel_pool[idx] = new RandPoolWrap(12345,sparta); + cmodel_pool[idx]->init(cmrand); + } +#endif +} + /* ---------------------------------------------------------------------- */ void SurfReactAdsorbKokkos::init_reactions_gs_kokkos() @@ -269,6 +342,19 @@ void SurfReactAdsorbKokkos::pre_react() fnum_ = update->fnum; + // cmodel scatter state: boltz, collide rot/vib styles, per-cmodel RNG + + boltz_ = update->boltz; + rotstyle_ = SRA_KK::NONE; + if (Pointers::collide) rotstyle_ = Pointers::collide->rotstyle; + vibstyle_ = SRA_KK::NONE; + if (Pointers::collide) vibstyle_ = Pointers::collide->vibstyle; + +#ifdef SPARTA_KOKKOS_EXACT + for (int idx = 0; idx < SRA_KK_MAXMODELS; idx++) + if (cmodel_pool[idx]) d_cmodel_rand[idx] = cmodel_pool[idx]->get_state(); +#endif + // copy current per-face state (changes only at sync) host->device auto h_total = Kokkos::create_mirror_view(d_total_state); diff --git a/src/KOKKOS/surf_react_adsorb_kokkos.h b/src/KOKKOS/surf_react_adsorb_kokkos.h index af3ba1384..96c219e13 100644 --- a/src/KOKKOS/surf_react_adsorb_kokkos.h +++ b/src/KOKKOS/surf_react_adsorb_kokkos.h @@ -24,6 +24,7 @@ SurfReactStyle(adsorb/kk,SurfReactAdsorbKokkos) #include "surf_react_adsorb.h" #include "kokkos_type.h" #include "math_extra_kokkos.h" +#include "math_const.h" #include "rand_pool_wrap.h" #include "Kokkos_Random.hpp" #include "particle_kokkos.h" @@ -37,12 +38,16 @@ namespace SRA_KK { enum{FACE,SURF}; enum{NOMODEL,SPECULAR,DIFFUSE,ADIABATIC,CLL,TD,IMPULSIVE,MAXMODELS}; enum{SIMPLE,ARRHENIUS}; + enum{NONE,DISCRETE,SMOOTH}; // rotstyle/vibstyle, must match collide.h } #define SRA_KK_MAXREACTANT 5 #define SRA_KK_MAXPRODUCT 5 #define SRA_KK_MAXCOEFF 4 #define SRA_KK_MAXPERSPECIES 16 // max GS reactions a single species can be in +#define SRA_KK_MAXMODELS 7 // = MAXMODELS +#define SRA_KK_MAXCMCOEFF 11 // max cmodel coeffs (impulsive) +#define SRA_KK_MAXCMFLAG 4 // max cmodel flags (impulsive) class SurfReactAdsorbKokkos : public SurfReactAdsorb { public: @@ -90,7 +95,25 @@ class SurfReactAdsorbKokkos : public SurfReactAdsorb { double fnum_; // update->fnum, set in pre_react + // post-reaction collision model (cmodel) state for bit-exact device scatter + // per-reaction flattened cmodel coeffs/flags (ip and jp), plus collide + // rot/vib styles and boltz captured in pre_react + + DAT::t_int_2d d_cmip_flags,d_cmjp_flags; // [nr][MAXCMFLAG] + DAT::t_float_2d d_cmip_coeffs,d_cmjp_coeffs; // [nr][MAXCMCOEFF] + double boltz_; + int rotstyle_,vibstyle_; + + // one RNG per cmodel type, wrapping that cmodel's RanKnuth, so device + // scatter draws match the host wrapper bit-for-bit (EXACT serial) + + RandPoolWrap *cmodel_pool[SRA_KK_MAXMODELS]; +#ifdef SPARTA_KOKKOS_EXACT + RandWrap d_cmodel_rand[SRA_KK_MAXMODELS]; +#endif + void init_reactions_gs_kokkos(); + void init_cmodels_kokkos(); #ifndef SPARTA_KOKKOS_EXACT Kokkos::Random_XorShift64_Pool rand_pool; @@ -311,16 +334,16 @@ class SurfReactAdsorbKokkos : public SurfReactAdsorb { if (nn == 1) { nn++; ip->ispecies = d_products(j,pj); - apply_cmodel(ip,norm,d_cmodel_ip(j)); + scatter_cmodel(ip,norm,d_cmodel_ip(j),j,0,rand_gen); if (d_pstoich(j,pj) == 2) { jp = create_particle(ip,d_products(j,pj),rand_gen,d_nlocal,d_retry); if (!jp) { rand_pool.free_state(rand_gen); return 0; } - apply_cmodel(jp,norm,d_cmodel_ip(j)); + scatter_cmodel(jp,norm,d_cmodel_ip(j),j,0,rand_gen); } } else { jp = create_particle(ip,d_products(j,pj),rand_gen,d_nlocal,d_retry); if (!jp) { rand_pool.free_state(rand_gen); return 0; } - apply_cmodel(jp,norm,d_cmodel_jp(j)); + scatter_cmodel(jp,norm,d_cmodel_jp(j),j,1,rand_gen); } } } @@ -333,7 +356,7 @@ class SurfReactAdsorbKokkos : public SurfReactAdsorb { case SRA_KK::LH1: case SRA_KK::ER: ip->ispecies = d_products(j,0); - apply_cmodel(ip,norm,d_cmodel_ip(j)); + scatter_cmodel(ip,norm,d_cmodel_ip(j),j,0,rand_gen); if (d_cmodel_ip(j) != SRA_KK::NOMODEL) velreset = 1; rand_pool.free_state(rand_gen); return (j + 1); @@ -341,16 +364,16 @@ class SurfReactAdsorbKokkos : public SurfReactAdsorb { case SRA_KK::CI: { ip->ispecies = d_products(j,0); - apply_cmodel(ip,norm,d_cmodel_ip(j)); + scatter_cmodel(ip,norm,d_cmodel_ip(j),j,0,rand_gen); if (d_nprod_g_tot(j) == 2) { if (d_pstoich(j,0) == 2) { jp = create_particle(ip,d_products(j,0),rand_gen,d_nlocal,d_retry); if (!jp) { rand_pool.free_state(rand_gen); return 0; } - apply_cmodel(jp,norm,d_cmodel_ip(j)); + scatter_cmodel(jp,norm,d_cmodel_ip(j),j,0,rand_gen); } else { jp = create_particle(ip,d_products(j,1),rand_gen,d_nlocal,d_retry); if (!jp) { rand_pool.free_state(rand_gen); return 0; } - apply_cmodel(jp,norm,d_cmodel_jp(j)); + scatter_cmodel(jp,norm,d_cmodel_jp(j),j,1,rand_gen); } } if (d_cmodel_ip(j) != SRA_KK::NOMODEL) velreset = 1; @@ -371,10 +394,311 @@ class SurfReactAdsorbKokkos : public SurfReactAdsorb { ------------------------------------------------------------------------- */ KOKKOS_INLINE_FUNCTION - void apply_cmodel(Particle::OnePart *p, const double *norm, int cmodel) const + void scatter_cmodel(Particle::OnePart *p, const double *norm, int cmodel, + int j, int useJp, rand_type &adsorb_rand) const + { + if (cmodel == SRA_KK::NOMODEL) return; + if (cmodel == SRA_KK::SPECULAR) { // SurfCollideSpecular::wrapper + MathExtraKokkos::reflect3(p->v,norm); + return; + } + + // RNG: bit-exact uses the cmodel's own RanKnuth (EXACT serial); on GPU + // builds fall back to the surf-react RNG (not bit-exact, not gated here) + +#ifdef SPARTA_KOKKOS_EXACT + rand_type rg = d_cmodel_rand[cmodel]; +#else + rand_type &rg = adsorb_rand; +#endif + + // gather this reaction's cmodel coeffs/flags (ip or jp slot) + + double cf[SRA_KK_MAXCMCOEFF]; + int fl[SRA_KK_MAXCMFLAG]; + for (int k = 0; k < SRA_KK_MAXCMCOEFF; k++) + cf[k] = useJp ? d_cmjp_coeffs(j,k) : d_cmip_coeffs(j,k); + for (int k = 0; k < SRA_KK_MAXCMFLAG; k++) + fl[k] = useJp ? d_cmjp_flags(j,k) : d_cmip_flags(j,k); + + if (cmodel == SRA_KK::DIFFUSE) { + diffuse_scatter(p,norm,cf[0],cf[1],rg); // tsurf, acc + } else if (cmodel == SRA_KK::CLL) { + double eccen = fl[0] ? cf[5] : 0.0; + cll_scatter(p,norm,cf[0],cf[1],cf[2],cf[3],cf[4],fl[0],eccen,rg); + } else if (cmodel == SRA_KK::TD) { + double tsurf = cf[0]; + int barrier_flag = fl[0], initen_flag = fl[1], bond_flag = fl[2]; + int m = 1; + double barrier_val = 0.0; + double initen_trans = 0.0,initen_rot = 0.0,initen_vib = 0.0; + double bond_trans = 0.0,bond_rot = 0.0,bond_vib = 0.0; + if (barrier_flag) barrier_val = cf[m++]; + if (initen_flag) { initen_trans = cf[m++]; initen_rot = cf[m++]; initen_vib = cf[m++]; } + if (bond_flag) { bond_trans = cf[m++]; bond_rot = cf[m++]; bond_vib = cf[m++]; } + td_scatter(p,norm,tsurf,barrier_flag,barrier_val,initen_flag,initen_trans, + initen_rot,initen_vib,bond_flag,bond_trans,bond_rot,bond_vib,rg); + } + } + + /* ---------------------------------------------------------------------- + replicas of the Kokkos surf-collide scatter device functions, drawing + from the cmodel's RNG so they match the host wrapper bit-for-bit; + cmodels never translate/rotate (trflag off) + ------------------------------------------------------------------------- */ + + KOKKOS_INLINE_FUNCTION + double erot_kk(int isp, double temp, rand_type &rg) const + { + double eng,a,erm,b; + if (rotstyle_ == SRA_KK::NONE) return 0.0; + if (d_species[isp].rotdof < 2) return 0.0; + if (rotstyle_ == SRA_KK::DISCRETE && d_species[isp].rotdof == 2) { + int irot = -log(rg.drand()) * temp / d_species[isp].rottemp[0]; + eng = irot * boltz_ * d_species[isp].rottemp[0]; + } else if (rotstyle_ == SRA_KK::SMOOTH && d_species[isp].rotdof == 2) { + eng = -log(rg.drand()) * boltz_ * temp; + } else { + a = 0.5*d_species[isp].rotdof - 1.0; + while (1) { + erm = 10.0*rg.drand(); + b = pow(erm/a,a) * exp(a-erm); + if (b > rg.drand()) break; + } + eng = erm * boltz_ * temp; + } + return eng; + } + + KOKKOS_INLINE_FUNCTION + double evib_kk(int isp, double temp, rand_type &rg) const + { + double eng,a,erm,b; + if (vibstyle_ == SRA_KK::NONE || d_species[isp].vibdof < 2) return 0.0; + eng = 0.0; + if (vibstyle_ == SRA_KK::DISCRETE && d_species[isp].vibdof == 2) { + int ivib = -log(rg.drand()) * temp / d_species[isp].vibtemp[0]; + eng = ivib * boltz_ * d_species[isp].vibtemp[0]; + } else if (vibstyle_ == SRA_KK::SMOOTH || d_species[isp].vibdof >= 2) { + if (d_species[isp].vibdof == 2) + eng = -log(rg.drand()) * boltz_ * temp; + else if (d_species[isp].vibdof > 2) { + a = 0.5*d_species[isp].vibdof - 1.0; + while (1) { + erm = 10.0*rg.drand(); + b = pow(erm/a,a) * exp(a-erm); + if (b > rg.drand()) break; + } + eng = erm * boltz_ * temp; + } + } + return eng; + } + + KOKKOS_INLINE_FUNCTION + void diffuse_scatter(Particle::OnePart *p, const double *norm, + double twall, double acc, rand_type &rg) const { - if (cmodel == SRA_KK::SPECULAR) + if (rg.drand() > acc) { MathExtraKokkos::reflect3(p->v,norm); + } else { + double tangent1[3],tangent2[3]; + int isp = p->ispecies; + double vrm = sqrt(2.0*boltz_*twall / d_species[isp].mass); + double vperp = vrm * sqrt(-log(rg.drand())); + double theta = MathConst::MY_2PI * rg.drand(); + double vtangent = vrm * sqrt(-log(rg.drand())); + double vtan1 = vtangent * sin(theta); + double vtan2 = vtangent * cos(theta); + double *v = p->v; + double dot = MathExtraKokkos::dot3(v,norm); + tangent1[0] = v[0] - dot*norm[0]; + tangent1[1] = v[1] - dot*norm[1]; + tangent1[2] = v[2] - dot*norm[2]; + if (MathExtraKokkos::lensq3(tangent1) == 0.0) { + tangent2[0] = rg.drand(); + tangent2[1] = rg.drand(); + tangent2[2] = rg.drand(); + MathExtraKokkos::cross3(norm,tangent2,tangent1); + } + MathExtraKokkos::norm3(tangent1); + MathExtraKokkos::cross3(norm,tangent1,tangent2); + v[0] = vperp*norm[0] + vtan1*tangent1[0] + vtan2*tangent2[0]; + v[1] = vperp*norm[1] + vtan1*tangent1[1] + vtan2*tangent2[1]; + v[2] = vperp*norm[2] + vtan1*tangent1[2] + vtan2*tangent2[2]; + p->erot = erot_kk(isp,twall,rg); + p->evib = evib_kk(isp,twall,rg); + } + } + + KOKKOS_INLINE_FUNCTION + void cll_scatter(Particle::OnePart *p, const double *norm, double twall, + double acc_n, double acc_t, double acc_rot, double acc_vib, + int pflag, double eccen, rand_type &rg) const + { + double tangent1[3],tangent2[3]; + int ispecies = p->ispecies; + double *v = p->v; + double dot = MathExtraKokkos::dot3(v,norm); + double vrm,vperp,vtan1,vtan2; + + tangent1[0] = v[0] - dot*norm[0]; + tangent1[1] = v[1] - dot*norm[1]; + tangent1[2] = v[2] - dot*norm[2]; + if (MathExtraKokkos::lensq3(tangent1) == 0.0) { + tangent2[0] = rg.drand(); + tangent2[1] = rg.drand(); + tangent2[2] = rg.drand(); + MathExtraKokkos::cross3(norm,tangent2,tangent1); + } + MathExtraKokkos::norm3(tangent1); + MathExtraKokkos::cross3(norm,tangent1,tangent2); + double tan1 = MathExtraKokkos::dot3(v,tangent1); + + vrm = sqrt(2.0*boltz_ * twall / d_species[ispecies].mass); + + double r_1 = sqrt(-acc_n*log(rg.drand())); + double theta_1 = MathConst::MY_2PI * rg.drand(); + double dot_norm = dot/vrm * sqrt(1-acc_n); + vperp = vrm * sqrt(r_1*r_1 + dot_norm*dot_norm + 2*r_1*dot_norm*cos(theta_1)); + + double r_2 = sqrt(-acc_t*log(rg.drand())); + double theta_2 = MathConst::MY_2PI * rg.drand(); + double vtangent = tan1/vrm * sqrt(1-acc_t); + vtan1 = vrm * (vtangent + r_2*cos(theta_2)); + vtan2 = vrm * r_2 * sin(theta_2); + + if (pflag) { + double tan2 = MathExtraKokkos::dot3(v,tangent2); + double phi_i,psi_i,theta_f,phi_f,psi_f,cos_beta; + psi_i = acos(dot*dot/MathExtraKokkos::lensq3(v)); + phi_i = atan2(tan2,tan1); + double v_mag = sqrt(vperp*vperp + vtan1*vtan1 + vtan2*vtan2); + double P = 0; + while (rg.drand() > P) { + phi_f = MathConst::MY_2PI*rg.drand(); + psi_f = acos(1-rg.drand()); + cos_beta = cos(psi_i)*cos(psi_f) + sin(psi_i)*sin(psi_f)*cos(phi_i - phi_f); + P = (1-eccen)/(1-eccen*cos_beta); + } + theta_f = acos(sqrt(cos(psi_f))); + vperp = v_mag * cos(theta_f); + vtan1 = v_mag * sin(theta_f) * cos(phi_f); + vtan2 = v_mag * sin(theta_f) * sin(phi_f); + } + + v[0] = vperp*norm[0] + vtan1*tangent1[0] + vtan2*tangent2[0]; + v[1] = vperp*norm[1] + vtan1*tangent1[1] + vtan2*tangent2[1]; + v[2] = vperp*norm[2] + vtan1*tangent1[2] + vtan2*tangent2[2]; + + // rotational component (CLL partial accommodation) + + if (rotstyle_ == SRA_KK::NONE || d_species[ispecies].rotdof < 2) p->erot = 0.0; + else { + double erot_mag = sqrt(p->erot*(1-acc_rot)/(boltz_*twall)); + double r_rot,cos_theta_rot,A_rot,X_rot; + if (d_species[ispecies].rotdof == 2) { + r_rot = sqrt(-acc_rot*log(rg.drand())); + cos_theta_rot = cos(MathConst::MY_2PI*rg.drand()); + } else { + A_rot = 0; + while (A_rot < rg.drand()) { + X_rot = 4*rg.drand(); + A_rot = 2.71828182845904523536028747*X_rot*X_rot*exp(-X_rot*X_rot); + } + r_rot = sqrt(acc_rot)*X_rot; + cos_theta_rot = 2*rg.drand() - 1; + } + p->erot = boltz_ * twall * + (r_rot*r_rot + erot_mag*erot_mag + 2*r_rot*erot_mag*cos_theta_rot); + } + + // vibrational component + + int vibdof = d_species[ispecies].vibdof; + double r_vib,cos_theta_vib,A_vib,X_vib,evib_mag,evib_val; + if (vibstyle_ == SRA_KK::NONE || vibdof < 2) p->evib = 0.0; + else if (vibstyle_ == SRA_KK::DISCRETE && vibdof == 2) { + double evib_star = -log(1 - rg.drand() * + (1 - exp(-boltz_*d_species[ispecies].vibtemp[0]))); + evib_val = p->evib + evib_star; + evib_mag = sqrt(evib_val*(1-acc_vib)/(boltz_*twall)); + r_vib = sqrt(-acc_vib*log(rg.drand())); + cos_theta_vib = cos(MathConst::MY_2PI*rg.drand()); + evib_val = boltz_ * twall * + (r_vib*r_vib + evib_mag*evib_mag + 2*r_vib*evib_mag*cos_theta_vib); + int ivib = evib_val / (boltz_*d_species[ispecies].vibtemp[0]); + p->evib = ivib * boltz_ * d_species[ispecies].vibtemp[0]; + } + else if (vibstyle_ == SRA_KK::SMOOTH || vibdof >= 2) { + evib_mag = sqrt(p->evib*(1-acc_vib)/(boltz_*twall)); + if (vibdof == 2) { + r_vib = sqrt(-acc_vib*log(rg.drand())); + cos_theta_vib = cos(MathConst::MY_2PI*rg.drand()); + } else { + A_vib = 0; + while (A_vib < rg.drand()) { + X_vib = 4*rg.drand(); + A_vib = 2.71828182845904523536028747*X_vib*X_vib*exp(-X_vib*X_vib); + } + r_vib = sqrt(acc_vib)*X_vib; + cos_theta_vib = 2*rg.drand() - 1; + } + p->evib = boltz_ * twall * + (r_vib*r_vib + evib_mag*evib_mag + 2*r_vib*evib_mag*cos_theta_vib); + } + } + + KOKKOS_INLINE_FUNCTION + void td_scatter(Particle::OnePart *p, const double *norm, double twall, + int barrier_flag, double barrier_val, + int initen_flag, double initen_trans, double initen_rot, double initen_vib, + int bond_flag, double bond_trans, double bond_rot, double bond_vib, + rand_type &rg) const + { + double tangent1[3],tangent2[3]; + int ispecies = p->ispecies; + double *v = p->v; + double dot = MathExtraKokkos::dot3(v,norm); + + tangent1[0] = v[0] - dot*norm[0]; + tangent1[1] = v[1] - dot*norm[1]; + tangent1[2] = v[2] - dot*norm[2]; + if (MathExtraKokkos::lensq3(tangent1) == 0.0) { + tangent2[0] = rg.drand(); + tangent2[1] = rg.drand(); + tangent2[2] = rg.drand(); + MathExtraKokkos::cross3(norm,tangent2,tangent1); + } + MathExtraKokkos::norm3(tangent1); + MathExtraKokkos::cross3(norm,tangent1,tangent2); + + double mass = d_species[ispecies].mass; + double E_i = 0.5 * mass * MathExtraKokkos::lensq3(v); + double E_t = boltz_ * twall; + if (bond_flag) E_t += boltz_*bond_trans; + if (initen_flag) E_t += E_i*initen_trans; + double E_n = E_t; + if (barrier_flag) E_n += boltz_*barrier_val; + + double vrm_n = sqrt(2.0*E_n / mass); + double vrm_t = sqrt(2.0*E_t / mass); + double vperp = vrm_n * sqrt(-log(rg.drand())); + double theta = MathConst::MY_2PI * rg.drand(); + double vtangent = vrm_t * sqrt(-log(rg.drand())); + double vtan1 = vtangent * sin(theta); + double vtan2 = vtangent * cos(theta); + + v[0] = vperp*norm[0] + vtan1*tangent1[0] + vtan2*tangent2[0]; + v[1] = vperp*norm[1] + vtan1*tangent1[1] + vtan2*tangent2[1]; + v[2] = vperp*norm[2] + vtan1*tangent1[2] + vtan2*tangent2[2]; + + double twall_rot = twall, twall_vib = twall; + if (bond_flag) { twall_rot += bond_rot; twall_vib += bond_vib; } + if (initen_flag) { twall_rot += E_i*initen_rot/boltz_; twall_vib += E_i*initen_vib/boltz_; } + + p->erot = erot_kk(ispecies,twall_rot,rg); + p->evib = evib_kk(ispecies,twall_vib,rg); } /* ---------------------------------------------------------------------- diff --git a/src/surf_collide.h b/src/surf_collide.h index e6f161a58..7b385935b 100644 --- a/src/surf_collide.h +++ b/src/surf_collide.h @@ -40,6 +40,7 @@ class SurfCollide : protected Pointers { virtual Particle::OnePart *collide(Particle::OnePart *&, double &, int, double *, int, int &) = 0; virtual void wrapper(Particle::OnePart *, double *, int *, double *) {} + virtual class RanKnuth *kokkos_random() { return NULL; } virtual void flags_and_coeffs(int *, double *) {} virtual void dynamic(); diff --git a/src/surf_collide_adiabatic.h b/src/surf_collide_adiabatic.h index 875e4842e..d90d052b7 100644 --- a/src/surf_collide_adiabatic.h +++ b/src/surf_collide_adiabatic.h @@ -35,6 +35,8 @@ class SurfCollideAdiabatic : public SurfCollide { void wrapper(Particle::OnePart *, double *, int *, double*); void flags_and_coeffs(int *, double *) {} + class RanKnuth *kokkos_random() { return random; } + protected: class RanKnuth *random; // RNG for particle reflection diff --git a/src/surf_collide_cll.h b/src/surf_collide_cll.h index b80bcdb4d..ffb401b0f 100644 --- a/src/surf_collide_cll.h +++ b/src/surf_collide_cll.h @@ -37,6 +37,8 @@ class SurfCollideCLL : public SurfCollide { void wrapper(Particle::OnePart *, double *, int *, double*); void flags_and_coeffs(int *, double *); + class RanKnuth *kokkos_random() { return random; } + protected: double acc_n,acc_t,acc_rot,acc_vib; // surface accomodation coeffs double vx,vy,vz; // translational velocity of surface diff --git a/src/surf_collide_diffuse.h b/src/surf_collide_diffuse.h index 76ae89742..584a4d9b9 100644 --- a/src/surf_collide_diffuse.h +++ b/src/surf_collide_diffuse.h @@ -37,6 +37,8 @@ class SurfCollideDiffuse : public SurfCollide { void wrapper(Particle::OnePart *, double *, int *, double*); void flags_and_coeffs(int *, double *); + class RanKnuth *kokkos_random() { return random; } + protected: double acc; // surface accomodation coeff double vx,vy,vz; // translational velocity of surface diff --git a/src/surf_collide_impulsive.h b/src/surf_collide_impulsive.h index 0ee87c53c..8c3a3cba9 100644 --- a/src/surf_collide_impulsive.h +++ b/src/surf_collide_impulsive.h @@ -36,6 +36,8 @@ class SurfCollideImpulsive : public SurfCollide { void wrapper(Particle::OnePart *, double *, int *, double*); void flags_and_coeffs(int *, double *); + class RanKnuth *kokkos_random() { return random; } + protected: double eng_ratio,eff_mass; // energy ratio and effective mass // of the surface for soft-sphere model diff --git a/src/surf_collide_td.h b/src/surf_collide_td.h index cea5ac30e..e98fbbc89 100644 --- a/src/surf_collide_td.h +++ b/src/surf_collide_td.h @@ -36,6 +36,8 @@ class SurfCollideTD : public SurfCollide { void wrapper(Particle::OnePart *, double *, int *, double*); void flags_and_coeffs(int *, double *); + class RanKnuth *kokkos_random() { return random; } + protected: double barrier_val; double initen_trans, initen_rot, initen_vib; From 108e4fcd0b191a4841c7c51b402c97ba3ad27531 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 21 Jun 2026 00:02:24 +0000 Subject: [PATCH 11/30] KOKKOS: adsorb - add SURF mode (explicit surf elements), bit-exact Generalize SurfReactAdsorbKokkos from box-face-only to also support surf mode (reactions on explicit surface elements), covering in.beam.surf.gs and in.circle.gs (2d/3d). - react_kokkos: state index is the box face (FACE) or the local surf index (SURF); in SURF mode mark the reacting surf for the periodic collate. - per-state device arrays sized nstate_ = nface (FACE) or nlocal+nghost surfs (SURF); add a per-surf mark dual view. - pre_react: in SURF mode refresh the host state pointers from the surf custom arrays (nstick_total/nstick_species/area/weight) before syncing to device. - tally_update: bring device per-surf deltas + mark to the host and reuse the host update_state_surf() (rendezvous collate_array + spread_custom), then re-sync the zeroed deltas/mark to device. - surf_custom: guard Surf::remove_custom() against a freed ewhich. In a Kokkos run SurfKokkos's destructor frees the Kokkos-managed custom data (and nulls ewhich) before the base Surf destructor deletes surf-react instances, whose destructors call remove_custom; without the guard that dereferenced NULL. Verified bit-for-bit CPU vs -sf kk (Serial+EXACT), identical stats and per-reaction tallies: in.beam.surf.gs (3d, 56634 reactions) and in.circle.gs (2d, 26588). FACE mode (in.beam.face.gs) remains bit-for-bit. Co-Authored-By: Claude Opus 4.8 Co-Authored-By: Stan Moore Claude-Session: https://claude.ai/code/session_01Q6nQVuxEiTUqSzDoDppCUM --- src/KOKKOS/surf_react_adsorb_kokkos.cpp | 62 ++++++++++++++++++------- src/KOKKOS/surf_react_adsorb_kokkos.h | 42 ++++++++++------- src/surf_custom.cpp | 7 +++ 3 files changed, 79 insertions(+), 32 deletions(-) diff --git a/src/KOKKOS/surf_react_adsorb_kokkos.cpp b/src/KOKKOS/surf_react_adsorb_kokkos.cpp index b10085c1e..827e6df2e 100644 --- a/src/KOKKOS/surf_react_adsorb_kokkos.cpp +++ b/src/KOKKOS/surf_react_adsorb_kokkos.cpp @@ -18,6 +18,7 @@ #include "input.h" #include "update.h" #include "collide.h" +#include "surf.h" #include "surf_collide.h" #include "random_knuth.h" #include "comm.h" @@ -111,8 +112,6 @@ void SurfReactAdsorbKokkos::init() error->all(FLERR,"Kokkos surf_react adsorb requires gas-surface (gs) chemistry"); if (psflag) error->all(FLERR,"Kokkos surf_react adsorb does not yet support on-surface (ps) chemistry"); - if (mode != SRA_KK::FACE) - error->all(FLERR,"Kokkos surf_react adsorb only supports the box-face (face) option"); for (int i = 0; i < nlist_gs; i++) { OneReaction_GS *r = &rlist_gs[i]; @@ -316,16 +315,23 @@ void SurfReactAdsorbKokkos::init_reactions_gs_kokkos() Kokkos::deep_copy(d_pad,h_pad); Kokkos::deep_copy(d_products,h_products); - // per-face state device storage (SRA_KK::FACE mode) + // per-state-slot device storage: FACE => 6 box faces, SURF => nlocal+nghost - d_total_state = DAT::t_int_1d("sra:total_state",nface); - d_area = DAT::t_float_1d("sra:area",nface); - d_weight = DAT::t_float_1d("sra:weight",nface); - d_species_state = DAT::t_int_2d("sra:species_state",nface,nspecies_surf); + nstate_ = (mode == SRA_KK::FACE) ? nface : (surf->nlocal + surf->nghost); + int ns = MAX(nstate_,1); - k_species_delta = DAT::tdual_int_2d("sra:species_delta",nface,nspecies_surf); + d_total_state = DAT::t_int_1d("sra:total_state",ns); + d_area = DAT::t_float_1d("sra:area",ns); + d_weight = DAT::t_float_1d("sra:weight",ns); + d_species_state = DAT::t_int_2d("sra:species_state",ns,nspecies_surf); + + k_species_delta = DAT::tdual_int_2d("sra:species_delta",ns,nspecies_surf); d_species_delta = k_species_delta.view_device(); Kokkos::deep_copy(d_species_delta,0); + + k_mark = DAT::tdual_int_1d("sra:mark",ns); + d_mark = k_mark.view_device(); + Kokkos::deep_copy(d_mark,0); } /* ---------------------------------------------------------------------- @@ -355,13 +361,23 @@ void SurfReactAdsorbKokkos::pre_react() if (cmodel_pool[idx]) d_cmodel_rand[idx] = cmodel_pool[idx]->get_state(); #endif - // copy current per-face state (changes only at sync) host->device + // SURF mode: refresh host state pointers to the current local custom arrays + // (they may have been reallocated/spread since last step) + + if (mode == SRA_KK::SURF) { + total_state = surf->eivec_local[surf->ewhich[total_state_index]]; + species_state = surf->eiarray_local[surf->ewhich[species_state_index]]; + area = surf->edvec_local[surf->ewhich[area_index]]; + weight = surf->edvec_local[surf->ewhich[weight_index]]; + } + + // copy current per-slot state (changes only at sync) host->device auto h_total = Kokkos::create_mirror_view(d_total_state); auto h_area = Kokkos::create_mirror_view(d_area); auto h_weight = Kokkos::create_mirror_view(d_weight); auto h_sstate = Kokkos::create_mirror_view(d_species_state); - for (int i = 0; i < nface; i++) { + for (int i = 0; i < nstate_; i++) { h_total(i) = total_state[i]; h_area(i) = area[i]; h_weight(i) = weight[i]; @@ -395,28 +411,42 @@ void SurfReactAdsorbKokkos::tally_update() nsingle = h_nsingle(); for (int i = 0; i < nlist_gs; i++) tally_single[i] = h_tally_single[i]; - // device -> host: per-face perspecies deltas accumulated since last sync + // device -> host: perspecies deltas (+ mark for SURF) accumulated since sync k_species_delta.modify_device(); k_species_delta.sync_host(); auto h_delta = k_species_delta.view_host(); - for (int i = 0; i < nface; i++) + for (int i = 0; i < nstate_; i++) for (int j = 0; j < nspecies_surf; j++) species_delta[i][j] = h_delta(i,j); - // host logic: accumulate tallies and (every nsync) MPI-sync per-face state; - // update_state_face() also re-zeros host species_delta + if (mode == SRA_KK::SURF) { + k_mark.modify_device(); + k_mark.sync_host(); + auto h_m = k_mark.view_host(); + for (int i = 0; i < nstate_; i++) mark[i] = h_m(i); + } + + // host logic: accumulate tallies and (every nsync) sync per-slot state; + // update_state_face()/update_state_surf() re-zero host species_delta + // (and update_state_surf clears mark) SurfReactAdsorb::tally_update(); - // mirror re-zeroed host deltas back to device (only changed on a sync step) + // mirror re-zeroed host deltas (+ mark) back to device (only on a sync step) if (update->ntimestep % nsync == 0) { - for (int i = 0; i < nface; i++) + for (int i = 0; i < nstate_; i++) for (int j = 0; j < nspecies_surf; j++) h_delta(i,j) = species_delta[i][j]; k_species_delta.modify_host(); k_species_delta.sync_device(); + if (mode == SRA_KK::SURF) { + auto h_m = k_mark.view_host(); + for (int i = 0; i < nstate_; i++) h_m(i) = mark[i]; + k_mark.modify_host(); + k_mark.sync_device(); + } Kokkos::deep_copy(d_scalars,0); } } diff --git a/src/KOKKOS/surf_react_adsorb_kokkos.h b/src/KOKKOS/surf_react_adsorb_kokkos.h index 96c219e13..51a92b23b 100644 --- a/src/KOKKOS/surf_react_adsorb_kokkos.h +++ b/src/KOKKOS/surf_react_adsorb_kokkos.h @@ -84,15 +84,19 @@ class SurfReactAdsorbKokkos : public SurfReactAdsorb { DAT::t_int_2d d_pstate,d_ppart,d_pstoich,d_pad; // product slots [j][MAXPRODUCT] DAT::t_int_2d d_products; // product species indices - // per-face state (FACE mode); small (nface <= 6) + // per-state-slot data: FACE mode => 6 box faces; SURF mode => nlocal+nghost surfs - DAT::t_int_1d d_total_state; // [nface] - DAT::t_float_1d d_area,d_weight; // [nface] - DAT::t_int_2d d_species_state; // [nface][nspecies_surf] - DAT::t_int_2d d_species_delta; // [nface][nspecies_surf] (atomic) + int nstate_; // # of state slots (nface or nall) + DAT::t_int_1d d_total_state; // [nstate] + DAT::t_float_1d d_area,d_weight; // [nstate] + DAT::t_int_2d d_species_state; // [nstate][nspecies_surf] + DAT::t_int_2d d_species_delta; // [nstate][nspecies_surf] (atomic) DAT::tdual_int_2d k_species_delta; + DAT::tdual_int_1d k_mark; // [nstate] reacted-surf mark (SURF) + DAT::t_int_1d d_mark; + double fnum_; // update->fnum, set in pre_react // post-reaction collision model (cmodel) state for bit-exact device scatter @@ -151,16 +155,18 @@ class SurfReactAdsorbKokkos : public SurfReactAdsorb { const DAT::t_int_scalar &d_retry, const DAT::t_int_scalar &d_nlocal) const { - // convert face index from negative value to 0..5 inclusive + // FACE: convert negative face code to 0..5; SURF: use local surf index - int iface = -(isurf+1); + int idx; + if (mode == SRA_KK::FACE) idx = -(isurf+1); + else idx = isurf; int n = d_reactions_n[ip->ispecies]; if (n == 0) return 0; double fnum = fnum_; - long int maxstick = ceil(max_cover*d_area[iface] / (fnum*d_weight[iface])); - double factor = fnum * d_weight[iface] / d_area[iface]; + long int maxstick = ceil(max_cover*d_area[idx] / (fnum*d_weight[idx])); + double factor = fnum * d_weight[idx] / d_area[idx]; double ms_inv = factor / max_cover; double prob_value[SRA_KK_MAXPERSPECIES]; @@ -189,7 +195,7 @@ class SurfReactAdsorbKokkos : public SurfReactAdsorb { case SRA_KK::LH1: case SRA_KK::LH3: case SRA_KK::CD: - surf_cover = d_total_state[iface] * ms_inv; + surf_cover = d_total_state[idx] * ms_inv; S_theta = 0.0; if (d_kisliuk_flag(j)) { K_ads = d_kisliuk(j,0) * pow(twall,d_kisliuk(j,1)) * @@ -208,7 +214,7 @@ class SurfReactAdsorbKokkos : public SurfReactAdsorb { double dot = 2.0; if (d_nreactant(j) == 1) prob_value[i] = 2.0 * d_kreact(j) * - (maxstick - d_total_state[iface]) * ms_inv / fabs(dot); + (maxstick - d_total_state[idx]) * ms_inv / fabs(dot); else prob_value[i] = 2.0 * d_kreact(j) / fabs(dot); break; @@ -230,10 +236,10 @@ class SurfReactAdsorbKokkos : public SurfReactAdsorb { for (int k = 1; k < d_nreactant(j); k++) { if (d_rstate(j,k) == 's') { if (d_rpart(j,k) == 0) - prob_value[i] *= stoich_pow_kk(d_total_state[iface],d_rstoich(j,k)) * + prob_value[i] *= stoich_pow_kk(d_total_state[idx],d_rstoich(j,k)) * pow(ms_inv,d_rstoich(j,k)); else - prob_value[i] *= stoich_pow_kk(d_species_state(iface,d_rad(j,k)), + prob_value[i] *= stoich_pow_kk(d_species_state(idx,d_rad(j,k)), d_rstoich(j,k)) * pow(ms_inv,d_rstoich(j,k)); } @@ -268,15 +274,19 @@ class SurfReactAdsorbKokkos : public SurfReactAdsorb { Kokkos::atomic_inc(&d_tally_single(j)); } - // update per-face perspecies deltas for participating surf reactants/products + // SURF mode: mark this surf element for the periodic state collate + + if (mode == SRA_KK::SURF) d_mark(idx) = 1; + + // update perspecies deltas for participating surf reactants/products auto a_species_delta = d_species_delta; for (int k = 0; k < d_nreactant(j); k++) if (d_rpart(j,k) == 1 && d_rstate(j,k) == 's') - Kokkos::atomic_add(&a_species_delta(iface,d_rad(j,k)),-d_rstoich(j,k)); + Kokkos::atomic_add(&a_species_delta(idx,d_rad(j,k)),-d_rstoich(j,k)); for (int k = 0; k < d_nproduct(j); k++) if (d_ppart(j,k) == 1 && d_pstate(j,k) == 's') - Kokkos::atomic_add(&a_species_delta(iface,d_pad(j,k)),d_pstoich(j,k)); + Kokkos::atomic_add(&a_species_delta(idx,d_pad(j,k)),d_pstoich(j,k)); // post-reaction particle handling, mirrors SurfReactAdsorb::react() // cmodel post-reaction scatter currently supports NOMODEL and SPECULAR diff --git a/src/surf_custom.cpp b/src/surf_custom.cpp index 0630b86d4..075eef491 100644 --- a/src/surf_custom.cpp +++ b/src/surf_custom.cpp @@ -229,6 +229,13 @@ void Surf::remove_custom(int index) { if (!ename || !ename[index]) return; + // ewhich may already be freed during Kokkos teardown: SurfKokkos's destructor + // runs (and frees the Kokkos-managed custom data + nulls ewhich) before the + // base Surf destructor deletes surf-react instances, whose destructors call + // remove_custom. The data is already freed there, so skip safely. + + if (!ewhich) return; + delete [] ename[index]; ename[index] = NULL; From 8349f2e240c753dd2196fd77ecf1e31bfdfbe81e Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 21 Jun 2026 00:08:27 +0000 Subject: [PATCH 12/30] KOKKOS: adsorb - enable PS (on-surface) chemistry; full adsorb support Enable periodic on-surface (PS) chemistry under -sf kk, completing adsorb Kokkos support across all shipped examples. PS_chemistry() runs on the host (it desorbs/inserts particles from the surface state every nsync steps). In a Kokkos run particles live on the device during the timestep, so SurfReactAdsorbKokkos::tally_update() now brings the particle list to the host (sync Host) before the base tally_update()/PS_chemistry() appends new particles via Particle::add_particle(), and marks the particle data host-modified afterward so the device picks them up on the next sync. - remove the gs/ps init guard; support gs, ps, and gs/ps modes. - init_reactions_gs_kokkos(): tolerate gsflag==0 (PS-only) where reactions_gs is not allocated (d_reactions_n set to zero so react_kokkos is a no-op). Verified bit-for-bit CPU vs -sf kk (Serial+EXACT) on all 9 surf_react_adsorb examples: in.{beam.face,beam.surf,circle}.{gs,gs_ps,ps} -- identical stats and per-reaction tallies (e.g. beam.surf.gs_ps 104924, circle.gs_ps 50713). Co-Authored-By: Claude Opus 4.8 Co-Authored-By: Stan Moore Claude-Session: https://claude.ai/code/session_01Q6nQVuxEiTUqSzDoDppCUM --- src/KOKKOS/surf_react_adsorb_kokkos.cpp | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/KOKKOS/surf_react_adsorb_kokkos.cpp b/src/KOKKOS/surf_react_adsorb_kokkos.cpp index 827e6df2e..3607384c9 100644 --- a/src/KOKKOS/surf_react_adsorb_kokkos.cpp +++ b/src/KOKKOS/surf_react_adsorb_kokkos.cpp @@ -108,10 +108,6 @@ void SurfReactAdsorbKokkos::init() // Kokkos GS adsorb currently supports a restricted feature set; // error clearly at init rather than silently producing wrong results - if (!gsflag) - error->all(FLERR,"Kokkos surf_react adsorb requires gas-surface (gs) chemistry"); - if (psflag) - error->all(FLERR,"Kokkos surf_react adsorb does not yet support on-surface (ps) chemistry"); for (int i = 0; i < nlist_gs; i++) { OneReaction_GS *r = &rlist_gs[i]; @@ -193,7 +189,7 @@ void SurfReactAdsorbKokkos::init_reactions_gs_kokkos() d_reactions_n = DAT::t_int_1d("surf_react_adsorb:reactions_n",nspecies); auto h_reactions_n = Kokkos::create_mirror_view(d_reactions_n); for (int i = 0; i < nspecies; i++) { - int n = reactions_gs[i].n; + int n = gsflag ? reactions_gs[i].n : 0; // PS-only: no GS reactions h_reactions_n(i) = n; nmax = MAX(nmax,n); } @@ -202,7 +198,8 @@ void SurfReactAdsorbKokkos::init_reactions_gs_kokkos() d_list = DAT::t_int_2d("surf_react_adsorb:list",nspecies,MAX(nmax,1)); auto h_list = Kokkos::create_mirror_view(d_list); - for (int i = 0; i < nspecies; i++) + if (gsflag) + for (int i = 0; i < nspecies; i++) for (int j = 0; j < reactions_gs[i].n; j++) h_list(i,j) = reactions_gs[i].list[j]; @@ -405,6 +402,14 @@ void SurfReactAdsorbKokkos::tally_reset() void SurfReactAdsorbKokkos::tally_update() { + // PS (on-surface) chemistry desorbs/inserts particles on the host inside the + // base tally_update(); make the host particle list current first so + // add_particle() appends to up-to-date data, and mark host-modified after + // so the device picks up the new particles on the next sync + + ParticleKokkos* particle_kk = (ParticleKokkos*) particle; + if (psflag) particle_kk->sync(Host,PARTICLE_MASK); + // device -> host: reaction counts Kokkos::deep_copy(h_scalars,d_scalars); @@ -433,6 +438,10 @@ void SurfReactAdsorbKokkos::tally_update() SurfReactAdsorb::tally_update(); + // PS chemistry may have appended particles on the host + + if (psflag) particle_kk->modify(Host,PARTICLE_MASK); + // mirror re-zeroed host deltas (+ mark) back to device (only on a sync step) if (update->ntimestep % nsync == 0) { From 3ba78446901e702a3c8a85a2ab91a189f4b2fd3c Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 21 Jun 2026 02:21:49 +0000 Subject: [PATCH 13/30] KOKKOS: port compute react/isurf/grid to Kokkos Device per-surf reaction tally for implicit surfaces, mirroring the compute isurf/grid Kokkos port. surf_tally_kk() counts reactions per surf (rpflag/reaction2col column mapping), tallied on-device from the move kernel's surface-collision loop; tallyinfo() syncs+compresses to the host, and the host base collates to per-grid. - ComputeReactISurfGridKokkos: surf_tally_kk + lifecycle (init/clear/pre_surf_ tally/post_surf_tally/tallyinfo/grow_tally), reaction2col flattened to device. post_process_isurf_grid() overridden to sync the device tally to the host first, since consumers like fix ablate read the compute directly (no fix ave/grid in between to call tallyinfo()). - base compute_react_isurf_grid: Kokkos copy ctor, virtual grow_tally/ post_process_isurf_grid, copy/copymode destructor guard. - update_kokkos: add a third surf-tally partition (react/isurf/grid) alongside compute surf and compute isurf/grid. Verified bit-for-bit CPU vs -sf kk (Serial+EXACT) on ablation/in.ablation.3d. reactions (reaction-count-driven ablation): identical np/nscoll/nsreact and f_ablate. Co-Authored-By: Claude Opus 4.8 Co-Authored-By: Stan Moore Claude-Session: https://claude.ai/code/session_01Q6nQVuxEiTUqSzDoDppCUM --- examples/ablation/binary.21x21x21 | Bin 9273 -> 0 bytes .../compute_react_isurf_grid_kokkos.cpp | 194 ++++++++++++++++++ src/KOKKOS/compute_react_isurf_grid_kokkos.h | 110 ++++++++++ src/KOKKOS/update_kokkos.cpp | 29 ++- src/KOKKOS/update_kokkos.h | 12 +- src/compute_react_isurf_grid.cpp | 2 + src/compute_react_isurf_grid.h | 5 +- 7 files changed, 343 insertions(+), 9 deletions(-) delete mode 100644 examples/ablation/binary.21x21x21 create mode 100644 src/KOKKOS/compute_react_isurf_grid_kokkos.cpp create mode 100644 src/KOKKOS/compute_react_isurf_grid_kokkos.h diff --git a/examples/ablation/binary.21x21x21 b/examples/ablation/binary.21x21x21 deleted file mode 100644 index 2c42f98a06a31ed4a69dcdef98896004e069f507..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9273 zcmeHs^;6YdwD*?oPC>dux;rGKkw!$Nr9mYnq@^1v=}@{G0g>)*r8`AJfxF+C_Yd&- z_0HWh%$Z@%_nf`=`qWy77yv;0|Nr32|E>Q$H}DE!LiHI>3@f!mG)JV-Iq9{HKLCsk z-1;bb7uRjzc)6RC{MBlF3jkulwG+c)D?&q2TuFiF7;Jy0m;ey{z@m*+M|8W_*Zr)t z(+fB2NjdM zk8WM(aaiS-L0Nl~z_@JqPt#iT_Hs)XxfV}3RD-?JtRqwaQn@W28g4eu5~}rLa1)J* z*&#jvAfR7g3xk}AKi2qKX@N8(qR2WJfF7pMq+ZrnD8o{@1J zYY36~;tC75u_j5}0&ugx8F`q?(maRA)B7!&T2B0r2msG2ilYmbb|_GHcAi<5+fquY z#Q?zGbonwaXD~ak0p)nJk}nI@LI{9pY0?4dXr!3_ULBfrb!7qbGD`sRn?~Yp{KASZ zt9M!KYMLkIHEjSeCpF7!70vw0jL1^DW2TX%i6IHV$2SJ%^xsS?VmAk9T@gCOu0`Je z7g)WUNcpKn#)Vm1-qAyYR|NEu3IWJCyVB4^rgF*?CoeDYW8jsaKL7yhXs2K7VuT+3 zc>>C>lE8QO+gkxZ9yU7E7v$9;V0Dj-qD5icq$>v?>f@12b>K5gKJvX>_93p~qyf13 zUkxr)va~_Nt-1D!Ltl!c`jg}UFx7axQJ3NBecvO=(Eo0S@N{4VfFS|eyovjWr`xiV zsxydbA~`7e0FXQ5zhr*mUX&9&Z^KY-d}M}b0KmED__H)ZzWY|%Q3Vq6DAX@DxdE_7 zOtRH6dt}X3cerhW@a=7QA^glhM_LRv=HpmYEt@Hua5-(F-gf{r$C1cu^%`6ZRD@RQ z-CuZf-W(5reYvEO*4P;ticy(U=f&sG42f9)@Sv35=JPFU>5mQ^t^(0Su{nt3j+5Ho zZghdC8QSD@)U`T}SGw@(3}4ziKgf%sy!xx4-{T+K^;OOrfRWXu!6FosT1mW=Z+A^D zXl3hx5KE-m$wN;jlcM-4>1|h+Pk6T6{!8K5HqNe$MfU6VoBlS9?K|w*{Qdwy9)g|% zqNWUm%%V{B4MH6qm-;RMKS=wDjqiR^#@d!FL>+Fh;_yO*RL(aSek`;so?fUgA$amy zo!4{<0QsYPb`2ef4+vt&ER^n>Iwlz41E5n0f{>PYLrEk0cl+1x4P!0e0Z^aTOtM5G z@uT$fGvm4p1(Y-&sh2{9^m9ocp){H2`~d&Ki=7Lo$tful|ru&q*sh zqJtMw)t4trBHM=?s;#V>%+4~`@e}~5#KS)49OK$Rl#&&17B!xk7F7V$Xq>J(jtKX6 zMrf9=TLbieeAI>}f>ftIi}${k})9aD|aLo0DqA`a41+x%WC-k z65znt?VV(D2H;C*GtX9XdXh6Ds~1rUcezy&e4P68gdqCz*Khr=&W1aVw_|XUef~?~ zkTl0W9d>ZlVJ)j%VzrSkjs-wv5H?{nr_b>*GB`WEeoDl;Xvp{r0Fh`Ldegq}1tbcG zJR614`2t$VxU`m*n3vQ0XKI2xWxpj{!(T-_10d$fcvpIEm@^ntDT+BN?86#y#6 zJVM``HB7eewB+S`w|VaCP69A{W{ZhAY|5@gzU$!67bJbWR1cxpO#Jj`zvfGNaK<&n z`8u3SkshAt*|e~Od4M36&UJ#TN}zWmrmheu~RZYHiYO7NY5 zs3wHs?~8B+6P4dfeL>zWYxSq8)6h`)0${hyG&##JlcD4tYEG>*GDu$e6M*#dBsa^%;#;OC>*eY(cueTC zkO-zaRRjneu||Z-a!qx}QxukV(4;6@EBY?iREvs4JGYa26=!w+pLe9YO?CvYTAE#J z(#=(T99g4KlqE&^PXZaQ`<&c~B$Oc+2 zCu*V&x;DX-9XVf}INAVMS#O|GslZS&8$eOWy{FOQ)+Px+9ZPmUo}$hwzRfM^FM@E_ zGG%uFmRL8)I7GrU+;P?JQ5QwMMd>65pvroM%n;)!LyvXch{nxi4ENg_0Jx=%oy%_d zNltVhf(@G1-amie0--ojT%CfSzw~8>@pb(vF~5Wi^i7-nlB%smRJ(%4Eq|a2mK*zm;_Q|Gvj}Y_!D+TTs#Pqw>=Q0wF2N#cb(j{I39a^cRj25Sk5=*A$9;z zJ@a$CAI5qBxtGmX*Re(mgXaLq+$HCW?K4&!3<+M(+zy@!2}%Iq!MQc1FGI)wNV5hv zdPE{yZo>}%ll_2~TojrpOB*v@M#}Wc8h+dW1pCq_Y;}B)s+X8kDyWbs11o`kBBAg?U*YOIF&d@0XTElo#`mpCrFmRz+2tL zS@qw9N#I(OC;LjKT+TG3+vl(lJp)Z56lR-p>ygOh48PEOOd6s*mCme%oB*iumHH#r zaLV3KiITpxq#FuwgR(JJ=jjBsXkVS%SzRY zfI=O~9@j>==piUVG&lC@uLzjz1mK|-QB-L5%&fjGQ=(aw+S39is7q0wscl@X@XM>D z!X{b~w1k(d+5tdpW8iw5CbQUd!QgIULB2(ejRZjdvNpG(Lxy~7%fWHmagP~_& zk?=CbQviUVxhfLPJ&LU5vK(|JaoJ*RBy7kyhbYa#gN@_oE*E&D3q#$=N3bRdg0@?Z9nq>*aU2B?}JqtK27K z^M^NB{p!5$)DM|V{!8K5>V3R7lLHo$HuD=p9wh$gIF|unkqr@DK&1Em%2Os~o^QwD zkug;OOisA?xv>(AJ8@v3IK=ICS|uU_!0MD-y7$zaL6Lj>Qy-pZp>QY$>}68(F~XmU z{SB`5S!eUE=3sai28AV0TxW_^L;GVrv*TjX;(V}z762hzihZ);*ybazK9Oh!Au5iN zL)heqa(;<57!E%$aGp}{>M2TH=77n6*$~{V=U%a>Td;91zPGlnu7<{UUt5hc=~n_a z{n7RF*YaiCiM0SAEJkAJw(t*SqKv5nTn9dK>E_tK4BVcMJk@mt;A#{5@FT;MlA~UI@9yLmKg3N?>T)OTBm_xR!U8`CkghjsG}ed2FvMz?d1K zCtaQj`3-cL1$^(fpZSucEJelxki6z7X1+jnw4m8cRd-~*n4+7X^h&NrW*Jh3!qQ!i zV~)wWAi};+n+T4l;JMG-Gjs?kAr@#_1s$=-*@bFGv?d=_cRFNlkuJBoA@JLtI5FdE#Nd1q)$-> z6Cf=Jj{5cyt@=rA@vx$!?OS~q$OUc1VvX_y;uDTLeuPA#`MBWQ0pRU%M--x==eW7T zrt+~eRj$k_$qv&~JSko7Fo?DYj@0|N~CQYG1nL1WlFrZr4YC1CnBoI4HO)Oz%-^~h}(>z82J{&GA3T@Hnu z3pJvApz+P2CS0D@wkji(_UarA*#?!@bP+Q))0#;6Gj#edb^ zkMR5y-nqMo*3Z?91pt=`cfrSuGrVvb73;31;O~|?@bT5Jo-gcF0}+pfrB{f(ML2o6 zp-C+s7pdvV7>?+6`yx6xS4gRGe+K|#G@``3w)FGBRS2R8%X{AZzDj7jA-{@jP$u&| zMvpk{J|TH6Jc8sZ8<;F^CzV)Dl6mfIN*M40XA%Yp!s|)VA4|O5WFw}M2R51JE0bHW zH=KHVm&_2}DEqy#CD9|{ZG@`?0Q%uVaTxqE7^q&U`Fj@0G;%m=0Cd)Fx1A3$J~#i+ zU2I=|LP)C#g?f0B56^`=?bIzchw_TRYgZqpU7>nW7qlZLDD(a&bq)Y@jDf=npLP&8u3u-;|vp==Uj7rAI|Fi*bz9_bhAF15jikUas=2w)u9`Ydk_I5l5X=NHKTs%@$ zg52;K^c-jt*R0|vtCNTr$U#SpTHaB|N=6j`JlBfgHTXlupQY?y$;>WNpMI+kfbYYL zs(kK|X*#;)u#TheEF87j04z6Zq^F->2e=l!cj0FLU=z^721hh~>PDN0&p7@ZZM~08 zZR%f2g(6V*DxhQc%cnb}dh#e4lc^`0S+H=~>5;n#ltSyjIh5T+1s@L6e_Vib279Cj z4w?LWad+>MO7apiWSgL^#+? zd_jhgH(+t>=jF8qZNgf`FU4sS)tSU(?ou~f6cZ&a?!Odnyq%YzO=2i=oWpqSfxq1m zcL6J8vGK!Jr^X*t2ftsV$=+vIh$1_IVkM27G`x@*j%$MawtNXSYvB zycd7X?mngKT7lE-=LP4~BLmfg2@^>x$Jm+ICNM9VC&_HxQyGzZsLs=}B)TwfX#u5D zq5L#fIy}xx0C@_OE30G0@pX3VKP~Qu+GMws}FFBa8Y2t?`0#5SJs?o|`qWgShsofOblQxC=$!=Gu zLDW?zY%CD6@cDbLr7USBq&AxIdX61Vdo*idAK!7ztyE4B%yl#*Yrip^Q=*eU3nJ9r zZeA$C694=9jP+W$Nj5Sje*1p4*MmdDOa&?crpOyE#qbR7-%+q*+1y4%PrsIXcW<28;`W3WYxs@@G{By1N9|1(XBqn$* z?1*p}jy{2IIy0e&I9IS+*^ycLiAoA~Qb#P@nlV3F&b+V#jVw1&RU(X_hLpb{jl?nd zDlN*59RIKxpDHT@21usaObo6PYaN_j?*!8SPrM5d(tRYXR#v1L05CB3IWsD!6rP;kBCz)fIRxh)8Ck^K|%>76!ebRj6 z4niLemOn%^>p?pUq*T%f!xTrD|67Yh-RV(lw5R&-Wdk5NR$lphf4(tR`~J~)+0_J6 zZP;^to5*sUw_+^VUYl_A$_?EsolS>B5+cdu)%6tlWXT|(qb;nl)Q>IztbJ!NkQV0u z@=WTzn%O)219=e`~#%a1p@HoZRW1G}D-L*8HY^r%d$ zszWtAdXf)xt6()hxaCAMk?YnB1}_$mX0K{W<-o8PDcN55CQ8NZg5h17-y8Blyr=MS z_O%IQbX+kGcN&blH9`YE1~^7@)b#PfAbM3ZVu`ebDxc&fmk#B(6Aev!aDLZ%_eB72 zw%iZxoyZpe6o|cZkwT>#Lek|Y_5bO3CN~VJ&C!bKX6tV8z;7}Du(}d@ebZ}9O1K>hHt6R#S`bv&yOYgt|i{` zOj>YKn$9{cm|oO8TuEn2{7WrisQ^;u?M$k!Wi@4C;49JsOpY7O10Q_=m^Qe-crnpE z#JiC=lMUd=S3zNf)g&M>Nf*!h(){nu-3qgsaWRE^i zH=G&~v_?3WG@HdpxTEiE&tO(<4}xHH9nv~^-0 z=ME>c*KALgITHg=sEv*Lme$|%7k}eh96V{7FE5}3QHkduy;NV*S-+KH<_n}_Gb4Ne zS!|tQ6cgM|BGt~;H(l)+>_F)NBS)$^7e<&Ah&lL-IY#v2hqPxf43I7h_i|p9u+}v` z_&(l7AfY@6+ZL3_KQ>*P&CTEi4`@sr;ZrY(fz-BX!F=W;c35+}B|P6iOiorp4S*Im zkvz-(5e@I5-lI{=-ZA`W*vHlTIn6CjZ1U^wgyURVJ67C8d%=P5eIiVJm-N6)x}Dk8 zRJ-zdT*wXzl1k8@mSiP`)sNsqLObE2)ee6mR0=V6PiL-tXxFZ+P)LWP{6kz&CP>O4t)wmb*kLw6*i7TZhN9M&W#fbw?;jhbz#%#tTp^f|T) zqjVK`0I;A?-F6%=V9)6=_mCPjm8C^HH z$ty4@&xPe5$a_Mo26f4JbDLTXB`2-fjYauHr@eX<&J|6@H8#>7(|W24-gCMU3{0-3JC~UbDgl7T;xQ+dUNcg9HR*V1*+x2B9Hfk1*S!`5LmWG? zV!L`SN}+_{6I%GQdestroy_kokkos(k_tally2surf,tally2surf); + memoryKK->destroy_kokkos(k_array_surf_tally,array_surf_tally); +} + +/* ---------------------------------------------------------------------- */ + +void ComputeReactISurfGridKokkos::init() +{ + ComputeReactISurfGrid::init(); + + // flatten reaction2col to device (only used when rpflag) + + if (rpflag) { + int nreaction = surf->sr[isr]->nlist; + d_reaction2col = DAT::t_int_2d("react/isurf/grid:reaction2col",nreaction,ntotal); + auto h_r2c = Kokkos::create_mirror_view(d_reaction2col); + for (int i = 0; i < nreaction; i++) + for (int j = 0; j < ntotal; j++) + h_r2c(i,j) = reaction2col[i][j]; + Kokkos::deep_copy(d_reaction2col,h_r2c); + } + + // size per-surf tally storage to nsurf (implicit-surf ablation only shrinks) + + resize_device(surf->nlocal + surf->nghost); +} + +/* ---------------------------------------------------------------------- */ + +void ComputeReactISurfGridKokkos::resize_device(int nsurf) +{ + if (nsurf < 1) nsurf = 1; + + memoryKK->grow_kokkos(k_tally2surf,tally2surf,nsurf,"react/isurf/grid:tally2surf"); + d_tally2surf = k_tally2surf.view_device(); + + d_surf2tally = DAT::t_int_1d("react/isurf/grid:surf2tally",nsurf); + Kokkos::deep_copy(d_surf2tally,-1); + + memoryKK->grow_kokkos(k_array_surf_tally,array_surf_tally,nsurf,ntotal, + "react/isurf/grid:array_surf_tally"); + d_array_surf_tally = k_array_surf_tally.view_device(); + + nsurf_tally_alloc = nsurf; +} + +/* ---------------------------------------------------------------------- */ + +void ComputeReactISurfGridKokkos::clear() +{ + // called by Update at beginning of timesteps surf tallying is done + + int nsurf = surf->nlocal + surf->nghost; + if (nsurf > nsurf_tally_alloc) resize_device(nsurf); + + Kokkos::deep_copy(d_array_surf_tally,0); + Kokkos::deep_copy(d_surf2tally,-1); + + ntally = 0; + combined = 0; +} + +/* ---------------------------------------------------------------------- */ + +void ComputeReactISurfGridKokkos::pre_surf_tally() +{ + SurfKokkos* surf_kk = (SurfKokkos*) surf; + surf_kk->sync(Device,ALL_MASK); + d_lines = surf_kk->k_lines.view_device(); + d_tris = surf_kk->k_tris.view_device(); + + need_dup = sparta->kokkos->need_dup(); + if (need_dup) + dup_array_surf_tally = Kokkos::Experimental::create_scatter_view(d_array_surf_tally); + else + ndup_array_surf_tally = Kokkos::Experimental::create_scatter_view(d_array_surf_tally); +} + +/* ---------------------------------------------------------------------- */ + +void ComputeReactISurfGridKokkos::post_surf_tally() +{ + if (need_dup) { + Kokkos::Experimental::contribute(d_array_surf_tally, dup_array_surf_tally); + dup_array_surf_tally = {}; + } + + k_tally2surf.modify_device(); + k_array_surf_tally.modify_device(); +} + +/* ---------------------------------------------------------------------- + sync device tallies to host and compress to dense list (ntally tallies) + matches ComputeISurfGridKokkos::tallyinfo() +------------------------------------------------------------------------- */ + +int ComputeReactISurfGridKokkos::tallyinfo(surfint *&ptr) +{ + k_tally2surf.sync_host(); + ptr = tally2surf; + + k_array_surf_tally.sync_host(); + auto h_surf2tally = Kokkos::create_mirror_view(d_surf2tally); + Kokkos::deep_copy(h_surf2tally,d_surf2tally); + + int nsurf = surf->nlocal + surf->nghost; + int istart = 0; + int iend = nsurf-1; + + while (1) { + while (h_surf2tally[istart] != -1 && istart < nsurf-2) istart++; + while (h_surf2tally[iend] == -1 && iend > 0) iend--; + if (istart >= iend) { + ntally = istart; + break; + } + for (int k = 0; k < ntotal; k++) + array_surf_tally[istart][k] = array_surf_tally[iend][k]; + h_surf2tally[istart] = h_surf2tally[iend]; + h_surf2tally[iend] = -1; + tally2surf[istart] = tally2surf[iend]; + } + + return ntally; +} + +/* ---------------------------------------------------------------------- + sync the device per-surf tally to the host (tallyinfo) before the host + base class collates it to per-grid; consumers (e.g. fix ablate) read the + compute directly via post_process_isurf_grid() rather than tallyinfo() +------------------------------------------------------------------------- */ + +void ComputeReactISurfGridKokkos::post_process_isurf_grid() +{ + if (combined) return; + surfint *dummy; + tallyinfo(dummy); + ComputeReactISurfGrid::post_process_isurf_grid(); +} + +/* ---------------------------------------------------------------------- */ + +void ComputeReactISurfGridKokkos::grow_tally() +{ + resize_device(surf->nlocal + surf->nghost); +} diff --git a/src/KOKKOS/compute_react_isurf_grid_kokkos.h b/src/KOKKOS/compute_react_isurf_grid_kokkos.h new file mode 100644 index 000000000..e897a3a5a --- /dev/null +++ b/src/KOKKOS/compute_react_isurf_grid_kokkos.h @@ -0,0 +1,110 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#ifdef COMPUTE_CLASS + +ComputeStyle(react/isurf/grid/kk,ComputeReactISurfGridKokkos) + +#else + +#ifndef SPARTA_COMPUTE_REACT_ISURF_GRID_KOKKOS_H +#define SPARTA_COMPUTE_REACT_ISURF_GRID_KOKKOS_H + +#include "compute_react_isurf_grid.h" +#include "kokkos_type.h" + +namespace SPARTA_NS { + +class ComputeReactISurfGridKokkos : public ComputeReactISurfGrid { + public: + ComputeReactISurfGridKokkos(class SPARTA *, int, char **); + ComputeReactISurfGridKokkos(class SPARTA *); + ~ComputeReactISurfGridKokkos(); + void init(); + void clear(); + int tallyinfo(surfint *&); + void post_process_isurf_grid(); + void pre_surf_tally(); + void post_surf_tally(); + +/* ---------------------------------------------------------------------- + tally a surface reaction for particle colliding with surf element isurf + mirrors ComputeReactISurfGrid::surf_tally(); per-surf tally, compressed to + the host in tallyinfo(); post-processing (collate to per-grid) is on host +------------------------------------------------------------------------- */ + + template + KOKKOS_INLINE_FUNCTION + void surf_tally_kk(double /*dtremain*/, int isurf, int /*icell*/, int reaction, + Particle::OnePart * /*iorig*/, + Particle::OnePart * /*ip*/, Particle::OnePart * /*jp*/) const + { + // skip if no reaction + + if (reaction == 0) return; + reaction--; + + // skip if isurf not in group or its reaction model is not a match + + surfint surfID; + if (dim == 2) { + if (!(d_lines[isurf].mask & groupbit)) return; + if (d_lines[isurf].isr != isr) return; + surfID = d_lines[isurf].id; + } else { + if (!(d_tris[isurf].mask & groupbit)) return; + if (d_tris[isurf].isr != isr) return; + surfID = d_tris[isurf].id; + } + + int itally = isurf; + d_tally2surf(itally) = surfID; + d_surf2tally(isurf) = isurf; + + auto v_array_surf_tally = ScatterViewHelper::value,decltype(dup_array_surf_tally),decltype(ndup_array_surf_tally)>::get(dup_array_surf_tally,ndup_array_surf_tally); + auto a_array_surf_tally = v_array_surf_tally.template access::value>(); + + if (rpflag) { + for (int i = 0; i < ntotal; i++) + if (d_reaction2col(reaction,i)) a_array_surf_tally(itally,i) += 1.0; + } else a_array_surf_tally(itally,reaction) += 1.0; + } + + private: + DAT::t_int_2d d_reaction2col; // [nreaction][ntotal], only if rpflag + + DAT::tdual_float_2d_lr k_array_surf_tally; + DAT::t_float_2d_lr d_array_surf_tally; + + int need_dup; + Kokkos::Experimental::ScatterView dup_array_surf_tally; + Kokkos::Experimental::ScatterView ndup_array_surf_tally; + + DAT::t_surfint_1d d_tally2surf; + DAT::tdual_surfint_1d k_tally2surf; + DAT::t_int_1d d_surf2tally; + + t_line_1d d_lines; + t_tri_1d d_tris; + + int nsurf_tally_alloc; // current device tally allocation (nsurf) + + void grow_tally(); + void resize_device(int); +}; + +} + +#endif +#endif diff --git a/src/KOKKOS/update_kokkos.cpp b/src/KOKKOS/update_kokkos.cpp index 364f3e2a5..3e45ea116 100644 --- a/src/KOKKOS/update_kokkos.cpp +++ b/src/KOKKOS/update_kokkos.cpp @@ -87,11 +87,13 @@ UpdateKokkos::UpdateKokkos(SPARTA *sparta) : Update(sparta), blist_active_copy{VAL_2(KKCopy(sparta))}, slist_active_copy{VAL_2(KKCopy(sparta))}, slist_active_isurf_copy{VAL_2(KKCopy(sparta))}, + slist_active_react_isurf_copy{VAL_2(KKCopy(sparta))}, tmp_compute_boundary_kk(sparta), tmp_compute_surf_kk(sparta), - tmp_compute_isurf_grid_kk(sparta) + tmp_compute_isurf_grid_kk(sparta), + tmp_compute_react_isurf_grid_kk(sparta) { - nslist_surf = nslist_isurf = 0; + nslist_surf = nslist_isurf = nslist_react_isurf = 0; // use 1D view for scalars to reduce GPU memory operations @@ -147,6 +149,7 @@ UpdateKokkos::~UpdateKokkos() tmp_compute_boundary_kk.uncopy = 1; tmp_compute_surf_kk.uncopy = 1; tmp_compute_isurf_grid_kk.uncopy = 1; + tmp_compute_react_isurf_grid_kk.uncopy = 1; for (int i=0; i void UpdateKokkos::move() ComputeISurfGridKokkos* compute_isurf_kk = (ComputeISurfGridKokkos*)(slist_active[m]); compute_isurf_kk->post_surf_tally(); + } else if (strcmp(slist_active[m]->style,"react/isurf/grid") == 0) { + ComputeReactISurfGridKokkos* compute_react_isurf_kk = + (ComputeReactISurfGridKokkos*)(slist_active[m]); + compute_react_isurf_kk->post_surf_tally(); } else { ComputeSurfKokkos* compute_surf_kk = (ComputeSurfKokkos*)(slist_active[m]); compute_surf_kk->post_surf_tally(); @@ -1507,6 +1515,9 @@ void UpdateKokkos::operator()(TagUpdateMove for (m = 0; m < nslist_isurf; m++) slist_active_isurf_copy[m].obj. surf_tally_kk(dtremain,minsurf,icell,reaction,&iorig,ipart,jpart); + for (m = 0; m < nslist_react_isurf; m++) + slist_active_react_isurf_copy[m].obj. + surf_tally_kk(dtremain,minsurf,icell,reaction,&iorig,ipart,jpart); } // stuck_iterate = consecutive iterations particle is immobile @@ -2075,7 +2086,7 @@ void UpdateKokkos::tally_set(bigint ntimestep) // "compute isurf/grid" (slist_active_isurf_copy); both tally on-device via // surf_tally_kk(), invoked from the move kernel's surface collision loop - nslist_surf = nslist_isurf = 0; + nslist_surf = nslist_isurf = nslist_react_isurf = 0; if (nsurf_tally) { for (i = 0; i < nsurf_tally; i++) { @@ -2089,6 +2100,16 @@ void UpdateKokkos::tally_set(bigint ntimestep) compute_isurf_kk->pre_surf_tally(); slist_active_isurf_copy[nslist_isurf].copy(compute_isurf_kk); nslist_isurf++; + } else if (strcmp(slist_active[i]->style,"react/isurf/grid") == 0) { + ComputeReactISurfGridKokkos* compute_react_isurf_kk = + dynamic_cast(slist_active[i]); + if (!compute_react_isurf_kk) + error->all(FLERR,"Must use Kokkos-enabled compute react/isurf/grid with Kokkos"); + if (nslist_react_isurf >= KOKKOS_MAX_SLIST) + error->all(FLERR,"Kokkos currently only supports two instances of compute react/isurf/grid"); + compute_react_isurf_kk->pre_surf_tally(); + slist_active_react_isurf_copy[nslist_react_isurf].copy(compute_react_isurf_kk); + nslist_react_isurf++; } else { ComputeSurfKokkos* compute_surf_kk = dynamic_cast(slist_active[i]); @@ -2110,6 +2131,8 @@ void UpdateKokkos::tally_set(bigint ntimestep) slist_active_copy[i].copy(&tmp_compute_surf_kk); for (i = nslist_isurf; i < KOKKOS_MAX_SLIST; i++) slist_active_isurf_copy[i].copy(&tmp_compute_isurf_grid_kk); + for (i = nslist_react_isurf; i < KOKKOS_MAX_SLIST; i++) + slist_active_react_isurf_copy[i].copy(&tmp_compute_react_isurf_grid_kk); if (ngas_tally) error->all(FLERR,"Kokkos does not (yet) support tallying gas/gas collisions or reactions"); diff --git a/src/KOKKOS/update_kokkos.h b/src/KOKKOS/update_kokkos.h index 01920afac..c94afefc9 100644 --- a/src/KOKKOS/update_kokkos.h +++ b/src/KOKKOS/update_kokkos.h @@ -33,6 +33,7 @@ #include "compute_boundary_kokkos.h" #include "compute_surf_kokkos.h" #include "compute_isurf_grid_kokkos.h" +#include "compute_react_isurf_grid_kokkos.h" namespace SPARTA_NS { @@ -149,18 +150,21 @@ class UpdateKokkos : public Update { //KKCopy blist_active_copy[KOKKOS_MAX_GLIST]; KKCopy slist_active_copy[KOKKOS_MAX_SLIST]; KKCopy slist_active_isurf_copy[KOKKOS_MAX_SLIST]; + KKCopy slist_active_react_isurf_copy[KOKKOS_MAX_SLIST]; KKCopy blist_active_copy[KOKKOS_MAX_BLIST]; // partition of slist_active (set in tally_set): - // nslist_surf = # of compute surf style tallies (slist_active_copy) - // nslist_isurf = # of compute isurf/grid tallies (slist_active_isurf_copy) - // nslist_surf + nslist_isurf == nsurf_tally + // nslist_surf = # of compute surf style tallies (slist_active_copy) + // nslist_isurf = # of compute isurf/grid tallies (slist_active_isurf_copy) + // nslist_react_isurf = # of compute react/isurf/grid tallies + // nslist_surf + nslist_isurf + nslist_react_isurf == nsurf_tally - int nslist_surf,nslist_isurf; + int nslist_surf,nslist_isurf,nslist_react_isurf; ComputeBoundaryKokkos tmp_compute_boundary_kk; ComputeSurfKokkos tmp_compute_surf_kk; ComputeISurfGridKokkos tmp_compute_isurf_grid_kk; + ComputeReactISurfGridKokkos tmp_compute_react_isurf_grid_kk; typedef Kokkos::DualView tdual_int_14; typedef tdual_int_14::t_dev t_int_14; diff --git a/src/compute_react_isurf_grid.cpp b/src/compute_react_isurf_grid.cpp index 0bb4fd5c8..5b1afb3e7 100644 --- a/src/compute_react_isurf_grid.cpp +++ b/src/compute_react_isurf_grid.cpp @@ -117,6 +117,8 @@ ComputeReactISurfGrid(SPARTA *sparta, int narg, char **arg) : ComputeReactISurfGrid::~ComputeReactISurfGrid() { + if (copy || copymode) return; + memory->destroy(reaction2col); memory->destroy(array_surf_tally); memory->destroy(tally2surf); diff --git a/src/compute_react_isurf_grid.h b/src/compute_react_isurf_grid.h index 370f649fb..528a739b9 100644 --- a/src/compute_react_isurf_grid.h +++ b/src/compute_react_isurf_grid.h @@ -31,6 +31,7 @@ namespace SPARTA_NS { class ComputeReactISurfGrid : public Compute { public: ComputeReactISurfGrid(class SPARTA *, int, char **); + ComputeReactISurfGrid(class SPARTA* sparta) : Compute(sparta) {} // needed for Kokkos ~ComputeReactISurfGrid(); virtual void init(); void compute_per_grid(); @@ -38,7 +39,7 @@ class ComputeReactISurfGrid : public Compute { virtual void surf_tally(double, int, int, int, Particle::OnePart *, Particle::OnePart *, Particle::OnePart *); virtual int tallyinfo(surfint *&); - void post_process_isurf_grid(); + virtual void post_process_isurf_grid(); bigint memory_usage(); protected: @@ -69,7 +70,7 @@ class ComputeReactISurfGrid : public Compute { Surf::Line *lines; Surf::Tri *tris; - void grow_tally(); + virtual void grow_tally(); }; } From 9a2a4f4b6d2fd73ee30bd2c7aa9b65db0862e5df Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 21 Jun 2026 02:29:23 +0000 Subject: [PATCH 14/30] KOKKOS: port compute react/surf to Kokkos Device per-surf reaction tally for explicit surfaces, mirroring compute react/isurf/grid but with per-surf output (post_process_surf + collate_array over owned surfs). surf_tally_kk() counts reactions per surf element (rpflag/reaction2col), tallied on-device from the move kernel surface-collision loop; tallyinfo() syncs+compresses to the host; post_process_surf() is overridden to sync first since consumers (dump surf, compute reduce, ...) read the compute directly. - ComputeReactSurfKokkos + base Kokkos copy ctor / virtual grow_tally / copy-guard destructor. - update_kokkos: 4th surf-tally partition (react/surf). Also fixes the nslist_react_surf counter not being reset in tally_set. Verified bit-for-bit CPU vs -sf kk (Serial+EXACT): explicit circle with diffuse collide + global surf react; compute react/surf summed via compute reduce in stats matches (per-step destroy/create reaction counts identical, sum == nsreact). Co-Authored-By: Claude Opus 4.8 Co-Authored-By: Stan Moore Claude-Session: https://claude.ai/code/session_01Q6nQVuxEiTUqSzDoDppCUM --- src/KOKKOS/compute_react_surf_kokkos.cpp | 186 +++++++++++++++++++++++ src/KOKKOS/compute_react_surf_kokkos.h | 105 +++++++++++++ src/KOKKOS/update_kokkos.cpp | 29 +++- src/KOKKOS/update_kokkos.h | 5 +- src/compute_react_surf.cpp | 2 + src/compute_react_surf.h | 3 +- 6 files changed, 325 insertions(+), 5 deletions(-) create mode 100644 src/KOKKOS/compute_react_surf_kokkos.cpp create mode 100644 src/KOKKOS/compute_react_surf_kokkos.h diff --git a/src/KOKKOS/compute_react_surf_kokkos.cpp b/src/KOKKOS/compute_react_surf_kokkos.cpp new file mode 100644 index 000000000..9e9b7c8ff --- /dev/null +++ b/src/KOKKOS/compute_react_surf_kokkos.cpp @@ -0,0 +1,186 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#include "string.h" +#include "compute_react_surf_kokkos.h" +#include "surf_kokkos.h" +#include "surf_react.h" +#include "grid.h" +#include "update.h" +#include "memory_kokkos.h" +#include "error.h" +#include "sparta_masks.h" +#include "kokkos.h" + +using namespace SPARTA_NS; + +/* ---------------------------------------------------------------------- */ + +ComputeReactSurfKokkos::ComputeReactSurfKokkos(SPARTA *sparta, int narg, char **arg) : + ComputeReactSurf(sparta, narg, arg) +{ + kokkos_flag = 1; +} + +ComputeReactSurfKokkos::ComputeReactSurfKokkos(SPARTA *sparta) : + ComputeReactSurf(sparta) +{ + copy = 1; + uncopy = 0; +} + +/* ---------------------------------------------------------------------- */ + +ComputeReactSurfKokkos::~ComputeReactSurfKokkos() +{ + if (copy) return; + + memoryKK->destroy_kokkos(k_tally2surf,tally2surf); + memoryKK->destroy_kokkos(k_array_surf_tally,array_surf_tally); +} + +/* ---------------------------------------------------------------------- */ + +void ComputeReactSurfKokkos::init() +{ + ComputeReactSurf::init(); + + if (rpflag) { + int nreaction = surf->sr[isr]->nlist; + d_reaction2col = DAT::t_int_2d("react/surf:reaction2col",nreaction,ntotal); + auto h_r2c = Kokkos::create_mirror_view(d_reaction2col); + for (int i = 0; i < nreaction; i++) + for (int j = 0; j < ntotal; j++) + h_r2c(i,j) = reaction2col[i][j]; + Kokkos::deep_copy(d_reaction2col,h_r2c); + } + + resize_device(surf->nlocal + surf->nghost); +} + +/* ---------------------------------------------------------------------- */ + +void ComputeReactSurfKokkos::resize_device(int nsurf) +{ + if (nsurf < 1) nsurf = 1; + + memoryKK->grow_kokkos(k_tally2surf,tally2surf,nsurf,"react/surf:tally2surf"); + d_tally2surf = k_tally2surf.view_device(); + + d_surf2tally = DAT::t_int_1d("react/surf:surf2tally",nsurf); + Kokkos::deep_copy(d_surf2tally,-1); + + memoryKK->grow_kokkos(k_array_surf_tally,array_surf_tally,nsurf,ntotal, + "react/surf:array_surf_tally"); + d_array_surf_tally = k_array_surf_tally.view_device(); + + nsurf_tally_alloc = nsurf; +} + +/* ---------------------------------------------------------------------- */ + +void ComputeReactSurfKokkos::clear() +{ + int nsurf = surf->nlocal + surf->nghost; + if (nsurf > nsurf_tally_alloc) resize_device(nsurf); + + Kokkos::deep_copy(d_array_surf_tally,0); + Kokkos::deep_copy(d_surf2tally,-1); + + ntally = 0; + combined = 0; +} + +/* ---------------------------------------------------------------------- */ + +void ComputeReactSurfKokkos::pre_surf_tally() +{ + SurfKokkos* surf_kk = (SurfKokkos*) surf; + surf_kk->sync(Device,ALL_MASK); + d_lines = surf_kk->k_lines.view_device(); + d_tris = surf_kk->k_tris.view_device(); + + need_dup = sparta->kokkos->need_dup(); + if (need_dup) + dup_array_surf_tally = Kokkos::Experimental::create_scatter_view(d_array_surf_tally); + else + ndup_array_surf_tally = Kokkos::Experimental::create_scatter_view(d_array_surf_tally); +} + +/* ---------------------------------------------------------------------- */ + +void ComputeReactSurfKokkos::post_surf_tally() +{ + if (need_dup) { + Kokkos::Experimental::contribute(d_array_surf_tally, dup_array_surf_tally); + dup_array_surf_tally = {}; + } + + k_tally2surf.modify_device(); + k_array_surf_tally.modify_device(); +} + +/* ---------------------------------------------------------------------- + sync device tallies to host and compress to dense list (ntally tallies) +------------------------------------------------------------------------- */ + +int ComputeReactSurfKokkos::tallyinfo(surfint *&ptr) +{ + k_tally2surf.sync_host(); + ptr = tally2surf; + + k_array_surf_tally.sync_host(); + auto h_surf2tally = Kokkos::create_mirror_view(d_surf2tally); + Kokkos::deep_copy(h_surf2tally,d_surf2tally); + + int nsurf = surf->nlocal + surf->nghost; + int istart = 0; + int iend = nsurf-1; + + while (1) { + while (h_surf2tally[istart] != -1 && istart < nsurf-2) istart++; + while (h_surf2tally[iend] == -1 && iend > 0) iend--; + if (istart >= iend) { + ntally = istart; + break; + } + for (int k = 0; k < ntotal; k++) + array_surf_tally[istart][k] = array_surf_tally[iend][k]; + h_surf2tally[istart] = h_surf2tally[iend]; + h_surf2tally[iend] = -1; + tally2surf[istart] = tally2surf[iend]; + } + + return ntally; +} + +/* ---------------------------------------------------------------------- + sync device tally to host before the host base collates to per-surf; + consumers (dump surf, compute reduce, ...) read via post_process_surf() +------------------------------------------------------------------------- */ + +void ComputeReactSurfKokkos::post_process_surf() +{ + if (combined) return; + surfint *dummy; + tallyinfo(dummy); + ComputeReactSurf::post_process_surf(); +} + +/* ---------------------------------------------------------------------- */ + +void ComputeReactSurfKokkos::grow_tally() +{ + resize_device(surf->nlocal + surf->nghost); +} diff --git a/src/KOKKOS/compute_react_surf_kokkos.h b/src/KOKKOS/compute_react_surf_kokkos.h new file mode 100644 index 000000000..91a9a38b2 --- /dev/null +++ b/src/KOKKOS/compute_react_surf_kokkos.h @@ -0,0 +1,105 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#ifdef COMPUTE_CLASS + +ComputeStyle(react/surf/kk,ComputeReactSurfKokkos) + +#else + +#ifndef SPARTA_COMPUTE_REACT_SURF_KOKKOS_H +#define SPARTA_COMPUTE_REACT_SURF_KOKKOS_H + +#include "compute_react_surf.h" +#include "kokkos_type.h" + +namespace SPARTA_NS { + +class ComputeReactSurfKokkos : public ComputeReactSurf { + public: + ComputeReactSurfKokkos(class SPARTA *, int, char **); + ComputeReactSurfKokkos(class SPARTA *); + ~ComputeReactSurfKokkos(); + void init(); + void clear(); + int tallyinfo(surfint *&); + void post_process_surf(); + void pre_surf_tally(); + void post_surf_tally(); + +/* ---------------------------------------------------------------------- + tally a surface reaction for particle colliding with surf element isurf + mirrors ComputeReactSurf::surf_tally(); per-surf tally compressed to host +------------------------------------------------------------------------- */ + + template + KOKKOS_INLINE_FUNCTION + void surf_tally_kk(double /*dtremain*/, int isurf, int /*icell*/, int reaction, + Particle::OnePart * /*iorig*/, + Particle::OnePart * /*ip*/, Particle::OnePart * /*jp*/) const + { + if (reaction == 0) return; + reaction--; + + surfint surfID; + if (dim == 2) { + if (!(d_lines[isurf].mask & groupbit)) return; + if (d_lines[isurf].isr != isr) return; + surfID = d_lines[isurf].id; + } else { + if (!(d_tris[isurf].mask & groupbit)) return; + if (d_tris[isurf].isr != isr) return; + surfID = d_tris[isurf].id; + } + + int itally = isurf; + d_tally2surf(itally) = surfID; + d_surf2tally(isurf) = isurf; + + auto v_array_surf_tally = ScatterViewHelper::value,decltype(dup_array_surf_tally),decltype(ndup_array_surf_tally)>::get(dup_array_surf_tally,ndup_array_surf_tally); + auto a_array_surf_tally = v_array_surf_tally.template access::value>(); + + if (rpflag) { + for (int i = 0; i < ntotal; i++) + if (d_reaction2col(reaction,i)) a_array_surf_tally(itally,i) += 1.0; + } else a_array_surf_tally(itally,reaction) += 1.0; + } + + private: + DAT::t_int_2d d_reaction2col; + + DAT::tdual_float_2d_lr k_array_surf_tally; + DAT::t_float_2d_lr d_array_surf_tally; + + int need_dup; + Kokkos::Experimental::ScatterView dup_array_surf_tally; + Kokkos::Experimental::ScatterView ndup_array_surf_tally; + + DAT::t_surfint_1d d_tally2surf; + DAT::tdual_surfint_1d k_tally2surf; + DAT::t_int_1d d_surf2tally; + + t_line_1d d_lines; + t_tri_1d d_tris; + + int nsurf_tally_alloc; + + void grow_tally(); + void resize_device(int); +}; + +} + +#endif +#endif diff --git a/src/KOKKOS/update_kokkos.cpp b/src/KOKKOS/update_kokkos.cpp index 3e45ea116..5c79a5902 100644 --- a/src/KOKKOS/update_kokkos.cpp +++ b/src/KOKKOS/update_kokkos.cpp @@ -88,12 +88,14 @@ UpdateKokkos::UpdateKokkos(SPARTA *sparta) : Update(sparta), slist_active_copy{VAL_2(KKCopy(sparta))}, slist_active_isurf_copy{VAL_2(KKCopy(sparta))}, slist_active_react_isurf_copy{VAL_2(KKCopy(sparta))}, + slist_active_react_surf_copy{VAL_2(KKCopy(sparta))}, tmp_compute_boundary_kk(sparta), tmp_compute_surf_kk(sparta), tmp_compute_isurf_grid_kk(sparta), - tmp_compute_react_isurf_grid_kk(sparta) + tmp_compute_react_isurf_grid_kk(sparta), + tmp_compute_react_surf_kk(sparta) { - nslist_surf = nslist_isurf = nslist_react_isurf = 0; + nslist_surf = nslist_isurf = nslist_react_isurf = nslist_react_surf = 0; // use 1D view for scalars to reduce GPU memory operations @@ -150,6 +152,7 @@ UpdateKokkos::~UpdateKokkos() tmp_compute_surf_kk.uncopy = 1; tmp_compute_isurf_grid_kk.uncopy = 1; tmp_compute_react_isurf_grid_kk.uncopy = 1; + tmp_compute_react_surf_kk.uncopy = 1; for (int i=0; i void UpdateKokkos::move() ComputeReactISurfGridKokkos* compute_react_isurf_kk = (ComputeReactISurfGridKokkos*)(slist_active[m]); compute_react_isurf_kk->post_surf_tally(); + } else if (strcmp(slist_active[m]->style,"react/surf") == 0) { + ComputeReactSurfKokkos* compute_react_surf_kk = + (ComputeReactSurfKokkos*)(slist_active[m]); + compute_react_surf_kk->post_surf_tally(); } else { ComputeSurfKokkos* compute_surf_kk = (ComputeSurfKokkos*)(slist_active[m]); compute_surf_kk->post_surf_tally(); @@ -1518,6 +1526,9 @@ void UpdateKokkos::operator()(TagUpdateMove for (m = 0; m < nslist_react_isurf; m++) slist_active_react_isurf_copy[m].obj. surf_tally_kk(dtremain,minsurf,icell,reaction,&iorig,ipart,jpart); + for (m = 0; m < nslist_react_surf; m++) + slist_active_react_surf_copy[m].obj. + surf_tally_kk(dtremain,minsurf,icell,reaction,&iorig,ipart,jpart); } // stuck_iterate = consecutive iterations particle is immobile @@ -2086,7 +2097,7 @@ void UpdateKokkos::tally_set(bigint ntimestep) // "compute isurf/grid" (slist_active_isurf_copy); both tally on-device via // surf_tally_kk(), invoked from the move kernel's surface collision loop - nslist_surf = nslist_isurf = nslist_react_isurf = 0; + nslist_surf = nslist_isurf = nslist_react_isurf = nslist_react_surf = 0; if (nsurf_tally) { for (i = 0; i < nsurf_tally; i++) { @@ -2110,6 +2121,16 @@ void UpdateKokkos::tally_set(bigint ntimestep) compute_react_isurf_kk->pre_surf_tally(); slist_active_react_isurf_copy[nslist_react_isurf].copy(compute_react_isurf_kk); nslist_react_isurf++; + } else if (strcmp(slist_active[i]->style,"react/surf") == 0) { + ComputeReactSurfKokkos* compute_react_surf_kk = + dynamic_cast(slist_active[i]); + if (!compute_react_surf_kk) + error->all(FLERR,"Must use Kokkos-enabled compute react/surf with Kokkos"); + if (nslist_react_surf >= KOKKOS_MAX_SLIST) + error->all(FLERR,"Kokkos currently only supports two instances of compute react/surf"); + compute_react_surf_kk->pre_surf_tally(); + slist_active_react_surf_copy[nslist_react_surf].copy(compute_react_surf_kk); + nslist_react_surf++; } else { ComputeSurfKokkos* compute_surf_kk = dynamic_cast(slist_active[i]); @@ -2133,6 +2154,8 @@ void UpdateKokkos::tally_set(bigint ntimestep) slist_active_isurf_copy[i].copy(&tmp_compute_isurf_grid_kk); for (i = nslist_react_isurf; i < KOKKOS_MAX_SLIST; i++) slist_active_react_isurf_copy[i].copy(&tmp_compute_react_isurf_grid_kk); + for (i = nslist_react_surf; i < KOKKOS_MAX_SLIST; i++) + slist_active_react_surf_copy[i].copy(&tmp_compute_react_surf_kk); if (ngas_tally) error->all(FLERR,"Kokkos does not (yet) support tallying gas/gas collisions or reactions"); diff --git a/src/KOKKOS/update_kokkos.h b/src/KOKKOS/update_kokkos.h index c94afefc9..ad054a071 100644 --- a/src/KOKKOS/update_kokkos.h +++ b/src/KOKKOS/update_kokkos.h @@ -34,6 +34,7 @@ #include "compute_surf_kokkos.h" #include "compute_isurf_grid_kokkos.h" #include "compute_react_isurf_grid_kokkos.h" +#include "compute_react_surf_kokkos.h" namespace SPARTA_NS { @@ -151,6 +152,7 @@ class UpdateKokkos : public Update { KKCopy slist_active_copy[KOKKOS_MAX_SLIST]; KKCopy slist_active_isurf_copy[KOKKOS_MAX_SLIST]; KKCopy slist_active_react_isurf_copy[KOKKOS_MAX_SLIST]; + KKCopy slist_active_react_surf_copy[KOKKOS_MAX_SLIST]; KKCopy blist_active_copy[KOKKOS_MAX_BLIST]; // partition of slist_active (set in tally_set): @@ -159,12 +161,13 @@ class UpdateKokkos : public Update { // nslist_react_isurf = # of compute react/isurf/grid tallies // nslist_surf + nslist_isurf + nslist_react_isurf == nsurf_tally - int nslist_surf,nslist_isurf,nslist_react_isurf; + int nslist_surf,nslist_isurf,nslist_react_isurf,nslist_react_surf; ComputeBoundaryKokkos tmp_compute_boundary_kk; ComputeSurfKokkos tmp_compute_surf_kk; ComputeISurfGridKokkos tmp_compute_isurf_grid_kk; ComputeReactISurfGridKokkos tmp_compute_react_isurf_grid_kk; + ComputeReactSurfKokkos tmp_compute_react_surf_kk; typedef Kokkos::DualView tdual_int_14; typedef tdual_int_14::t_dev t_int_14; diff --git a/src/compute_react_surf.cpp b/src/compute_react_surf.cpp index 4b49c3e45..a56ed9ef1 100644 --- a/src/compute_react_surf.cpp +++ b/src/compute_react_surf.cpp @@ -108,6 +108,8 @@ ComputeReactSurf::ComputeReactSurf(SPARTA *sparta, int narg, char **arg) : ComputeReactSurf::~ComputeReactSurf() { + if (copy || copymode) return; + memory->destroy(reaction2col); memory->destroy(array_surf_tally); memory->destroy(tally2surf); diff --git a/src/compute_react_surf.h b/src/compute_react_surf.h index c91184331..c041e738a 100644 --- a/src/compute_react_surf.h +++ b/src/compute_react_surf.h @@ -30,6 +30,7 @@ namespace SPARTA_NS { class ComputeReactSurf : public Compute { public: ComputeReactSurf(class SPARTA *, int, char **); + ComputeReactSurf(class SPARTA* sparta) : Compute(sparta) {} // needed for Kokkos ~ComputeReactSurf(); virtual void init(); void compute_per_surf(); @@ -68,7 +69,7 @@ class ComputeReactSurf : public Compute { Surf::Line *lines; Surf::Tri *tris; - void grow_tally(); + virtual void grow_tally(); }; } From a208a05b1b5b98074de2583eeb58dd2298f5b518 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 21 Jun 2026 02:33:56 +0000 Subject: [PATCH 15/30] KOKKOS: port compute property/surf to Kokkos Device per-owned-surf geometry extraction (id, vertices, centroid, area, normal), mirroring compute property/grid. A RangePolicy over the owned in-group surfs (cglobal) packs each requested field on-device via an index switch, then results sync to the host vector_surf/array_surf for consumers (dump surf, compute reduce, ...). Verified bit-for-bit CPU vs -sf kk (Serial+EXACT): 2d circle, dump surf of id/area/xc/yc/normx/normy -- identical dump files. Co-Authored-By: Claude Opus 4.8 Co-Authored-By: Stan Moore Claude-Session: https://claude.ai/code/session_01Q6nQVuxEiTUqSzDoDppCUM --- src/KOKKOS/compute_property_surf_kokkos.cpp | 143 ++++++++++++++++++++ src/KOKKOS/compute_property_surf_kokkos.h | 115 ++++++++++++++++ 2 files changed, 258 insertions(+) create mode 100644 src/KOKKOS/compute_property_surf_kokkos.cpp create mode 100644 src/KOKKOS/compute_property_surf_kokkos.h diff --git a/src/KOKKOS/compute_property_surf_kokkos.cpp b/src/KOKKOS/compute_property_surf_kokkos.cpp new file mode 100644 index 000000000..b9eb20a26 --- /dev/null +++ b/src/KOKKOS/compute_property_surf_kokkos.cpp @@ -0,0 +1,143 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#include "string.h" +#include "compute_property_surf_kokkos.h" +#include "surf_kokkos.h" +#include "domain.h" +#include "update.h" +#include "memory_kokkos.h" +#include "error.h" +#include "sparta_masks.h" +#include "kokkos.h" + +using namespace SPARTA_NS; + +/* ---------------------------------------------------------------------- */ + +ComputePropertySurfKokkos::ComputePropertySurfKokkos(SPARTA *sparta, int narg, char **arg) : + ComputePropertySurf(sparta, narg, arg) +{ + kokkos_flag = 1; + + // map field keywords to device index enum (must match base parse order) + + d_index = DAT::t_int_1d("property/surf:index",nvalues); + auto h_index = Kokkos::create_mirror_view(d_index); + for (int i = 0; i < nvalues; i++) { + char *a = arg[3+i]; + int idx = -1; + if (strcmp(a,"id") == 0) idx = ID; + else if (strcmp(a,"v1x") == 0) idx = V1X; + else if (strcmp(a,"v1y") == 0) idx = V1Y; + else if (strcmp(a,"v1z") == 0) idx = V1Z; + else if (strcmp(a,"v2x") == 0) idx = V2X; + else if (strcmp(a,"v2y") == 0) idx = V2Y; + else if (strcmp(a,"v2z") == 0) idx = V2Z; + else if (strcmp(a,"v3x") == 0) idx = V3X; + else if (strcmp(a,"v3y") == 0) idx = V3Y; + else if (strcmp(a,"v3z") == 0) idx = V3Z; + else if (strcmp(a,"xc") == 0) idx = XC; + else if (strcmp(a,"yc") == 0) idx = YC; + else if (strcmp(a,"zc") == 0) idx = ZC; + else if (strcmp(a,"area") == 0) idx = AREA; + else if (strcmp(a,"normx") == 0) idx = NORMX; + else if (strcmp(a,"normy") == 0) idx = NORMY; + else if (strcmp(a,"normz") == 0) idx = NORMZ; + h_index(i) = idx; + } + Kokkos::deep_copy(d_index,h_index); +} + +/* ---------------------------------------------------------------------- */ + +ComputePropertySurfKokkos::~ComputePropertySurfKokkos() +{ +} + +/* ---------------------------------------------------------------------- */ + +void ComputePropertySurfKokkos::init() +{ + ComputePropertySurf::init(); + + // copy cglobal (owned-in-group surf indices) to device + + d_cglobal = DAT::t_int_1d("property/surf:cglobal",MAX(nchoose,1)); + auto h_cglobal = Kokkos::create_mirror_view(d_cglobal); + for (int i = 0; i < nchoose; i++) h_cglobal(i) = cglobal[i]; + Kokkos::deep_copy(d_cglobal,h_cglobal); + + // device output storage (sized nsown to match host vector_surf/array_surf) + + int n = MAX(nsown,1); + if (nvalues == 1) { + d_vector_surf = DAT::t_float_1d("property/surf:vector_surf",n); + k_vector_surf = DAT::tdual_float_1d("property/surf:vector_surf",n); + } else { + d_array_surf = DAT::t_float_2d_lr("property/surf:array_surf",n,nvalues); + k_array_surf = DAT::tdual_float_2d_lr("property/surf:array_surf",n,nvalues); + } +} + +/* ---------------------------------------------------------------------- */ + +void ComputePropertySurfKokkos::compute_per_surf() +{ + if (sparta->kokkos->prewrap) { + ComputePropertySurf::compute_per_surf(); + } else { + compute_per_surf_kokkos(); + if (nvalues == 1) { + Kokkos::deep_copy(k_vector_surf.view_device(),d_vector_surf); + k_vector_surf.modify_device(); + k_vector_surf.sync_host(); + auto h = k_vector_surf.view_host(); + for (int i = 0; i < nsown; i++) vector_surf[i] = h(i); + } else { + Kokkos::deep_copy(k_array_surf.view_device(),d_array_surf); + k_array_surf.modify_device(); + k_array_surf.sync_host(); + auto h = k_array_surf.view_host(); + for (int i = 0; i < nsown; i++) + for (int n = 0; n < nvalues; n++) array_surf[i][n] = h(i,n); + } + } +} + +/* ---------------------------------------------------------------------- */ + +void ComputePropertySurfKokkos::compute_per_surf_kokkos() +{ + invoked_per_surf = update->ntimestep; + + dim = domain->dimension; + + SurfKokkos* surf_kk = (SurfKokkos*) surf; + surf_kk->sync(Device,ALL_MASK); + if (distributed) { + d_lines = surf_kk->k_mylines.view_device(); + d_tris = surf_kk->k_mytris.view_device(); + } else { + d_lines = surf_kk->k_lines.view_device(); + d_tris = surf_kk->k_tris.view_device(); + } + + if (nvalues == 1) Kokkos::deep_copy(d_vector_surf,0.0); + else Kokkos::deep_copy(d_array_surf,0.0); + + copymode = 1; + Kokkos::parallel_for(Kokkos::RangePolicy(0,nchoose),*this); + copymode = 0; +} diff --git a/src/KOKKOS/compute_property_surf_kokkos.h b/src/KOKKOS/compute_property_surf_kokkos.h new file mode 100644 index 000000000..38ebc9013 --- /dev/null +++ b/src/KOKKOS/compute_property_surf_kokkos.h @@ -0,0 +1,115 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#ifdef COMPUTE_CLASS + +ComputeStyle(property/surf/kk,ComputePropertySurfKokkos) + +#else + +#ifndef SPARTA_COMPUTE_PROPERTY_SURF_KOKKOS_H +#define SPARTA_COMPUTE_PROPERTY_SURF_KOKKOS_H + +#include "compute_property_surf.h" +#include "kokkos_base.h" +#include "kokkos_type.h" +#include "math_extra_kokkos.h" + +namespace SPARTA_NS { + +class ComputePropertySurfKokkos : public ComputePropertySurf, public KokkosBase { + public: + enum{ID,V1X,V1Y,V1Z,V2X,V2Y,V2Z,V3X,V3Y,V3Z,XC,YC,ZC,AREA,NORMX,NORMY,NORMZ}; + + ComputePropertySurfKokkos(class SPARTA *, int, char **); + ~ComputePropertySurfKokkos(); + void init(); + void compute_per_surf(); + void compute_per_surf_kokkos(); + + KOKKOS_INLINE_FUNCTION + double pack_one(int m, int field) const + { + const double THIRD = 1.0/3.0; + if (dim == 2) { + const auto &L = d_lines[m]; + switch (field) { + case ID: return (double) L.id; + case V1X: return L.p1[0]; + case V1Y: return L.p1[1]; + case V2X: return L.p2[0]; + case V2Y: return L.p2[1]; + case XC: return 0.5*(L.p1[0]+L.p2[0]); + case YC: return 0.5*(L.p1[1]+L.p2[1]); + case AREA: { double p12[3]; MathExtraKokkos::sub3(L.p2,L.p1,p12); + return MathExtraKokkos::len3(p12); } + case NORMX: return L.norm[0]; + case NORMY: return L.norm[1]; + } + } else { + const auto &T = d_tris[m]; + switch (field) { + case ID: return (double) T.id; + case V1X: return T.p1[0]; + case V1Y: return T.p1[1]; + case V1Z: return T.p1[2]; + case V2X: return T.p2[0]; + case V2Y: return T.p2[1]; + case V2Z: return T.p2[2]; + case V3X: return T.p3[0]; + case V3Y: return T.p3[1]; + case V3Z: return T.p3[2]; + case XC: return THIRD*(T.p1[0]+T.p2[0]+T.p3[0]); + case YC: return THIRD*(T.p1[1]+T.p2[1]+T.p3[1]); + case ZC: return THIRD*(T.p1[2]+T.p2[2]+T.p3[2]); + case AREA: { double p12[3],p13[3],cross[3]; + MathExtraKokkos::sub3(T.p2,T.p1,p12); + MathExtraKokkos::sub3(T.p3,T.p1,p13); + MathExtraKokkos::cross3(p12,p13,cross); + return 0.5*MathExtraKokkos::len3(cross); } + case NORMX: return T.norm[0]; + case NORMY: return T.norm[1]; + case NORMZ: return T.norm[2]; + } + } + return 0.0; + } + + KOKKOS_INLINE_FUNCTION + void operator()(const int &i) const + { + int m = d_cglobal[i]; + if (nvalues == 1) d_vector_surf[i] = pack_one(m,d_index[0]); + else + for (int n = 0; n < nvalues; n++) + d_array_surf(i,n) = pack_one(m,d_index[n]); + } + + DAT::tdual_float_1d k_vector_surf; + DAT::tdual_float_2d_lr k_array_surf; + + private: + int dim; + DAT::t_int_1d d_index; + DAT::t_int_1d d_cglobal; + DAT::t_float_1d d_vector_surf; + DAT::t_float_2d_lr d_array_surf; + t_line_1d d_lines; + t_tri_1d d_tris; +}; + +} + +#endif +#endif From da60837f45c105bfb4d2944ed9040d9340c9f08a Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 21 Jun 2026 04:03:28 +0000 Subject: [PATCH 16/30] KOKKOS: port react qk and react tce/qk (quantum-kinetic gas reactions) Add device gas-phase quantum-kinetic chemistry, dispatched from the collide vss/kk kernel. - ReactQKKokkos (qk/kk): device attempt_kk mirroring ReactQK::attempt -- DISSOCIATION + EXCHANGE via discrete vibrational-level thresholds and (for exchange) the rejection-sampling loop, drawing one random_prob per attempt. - ReactTCEQKKokkos (tce/qk/kk): device attempt_kk mirroring ReactTCEQK::attempt -- per reaction, ARRHENIUS style uses the simple TCE probability, QUANTUM style uses the QK model; each evaluated reaction draws its own random number before its inner energy screen, matching the host RNG order exactly. - Both flatten the VSS omega for all species pairs to a device array at init (collide->extract) and reuse the ReactBirdKokkos rand_pool / d_rlist / d_reactions / d_tally_reactions; recombination + compute_chem_rates rejected at init as on the host. - collide_vss_kokkos: dispatch the per-collision react attempt by style (TCE / QK / TCE-QK) via dynamic_cast at setup + a react_style branch in perform_collision_kokkos; previously the kernel hardcoded ReactTCEKokkos. Verified bit-for-bit CPU vs -sf kk (Serial+EXACT) on an N2/N dissociation chem box: react qk (identical np/nreact/temp trajectory) and react tce/qk with a mixed A/Q reaction file (QK path firing). Both run clean and reproducible at OMP_NUM_THREADS=4. Co-Authored-By: Claude Opus 4.8 Co-Authored-By: Stan Moore Claude-Session: https://claude.ai/code/session_01Q6nQVuxEiTUqSzDoDppCUM --- src/KOKKOS/collide_vss_kokkos.cpp | 52 +++++++-- src/KOKKOS/collide_vss_kokkos.h | 5 + src/KOKKOS/react_qk_kokkos.cpp | 59 ++++++++++ src/KOKKOS/react_qk_kokkos.h | 164 ++++++++++++++++++++++++++ src/KOKKOS/react_tce_qk_kokkos.cpp | 59 ++++++++++ src/KOKKOS/react_tce_qk_kokkos.h | 179 +++++++++++++++++++++++++++++ 6 files changed, 510 insertions(+), 8 deletions(-) create mode 100644 src/KOKKOS/react_qk_kokkos.cpp create mode 100644 src/KOKKOS/react_qk_kokkos.h create mode 100644 src/KOKKOS/react_tce_qk_kokkos.cpp create mode 100644 src/KOKKOS/react_tce_qk_kokkos.h diff --git a/src/KOKKOS/collide_vss_kokkos.cpp b/src/KOKKOS/collide_vss_kokkos.cpp index 6d4f5384e..1f97d50ae 100644 --- a/src/KOKKOS/collide_vss_kokkos.cpp +++ b/src/KOKKOS/collide_vss_kokkos.cpp @@ -59,9 +59,12 @@ CollideVSSKokkos::CollideVSSKokkos(SPARTA *sparta, int narg, char **arg) : #endif ), grid_kk_copy(sparta), - react_kk_copy(sparta) + react_kk_copy(sparta), + react_qk_kk_copy(sparta), + react_tceqk_kk_copy(sparta) { kokkos_flag = 1; + react_style = 0; // use 1D view for scalars to reduce GPU memory operations @@ -106,6 +109,8 @@ CollideVSSKokkos::~CollideVSSKokkos() grid_kk_copy.uncopy(); react_kk_copy.uncopy(); + react_qk_kk_copy.uncopy(); + react_tceqk_kk_copy.uncopy(); memoryKK->destroy_kokkos(k_dellist,dellist); @@ -552,8 +557,18 @@ template < int NEARCP, int GASTALLY > void CollideVSSKokkos::collisions_one(COLL grid_kk_copy.copy(grid_kk); if (react) { - ReactTCEKokkos* react_kk = (ReactTCEKokkos*) react; - react_kk_copy.copy(react_kk); + ReactQKKokkos* react_qk = dynamic_cast(react); + ReactTCEQKKokkos* react_tceqk = dynamic_cast(react); + if (react_tceqk) { + react_style = 2; + react_tceqk_kk_copy.copy(react_tceqk); + } else if (react_qk) { + react_style = 1; + react_qk_kk_copy.copy(react_qk); + } else { + react_style = 0; + react_kk_copy.copy((ReactTCEKokkos*) react); + } } if (sparta->kokkos->atomic_reduction) { @@ -911,8 +926,18 @@ void CollideVSSKokkos::collisions_one_ambipolar(COLLIDE_REDUCE &reduce) grid_kk_copy.copy(grid_kk); if (react) { - ReactTCEKokkos* react_kk = (ReactTCEKokkos*) react; - react_kk_copy.copy(react_kk); + ReactQKKokkos* react_qk = dynamic_cast(react); + ReactTCEQKKokkos* react_tceqk = dynamic_cast(react); + if (react_tceqk) { + react_style = 2; + react_tceqk_kk_copy.copy(react_tceqk); + } else if (react_qk) { + react_style = 1; + react_qk_kk_copy.copy(react_qk); + } else { + react_style = 0; + react_kk_copy.copy((ReactTCEKokkos*) react); + } } if (sparta->kokkos->atomic_reduction) { @@ -1463,12 +1488,23 @@ int CollideVSSKokkos::perform_collision_kokkos(Particle::OnePart *&ip, // reaction = 1 to N for which reaction occurs // reaction is returned to caller - if (react_defined) - reaction = react_kk_copy.obj.attempt_kk(ip,jp, + if (react_defined) { + if (react_style == 1) + reaction = react_qk_kk_copy.obj.attempt_kk(ip,jp, + precoln.etrans,precoln.erot, + precoln.evib,postcoln.etotal,kspecies, + recomb_species,recomb_density,d_species); + else if (react_style == 2) + reaction = react_tceqk_kk_copy.obj.attempt_kk(ip,jp, + precoln.etrans,precoln.erot, + precoln.evib,postcoln.etotal,kspecies, + recomb_species,recomb_density,d_species); + else + reaction = react_kk_copy.obj.attempt_kk(ip,jp, precoln.etrans,precoln.erot, precoln.evib,postcoln.etotal,kspecies, recomb_species,recomb_density,d_species); - else reaction = 0; + } else reaction = 0; // just collision, no reaction diff --git a/src/KOKKOS/collide_vss_kokkos.h b/src/KOKKOS/collide_vss_kokkos.h index 00fd1148a..edab98b42 100644 --- a/src/KOKKOS/collide_vss_kokkos.h +++ b/src/KOKKOS/collide_vss_kokkos.h @@ -26,6 +26,8 @@ CollideStyle(vss/kk,CollideVSSKokkos) #include "particle_kokkos.h" #include "grid_kokkos.h" #include "react_tce_kokkos.h" +#include "react_qk_kokkos.h" +#include "react_tce_qk_kokkos.h" #include "kokkos_type.h" #include "Kokkos_Random.hpp" #include "rand_pool_wrap.h" @@ -138,6 +140,9 @@ class CollideVSSKokkos : public CollideVSS { KKCopy grid_kk_copy; KKCopy react_kk_copy; + KKCopy react_qk_kk_copy; + KKCopy react_tceqk_kk_copy; + int react_style; // 0=TCE, 1=QK, 2=TCEQK (set in setup) t_particle_1d d_particles; t_species_1d_const d_species; diff --git a/src/KOKKOS/react_qk_kokkos.cpp b/src/KOKKOS/react_qk_kokkos.cpp new file mode 100644 index 000000000..1e3b276eb --- /dev/null +++ b/src/KOKKOS/react_qk_kokkos.cpp @@ -0,0 +1,59 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#include "string.h" +#include "react_qk_kokkos.h" +#include "particle.h" +#include "collide.h" +#include "update.h" +#include "error.h" + +using namespace SPARTA_NS; + +/* ---------------------------------------------------------------------- */ + +ReactQKKokkos::ReactQKKokkos(SPARTA *sparta, int narg, char **arg) : + ReactBirdKokkos(sparta, narg, arg) {} + +/* ---------------------------------------------------------------------- */ + +void ReactQKKokkos::init() +{ + if (!collide || (strcmp(collide->style,"vss") != 0 && + strcmp(collide->style,"vss/kk") != 0)) + error->all(FLERR,"React qk can only be used with collide vss"); + + ReactBirdKokkos::init(); + + // do not allow recombination reactions (not supported by QK) + + for (int i = 0; i < nlist; i++) + if (rlist[i].active && rlist[i].type == RECOMBINATION) + error->all(FLERR,"React qk does not currently support recombination reactions"); + if (computeChemRates) + error->all(FLERR,"React qk does not currently support the " + "'react_modify compute_chem_rates' option"); + + boltz = update->boltz; + + // flatten VSS omega for all species pairs to device + + int nspecies = particle->nspecies; + d_omega = DAT::t_float_2d("react/qk:omega",nspecies,nspecies); + auto h_omega = Kokkos::create_mirror_view(d_omega); + for (int i = 0; i < nspecies; i++) + for (int j = 0; j < nspecies; j++) + h_omega(i,j) = collide->extract(i,j,"omega"); + Kokkos::deep_copy(d_omega,h_omega); +} diff --git a/src/KOKKOS/react_qk_kokkos.h b/src/KOKKOS/react_qk_kokkos.h new file mode 100644 index 000000000..e2e592d72 --- /dev/null +++ b/src/KOKKOS/react_qk_kokkos.h @@ -0,0 +1,164 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#ifdef REACT_CLASS + +ReactStyle(qk/kk,ReactQKKokkos) + +#else + +#ifndef SPARTA_REACT_QK_KOKKOS_H +#define SPARTA_REACT_QK_KOKKOS_H + +#include "math.h" +#include "react_bird_kokkos.h" +#include "kokkos_type.h" + +namespace SPARTA_NS { + +class ReactQKKokkos : public ReactBirdKokkos { + public: + ReactQKKokkos(class SPARTA *, int, char **); + ReactQKKokkos(class SPARTA* sparta) : ReactBirdKokkos(sparta) {copy = 1;} + void init(); + int attempt(Particle::OnePart *, Particle::OnePart *, + double, double, double, double &, int &) {return 0;} + + enum{DISSOCIATION,EXCHANGE,IONIZATION,RECOMBINATION}; // other files + +/* ---------------------------------------------------------------------- + quantum-kinetic (QK) reaction attempt, device version + mirrors ReactQK::attempt(); supports DISSOCIATION and EXCHANGE + recomb args are accepted for a uniform collide dispatch but unused (QK has + no recombination) +------------------------------------------------------------------------- */ + +KOKKOS_INLINE_FUNCTION +int attempt_kk(Particle::OnePart *ip, Particle::OnePart *jp, + double pre_etrans, double pre_erot, double pre_evib, + double &post_etotal, int &kspecies, + int & /*recomb_species*/, double & /*recomb_density*/, + const t_species_1d_const &d_species) const +{ + const int isp = ip->ispecies; + const int jsp = jp->ispecies; + + const double pre_ave_rotdof = (d_species[isp].rotdof + d_species[jsp].rotdof)/2.0; + const double omega = d_omega(isp,jsp); + + const int n = d_reactions(isp,jsp).n; + if (n == 0) return 0; + auto& d_list = d_reactions(isp,jsp).d_list; + + double react_prob = 0.0; + rand_type rand_gen = rand_pool.get_state(); + const double random_prob = rand_gen.drand(); + + for (int i = 0; i < n; i++) { + OneReactionKokkos *r = &d_rlist[d_list[i]]; + + const double pre_etotal = pre_etrans + pre_erot + pre_evib; + + double ecc = pre_etrans; + if (pre_ave_rotdof > 0.1) ecc += pre_erot*r->d_coeff[0]/pre_ave_rotdof; + + double e_excess = ecc - r->d_coeff[1]; + if (e_excess <= 0.0) continue; + + const double inverse_kT = 1.0 / (boltz * d_species[isp].vibtemp[0]); + + int iv = 0,ilevel,maxlev,limlev; + + switch (r->type) { + case DISSOCIATION: + { + ecc = pre_etrans + ip->evib; + maxlev = static_cast (ecc * inverse_kT); + limlev = static_cast (fabs(r->d_coeff[1]) * inverse_kT); + if (maxlev > limlev) react_prob = 1.0; + break; + } + case EXCHANGE: + { + if (r->d_coeff[4] < 0.0 && d_species[isp].rotdof > 0) { + + // endothermic reaction + + ecc = pre_etrans + ip->evib; + maxlev = static_cast (ecc * inverse_kT); + if (ecc > r->d_coeff[1]) { + do { + iv = static_cast (rand_gen.drand()*(maxlev+0.99999999)); + double evib = static_cast (iv / inverse_kT); + if (evib < ecc) react_prob = pow(1.0-evib/ecc,1.5-omega); + } while (rand_gen.drand() < react_prob); + + ilevel = static_cast (fabs(r->d_coeff[4]) * inverse_kT); + if (iv >= ilevel) react_prob = 1.0; + } + + } else if (r->d_coeff[4] > 0.0 && d_species[isp].rotdof > 0) { + + ecc = pre_etrans + ip->evib; + + // mspec = post-collision molecular species + + int mspec = r->d_products[0]; + if (d_species[mspec].rotdof < 2.0) mspec = r->d_products[1]; + + ecc += r->d_coeff[4]; + maxlev = static_cast (ecc * inverse_kT); + double prob = 0.0; + do { + iv = rand_gen.drand()*(maxlev+0.99999999); + double evib = static_cast (iv * boltz*d_species[mspec].vibtemp[0]); + if (evib < ecc) prob = pow(1.0-evib/ecc,1.5 - r->d_coeff[6]); + } while (rand_gen.drand() < prob); + + ilevel = static_cast (fabs(r->d_coeff[4]/boltz/d_species[mspec].vibtemp[0])); + if (iv >= ilevel) react_prob = 1.0; + } + + break; + } + default: + Kokkos::abort("ReactQKKokkos: Unknown outcome in reaction\n"); + break; + } + + if (react_prob > random_prob) { + Kokkos::atomic_inc(&d_tally_reactions[d_list[i]]); + ip->ispecies = r->d_products[0]; + jp->ispecies = r->d_products[1]; + post_etotal = pre_etotal + r->d_coeff[4]; + if (r->nproduct > 2) kspecies = r->d_products[2]; + else kspecies = -1; + rand_pool.free_state(rand_gen); + return d_list[i] + 1; + } + } + + rand_pool.free_state(rand_gen); + return 0; +} + + protected: + double boltz; + DAT::t_float_2d d_omega; // VSS omega for each species pair +}; + +} + +#endif +#endif diff --git a/src/KOKKOS/react_tce_qk_kokkos.cpp b/src/KOKKOS/react_tce_qk_kokkos.cpp new file mode 100644 index 000000000..0cc06574b --- /dev/null +++ b/src/KOKKOS/react_tce_qk_kokkos.cpp @@ -0,0 +1,59 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#include "string.h" +#include "react_tce_qk_kokkos.h" +#include "particle.h" +#include "collide.h" +#include "update.h" +#include "error.h" + +using namespace SPARTA_NS; + +/* ---------------------------------------------------------------------- */ + +ReactTCEQKKokkos::ReactTCEQKKokkos(SPARTA *sparta, int narg, char **arg) : + ReactBirdKokkos(sparta, narg, arg) {} + +/* ---------------------------------------------------------------------- */ + +void ReactTCEQKKokkos::init() +{ + if (!collide || (strcmp(collide->style,"vss") != 0 && + strcmp(collide->style,"vss/kk") != 0)) + error->all(FLERR,"React tce/qk can only be used with collide vss"); + + ReactBirdKokkos::init(); + + // do not allow recombination reactions (not supported) + + for (int i = 0; i < nlist; i++) + if (rlist[i].active && rlist[i].type == RECOMBINATION) + error->all(FLERR,"React tce/qk does not currently support recombination reactions"); + if (computeChemRates) + error->all(FLERR,"React tce/qk does not currently support the " + "'react_modify compute_chem_rates' option"); + + boltz = update->boltz; + + // flatten VSS omega for all species pairs to device + + int nspecies = particle->nspecies; + d_omega = DAT::t_float_2d("react/tce/qk:omega",nspecies,nspecies); + auto h_omega = Kokkos::create_mirror_view(d_omega); + for (int i = 0; i < nspecies; i++) + for (int j = 0; j < nspecies; j++) + h_omega(i,j) = collide->extract(i,j,"omega"); + Kokkos::deep_copy(d_omega,h_omega); +} diff --git a/src/KOKKOS/react_tce_qk_kokkos.h b/src/KOKKOS/react_tce_qk_kokkos.h new file mode 100644 index 000000000..01869f584 --- /dev/null +++ b/src/KOKKOS/react_tce_qk_kokkos.h @@ -0,0 +1,179 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#ifdef REACT_CLASS + +ReactStyle(tce/qk/kk,ReactTCEQKKokkos) + +#else + +#ifndef SPARTA_REACT_TCE_QK_KOKKOS_H +#define SPARTA_REACT_TCE_QK_KOKKOS_H + +#include "math.h" +#include "react_bird_kokkos.h" +#include "kokkos_type.h" + +namespace SPARTA_NS { + +class ReactTCEQKKokkos : public ReactBirdKokkos { + public: + ReactTCEQKKokkos(class SPARTA *, int, char **); + ReactTCEQKKokkos(class SPARTA* sparta) : ReactBirdKokkos(sparta) {copy = 1;} + void init(); + int attempt(Particle::OnePart *, Particle::OnePart *, + double, double, double, double &, int &) {return 0;} + + enum{DISSOCIATION,EXCHANGE,IONIZATION,RECOMBINATION}; // reaction types + enum{ARRHENIUS,QUANTUM}; // reaction styles + +/* ---------------------------------------------------------------------- + hybrid TCE/QK reaction attempt, device version + mirrors ReactTCEQK::attempt(): per reaction, ARRHENIUS style uses the simple + TCE probability, QUANTUM style uses the QK model; each evaluated reaction + draws its own random number (matching the host RNG order) +------------------------------------------------------------------------- */ + +KOKKOS_INLINE_FUNCTION +int attempt_kk(Particle::OnePart *ip, Particle::OnePart *jp, + double pre_etrans, double pre_erot, double pre_evib, + double &post_etotal, int &kspecies, + int & /*recomb_species*/, double & /*recomb_density*/, + const t_species_1d_const &d_species) const +{ + const int isp = ip->ispecies; + const int jsp = jp->ispecies; + + const int n = d_reactions(isp,jsp).n; + if (n == 0) return 0; + auto& d_list = d_reactions(isp,jsp).d_list; + + const double pre_ave_rotdof = (d_species[isp].rotdof + d_species[jsp].rotdof)/2.0; + const double omega = d_omega(isp,jsp); + + rand_type rand_gen = rand_pool.get_state(); + + for (int i = 0; i < n; i++) { + OneReactionKokkos *r = &d_rlist[d_list[i]]; + + const double pre_etotal = pre_etrans + pre_erot + pre_evib; + + // top-level energetic-possibility screen (uses total energy) + + double ecc = pre_etotal; + if (ecc - r->d_coeff[1] <= 0.0) continue; + + int fired = 0; + + // per-reaction probability + its own random draw (helper semantics) + + const double random_prob = rand_gen.drand(); + + double react_prob = 0.0; + double ecc2 = pre_etrans; + if (pre_ave_rotdof > 0.1) ecc2 += pre_erot*r->d_coeff[0]/pre_ave_rotdof; + const double e_excess = ecc2 - r->d_coeff[1]; + + if (e_excess > 0.0) { + if (r->style == ARRHENIUS) { // attempt_tce + switch (r->type) { + case DISSOCIATION: + case EXCHANGE: + react_prob += r->d_coeff[2] * + pow(ecc2-r->d_coeff[1],r->d_coeff[3]) * + pow(1.0-r->d_coeff[1]/ecc2,r->d_coeff[5]); + break; + default: + Kokkos::abort("ReactTCEQKKokkos: Unknown outcome in reaction\n"); + break; + } + if (react_prob > random_prob) fired = 1; + + } else { // attempt_qk + const double inverse_kT = 1.0 / (boltz * d_species[isp].vibtemp[0]); + int iv = 0,ilevel,maxlev,limlev; + double eccq; + switch (r->type) { + case DISSOCIATION: + { + eccq = pre_etrans + ip->evib; + maxlev = static_cast (eccq * inverse_kT); + limlev = static_cast (fabs(r->d_coeff[1]) * inverse_kT); + if (maxlev > limlev) react_prob = 1.0; + break; + } + case EXCHANGE: + { + if (r->d_coeff[4] < 0.0 && d_species[isp].rotdof > 0) { + eccq = pre_etrans + ip->evib; + maxlev = static_cast (eccq * inverse_kT); + if (eccq > r->d_coeff[1]) { + do { + iv = static_cast (rand_gen.drand()*(maxlev+0.99999999)); + double evib = static_cast (iv / inverse_kT); + if (evib < eccq) react_prob = pow(1.0-evib/eccq,1.5-omega); + } while (rand_gen.drand() < react_prob); + ilevel = static_cast (fabs(r->d_coeff[4]) * inverse_kT); + if (iv >= ilevel) react_prob = 1.0; + } + } else if (r->d_coeff[4] > 0.0 && d_species[isp].rotdof > 0) { + eccq = pre_etrans + ip->evib; + int mspec = r->d_products[0]; + if (d_species[mspec].rotdof < 2.0) mspec = r->d_products[1]; + eccq += r->d_coeff[4]; + maxlev = static_cast (eccq * inverse_kT); + double prob = 0.0; + do { + iv = rand_gen.drand()*(maxlev+0.99999999); + double evib = static_cast (iv * boltz*d_species[mspec].vibtemp[0]); + if (evib < eccq) prob = pow(1.0-evib/eccq,1.5 - r->d_coeff[6]); + } while (rand_gen.drand() < prob); + ilevel = static_cast (fabs(r->d_coeff[4]/boltz/d_species[mspec].vibtemp[0])); + if (iv >= ilevel) react_prob = 1.0; + } + break; + } + default: + Kokkos::abort("ReactTCEQKKokkos: Unknown outcome in reaction\n"); + break; + } + if (react_prob > random_prob) fired = 1; + } + } + + if (fired) { + Kokkos::atomic_inc(&d_tally_reactions[d_list[i]]); + ip->ispecies = r->d_products[0]; + jp->ispecies = r->d_products[1]; + post_etotal = pre_etotal + r->d_coeff[4]; + if (r->nproduct > 2) kspecies = r->d_products[2]; + else kspecies = -1; + rand_pool.free_state(rand_gen); + return d_list[i] + 1; + } + } + + rand_pool.free_state(rand_gen); + return 0; +} + + protected: + double boltz; + DAT::t_float_2d d_omega; // VSS omega for each species pair +}; + +} + +#endif +#endif From 0a179914e4543b8be460efcf573f92bf0241eeeb Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 22 Jun 2026 13:31:19 +0000 Subject: [PATCH 17/30] doc: document new KOKKOS styles (/kk suffixes + command list) Add Kokkos accelerator documentation for the styles ported in this branch. In Section_commands.txt, append the "(k)" marker to: fix temp/global/rescale compute isurf/grid, property/surf, react/surf, react/isurf/grid surf_collide adiabatic, cll, impulsive, td surf_react adsorb In each style's doc page, add the "/kk" style name (syntax heading and/or style/args list) and the standard KOKKOS accelerator boilerplate paragraph: surf_collide.txt - cll/kk, adiabatic/kk, impulsive/kk, td/kk (also transparent/kk in the style line) surf_react_adsorb.txt - adsorb/kk + boilerplate react.txt - qk/kk, tce/qk/kk compute_isurf_grid.txt, compute_property_surf.txt, compute_react_surf.txt, compute_react_isurf_grid.txt, fix_temp_global_rescale.txt - /kk heading + boilerplate Co-Authored-By: Claude Opus 4.8 Co-Authored-By: Stan Moore Claude-Session: https://claude.ai/code/session_01Q6nQVuxEiTUqSzDoDppCUM --- doc/Section_commands.txt | 20 ++++++++++---------- doc/compute_isurf_grid.txt | 24 ++++++++++++++++++++++++ doc/compute_property_surf.txt | 24 ++++++++++++++++++++++++ doc/compute_react_isurf_grid.txt | 24 ++++++++++++++++++++++++ doc/compute_react_surf.txt | 24 ++++++++++++++++++++++++ doc/fix_temp_global_rescale.txt | 24 ++++++++++++++++++++++++ doc/react.txt | 4 ++++ doc/surf_collide.txt | 10 +++++----- doc/surf_react_adsorb.txt | 26 +++++++++++++++++++++++++- 9 files changed, 164 insertions(+), 16 deletions(-) diff --git a/doc/Section_commands.txt b/doc/Section_commands.txt index acf777483..34f395527 100644 --- a/doc/Section_commands.txt +++ b/doc/Section_commands.txt @@ -415,7 +415,7 @@ This is indicated by additional letters in parenthesis: k = KOKKOS. "move/surf (k)"_fix_move_surf.html, "print"_fix_print.html, "surf/temp"_fix_surf_temp.html, -"temp/global/rescale"_fix_temp_global_rescale.html, +"temp/global/rescale (k)"_fix_temp_global_rescale.html, "temp/rescale (k)"_fix_temp_rescale.html, "vibmode (k)"_fix_vibmode.html :tb(c=6,ea=c) @@ -441,15 +441,15 @@ letters in parenthesis: k = KOKKOS. "gas/reaction/grid"_compute_gas_reaction_grid.html, "gas/reaction/tally"_compute_gas_reaction_tally.html, "grid (k)"_compute_grid.html, -"isurf/grid"_compute_isurf_grid.html, +"isurf/grid (k)"_compute_isurf_grid.html, "ke/particle (k)"_compute_ke_particle.html, "lambda/grid (k)"_compute_lambda_grid.html, "pflux/grid (k)"_compute_pflux_grid.html, "property/grid (k)"_compute_property_grid.html, -"property/surf"_compute_property_surf.html, +"property/surf (k)"_compute_property_surf.html, "react/boundary"_compute_react_boundary.html, -"react/surf"_compute_react_surf.html, -"react/isurf/grid"_compute_react_isurf_grid.html, +"react/surf (k)"_compute_react_surf.html, +"react/isurf/grid (k)"_compute_react_isurf_grid.html, "reduce"_compute_reduce.html, "sonine/grid (k)"_compute_sonine_grid.html, "surf (k)"_compute_surf.html, @@ -481,13 +481,13 @@ used if SPARTA is built with the "appropriate accelerated package"_Section_accelerate.html. This is indicated by additional letters in parenthesis: k = KOKKOS. -"adiabatic"_surf_collide.html, -"cll"_surf_collide.html, +"adiabatic (k)"_surf_collide.html, +"cll (k)"_surf_collide.html, "diffuse (k)"_surf_collide.html, -"impulsive"_surf_collide.html, +"impulsive (k)"_surf_collide.html, "piston (k)"_surf_collide.html, "specular (k)"_surf_collide.html, -"td"_surf_collide.html, +"td (k)"_surf_collide.html, "transparent (k)"_surf_collide.html, "vanish (k)"_surf_collide.html :tb(c=3,ea=c) @@ -501,6 +501,6 @@ used if SPARTA is built with the "appropriate accelerated package"_Section_accelerate.html. This is indicated by additional letters in parenthesis: k = KOKKOS. -"adsorb"_surf_react_adsorb.html, +"adsorb (k)"_surf_react_adsorb.html, "global (k)"_surf_react.html, "prob (k)"_surf_react.html :tb(c=2,ea=c) diff --git a/doc/compute_isurf_grid.txt b/doc/compute_isurf_grid.txt index 40a054de5..c1ce3e35e 100644 --- a/doc/compute_isurf_grid.txt +++ b/doc/compute_isurf_grid.txt @@ -7,6 +7,7 @@ :line compute isurf/grid command :h3 +compute isurf/grid/kk command :h3 [Syntax:] @@ -149,6 +150,29 @@ for 2d simulations. :line +Styles with a {kk} suffix are functionally the same as the +corresponding style without the suffix. They have been optimized to +run faster, depending on your available hardware, as discussed in the +"Accelerating SPARTA"_Section_accelerate.html section of the manual. +The accelerated styles take the same arguments and should produce the +same results, except for different random number, round-off and +precision issues. + +These accelerated styles are part of the KOKKOS package. They are only +enabled if SPARTA was built with that package. See the "Making +SPARTA"_Section_start.html#start_3 section for more info. + +You can specify the accelerated styles explicitly in your input script +by including their suffix, or you can use the "-suffix command-line +switch"_Section_start.html#start_7 when you invoke SPARTA, or you can +use the "suffix"_suffix.html command in your input script. + +See the "Accelerating SPARTA"_Section_accelerate.html section of the +manual for more instructions on how to use the accelerated styles +effectively. + +:line + [Restrictions:] none [Related commands:] diff --git a/doc/compute_property_surf.txt b/doc/compute_property_surf.txt index 884cab472..1fd2d533f 100644 --- a/doc/compute_property_surf.txt +++ b/doc/compute_property_surf.txt @@ -7,6 +7,7 @@ :line compute property/surf command :h3 +compute property/surf/kk command :h3 [Syntax:] @@ -101,6 +102,29 @@ length units for {area} in 2d, area units for {area} in 3d. :line +Styles with a {kk} suffix are functionally the same as the +corresponding style without the suffix. They have been optimized to +run faster, depending on your available hardware, as discussed in the +"Accelerating SPARTA"_Section_accelerate.html section of the manual. +The accelerated styles take the same arguments and should produce the +same results, except for different random number, round-off and +precision issues. + +These accelerated styles are part of the KOKKOS package. They are only +enabled if SPARTA was built with that package. See the "Making +SPARTA"_Section_start.html#start_3 section for more info. + +You can specify the accelerated styles explicitly in your input script +by including their suffix, or you can use the "-suffix command-line +switch"_Section_start.html#start_7 when you invoke SPARTA, or you can +use the "suffix"_suffix.html command in your input script. + +See the "Accelerating SPARTA"_Section_accelerate.html section of the +manual for more instructions on how to use the accelerated styles +effectively. + +:line + [Restrictions:] For 2d simulations, none of the attributes which refer to the 3rd diff --git a/doc/compute_react_isurf_grid.txt b/doc/compute_react_isurf_grid.txt index 0922b3d04..99bf63952 100644 --- a/doc/compute_react_isurf_grid.txt +++ b/doc/compute_react_isurf_grid.txt @@ -7,6 +7,7 @@ :line compute react/isurf/grid command :h3 +compute react/isurf/grid/kk command :h3 [Syntax:] @@ -106,6 +107,29 @@ occurred on surface elements in that grid cell. :line +Styles with a {kk} suffix are functionally the same as the +corresponding style without the suffix. They have been optimized to +run faster, depending on your available hardware, as discussed in the +"Accelerating SPARTA"_Section_accelerate.html section of the manual. +The accelerated styles take the same arguments and should produce the +same results, except for different random number, round-off and +precision issues. + +These accelerated styles are part of the KOKKOS package. They are only +enabled if SPARTA was built with that package. See the "Making +SPARTA"_Section_start.html#start_3 section for more info. + +You can specify the accelerated styles explicitly in your input script +by including their suffix, or you can use the "-suffix command-line +switch"_Section_start.html#start_7 when you invoke SPARTA, or you can +use the "suffix"_suffix.html command in your input script. + +See the "Accelerating SPARTA"_Section_accelerate.html section of the +manual for more instructions on how to use the accelerated styles +effectively. + +:line + [Restrictions:] none [Related commands:] diff --git a/doc/compute_react_surf.txt b/doc/compute_react_surf.txt index 2d3ac3fa3..a888ec231 100644 --- a/doc/compute_react_surf.txt +++ b/doc/compute_react_surf.txt @@ -7,6 +7,7 @@ :line compute react/surf command :h3 +compute react/surf/kk command :h3 [Syntax:] @@ -104,6 +105,29 @@ occurred. :line +Styles with a {kk} suffix are functionally the same as the +corresponding style without the suffix. They have been optimized to +run faster, depending on your available hardware, as discussed in the +"Accelerating SPARTA"_Section_accelerate.html section of the manual. +The accelerated styles take the same arguments and should produce the +same results, except for different random number, round-off and +precision issues. + +These accelerated styles are part of the KOKKOS package. They are only +enabled if SPARTA was built with that package. See the "Making +SPARTA"_Section_start.html#start_3 section for more info. + +You can specify the accelerated styles explicitly in your input script +by including their suffix, or you can use the "-suffix command-line +switch"_Section_start.html#start_7 when you invoke SPARTA, or you can +use the "suffix"_suffix.html command in your input script. + +See the "Accelerating SPARTA"_Section_accelerate.html section of the +manual for more instructions on how to use the accelerated styles +effectively. + +:line + [Restrictions:] none [Related commands:] diff --git a/doc/fix_temp_global_rescale.txt b/doc/fix_temp_global_rescale.txt index b7d7d6bb3..ccb9a0bd1 100644 --- a/doc/fix_temp_global_rescale.txt +++ b/doc/fix_temp_global_rescale.txt @@ -7,6 +7,7 @@ :line fix temp/global/rescale command :h3 +fix temp/global/rescale/kk command :h3 [Syntax:] @@ -76,6 +77,29 @@ details of how to do this. :line +Styles with a {kk} suffix are functionally the same as the +corresponding style without the suffix. They have been optimized to +run faster, depending on your available hardware, as discussed in the +"Accelerating SPARTA"_Section_accelerate.html section of the manual. +The accelerated styles take the same arguments and should produce the +same results, except for different random number, round-off and +precision issues. + +These accelerated styles are part of the KOKKOS package. They are only +enabled if SPARTA was built with that package. See the "Making +SPARTA"_Section_start.html#start_3 section for more info. + +You can specify the accelerated styles explicitly in your input script +by including their suffix, or you can use the "-suffix command-line +switch"_Section_start.html#start_7 when you invoke SPARTA, or you can +use the "suffix"_suffix.html command in your input script. + +See the "Accelerating SPARTA"_Section_accelerate.html section of the +manual for more instructions on how to use the accelerated styles +effectively. + +:line + [Restrictions:] none [Related commands:] diff --git a/doc/react.txt b/doc/react.txt index bae04aa92..58d2aebd5 100644 --- a/doc/react.txt +++ b/doc/react.txt @@ -22,6 +22,10 @@ args = arguments for that style :l {tce/qk} args = infile infile = file with list of gas-phase chemistry reactions {tce/kk} args = infile + infile = file with list of gas-phase chemistry reactions + {qk/kk} args = infile + infile = file with list of gas-phase chemistry reactions + {tce/qk/kk} args = infile infile = file with list of gas-phase chemistry reactions :pre :ule diff --git a/doc/surf_collide.txt b/doc/surf_collide.txt index 5e7f1911a..ce25eb29f 100644 --- a/doc/surf_collide.txt +++ b/doc/surf_collide.txt @@ -13,7 +13,7 @@ surf_collide command :h3 surf_collide ID style args keyword values ... :pre ID = user-assigned name for the surface collision model :ulb,l -style = {specular} or {diffuse} or {cll} or {adiabatic} or {impulsive} or {td} or {piston} or {transparent} or {vanish} or {specular/kk} or {diffuse/kk} or {piston/kk} or {vanish/kk} :l +style = {specular} or {diffuse} or {cll} or {adiabatic} or {impulsive} or {td} or {piston} or {transparent} or {vanish} or {specular/kk} or {diffuse/kk} or {cll/kk} or {adiabatic/kk} or {impulsive/kk} or {td/kk} or {piston/kk} or {transparent/kk} or {vanish/kk} :l args = arguments for specific style :l {specular} or {specular/kk} args = noslip (optional) noslip = reflect all velocity components off surface (not just normal component) @@ -21,15 +21,15 @@ args = arguments for specific style :l Tsurf = temperature of surface (temperature units) Tsurf can be a variable or custom per-surf attribute (see below) acc = accommodation coefficient - {cll} args = Tsurf acc_n acc_t acc_rot acc_vib + {cll} or {cll/kk} args = Tsurf acc_n acc_t acc_rot acc_vib Tsurf = temperature of surface (temperature units) Tsurf can be a variable or custom per-surf attribute (see below) acc_n = accommodation coefficient in the surface normal direction acc_t = accommodation coefficient in the surface tangential direction acc_rot = accommodation coefficient for the rotational modes acc_vib = accommodation coefficient for the vibrational modes - {adiabatic} args = none - {impulsive} args = Tsurf model param1 param2 var theta_peak pol_pow azi_pow + {adiabatic} or {adiabatic/kk} args = none + {impulsive} or {impulsive/kk} args = Tsurf model param1 param2 var theta_peak pol_pow azi_pow Tsurf = temperature of surface (temperature units) Tsurf can be a variable or custom per-surf attribute (see below) model can be {softsphere} or {tempvar} @@ -43,7 +43,7 @@ args = arguments for specific style :l theta_peak = peak location of the polar angle distribution pol_pow = cosine power represeting the polar angular distribution azi_pow = cosine power represeting the azimuthal angular distribution - {td} arg = Tsurf + {td} or {td/kk} arg = Tsurf Tsurf = temperature of surface (temperature units) Tsurf can be a variable or custom per-surf attribute (see below) {piston} or {piston/kk} args = Vwall diff --git a/doc/surf_react_adsorb.txt b/doc/surf_react_adsorb.txt index 3fb695b75..d4ae63502 100644 --- a/doc/surf_react_adsorb.txt +++ b/doc/surf_react_adsorb.txt @@ -7,13 +7,14 @@ :line surf_react adsorb command :h3 +surf_react adsorb/kk command :h3 [Syntax:] surf_react ID adsorb model infile(s) Nsync type temp n_sites adsp1 adsp2 ... :pre ID = user-assigned name for the surface reaction model :ulb,l -style = {adsorb} :l +style = {adsorb} or {adsorb/kk} :l model = {gs} or {ps} or {gs/ps} :l gs = gas-surface reactions ps = pure-surface reactions @@ -356,6 +357,29 @@ reaction since the beginning of the current run. :line +Styles with a {kk} suffix are functionally the same as the +corresponding style without the suffix. They have been optimized to +run faster, depending on your available hardware, as discussed in the +"Accelerating SPARTA"_Section_accelerate.html section of the manual. +The accelerated styles take the same arguments and should produce the +same results, except for different random number, round-off and +precision issues. + +These accelerated styles are part of the KOKKOS package. They are only +enabled if SPARTA was built with that package. See the "Making +SPARTA"_Section_start.html#start_3 section for more info. + +You can specify the accelerated styles explicitly in your input script +by including their suffix, or you can use the "-suffix command-line +switch"_Section_start.html#start_7 when you invoke SPARTA, or you can +use the "suffix"_suffix.html command in your input script. + +See the "Accelerating SPARTA"_Section_accelerate.html section of the +manual for more instructions on how to use the accelerated styles +effectively. + +:line + [Restrictions:] If the following conditions are met: From 415488cf14a652db6f5748d52280961bdbf3494d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 22 Jun 2026 20:17:48 +0000 Subject: [PATCH 18/30] Restore examples/ablation/binary.21x21x21 (accidental deletion) This binary read_isurf data file was unintentionally removed during the KOKKOS porting work and is still referenced by examples/ablation/in.ablation.3d.reactions. Restore it from master. Co-Authored-By: Claude Opus 4.8 Co-Authored-By: Stan Moore Claude-Session: https://claude.ai/code/session_01Q6nQVuxEiTUqSzDoDppCUM --- examples/ablation/binary.21x21x21 | Bin 0 -> 9273 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 examples/ablation/binary.21x21x21 diff --git a/examples/ablation/binary.21x21x21 b/examples/ablation/binary.21x21x21 new file mode 100644 index 0000000000000000000000000000000000000000..2c42f98a06a31ed4a69dcdef98896004e069f507 GIT binary patch literal 9273 zcmeHs^;6YdwD*?oPC>dux;rGKkw!$Nr9mYnq@^1v=}@{G0g>)*r8`AJfxF+C_Yd&- z_0HWh%$Z@%_nf`=`qWy77yv;0|Nr32|E>Q$H}DE!LiHI>3@f!mG)JV-Iq9{HKLCsk z-1;bb7uRjzc)6RC{MBlF3jkulwG+c)D?&q2TuFiF7;Jy0m;ey{z@m*+M|8W_*Zr)t z(+fB2NjdM zk8WM(aaiS-L0Nl~z_@JqPt#iT_Hs)XxfV}3RD-?JtRqwaQn@W28g4eu5~}rLa1)J* z*&#jvAfR7g3xk}AKi2qKX@N8(qR2WJfF7pMq+ZrnD8o{@1J zYY36~;tC75u_j5}0&ugx8F`q?(maRA)B7!&T2B0r2msG2ilYmbb|_GHcAi<5+fquY z#Q?zGbonwaXD~ak0p)nJk}nI@LI{9pY0?4dXr!3_ULBfrb!7qbGD`sRn?~Yp{KASZ zt9M!KYMLkIHEjSeCpF7!70vw0jL1^DW2TX%i6IHV$2SJ%^xsS?VmAk9T@gCOu0`Je z7g)WUNcpKn#)Vm1-qAyYR|NEu3IWJCyVB4^rgF*?CoeDYW8jsaKL7yhXs2K7VuT+3 zc>>C>lE8QO+gkxZ9yU7E7v$9;V0Dj-qD5icq$>v?>f@12b>K5gKJvX>_93p~qyf13 zUkxr)va~_Nt-1D!Ltl!c`jg}UFx7axQJ3NBecvO=(Eo0S@N{4VfFS|eyovjWr`xiV zsxydbA~`7e0FXQ5zhr*mUX&9&Z^KY-d}M}b0KmED__H)ZzWY|%Q3Vq6DAX@DxdE_7 zOtRH6dt}X3cerhW@a=7QA^glhM_LRv=HpmYEt@Hua5-(F-gf{r$C1cu^%`6ZRD@RQ z-CuZf-W(5reYvEO*4P;ticy(U=f&sG42f9)@Sv35=JPFU>5mQ^t^(0Su{nt3j+5Ho zZghdC8QSD@)U`T}SGw@(3}4ziKgf%sy!xx4-{T+K^;OOrfRWXu!6FosT1mW=Z+A^D zXl3hx5KE-m$wN;jlcM-4>1|h+Pk6T6{!8K5HqNe$MfU6VoBlS9?K|w*{Qdwy9)g|% zqNWUm%%V{B4MH6qm-;RMKS=wDjqiR^#@d!FL>+Fh;_yO*RL(aSek`;so?fUgA$amy zo!4{<0QsYPb`2ef4+vt&ER^n>Iwlz41E5n0f{>PYLrEk0cl+1x4P!0e0Z^aTOtM5G z@uT$fGvm4p1(Y-&sh2{9^m9ocp){H2`~d&Ki=7Lo$tful|ru&q*sh zqJtMw)t4trBHM=?s;#V>%+4~`@e}~5#KS)49OK$Rl#&&17B!xk7F7V$Xq>J(jtKX6 zMrf9=TLbieeAI>}f>ftIi}${k})9aD|aLo0DqA`a41+x%WC-k z65znt?VV(D2H;C*GtX9XdXh6Ds~1rUcezy&e4P68gdqCz*Khr=&W1aVw_|XUef~?~ zkTl0W9d>ZlVJ)j%VzrSkjs-wv5H?{nr_b>*GB`WEeoDl;Xvp{r0Fh`Ldegq}1tbcG zJR614`2t$VxU`m*n3vQ0XKI2xWxpj{!(T-_10d$fcvpIEm@^ntDT+BN?86#y#6 zJVM``HB7eewB+S`w|VaCP69A{W{ZhAY|5@gzU$!67bJbWR1cxpO#Jj`zvfGNaK<&n z`8u3SkshAt*|e~Od4M36&UJ#TN}zWmrmheu~RZYHiYO7NY5 zs3wHs?~8B+6P4dfeL>zWYxSq8)6h`)0${hyG&##JlcD4tYEG>*GDu$e6M*#dBsa^%;#;OC>*eY(cueTC zkO-zaRRjneu||Z-a!qx}QxukV(4;6@EBY?iREvs4JGYa26=!w+pLe9YO?CvYTAE#J z(#=(T99g4KlqE&^PXZaQ`<&c~B$Oc+2 zCu*V&x;DX-9XVf}INAVMS#O|GslZS&8$eOWy{FOQ)+Px+9ZPmUo}$hwzRfM^FM@E_ zGG%uFmRL8)I7GrU+;P?JQ5QwMMd>65pvroM%n;)!LyvXch{nxi4ENg_0Jx=%oy%_d zNltVhf(@G1-amie0--ojT%CfSzw~8>@pb(vF~5Wi^i7-nlB%smRJ(%4Eq|a2mK*zm;_Q|Gvj}Y_!D+TTs#Pqw>=Q0wF2N#cb(j{I39a^cRj25Sk5=*A$9;z zJ@a$CAI5qBxtGmX*Re(mgXaLq+$HCW?K4&!3<+M(+zy@!2}%Iq!MQc1FGI)wNV5hv zdPE{yZo>}%ll_2~TojrpOB*v@M#}Wc8h+dW1pCq_Y;}B)s+X8kDyWbs11o`kBBAg?U*YOIF&d@0XTElo#`mpCrFmRz+2tL zS@qw9N#I(OC;LjKT+TG3+vl(lJp)Z56lR-p>ygOh48PEOOd6s*mCme%oB*iumHH#r zaLV3KiITpxq#FuwgR(JJ=jjBsXkVS%SzRY zfI=O~9@j>==piUVG&lC@uLzjz1mK|-QB-L5%&fjGQ=(aw+S39is7q0wscl@X@XM>D z!X{b~w1k(d+5tdpW8iw5CbQUd!QgIULB2(ejRZjdvNpG(Lxy~7%fWHmagP~_& zk?=CbQviUVxhfLPJ&LU5vK(|JaoJ*RBy7kyhbYa#gN@_oE*E&D3q#$=N3bRdg0@?Z9nq>*aU2B?}JqtK27K z^M^NB{p!5$)DM|V{!8K5>V3R7lLHo$HuD=p9wh$gIF|unkqr@DK&1Em%2Os~o^QwD zkug;OOisA?xv>(AJ8@v3IK=ICS|uU_!0MD-y7$zaL6Lj>Qy-pZp>QY$>}68(F~XmU z{SB`5S!eUE=3sai28AV0TxW_^L;GVrv*TjX;(V}z762hzihZ);*ybazK9Oh!Au5iN zL)heqa(;<57!E%$aGp}{>M2TH=77n6*$~{V=U%a>Td;91zPGlnu7<{UUt5hc=~n_a z{n7RF*YaiCiM0SAEJkAJw(t*SqKv5nTn9dK>E_tK4BVcMJk@mt;A#{5@FT;MlA~UI@9yLmKg3N?>T)OTBm_xR!U8`CkghjsG}ed2FvMz?d1K zCtaQj`3-cL1$^(fpZSucEJelxki6z7X1+jnw4m8cRd-~*n4+7X^h&NrW*Jh3!qQ!i zV~)wWAi};+n+T4l;JMG-Gjs?kAr@#_1s$=-*@bFGv?d=_cRFNlkuJBoA@JLtI5FdE#Nd1q)$-> z6Cf=Jj{5cyt@=rA@vx$!?OS~q$OUc1VvX_y;uDTLeuPA#`MBWQ0pRU%M--x==eW7T zrt+~eRj$k_$qv&~JSko7Fo?DYj@0|N~CQYG1nL1WlFrZr4YC1CnBoI4HO)Oz%-^~h}(>z82J{&GA3T@Hnu z3pJvApz+P2CS0D@wkji(_UarA*#?!@bP+Q))0#;6Gj#edb^ zkMR5y-nqMo*3Z?91pt=`cfrSuGrVvb73;31;O~|?@bT5Jo-gcF0}+pfrB{f(ML2o6 zp-C+s7pdvV7>?+6`yx6xS4gRGe+K|#G@``3w)FGBRS2R8%X{AZzDj7jA-{@jP$u&| zMvpk{J|TH6Jc8sZ8<;F^CzV)Dl6mfIN*M40XA%Yp!s|)VA4|O5WFw}M2R51JE0bHW zH=KHVm&_2}DEqy#CD9|{ZG@`?0Q%uVaTxqE7^q&U`Fj@0G;%m=0Cd)Fx1A3$J~#i+ zU2I=|LP)C#g?f0B56^`=?bIzchw_TRYgZqpU7>nW7qlZLDD(a&bq)Y@jDf=npLP&8u3u-;|vp==Uj7rAI|Fi*bz9_bhAF15jikUas=2w)u9`Ydk_I5l5X=NHKTs%@$ zg52;K^c-jt*R0|vtCNTr$U#SpTHaB|N=6j`JlBfgHTXlupQY?y$;>WNpMI+kfbYYL zs(kK|X*#;)u#TheEF87j04z6Zq^F->2e=l!cj0FLU=z^721hh~>PDN0&p7@ZZM~08 zZR%f2g(6V*DxhQc%cnb}dh#e4lc^`0S+H=~>5;n#ltSyjIh5T+1s@L6e_Vib279Cj z4w?LWad+>MO7apiWSgL^#+? zd_jhgH(+t>=jF8qZNgf`FU4sS)tSU(?ou~f6cZ&a?!Odnyq%YzO=2i=oWpqSfxq1m zcL6J8vGK!Jr^X*t2ftsV$=+vIh$1_IVkM27G`x@*j%$MawtNXSYvB zycd7X?mngKT7lE-=LP4~BLmfg2@^>x$Jm+ICNM9VC&_HxQyGzZsLs=}B)TwfX#u5D zq5L#fIy}xx0C@_OE30G0@pX3VKP~Qu+GMws}FFBa8Y2t?`0#5SJs?o|`qWgShsofOblQxC=$!=Gu zLDW?zY%CD6@cDbLr7USBq&AxIdX61Vdo*idAK!7ztyE4B%yl#*Yrip^Q=*eU3nJ9r zZeA$C694=9jP+W$Nj5Sje*1p4*MmdDOa&?crpOyE#qbR7-%+q*+1y4%PrsIXcW<28;`W3WYxs@@G{By1N9|1(XBqn$* z?1*p}jy{2IIy0e&I9IS+*^ycLiAoA~Qb#P@nlV3F&b+V#jVw1&RU(X_hLpb{jl?nd zDlN*59RIKxpDHT@21usaObo6PYaN_j?*!8SPrM5d(tRYXR#v1L05CB3IWsD!6rP;kBCz)fIRxh)8Ck^K|%>76!ebRj6 z4niLemOn%^>p?pUq*T%f!xTrD|67Yh-RV(lw5R&-Wdk5NR$lphf4(tR`~J~)+0_J6 zZP;^to5*sUw_+^VUYl_A$_?EsolS>B5+cdu)%6tlWXT|(qb;nl)Q>IztbJ!NkQV0u z@=WTzn%O)219=e`~#%a1p@HoZRW1G}D-L*8HY^r%d$ zszWtAdXf)xt6()hxaCAMk?YnB1}_$mX0K{W<-o8PDcN55CQ8NZg5h17-y8Blyr=MS z_O%IQbX+kGcN&blH9`YE1~^7@)b#PfAbM3ZVu`ebDxc&fmk#B(6Aev!aDLZ%_eB72 zw%iZxoyZpe6o|cZkwT>#Lek|Y_5bO3CN~VJ&C!bKX6tV8z;7}Du(}d@ebZ}9O1K>hHt6R#S`bv&yOYgt|i{` zOj>YKn$9{cm|oO8TuEn2{7WrisQ^;u?M$k!Wi@4C;49JsOpY7O10Q_=m^Qe-crnpE z#JiC=lMUd=S3zNf)g&M>Nf*!h(){nu-3qgsaWRE^i zH=G&~v_?3WG@HdpxTEiE&tO(<4}xHH9nv~^-0 z=ME>c*KALgITHg=sEv*Lme$|%7k}eh96V{7FE5}3QHkduy;NV*S-+KH<_n}_Gb4Ne zS!|tQ6cgM|BGt~;H(l)+>_F)NBS)$^7e<&Ah&lL-IY#v2hqPxf43I7h_i|p9u+}v` z_&(l7AfY@6+ZL3_KQ>*P&CTEi4`@sr;ZrY(fz-BX!F=W;c35+}B|P6iOiorp4S*Im zkvz-(5e@I5-lI{=-ZA`W*vHlTIn6CjZ1U^wgyURVJ67C8d%=P5eIiVJm-N6)x}Dk8 zRJ-zdT*wXzl1k8@mSiP`)sNsqLObE2)ee6mR0=V6PiL-tXxFZ+P)LWP{6kz&CP>O4t)wmb*kLw6*i7TZhN9M&W#fbw?;jhbz#%#tTp^f|T) zqjVK`0I;A?-F6%=V9)6=_mCPjm8C^HH z$ty4@&xPe5$a_Mo26f4JbDLTXB`2-fjYauHr@eX<&J|6@H8#>7(|W24-gCMU3{0-3JC~UbDgl7T;xQ+dUNcg9HR*V1*+x2B9Hfk1*S!`5LmWG? zV!L`SN}+_{6I%GQ Date: Mon, 22 Jun 2026 22:24:51 +0000 Subject: [PATCH 19/30] doc: regenerate html for new KOKKOS style pages Sync the generated .html doc pages with the .txt changes from this branch (the /kk style names, accelerator boilerplate, and "(k)" command list markers), so the published html matches the source: - surf_collide.html: cll/kk, adiabatic/kk, impulsive/kk, td/kk (and transparent/kk) in the style line and per-style args - surf_react_adsorb.html: adsorb/kk heading + style + boilerplate - react.html: qk/kk and tce/qk/kk args - compute_isurf_grid.html, compute_property_surf.html, compute_react_surf.html, compute_react_isurf_grid.html, fix_temp_global_rescale.html: /kk heading + boilerplate - Section_commands.html: "(k)" markers for the ported styles Co-Authored-By: Claude Opus 4.8 Co-Authored-By: Stan Moore Claude-Session: https://claude.ai/code/session_01Q6nQVuxEiTUqSzDoDppCUM --- doc/Section_commands.html | 16 ++++++++-------- doc/compute_isurf_grid.html | 25 +++++++++++++++++++++++++ doc/compute_property_surf.html | 25 +++++++++++++++++++++++++ doc/compute_react_isurf_grid.html | 25 +++++++++++++++++++++++++ doc/compute_react_surf.html | 25 +++++++++++++++++++++++++ doc/fix_temp_global_rescale.html | 25 +++++++++++++++++++++++++ doc/react.html | 4 ++++ doc/surf_collide.html | 10 +++++----- doc/surf_react_adsorb.html | 27 ++++++++++++++++++++++++++- 9 files changed, 168 insertions(+), 14 deletions(-) diff --git a/doc/Section_commands.html b/doc/Section_commands.html index f6a7dfae2..31e03bfe3 100644 --- a/doc/Section_commands.html +++ b/doc/Section_commands.html @@ -345,7 +345,7 @@

Fix styles ablateadapt (k)ambipolar (k)ave/grid (k)ave/histo (k)ave/histo/weight (k) ave/surfave/timebalance (k)customdt/reset (k)emit/face (k) emit/face/fileemit/surffield/gridfield/particlegrid/check (k)halt -move/surf (k)printsurf/temptemp/global/rescaletemp/rescale (k)vibmode (k) +move/surf (k)printsurf/temptemp/global/rescale (k)temp/rescale (k)vibmode (k)
@@ -361,9 +361,9 @@

Compute styles

@@ -392,9 +392,9 @@

Surface collide styles letters in parenthesis: k = KOKKOS.

@@ -408,7 +408,7 @@

Surface reaction styles letters in parenthesis: k = KOKKOS.

diff --git a/doc/compute_isurf_grid.html b/doc/compute_isurf_grid.html index fa8a45b46..feba21c84 100644 --- a/doc/compute_isurf_grid.html +++ b/doc/compute_isurf_grid.html @@ -11,6 +11,8 @@

compute isurf/grid command

+

compute isurf/grid/kk command +

Syntax: 

compute ID isurf/grid group-ID mix-ID value1 value2 ... 
@@ -159,6 +161,29 @@ 
compute isurf/grid command
 

 

 
+
Styles with a kk suffix are functionally the same as the
+corresponding style without the suffix.  They have been optimized to
+run faster, depending on your available hardware, as discussed in the
+Accelerating SPARTA section of the manual.
+The accelerated styles take the same arguments and should produce the
+same results, except for different random number, round-off and
+precision issues.
+

+
These accelerated styles are part of the KOKKOS package. They are only
+enabled if SPARTA was built with that package.  See the Making
+SPARTA section for more info.
+

+
You can specify the accelerated styles explicitly in your input script
+by including their suffix, or you can use the -suffix command-line
+switch when you invoke SPARTA, or you can
+use the suffix command in your input script.
+

+
See the Accelerating SPARTA section of the
+manual for more instructions on how to use the accelerated styles
+effectively.
+

+

+
 
Restrictions: none
 

 
Related commands:
diff --git a/doc/compute_property_surf.html b/doc/compute_property_surf.html
index 196fbeef8..7cdce62bf 100644
--- a/doc/compute_property_surf.html
+++ b/doc/compute_property_surf.html
@@ -11,6 +11,8 @@
 
 
compute property/surf command 
 

+
compute property/surf/kk command 
+

 
Syntax:
 

 
compute ID property/surf group-ID input1 input2 ... 
@@ -109,6 +111,29 @@ 
compute property/surf command
 

 

 
+
Styles with a kk suffix are functionally the same as the
+corresponding style without the suffix.  They have been optimized to
+run faster, depending on your available hardware, as discussed in the
+Accelerating SPARTA section of the manual.
+The accelerated styles take the same arguments and should produce the
+same results, except for different random number, round-off and
+precision issues.
+

+
These accelerated styles are part of the KOKKOS package. They are only
+enabled if SPARTA was built with that package.  See the Making
+SPARTA section for more info.
+

+
You can specify the accelerated styles explicitly in your input script
+by including their suffix, or you can use the -suffix command-line
+switch when you invoke SPARTA, or you can
+use the suffix command in your input script.
+

+
See the Accelerating SPARTA section of the
+manual for more instructions on how to use the accelerated styles
+effectively.
+

+

+
 
Restrictions:
 

 
For 2d simulations, none of the attributes which refer to the 3rd
diff --git a/doc/compute_react_isurf_grid.html b/doc/compute_react_isurf_grid.html
index 53883a3d6..749840194 100644
--- a/doc/compute_react_isurf_grid.html
+++ b/doc/compute_react_isurf_grid.html
@@ -11,6 +11,8 @@
 
 
compute react/isurf/grid command 
 

+
compute react/isurf/grid/kk command 
+

 
Syntax:
 

 
compute ID react/isurf/grid group-ID reaction-ID value1 value2 ... 
@@ -116,6 +118,29 @@ 
compute react/isurf/grid command
 

 

 
+
Styles with a kk suffix are functionally the same as the
+corresponding style without the suffix.  They have been optimized to
+run faster, depending on your available hardware, as discussed in the
+Accelerating SPARTA section of the manual.
+The accelerated styles take the same arguments and should produce the
+same results, except for different random number, round-off and
+precision issues.
+

+
These accelerated styles are part of the KOKKOS package. They are only
+enabled if SPARTA was built with that package.  See the Making
+SPARTA section for more info.
+

+
You can specify the accelerated styles explicitly in your input script
+by including their suffix, or you can use the -suffix command-line
+switch when you invoke SPARTA, or you can
+use the suffix command in your input script.
+

+
See the Accelerating SPARTA section of the
+manual for more instructions on how to use the accelerated styles
+effectively.
+

+

+
 
Restrictions: none
 

 
Related commands:
diff --git a/doc/compute_react_surf.html b/doc/compute_react_surf.html
index 893eb9e68..fcba14891 100644
--- a/doc/compute_react_surf.html
+++ b/doc/compute_react_surf.html
@@ -11,6 +11,8 @@
 
 
compute react/surf command 
 

+
compute react/surf/kk command 
+

 
Syntax:
 

 
compute ID react/surf group-ID reaction-ID value1 value2 ... 
@@ -114,6 +116,29 @@ 
compute react/surf command
 

 

 
+
Styles with a kk suffix are functionally the same as the
+corresponding style without the suffix.  They have been optimized to
+run faster, depending on your available hardware, as discussed in the
+Accelerating SPARTA section of the manual.
+The accelerated styles take the same arguments and should produce the
+same results, except for different random number, round-off and
+precision issues.
+

+
These accelerated styles are part of the KOKKOS package. They are only
+enabled if SPARTA was built with that package.  See the Making
+SPARTA section for more info.
+

+
You can specify the accelerated styles explicitly in your input script
+by including their suffix, or you can use the -suffix command-line
+switch when you invoke SPARTA, or you can
+use the suffix command in your input script.
+

+
See the Accelerating SPARTA section of the
+manual for more instructions on how to use the accelerated styles
+effectively.
+

+

+
 
Restrictions: none
 

 
Related commands:
diff --git a/doc/fix_temp_global_rescale.html b/doc/fix_temp_global_rescale.html
index 0d341f7f2..8103b33bf 100644
--- a/doc/fix_temp_global_rescale.html
+++ b/doc/fix_temp_global_rescale.html
@@ -11,6 +11,8 @@
 
 
fix temp/global/rescale command 
 

+
fix temp/global/rescale/kk command 
+

 
Syntax:
 

 
fix ID temp/global/rescale N Tstart Tstop fraction 
@@ -79,6 +81,29 @@ 
fix temp/global/rescale command
 

 

 
+
Styles with a kk suffix are functionally the same as the
+corresponding style without the suffix.  They have been optimized to
+run faster, depending on your available hardware, as discussed in the
+Accelerating SPARTA section of the manual.
+The accelerated styles take the same arguments and should produce the
+same results, except for different random number, round-off and
+precision issues.
+

+
These accelerated styles are part of the KOKKOS package. They are only
+enabled if SPARTA was built with that package.  See the Making
+SPARTA section for more info.
+

+
You can specify the accelerated styles explicitly in your input script
+by including their suffix, or you can use the -suffix command-line
+switch when you invoke SPARTA, or you can
+use the suffix command in your input script.
+

+
See the Accelerating SPARTA section of the
+manual for more instructions on how to use the accelerated styles
+effectively.
+

+

+
 
Restrictions: none
 

 
Related commands:
diff --git a/doc/react.html b/doc/react.html
index 8b6a4d470..82ea2b10e 100644
--- a/doc/react.html
+++ b/doc/react.html
@@ -27,6 +27,10 @@ 
react command
   tce/qk args = infile
     infile = file with list of gas-phase chemistry reactions
   tce/kk args = infile
+    infile = file with list of gas-phase chemistry reactions
+  qk/kk args = infile
+    infile = file with list of gas-phase chemistry reactions
+  tce/qk/kk args = infile
     infile = file with list of gas-phase chemistry reactions 
 

 
diff --git a/doc/surf_collide.html b/doc/surf_collide.html
index d7cf82886..f51845d6d 100644
--- a/doc/surf_collide.html
+++ b/doc/surf_collide.html
@@ -17,7 +17,7 @@ 
surf_collide command
 

 
  • ID = user-assigned name for the surface collision model 
 
-
  • style = specular or diffuse or cll or adiabatic or impulsive or td or piston or transparent or vanish or specular/kk or diffuse/kk or piston/kk or vanish/kk 
+
  • style = specular or diffuse or cll or adiabatic or impulsive or td or piston or transparent or vanish or specular/kk or diffuse/kk or cll/kk or adiabatic/kk or impulsive/kk or td/kk or piston/kk or transparent/kk or vanish/kk 
 
 
  • args = arguments for specific style 
 
@@ -27,15 +27,15 @@ 
    surf_collide command
     Tsurf = temperature of surface (temperature units)
             Tsurf can be a variable or custom per-surf attribute (see below)
     acc = accommodation coefficient
-  cll args = Tsurf acc_n acc_t acc_rot acc_vib
+  cll or cll/kk args = Tsurf acc_n acc_t acc_rot acc_vib
     Tsurf = temperature of surface (temperature units)
             Tsurf can be a variable or custom per-surf attribute (see below)
     acc_n = accommodation coefficient in the surface normal direction
     acc_t = accommodation coefficient in the surface tangential direction
     acc_rot = accommodation coefficient for the rotational modes
     acc_vib = accommodation coefficient for the vibrational modes
-  adiabatic args = none
-  impulsive args = Tsurf model param1 param2 var theta_peak pol_pow azi_pow
+  adiabatic or adiabatic/kk args = none
+  impulsive or impulsive/kk args = Tsurf model param1 param2 var theta_peak pol_pow azi_pow
     Tsurf = temperature of surface (temperature units)
             Tsurf can be a variable or custom per-surf attribute (see below)
     model can be softsphere or tempvar
@@ -49,7 +49,7 @@ 
    surf_collide command
     theta_peak = peak location of the polar angle distribution
     pol_pow = cosine power represeting the polar angular distribution
     azi_pow = cosine power represeting the azimuthal angular distribution
-  td arg = Tsurf 
+  td or td/kk arg = Tsurf
     Tsurf = temperature of surface (temperature units)
             Tsurf can be a variable or custom per-surf attribute (see below)
   piston or piston/kk args = Vwall
diff --git a/doc/surf_react_adsorb.html b/doc/surf_react_adsorb.html
index e8d770703..ca85734ad 100644
--- a/doc/surf_react_adsorb.html
+++ b/doc/surf_react_adsorb.html
@@ -11,13 +11,15 @@
 
 
    surf_react adsorb command 
 
    
+
    surf_react adsorb/kk command 
+
    
 
    Syntax:
 
    
 
    surf_react ID adsorb model infile(s) Nsync type temp n_sites adsp1 adsp2 ... 
 
    
 
    • ID = user-assigned name for the surface reaction model 
 
-
    • style = adsorb 
+
    • style = adsorb or adsorb/kk 
 
 
    • model = gs or ps or gs/ps 
 
@@ -372,6 +374,29 @@ 
      surf_react adsorb command
 
      
 
      
 
+
      Styles with a kk suffix are functionally the same as the
+corresponding style without the suffix.  They have been optimized to
+run faster, depending on your available hardware, as discussed in the
+Accelerating SPARTA section of the manual.
+The accelerated styles take the same arguments and should produce the
+same results, except for different random number, round-off and
+precision issues.
+
      
+
      These accelerated styles are part of the KOKKOS package. They are only
+enabled if SPARTA was built with that package.  See the Making
+SPARTA section for more info.
+
      
+
      You can specify the accelerated styles explicitly in your input script
+by including their suffix, or you can use the -suffix command-line
+switch when you invoke SPARTA, or you can
+use the suffix command in your input script.
+
      
+
      See the Accelerating SPARTA section of the
+manual for more instructions on how to use the accelerated styles
+effectively.
+
      
+
      
+
 
      Restrictions:
 
      
 
      If the following conditions are met:

From c90ba37ebc59c4d38f44b37f5d4602595d21a73e Mon Sep 17 00:00:00 2001
From: Claude 
Date: Tue, 23 Jun 2026 03:11:21 +0000
Subject: [PATCH 20/30] KOKKOS: support non-reacting multigroup collisions in
 collide vss/kk

Add a Kokkos NTC collision path for mixtures with ngroup > 1, mirroring
the non-Kokkos Collide::collisions_group algorithm. Previously the vss/kk
style errored out whenever the collision mixture defined more than one
group.

Scope is limited to the case where group membership is static within a
timestep, so collisions can run in a parallel per-cell kernel and the
result is bit-for-bit identical to the non-Kokkos version: no gas-phase
reactions, no ambipolar approximation, and no near-neighbor (nearcp)
selection. Those combinations still raise a clear "not (yet) supported"
error.

Implementation:
 - new TagCollideCollisionsGroup kernel and collisions_group() launcher
 - per-cell group partitioning built on device into d_glist with
   per-group offsets, in the same per-group order as the CPU version
 - attempt counts pre-computed per group pair into d_nattempt_pair, with
   RN drawn for every pair to preserve the CPU collision RNG ordering
 - attempt_collision_kokkos() overload for a pair of groups
 - device copy of mixture species-to-group map (d_species2group)

Verified bit-for-bit identical to the non-Kokkos path (Serial build with
SPARTA_KOKKOS_EXACT, 1 thread) over 500 steps of a 2-group air case, and
statistically consistent with 4 OpenMP threads. Adds examples/collide/
in.collide.group as a 2-group regression case.

Co-Authored-By: Claude Opus 4.8 
Claude-Session: https://claude.ai/code/session_01Q6nQVuxEiTUqSzDoDppCUM
Co-authored-by: stanmoore1 
---
 examples/collide/in.collide.group |  47 +++++
 src/KOKKOS/collide_vss_kokkos.cpp | 295 ++++++++++++++++++++++++++++--
 src/KOKKOS/collide_vss_kokkos.h   |  19 ++
 3 files changed, 342 insertions(+), 19 deletions(-)
 create mode 100644 examples/collide/in.collide.group

diff --git a/examples/collide/in.collide.group b/examples/collide/in.collide.group
new file mode 100644
index 000000000..c6c492177
--- /dev/null
+++ b/examples/collide/in.collide.group
@@ -0,0 +1,47 @@
+################################################################################
+# thermal gas in a 3d box with collisions, multiple collision groups
+# particles reflect off global box boundaries
+#
+# Demonstrates/verifies non-reacting multigroup (ngroup > 1) collisions.
+# The species are split into two collision groups: "heavy" and "light".
+#
+# Note:
+#  - The "comm/sort" option to the "global" command is used to match MPI runs.
+#  - The "twopass" option is used to match Kokkos runs.
+# The "comm/sort" and "twopass" options should not be used for production runs.
+################################################################################
+
+seed	    	    12345
+dimension   	    3
+global              gridcut 1.0e-5 comm/sort yes
+
+boundary	    rr rr rr
+
+create_box  	    0 0.0001 0 0.0001 0 0.0001
+create_grid 	    10 10 10
+
+balance_grid        rcb part
+
+species		    6SpeciesAir.species N2 O2 NO N O Ar
+
+mixture		    air O2 N2 O N vstream 0.0 0.0 0.0 temp 273.1
+mixture		    air O2 frac 0.21 group heavy
+mixture		    air N2 frac 0.78 group heavy
+mixture		    air NO group heavy
+mixture		    air Ar frac 0.009 group heavy
+mixture		    air O group light
+mixture		    air N group light
+
+global              nrho 7.07043E22
+global              fnum 7.07043E6
+
+collide		    vss air 6SpeciesAirII.vss
+
+create_particles    air n 10000 twopass
+
+stats		    100
+compute             temp temp
+stats_style	    step cpu np nattempt ncoll c_temp
+
+timestep 	    7.00E-9
+run 		    1000
diff --git a/src/KOKKOS/collide_vss_kokkos.cpp b/src/KOKKOS/collide_vss_kokkos.cpp
index 1f97d50ae..065f25234 100644
--- a/src/KOKKOS/collide_vss_kokkos.cpp
+++ b/src/KOKKOS/collide_vss_kokkos.cpp
@@ -44,6 +44,7 @@ enum{CONSTANT,VARIABLE};
 #define DELTADELETE 1024
 #define DELTAELECTRON 128
 #define DELTACELLCOUNT 2
+#define MAXGROUP 16               // max # of collision groups for Kokkos group collisions
 
 #define MAXLINE 1024
 #define EPSZERO 1.0e-14
@@ -238,6 +239,17 @@ void CollideVSSKokkos::init()
     d_vremax_initial = k_vremax_initial.view_device();
   }
 
+  // device copy of species-to-group mapping for group collisions
+
+  if (ngroups > 1) {
+    int nspecies = particle->nspecies;
+    int *species2group = mixture->species2group;
+    d_species2group = DAT::t_int_1d("collide:species2group",nspecies);
+    auto h_species2group = Kokkos::create_mirror_view(d_species2group);
+    for (int i = 0; i < nspecies; i++) h_species2group(i) = species2group[i];
+    Kokkos::deep_copy(d_species2group,h_species2group);
+  }
+
   // if recombination reactions exist, set flags per species pair
 
   recombflag = 0;
@@ -399,31 +411,43 @@ void CollideVSSKokkos::collisions()
   if (ngas_tally)
     error->all(FLERR,"Kokkos does not (yet) support tallying gas/gas collisions or reactions");
 
-  if (ngroups != 1)
-    error->all(FLERR,"Group collisions not yet supported with Kokkos");
-
   COLLIDE_REDUCE reduce;
 
-  if (!ambiflag) {
-    if (!nearcp) {
-      if (!ngas_tally) {
-        collisions_one<0,0>(reduce);
-      } else if (ngas_tally) {
-        collisions_one<0,1>(reduce);
+  if (ngroups == 1) {
+    if (!ambiflag) {
+      if (!nearcp) {
+        if (!ngas_tally) {
+          collisions_one<0,0>(reduce);
+        } else if (ngas_tally) {
+          collisions_one<0,1>(reduce);
+        }
+      } else if (nearcp) {
+        if (!ngas_tally) {
+          collisions_one<1,0>(reduce);
+        } else if (ngas_tally) {
+          collisions_one<1,1>(reduce);
+        }
       }
-    } else if (nearcp) {
+    } else if (ambiflag) {
       if (!ngas_tally) {
-        collisions_one<1,0>(reduce);
-      } else if (ngas_tally) {
-        collisions_one<1,1>(reduce);
+        collisions_one_ambipolar<0>(reduce);
+      } else if (!ngas_tally) {
+        collisions_one_ambipolar<1>(reduce);
       }
     }
-  } else if (ambiflag) {
-    if (!ngas_tally) {
-      collisions_one_ambipolar<0>(reduce);
-    } else if (!ngas_tally) {
-      collisions_one_ambipolar<1>(reduce);
-    }
+
+  // multiple groups
+  // Kokkos currently supports only non-reacting, non-ambipolar,
+  //   non-near-neighbor group collisions
+
+  } else {
+    if (react)
+      error->all(FLERR,"Kokkos does not (yet) support reacting group collisions");
+    if (ambiflag)
+      error->all(FLERR,"Kokkos does not (yet) support multigroup ambipolar collisions");
+    if (nearcp)
+      error->all(FLERR,"Kokkos does not (yet) support near-neighbor group collisions");
+    collisions_group<0,0>(reduce);
   }
 
   // remove any particles deleted in chemistry reactions
@@ -820,6 +844,211 @@ void CollideVSSKokkos::operator()(TagCollideCollisionsOne< NEARCP, GASTALLY, ATO
   rand_pool.free_state(rand_gen);
 }
 
+/* ----------------------------------------------------------------------
+   NTC algorithm for multiple groups
+   Kokkos version supports only non-reacting, non-ambipolar, non-nearcp
+     collisions, so group membership is static within the timestep
+     and no particles are created or destroyed
+------------------------------------------------------------------------- */
+
+template < int NEARCP, int GASTALLY >
+void CollideVSSKokkos::collisions_group(COLLIDE_REDUCE &reduce)
+{
+  if (ngroups > MAXGROUP)
+    error->all(FLERR,"Too many collision groups for Kokkos group collisions");
+
+  // loop over cells I own
+
+  this->sync(Device,ALL_MASK);
+
+  ParticleKokkos* particle_kk = (ParticleKokkos*) particle;
+  particle_kk->sync(Device,PARTICLE_MASK|SPECIES_MASK);
+  if (vibstyle == DISCRETE) particle_kk->sync(Device,CUSTOM_MASK);
+  d_particles = particle_kk->k_particles.view_device();
+  d_species = particle_kk->k_species.view_device();
+  d_ewhich = particle_kk->k_ewhich.view_device();
+  k_eiarray = particle_kk->k_eiarray;
+
+  GridKokkos* grid_kk = (GridKokkos*) grid;
+  grid_kk->sync(Device,CINFO_MASK);
+  d_plist = grid_kk->d_plist;
+
+  // allocate per-cell group scratch arrays
+  // d_glist holds plist indices laid out group-contiguous per cell
+  // d_nattempt_pair holds the pre-computed attempt count per group pair
+
+  if (int(d_glist.extent(0)) < nglocal ||
+      int(d_glist.extent(1)) < int(d_plist.extent(1)))
+    MemKK::realloc_kokkos(d_glist,"collide:glist",nglocal,d_plist.extent(1));
+  if (int(d_nattempt_pair.extent(0)) < nglocal)
+    MemKK::realloc_kokkos(d_nattempt_pair,"collide:nattempt_pair",nglocal,ngroups,ngroups);
+
+  copymode = 1;
+
+  // no particles are created or destroyed for non-reacting group collisions
+
+  ndelete = 0;
+
+  h_error_flag() = 0;
+  Kokkos::deep_copy(d_scalars,h_scalars);
+
+  grid_kk_copy.copy(grid_kk);
+
+  if (sparta->kokkos->atomic_reduction) {
+    if (sparta->kokkos->need_atomics)
+      Kokkos::parallel_for(Kokkos::RangePolicy >(0,nglocal),*this);
+    else
+      Kokkos::parallel_for(Kokkos::RangePolicy >(0,nglocal),*this);
+  } else
+    Kokkos::parallel_reduce(Kokkos::RangePolicy >(0,nglocal),*this,reduce);
+
+  Kokkos::deep_copy(h_scalars,d_scalars);
+
+  copymode = 0;
+
+  if (h_error_flag())
+    error->one(FLERR,"Collision cell volume is zero");
+
+  this->modified(Device,ALL_MASK);
+  particle_kk->modify(Device,PARTICLE_MASK);
+  if (vibstyle == DISCRETE) particle_kk->modify(Device,CUSTOM_MASK);
+
+  d_particles = t_particle_1d(); // destroy reference to reduce memory use
+  d_plist = {};
+}
+
+template < int NEARCP, int GASTALLY, int ATOMIC_REDUCTION >
+KOKKOS_INLINE_FUNCTION
+void CollideVSSKokkos::operator()(TagCollideCollisionsGroup< NEARCP, GASTALLY, ATOMIC_REDUCTION >, const int &icell) const {
+  COLLIDE_REDUCE reduce;
+  this->template operator()< NEARCP, GASTALLY, ATOMIC_REDUCTION >(TagCollideCollisionsGroup< NEARCP, GASTALLY, ATOMIC_REDUCTION >(), icell, reduce);
+}
+
+template < int NEARCP, int GASTALLY, int ATOMIC_REDUCTION >
+KOKKOS_INLINE_FUNCTION
+void CollideVSSKokkos::operator()(TagCollideCollisionsGroup< NEARCP, GASTALLY, ATOMIC_REDUCTION >, const int &icell, COLLIDE_REDUCE &reduce) const {
+
+  int np = grid_kk_copy.obj.d_cellcount[icell];
+  if (np <= 1) return;
+
+  const double volume = grid_kk_copy.obj.k_cinfo.view_device()[icell].volume / grid_kk_copy.obj.k_cinfo.view_device()[icell].weight;
+  if (volume == 0.0) d_error_flag() = 1;
+
+  // build per-group particle lists for this cell
+  // gcount[g] = # of particles in group g
+  // gstart[g] = offset of group g within d_glist(icell,*)
+  // d_glist(icell,k) = plist index of kth particle, laid out group-contiguous
+  //   in the same per-group order as the non-Kokkos version
+
+  int gcount[MAXGROUP];
+  int gstart[MAXGROUP];
+  int gcursor[MAXGROUP];
+
+  for (int g = 0; g < ngroups; g++) gcount[g] = 0;
+  for (int n = 0; n < np; n++) {
+    const int isp = d_particles[d_plist(icell,n)].ispecies;
+    gcount[d_species2group[isp]]++;
+  }
+  int offset = 0;
+  for (int g = 0; g < ngroups; g++) {
+    gstart[g] = offset;
+    gcursor[g] = offset;
+    offset += gcount[g];
+  }
+  for (int n = 0; n < np; n++) {
+    const int isp = d_particles[d_plist(icell,n)].ispecies;
+    const int g = d_species2group[isp];
+    d_glist(icell,gcursor[g]++) = n;
+  }
+
+  struct State precoln;       // state before collision
+  struct State postcoln;      // state after collision
+
+  rand_type rand_gen = rand_pool.get_state();
+
+  // pre-compute # of attempts for each pair of groups
+  // double loop over N^2 / 2 pairs of groups
+  // draw RN for every pair to match non-Kokkos collision ordering
+
+  for (int ig = 0; ig < ngroups; ig++)
+    for (int jg = ig; jg < ngroups; jg++) {
+      const double attempt =
+        attempt_collision_kokkos(icell,ig,jg,gcount[ig],gcount[jg],volume,rand_gen);
+      const int nattempt = static_cast (attempt);
+      d_nattempt_pair(icell,ig,jg) = nattempt;
+      if (nattempt) {
+        if (ATOMIC_REDUCTION == 1)
+          Kokkos::atomic_add(&d_nattempt_one(),nattempt);
+        else if (ATOMIC_REDUCTION == 0)
+          d_nattempt_one() += nattempt;
+        else
+          reduce.nattempt_one += nattempt;
+      }
+    }
+
+  // perform collisions for each pair of groups
+  // select random particle in each group, cannot be same if igroup == jgroup
+  // test if collision actually occurs, then perform it
+
+  for (int ig = 0; ig < ngroups; ig++)
+    for (int jg = ig; jg < ngroups; jg++) {
+      const int nattempt = d_nattempt_pair(icell,ig,jg);
+      if (!nattempt) continue;
+      const int ni = gcount[ig];
+      const int nj = gcount[jg];
+      if (ni == 0 || nj == 0) continue;
+      if (ig == jg && ni == 1) continue;
+
+      for (int iattempt = 0; iattempt < nattempt; iattempt++) {
+        int i = ni * rand_gen.drand();
+        int j = nj * rand_gen.drand();
+        if (ig == jg)
+          while (i == j) j = nj * rand_gen.drand();
+
+        Particle::OnePart* ipart = &d_particles[d_plist(icell,d_glist(icell,gstart[ig]+i))];
+        Particle::OnePart* jpart = &d_particles[d_plist(icell,d_glist(icell,gstart[jg]+j))];
+
+        // test if collision actually occurs
+
+        if (!test_collision_kokkos(icell,ig,jg,ipart,jpart,precoln,rand_gen)) continue;
+
+        // perform collision
+        // non-reacting: no chemistry, no 3rd particle, no create/delete
+        // if GASTALLY: save iorig/jorig for tally (tally hook deferred)
+
+        Particle::OnePart iorig,jorig;
+        if (GASTALLY) {
+          iorig = *ipart;
+          jorig = *jpart;
+        }
+
+        Particle::OnePart* kpart = NULL;
+        Particle::OnePart* recomb_part3 = NULL;
+        int recomb_species = -1;
+        double recomb_density = 0.0;
+        int index_kpart = 0;
+
+        setup_collision_kokkos(ipart,jpart,precoln,postcoln);
+        perform_collision_kokkos(ipart,jpart,kpart,precoln,postcoln,rand_gen,
+                                 recomb_part3,recomb_species,recomb_density,index_kpart);
+
+        if (ATOMIC_REDUCTION == 1)
+          Kokkos::atomic_inc(&d_ncollide_one());
+        else if (ATOMIC_REDUCTION == 0)
+          d_ncollide_one()++;
+        else
+          reduce.ncollide_one++;
+
+        //if (GASTALLY)
+        //  for (int m = 0; m < ngas_tally; m++)
+        //    glist_active[m]->gas_tally(icell,reactflag,
+        //                               &iorig,&jorig,ipart,jpart,kpart);
+      }
+    }
+
+  rand_pool.free_state(rand_gen);
+}
+
 /* ----------------------------------------------------------------------
    NTC algorithm for a single group with ambipolar approximation
 ------------------------------------------------------------------------- */
@@ -1390,6 +1619,34 @@ double CollideVSSKokkos::attempt_collision_kokkos(int icell, int np, double volu
   return nattempt;
 }
 
+/* ----------------------------------------------------------------------
+   attempt count for a pair of groups
+   ni,nj = particle counts in igroup,jgroup
+------------------------------------------------------------------------- */
+
+KOKKOS_INLINE_FUNCTION
+double CollideVSSKokkos::attempt_collision_kokkos(int icell, int igroup, int jgroup,
+                                                  int ni, int nj, double volume,
+                                                  rand_type &rand_gen) const
+{
+  double nattempt;
+
+  // return 2x the value for igroup != jgroup, since no J,I pairing
+
+  double npairs;
+  if (igroup == jgroup) npairs = 0.5 * ni * (ni-1);
+  else npairs = ni * nj;
+
+  nattempt = npairs * d_vremax(icell,igroup,jgroup) * dt * fnum / volume;
+
+  if (remainflag) {
+    nattempt += d_remain(icell,igroup,jgroup);
+    d_remain(icell,igroup,jgroup) = nattempt - static_cast (nattempt);
+  } else nattempt += rand_gen.drand();
+
+  return nattempt;
+}
+
 /* ----------------------------------------------------------------------
    determine if collision actually occurs
    1 = yes, 0 = no
diff --git a/src/KOKKOS/collide_vss_kokkos.h b/src/KOKKOS/collide_vss_kokkos.h
index edab98b42..0c80f6e69 100644
--- a/src/KOKKOS/collide_vss_kokkos.h
+++ b/src/KOKKOS/collide_vss_kokkos.h
@@ -62,6 +62,9 @@ struct TagCollideCollisionsOne{};
 template < int GASTALLY, int ATOMIC_REDUCTION >
 struct TagCollideCollisionsOneAmbipolar{};
 
+template < int NEARCP, int GASTALLY, int ATOMIC_REDUCTION >
+struct TagCollideCollisionsGroup{};
+
 class CollideVSSKokkos : public CollideVSS {
  public:
   typedef COLLIDE_REDUCE value_type;
@@ -87,6 +90,8 @@ class CollideVSSKokkos : public CollideVSS {
   KOKKOS_INLINE_FUNCTION
   double attempt_collision_kokkos(int, int, double, rand_type &) const;
   KOKKOS_INLINE_FUNCTION
+  double attempt_collision_kokkos(int, int, int, int, int, double, rand_type &) const;
+  KOKKOS_INLINE_FUNCTION
   int test_collision_kokkos(int, int, int, Particle::OnePart *, Particle::OnePart *, struct State &, rand_type &) const;
   KOKKOS_INLINE_FUNCTION
   void setup_collision_kokkos(Particle::OnePart *, Particle::OnePart *, struct State &, struct State &) const;
@@ -118,6 +123,14 @@ class CollideVSSKokkos : public CollideVSS {
   KOKKOS_INLINE_FUNCTION
   void operator()(TagCollideCollisionsOneAmbipolar< GASTALLY, ATOMIC_REDUCTION >, const int&, COLLIDE_REDUCE&) const;
 
+  template < int NEARCP, int GASTALLY, int ATOMIC_REDUCTION >
+  KOKKOS_INLINE_FUNCTION
+  void operator()(TagCollideCollisionsGroup< NEARCP, GASTALLY, ATOMIC_REDUCTION >, const int&) const;
+
+  template < int NEARCP, int GASTALLY, int ATOMIC_REDUCTION >
+  KOKKOS_INLINE_FUNCTION
+  void operator()(TagCollideCollisionsGroup< NEARCP, GASTALLY, ATOMIC_REDUCTION >, const int&, COLLIDE_REDUCE&) const;
+
   typedef Kokkos::
     DualView tdual_params_2d;
   typedef tdual_params_2d::t_dev t_params_2d;
@@ -148,6 +161,11 @@ class CollideVSSKokkos : public CollideVSS {
   t_species_1d_const d_species;
   DAT::t_int_2d d_plist;
 
+  // group collision scratch (ngroups > 1)
+  DAT::t_int_1d d_species2group;
+  DAT::t_int_2d d_glist;
+  Kokkos::View d_nattempt_pair;
+
   DAT::t_int_1d d_ewhich;
   tdual_struct_tdual_int_1d_1d k_eivec;
   tdual_struct_tdual_int_2d_1d k_eiarray;
@@ -211,6 +229,7 @@ class CollideVSSKokkos : public CollideVSS {
 
   template < int NEARCP, int GASTALLY > void collisions_one(COLLIDE_REDUCE&);
   template < int GASTALLY > void collisions_one_ambipolar(COLLIDE_REDUCE&);
+  template < int NEARCP, int GASTALLY > void collisions_group(COLLIDE_REDUCE&);
 
   // VSS specific
 

From 137745af391d16c8eb2e679c5ffa07ef51b66480 Mon Sep 17 00:00:00 2001
From: Claude 
Date: Tue, 23 Jun 2026 04:27:26 +0000
Subject: [PATCH 21/30] KOKKOS: support per-grid gas collision/reaction tallies
 in collide vss/kk

Add Kokkos-enabled compute gas/collision/grid and compute gas/reaction/grid
(the per-grid gas tally computes) and wire them into the vss/kk collision
kernels. Previously any active gas/gas tally compute caused the Kokkos path
to error out.

New computes (compute_gas_collision_grid_kokkos, compute_gas_reaction_grid_kokkos):
 - subclass the CPU computes, allocate the per-grid output as a DualView
   (vector_grid, or array_grid for reaction every/select modes) and override
   clear()/reallocate() to manage it on device
 - provide an on-device gas_tally_kk() that mirrors the CPU gas_tally()
   filtering (grid group mask + mixture species2group) and accumulates the
   tally directly into the per-cell slot
 - pre_gas_tally()/post_gas_tally() bind device views before the kernel and
   sync the result back to the host array afterward

Because Collide parallelizes over grid cells (one icell per work item), the
per-cell tally has no cross-thread write contention, so it is written
directly with no atomics or ScatterView duplication.

collide_vss_kokkos:
 - setup_gas_tally()/finish_gas_tally() partition update->glist_active into
   typed KKCopy lists, validate each is a supported per-grid Kokkos compute
   (the per-event gas/collision/tally and gas/reaction/tally are rejected
   with a clear error), and call pre/post hooks
 - the collisions_one, collisions_group, and ambipolar kernels invoke
   gas_tally_kk() on each active compute when GASTALLY is set

Base ComputeGasCollisionGrid/ComputeGasReactionGrid get a SPARTA-only ctor
for the Kokkos copy pattern. UpdateKokkos::tally_set no longer errors on
gas tallies (validation moved to the collide setup).

Verified bit-for-bit identical to the non-Kokkos path (Serial build with
SPARTA_KOKKOS_EXACT, 1 thread) over 500 steps of a reacting air case, both
for stats reductions and a per-cell grid dump covering vector output and the
reaction every/select array modes; also verified for non-reacting group
collisions. With 4 OpenMP threads the per-cell tally identities
(sum gas/collision/grid == ncoll-nreact, sum gas/reaction/grid == nreact)
hold exactly. Adds examples/chem/in.chem.gastally.

Co-Authored-By: Claude Opus 4.8 
Claude-Session: https://claude.ai/code/session_01Q6nQVuxEiTUqSzDoDppCUM
Co-authored-by: stanmoore1 
---
 doc/Section_commands.html                     |   2 +-
 doc/Section_commands.txt                      |   4 +-
 examples/chem/in.chem.gastally                |  51 +++++++
 src/KOKKOS/collide_vss_kokkos.cpp             | 130 ++++++++++++++---
 src/KOKKOS/collide_vss_kokkos.h               |  13 ++
 .../compute_gas_collision_grid_kokkos.cpp     | 106 ++++++++++++++
 .../compute_gas_collision_grid_kokkos.h       |  88 +++++++++++
 .../compute_gas_reaction_grid_kokkos.cpp      | 138 ++++++++++++++++++
 src/KOKKOS/compute_gas_reaction_grid_kokkos.h | 104 +++++++++++++
 src/KOKKOS/update_kokkos.cpp                  |   4 +-
 src/compute_gas_collision_grid.h              |   1 +
 src/compute_gas_reaction_grid.h               |   1 +
 12 files changed, 619 insertions(+), 23 deletions(-)
 create mode 100644 examples/chem/in.chem.gastally
 create mode 100644 src/KOKKOS/compute_gas_collision_grid_kokkos.cpp
 create mode 100644 src/KOKKOS/compute_gas_collision_grid_kokkos.h
 create mode 100644 src/KOKKOS/compute_gas_reaction_grid_kokkos.cpp
 create mode 100644 src/KOKKOS/compute_gas_reaction_grid_kokkos.h

diff --git a/doc/Section_commands.html b/doc/Section_commands.html
index 31e03bfe3..546759339 100644
--- a/doc/Section_commands.html
+++ b/doc/Section_commands.html
@@ -361,7 +361,7 @@ 
      Compute styles
 
      
 
      
-
+
      
 
      boundary (k)count (k)distsurf/grid (k)dt/grid (k)eflux/grid (k)fft/grid (k)
      gas/collision/gridgas/collision/tallygas/reaction/gridgas/reaction/tallygrid (k)isurf/grid (k)
      gas/collision/grid (k)gas/collision/tallygas/reaction/grid (k)gas/reaction/tallygrid (k)isurf/grid (k)
      
 
      ke/particle (k)lambda/grid (k)pflux/grid (k)property/grid (k)property/surf (k)react/boundary
      
 
      react/surf (k)react/isurf/grid (k)reducesonine/grid (k)surf (k)surf/collision/tally
      
 
      surf/reaction/tallytemp (k)thermal/grid (k)tvib/grid (k) 
diff --git a/doc/Section_commands.txt b/doc/Section_commands.txt
index 34f395527..44ef3e9bf 100644
--- a/doc/Section_commands.txt
+++ b/doc/Section_commands.txt
@@ -436,9 +436,9 @@ letters in parenthesis: k = KOKKOS.
 "dt/grid (k)"_compute_dt_grid.html,
 "eflux/grid (k)"_compute_eflux_grid.html,
 "fft/grid (k)"_compute_fft_grid.html,
-"gas/collision/grid"_compute_gas_collision_grid.html,
+"gas/collision/grid (k)"_compute_gas_collision_grid.html,
 "gas/collision/tally"_compute_gas_collision_tally.html,
-"gas/reaction/grid"_compute_gas_reaction_grid.html,
+"gas/reaction/grid (k)"_compute_gas_reaction_grid.html,
 "gas/reaction/tally"_compute_gas_reaction_tally.html,
 "grid (k)"_compute_grid.html,
 "isurf/grid (k)"_compute_isurf_grid.html,
diff --git a/examples/chem/in.chem.gastally b/examples/chem/in.chem.gastally
new file mode 100644
index 000000000..dfd522925
--- /dev/null
+++ b/examples/chem/in.chem.gastally
@@ -0,0 +1,51 @@
+################################################################################
+# thermal gas in a 3d box with collisions and reactions
+# tally per-grid-cell gas collisions and reactions
+#
+# Demonstrates/verifies compute gas/collision/grid and compute gas/reaction/grid
+# (the latter in all/every/select modes).
+#
+# Note:
+#  - The "comm/sort" option to the "global" command is used to match MPI runs.
+#  - The "twopass" option is used to match Kokkos runs.
+# The "comm/sort" and "twopass" options should not be used for production runs.
+################################################################################
+
+seed	    	    12345
+dimension   	    3
+global              gridcut 1.0e-5 comm/sort yes
+boundary	    rr rr rr
+create_box  	    0 0.0001 0 0.0001 0 0.0001
+create_grid 	    10 10 10
+balance_grid        rcb part
+
+species		    air.species N2 N
+mixture		    air N2 N vstream 0.0 0.0 0.0 temp 20000.0
+mixture             air N2 frac 1.0
+mixture             air N frac 0.0
+
+global              nrho 7.07043E22
+global              fnum 7.07043E5
+
+collide		    vss air air.vss
+react               tce air.tce
+
+create_particles    air n 10000 twopass
+
+# per-grid-cell tallies of gas collisions and reactions
+
+compute             cc gas/collision/grid all all
+compute             cr gas/reaction/grid all all all
+compute             cre gas/reaction/grid all all every
+
+# sums over all cells: c_sumcc == ncoll-nreact and c_sumcr == nreact each step
+
+compute             sumcc reduce sum c_cc
+compute             sumcr reduce sum c_cr
+
+stats		    100
+compute             temp temp
+stats_style	    step np nattempt ncoll nreact c_temp c_sumcc c_sumcr
+
+timestep 	    7.00E-9
+run 		    500
diff --git a/src/KOKKOS/collide_vss_kokkos.cpp b/src/KOKKOS/collide_vss_kokkos.cpp
index 065f25234..14bd975d6 100644
--- a/src/KOKKOS/collide_vss_kokkos.cpp
+++ b/src/KOKKOS/collide_vss_kokkos.cpp
@@ -37,6 +37,10 @@
 using namespace SPARTA_NS;
 using namespace MathConst;
 
+#define VAL_1(X) X
+#define VAL_2(X) VAL_1(X), VAL_1(X)
+#define VAL_4(X) VAL_2(X), VAL_2(X)
+
 enum{NONE,DISCRETE,SMOOTH};            // several files
 enum{CONSTANT,VARIABLE};
 
@@ -62,10 +66,15 @@ CollideVSSKokkos::CollideVSSKokkos(SPARTA *sparta, int narg, char **arg) :
   grid_kk_copy(sparta),
   react_kk_copy(sparta),
   react_qk_kk_copy(sparta),
-  react_tceqk_kk_copy(sparta)
+  react_tceqk_kk_copy(sparta),
+  glist_collision_copy{VAL_4(KKCopy(sparta))},
+  glist_reaction_copy{VAL_4(KKCopy(sparta))},
+  tmp_compute_gas_collision_kk(sparta),
+  tmp_compute_gas_reaction_kk(sparta)
 {
   kokkos_flag = 1;
   react_style = 0;
+  nglist_collision = nglist_reaction = 0;
 
   // use 1D view for scalars to reduce GPU memory operations
 
@@ -113,6 +122,11 @@ CollideVSSKokkos::~CollideVSSKokkos()
   react_qk_kk_copy.uncopy();
   react_tceqk_kk_copy.uncopy();
 
+  for (int i = 0; i < KOKKOS_MAX_GLIST; i++) {
+    glist_collision_copy[i].uncopy();
+    glist_reaction_copy[i].uncopy();
+  }
+
   memoryKK->destroy_kokkos(k_dellist,dellist);
 
 #ifdef SPARTA_KOKKOS_EXACT
@@ -408,8 +422,11 @@ void CollideVSSKokkos::collisions()
   // variant for ngas_tally active or not
   // variant for single group or multiple groups
 
-  if (ngas_tally)
-    error->all(FLERR,"Kokkos does not (yet) support tallying gas/gas collisions or reactions");
+  // partition active gas/gas tally computes by type into typed KKCopy lists
+  // each must be a supported Kokkos per-grid compute; call pre_gas_tally()
+  // the per-event gas/collision/tally and gas/reaction/tally are not supported
+
+  if (ngas_tally) setup_gas_tally();
 
   COLLIDE_REDUCE reduce;
 
@@ -431,7 +448,7 @@ void CollideVSSKokkos::collisions()
     } else if (ambiflag) {
       if (!ngas_tally) {
         collisions_one_ambipolar<0>(reduce);
-      } else if (!ngas_tally) {
+      } else if (ngas_tally) {
         collisions_one_ambipolar<1>(reduce);
       }
     }
@@ -447,9 +464,17 @@ void CollideVSSKokkos::collisions()
       error->all(FLERR,"Kokkos does not (yet) support multigroup ambipolar collisions");
     if (nearcp)
       error->all(FLERR,"Kokkos does not (yet) support near-neighbor group collisions");
-    collisions_group<0,0>(reduce);
+    if (!ngas_tally) {
+      collisions_group<0,0>(reduce);
+    } else if (ngas_tally) {
+      collisions_group<0,1>(reduce);
+    }
   }
 
+  // finalize active gas/gas tally computes: contribute and sync to host
+
+  if (ngas_tally) finish_gas_tally();
+
   // remove any particles deleted in chemistry reactions
   // if particles deleted/created by chemistry, particles are no longer sorted
 
@@ -486,6 +511,69 @@ void CollideVSSKokkos::collisions()
   nreact_running += nreact_one;
 }
 
+/* ----------------------------------------------------------------------
+   partition the active gas/gas tally computes (update->glist_active) into
+     typed KKCopy lists and call pre_gas_tally() on each
+   only the per-grid Kokkos computes are supported; the per-event
+     gas/collision/tally and gas/reaction/tally computes are not
+------------------------------------------------------------------------- */
+
+void CollideVSSKokkos::setup_gas_tally()
+{
+  nglist_collision = nglist_reaction = 0;
+
+  for (int i = 0; i < ngas_tally; i++) {
+    Compute *c = update->glist_active[i];
+    if (strcmp(c->style,"gas/collision/grid") == 0) {
+      ComputeGasCollisionGridKokkos *ckk =
+        dynamic_cast(c);
+      if (!ckk)
+        error->all(FLERR,"Must use Kokkos-enabled compute gas/collision/grid with Kokkos");
+      if (nglist_collision >= KOKKOS_MAX_GLIST)
+        error->all(FLERR,"Kokkos currently only supports two instances of compute gas/collision/grid");
+      ckk->pre_gas_tally();
+      glist_collision_copy[nglist_collision].copy(ckk);
+      nglist_collision++;
+    } else if (strcmp(c->style,"gas/reaction/grid") == 0) {
+      ComputeGasReactionGridKokkos *ckk =
+        dynamic_cast(c);
+      if (!ckk)
+        error->all(FLERR,"Must use Kokkos-enabled compute gas/reaction/grid with Kokkos");
+      if (nglist_reaction >= KOKKOS_MAX_GLIST)
+        error->all(FLERR,"Kokkos currently only supports two instances of compute gas/reaction/grid");
+      ckk->pre_gas_tally();
+      glist_reaction_copy[nglist_reaction].copy(ckk);
+      nglist_reaction++;
+    } else {
+      error->all(FLERR,"Kokkos does not (yet) support compute gas/collision/tally or compute gas/reaction/tally");
+    }
+  }
+
+  // fill unused slots of each typed copy list with the temporary
+  //   to avoid the copy getting stale leading to an issue with view ref counting
+
+  for (int i = nglist_collision; i < KOKKOS_MAX_GLIST; i++)
+    glist_collision_copy[i].copy(&tmp_compute_gas_collision_kk);
+  for (int i = nglist_reaction; i < KOKKOS_MAX_GLIST; i++)
+    glist_reaction_copy[i].copy(&tmp_compute_gas_reaction_kk);
+}
+
+/* ----------------------------------------------------------------------
+   finalize the active gas/gas tally computes
+   call post_gas_tally() on the real compute objects (not the copies)
+------------------------------------------------------------------------- */
+
+void CollideVSSKokkos::finish_gas_tally()
+{
+  for (int i = 0; i < ngas_tally; i++) {
+    Compute *c = update->glist_active[i];
+    if (strcmp(c->style,"gas/collision/grid") == 0)
+      ((ComputeGasCollisionGridKokkos*)c)->post_gas_tally();
+    else if (strcmp(c->style,"gas/reaction/grid") == 0)
+      ((ComputeGasReactionGridKokkos*)c)->post_gas_tally();
+  }
+}
+
 /* ----------------------------------------------------------------------
    NTC algorithm for a single group
 ------------------------------------------------------------------------- */
@@ -786,10 +874,12 @@ void CollideVSSKokkos::operator()(TagCollideCollisionsOne< NEARCP, GASTALLY, ATO
     else
       reduce.ncollide_one++;
 
-    //if (GASTALLY)
-    //  for (int m = 0; m < ngas_tally; m++)
-    //    glist_active[m]->gas_tally(icell,reactflag,
-    //                               &iorig,&jorig,ipart,jpart,kpart); //////
+    if (GASTALLY) {
+      for (int m = 0; m < nglist_collision; m++)
+        glist_collision_copy[m].obj.template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart);
+      for (int m = 0; m < nglist_reaction; m++)
+        glist_reaction_copy[m].obj.template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart);
+    }
 
     if (reactflag) {
       if (ATOMIC_REDUCTION == 1)
@@ -1029,7 +1119,7 @@ void CollideVSSKokkos::operator()(TagCollideCollisionsGroup< NEARCP, GASTALLY, A
         int index_kpart = 0;
 
         setup_collision_kokkos(ipart,jpart,precoln,postcoln);
-        perform_collision_kokkos(ipart,jpart,kpart,precoln,postcoln,rand_gen,
+        const int reactflag = perform_collision_kokkos(ipart,jpart,kpart,precoln,postcoln,rand_gen,
                                  recomb_part3,recomb_species,recomb_density,index_kpart);
 
         if (ATOMIC_REDUCTION == 1)
@@ -1039,10 +1129,12 @@ void CollideVSSKokkos::operator()(TagCollideCollisionsGroup< NEARCP, GASTALLY, A
         else
           reduce.ncollide_one++;
 
-        //if (GASTALLY)
-        //  for (int m = 0; m < ngas_tally; m++)
-        //    glist_active[m]->gas_tally(icell,reactflag,
-        //                               &iorig,&jorig,ipart,jpart,kpart);
+        if (GASTALLY) {
+          for (int m = 0; m < nglist_collision; m++)
+            glist_collision_copy[m].obj.template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart);
+          for (int m = 0; m < nglist_reaction; m++)
+            glist_reaction_copy[m].obj.template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart);
+        }
       }
     }
 
@@ -1411,10 +1503,12 @@ void CollideVSSKokkos::operator()(TagCollideCollisionsOneAmbipolar< GASTALLY, AT
     else
       reduce.ncollide_one++;
 
-    //if (GASTALLY)
-    //  for (int m = 0; m < ngas_tally; m++)
-    //    glist_active[m]->gas_tally(icell,reactflag,
-    //                               &iorig,&jorig,ipart,jpart,kpart); //////
+    if (GASTALLY) {
+      for (int m = 0; m < nglist_collision; m++)
+        glist_collision_copy[m].obj.template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart);
+      for (int m = 0; m < nglist_reaction; m++)
+        glist_reaction_copy[m].obj.template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart);
+    }
 
     if (reactflag) {
       if (ATOMIC_REDUCTION == 1)
diff --git a/src/KOKKOS/collide_vss_kokkos.h b/src/KOKKOS/collide_vss_kokkos.h
index 0c80f6e69..ec62b333f 100644
--- a/src/KOKKOS/collide_vss_kokkos.h
+++ b/src/KOKKOS/collide_vss_kokkos.h
@@ -32,6 +32,10 @@ CollideStyle(vss/kk,CollideVSSKokkos)
 #include "Kokkos_Random.hpp"
 #include "rand_pool_wrap.h"
 #include "kokkos_copy.h"
+#include "compute_gas_collision_grid_kokkos.h"
+#include "compute_gas_reaction_grid_kokkos.h"
+
+#define KOKKOS_MAX_GLIST 4
 
 namespace SPARTA_NS {
 
@@ -157,6 +161,15 @@ class CollideVSSKokkos : public CollideVSS {
   KKCopy react_tceqk_kk_copy;
   int react_style;   // 0=TCE, 1=QK, 2=TCEQK (set in setup)
 
+  // active gas/gas per-grid tally computes, partitioned by type
+  KKCopy glist_collision_copy[KOKKOS_MAX_GLIST];
+  KKCopy glist_reaction_copy[KOKKOS_MAX_GLIST];
+  int nglist_collision,nglist_reaction;
+  ComputeGasCollisionGridKokkos tmp_compute_gas_collision_kk;
+  ComputeGasReactionGridKokkos tmp_compute_gas_reaction_kk;
+  void setup_gas_tally();
+  void finish_gas_tally();
+
   t_particle_1d d_particles;
   t_species_1d_const d_species;
   DAT::t_int_2d d_plist;
diff --git a/src/KOKKOS/compute_gas_collision_grid_kokkos.cpp b/src/KOKKOS/compute_gas_collision_grid_kokkos.cpp
new file mode 100644
index 000000000..9df6b374a
--- /dev/null
+++ b/src/KOKKOS/compute_gas_collision_grid_kokkos.cpp
@@ -0,0 +1,106 @@
+/* ----------------------------------------------------------------------
+   SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer
+   http://sparta.github.io
+   Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov
+   Sandia National Laboratories
+
+   Copyright (2014) Sandia Corporation.  Under the terms of Contract
+   DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains
+   certain rights in this software.  This software is distributed under
+   the GNU General Public License.
+
+   See the README file in the top-level SPARTA directory.
+------------------------------------------------------------------------- */
+
+#include "compute_gas_collision_grid_kokkos.h"
+#include "particle_kokkos.h"
+#include "grid_kokkos.h"
+#include "memory_kokkos.h"
+#include "sparta_masks.h"
+#include "kokkos.h"
+
+using namespace SPARTA_NS;
+
+/* ---------------------------------------------------------------------- */
+
+ComputeGasCollisionGridKokkos::ComputeGasCollisionGridKokkos(SPARTA *sparta, int narg, char **arg) :
+  ComputeGasCollisionGrid(sparta, narg, arg)
+{
+  kokkos_flag = 1;
+}
+
+/* ---------------------------------------------------------------------- */
+
+ComputeGasCollisionGridKokkos::ComputeGasCollisionGridKokkos(SPARTA *sparta) :
+  ComputeGasCollisionGrid(sparta)
+{
+  copy = 1;
+  uncopy = 0;
+}
+
+/* ---------------------------------------------------------------------- */
+
+ComputeGasCollisionGridKokkos::~ComputeGasCollisionGridKokkos()
+{
+  if (copy || copymode) return;
+
+  memoryKK->destroy_kokkos(k_vector_grid,vector_grid);
+  vector_grid = NULL;
+}
+
+/* ----------------------------------------------------------------------
+   zero the tally array on device
+   called by Update at beginning of timesteps gas tallying is done
+------------------------------------------------------------------------- */
+
+void ComputeGasCollisionGridKokkos::clear()
+{
+  Kokkos::deep_copy(d_vector_grid,0.0);
+}
+
+/* ----------------------------------------------------------------------
+   setup device views and scatter view before gas tallying
+   called by Collide before the collision kernel
+------------------------------------------------------------------------- */
+
+void ComputeGasCollisionGridKokkos::pre_gas_tally()
+{
+  ParticleKokkos* particle_kk = (ParticleKokkos*) particle;
+  d_s2g = particle_kk->k_species2group.view_device();
+
+  GridKokkos* grid_kk = (GridKokkos*) grid;
+  grid_kk->sync(Device,CINFO_MASK);
+  d_cinfo = grid_kk->k_cinfo.view_device();
+}
+
+/* ----------------------------------------------------------------------
+   finalize gas tallying, sync result to host
+   called by Collide after the collision kernel
+------------------------------------------------------------------------- */
+
+void ComputeGasCollisionGridKokkos::post_gas_tally()
+{
+  k_vector_grid.modify_device();
+  k_vector_grid.sync_host();
+}
+
+/* ----------------------------------------------------------------------
+   reallocate data storage if nglocal has changed
+   called by init() and whenever grid changes
+------------------------------------------------------------------------- */
+
+void ComputeGasCollisionGridKokkos::reallocate()
+{
+  if (grid->nlocal == nglocal) return;
+
+  memoryKK->destroy_kokkos(k_vector_grid,vector_grid);
+  nglocal = grid->nlocal;
+  memoryKK->create_kokkos(k_vector_grid,vector_grid,nglocal,"gas/collision/grid:vector_grid");
+  d_vector_grid = k_vector_grid.view_device();
+
+  // clear counts b/c may be accessed before tallying is done
+
+  Kokkos::deep_copy(d_vector_grid,0.0);
+  k_vector_grid.modify_device();
+  k_vector_grid.sync_host();
+}
diff --git a/src/KOKKOS/compute_gas_collision_grid_kokkos.h b/src/KOKKOS/compute_gas_collision_grid_kokkos.h
new file mode 100644
index 000000000..d0cbd9769
--- /dev/null
+++ b/src/KOKKOS/compute_gas_collision_grid_kokkos.h
@@ -0,0 +1,88 @@
+/* ----------------------------------------------------------------------
+   SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer
+   http://sparta.github.io
+   Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov
+   Sandia National Laboratories
+
+   Copyright (2014) Sandia Corporation.  Under the terms of Contract
+   DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains
+   certain rights in this software.  This software is distributed under
+   the GNU General Public License.
+
+   See the README file in the top-level SPARTA directory.
+------------------------------------------------------------------------- */
+
+#ifdef COMPUTE_CLASS
+
+ComputeStyle(gas/collision/grid/kk,ComputeGasCollisionGridKokkos)
+
+#else
+
+#ifndef SPARTA_COMPUTE_GAS_COLLISION_GRID_KOKKOS_H
+#define SPARTA_COMPUTE_GAS_COLLISION_GRID_KOKKOS_H
+
+#include "compute_gas_collision_grid.h"
+#include "kokkos_type.h"
+#include "particle.h"
+
+namespace SPARTA_NS {
+
+class ComputeGasCollisionGridKokkos : public ComputeGasCollisionGrid {
+ public:
+  ComputeGasCollisionGridKokkos(class SPARTA *, int, char **);
+  ComputeGasCollisionGridKokkos(class SPARTA *);
+  ~ComputeGasCollisionGridKokkos();
+  void clear();
+  void pre_gas_tally();
+  void post_gas_tally();
+  void reallocate();
+
+  // tally a single gas collision in icell on device
+  // reaction = 0 for a collision that did not induce a reaction
+  // this compute only tallies non-reacting collisions
+  // Collide parallelizes over grid cells (one icell per thread), so the
+  //   per-cell tally has no write contention and needs no atomics/duplication
+  // ATOMIC_REDUCTION template arg is unused, kept for a uniform call interface
+
+  template
+  KOKKOS_INLINE_FUNCTION
+  void gas_tally_kk(int icell, int reaction,
+                    Particle::OnePart *iorig, Particle::OnePart *jorig,
+                    Particle::OnePart * /*ip*/, Particle::OnePart * /*jp*/,
+                    Particle::OnePart * /*kp*/) const
+  {
+    // skip if a reaction (reactions tallied by compute gas/reaction/grid)
+
+    if (reaction) return;
+
+    // skip if icell not in grid group
+
+    if (!(d_cinfo[icell].mask & groupbit)) return;
+
+    // skip if either particle species not in mixture group
+
+    int igroup = d_s2g(imix,iorig->ispecies);
+    int jgroup = d_s2g(imix,jorig->ispecies);
+    if (igroup < 0 || jgroup < 0) return;
+
+    // tally the collision to its grid cell
+
+    d_vector_grid(icell) += 1.0;
+  }
+
+ private:
+  DAT::tdual_float_1d k_vector_grid;
+  DAT::t_float_1d d_vector_grid;
+
+  t_cinfo_1d d_cinfo;
+  DAT::t_int_2d d_s2g;
+};
+
+}
+
+#endif
+#endif
+
+/* ERROR/WARNING messages:
+
+*/
diff --git a/src/KOKKOS/compute_gas_reaction_grid_kokkos.cpp b/src/KOKKOS/compute_gas_reaction_grid_kokkos.cpp
new file mode 100644
index 000000000..4ddf8bcb4
--- /dev/null
+++ b/src/KOKKOS/compute_gas_reaction_grid_kokkos.cpp
@@ -0,0 +1,138 @@
+/* ----------------------------------------------------------------------
+   SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer
+   http://sparta.github.io
+   Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov
+   Sandia National Laboratories
+
+   Copyright (2014) Sandia Corporation.  Under the terms of Contract
+   DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains
+   certain rights in this software.  This software is distributed under
+   the GNU General Public License.
+
+   See the README file in the top-level SPARTA directory.
+------------------------------------------------------------------------- */
+
+#include "compute_gas_reaction_grid_kokkos.h"
+#include "particle_kokkos.h"
+#include "grid_kokkos.h"
+#include "react.h"
+#include "memory_kokkos.h"
+#include "sparta_masks.h"
+#include "kokkos.h"
+
+using namespace SPARTA_NS;
+
+/* ---------------------------------------------------------------------- */
+
+ComputeGasReactionGridKokkos::ComputeGasReactionGridKokkos(SPARTA *sparta, int narg, char **arg) :
+  ComputeGasReactionGrid(sparta, narg, arg)
+{
+  kokkos_flag = 1;
+}
+
+/* ---------------------------------------------------------------------- */
+
+ComputeGasReactionGridKokkos::ComputeGasReactionGridKokkos(SPARTA *sparta) :
+  ComputeGasReactionGrid(sparta)
+{
+  copy = 1;
+  uncopy = 0;
+}
+
+/* ---------------------------------------------------------------------- */
+
+ComputeGasReactionGridKokkos::~ComputeGasReactionGridKokkos()
+{
+  if (copy || copymode) return;
+
+  if (ncol == 0) memoryKK->destroy_kokkos(k_vector_grid,vector_grid);
+  else memoryKK->destroy_kokkos(k_array_grid,array_grid);
+  vector_grid = NULL;
+  array_grid = NULL;
+}
+
+/* ---------------------------------------------------------------------- */
+
+void ComputeGasReactionGridKokkos::init()
+{
+  ComputeGasReactionGrid::init();
+
+  // device copy of reaction -> column map for SELECT mode
+
+  if (mode == SELECT) {
+    int n = react->nlist + 1;
+    d_reaction2col = DAT::t_int_1d("gas/reaction/grid:reaction2col",n);
+    auto h_reaction2col = Kokkos::create_mirror_view(d_reaction2col);
+    for (int i = 0; i < n; i++) h_reaction2col(i) = reaction2col[i];
+    Kokkos::deep_copy(d_reaction2col,h_reaction2col);
+  }
+}
+
+/* ----------------------------------------------------------------------
+   zero the tally array on device
+   called by Update at beginning of timesteps gas tallying is done
+------------------------------------------------------------------------- */
+
+void ComputeGasReactionGridKokkos::clear()
+{
+  if (ncol == 0) Kokkos::deep_copy(d_vector_grid,0.0);
+  else Kokkos::deep_copy(d_array_grid,0.0);
+}
+
+/* ----------------------------------------------------------------------
+   setup device views and scatter view before gas tallying
+   called by Collide before the collision kernel
+------------------------------------------------------------------------- */
+
+void ComputeGasReactionGridKokkos::pre_gas_tally()
+{
+  ParticleKokkos* particle_kk = (ParticleKokkos*) particle;
+  d_s2g = particle_kk->k_species2group.view_device();
+
+  GridKokkos* grid_kk = (GridKokkos*) grid;
+  grid_kk->sync(Device,CINFO_MASK);
+  d_cinfo = grid_kk->k_cinfo.view_device();
+}
+
+/* ----------------------------------------------------------------------
+   finalize gas tallying, sync result to host
+   called by Collide after the collision kernel
+------------------------------------------------------------------------- */
+
+void ComputeGasReactionGridKokkos::post_gas_tally()
+{
+  if (ncol == 0) {
+    k_vector_grid.modify_device();
+    k_vector_grid.sync_host();
+  } else {
+    k_array_grid.modify_device();
+    k_array_grid.sync_host();
+  }
+}
+
+/* ----------------------------------------------------------------------
+   reallocate data storage if nglocal has changed
+   called by init() and whenever grid changes
+------------------------------------------------------------------------- */
+
+void ComputeGasReactionGridKokkos::reallocate()
+{
+  if (grid->nlocal == nglocal) return;
+
+  if (ncol == 0) memoryKK->destroy_kokkos(k_vector_grid,vector_grid);
+  else memoryKK->destroy_kokkos(k_array_grid,array_grid);
+  nglocal = grid->nlocal;
+  if (ncol == 0) {
+    memoryKK->create_kokkos(k_vector_grid,vector_grid,nglocal,"gas/reaction/grid:vector_grid");
+    d_vector_grid = k_vector_grid.view_device();
+    Kokkos::deep_copy(d_vector_grid,0.0);
+    k_vector_grid.modify_device();
+    k_vector_grid.sync_host();
+  } else {
+    memoryKK->create_kokkos(k_array_grid,array_grid,nglocal,ncol,"gas/reaction/grid:array_grid");
+    d_array_grid = k_array_grid.view_device();
+    Kokkos::deep_copy(d_array_grid,0.0);
+    k_array_grid.modify_device();
+    k_array_grid.sync_host();
+  }
+}
diff --git a/src/KOKKOS/compute_gas_reaction_grid_kokkos.h b/src/KOKKOS/compute_gas_reaction_grid_kokkos.h
new file mode 100644
index 000000000..f0d6313df
--- /dev/null
+++ b/src/KOKKOS/compute_gas_reaction_grid_kokkos.h
@@ -0,0 +1,104 @@
+/* ----------------------------------------------------------------------
+   SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer
+   http://sparta.github.io
+   Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov
+   Sandia National Laboratories
+
+   Copyright (2014) Sandia Corporation.  Under the terms of Contract
+   DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains
+   certain rights in this software.  This software is distributed under
+   the GNU General Public License.
+
+   See the README file in the top-level SPARTA directory.
+------------------------------------------------------------------------- */
+
+#ifdef COMPUTE_CLASS
+
+ComputeStyle(gas/reaction/grid/kk,ComputeGasReactionGridKokkos)
+
+#else
+
+#ifndef SPARTA_COMPUTE_GAS_REACTION_GRID_KOKKOS_H
+#define SPARTA_COMPUTE_GAS_REACTION_GRID_KOKKOS_H
+
+#include "compute_gas_reaction_grid.h"
+#include "kokkos_type.h"
+#include "particle.h"
+
+namespace SPARTA_NS {
+
+class ComputeGasReactionGridKokkos : public ComputeGasReactionGrid {
+ public:
+  enum{ALL,EVERY,SELECT};        // must match compute_gas_reaction_grid.cpp
+
+  ComputeGasReactionGridKokkos(class SPARTA *, int, char **);
+  ComputeGasReactionGridKokkos(class SPARTA *);
+  ~ComputeGasReactionGridKokkos();
+  void init();
+  void clear();
+  void pre_gas_tally();
+  void post_gas_tally();
+  void reallocate();
+
+  // tally a single gas reaction in icell on device
+  // reaction = 1 to N for which reaction, 0 = collision only (skipped)
+  // this compute only tallies reacting collisions
+  // Collide parallelizes over grid cells (one icell per thread), so the
+  //   per-cell tally has no write contention and needs no atomics/duplication
+  // ATOMIC_REDUCTION template arg is unused, kept for a uniform call interface
+
+  template
+  KOKKOS_INLINE_FUNCTION
+  void gas_tally_kk(int icell, int reaction,
+                    Particle::OnePart *iorig, Particle::OnePart *jorig,
+                    Particle::OnePart * /*ip*/, Particle::OnePart * /*jp*/,
+                    Particle::OnePart * /*kp*/) const
+  {
+    // skip if not a reaction (collisions tallied by compute gas/collision/grid)
+
+    if (!reaction) return;
+
+    // skip if icell not in grid group
+
+    if (!(d_cinfo[icell].mask & groupbit)) return;
+
+    // skip if either particle species not in mixture group
+
+    int igroup = d_s2g(imix,iorig->ispecies);
+    int jgroup = d_s2g(imix,jorig->ispecies);
+    if (igroup < 0 || jgroup < 0) return;
+
+    // tally the reaction to its grid cell
+    // for EVERY and SELECT mode, reaction index determines column of array_grid
+
+    if (mode == ALL) {
+      d_vector_grid(icell) += 1.0;
+    } else if (mode == EVERY) {
+      int icol = reaction - 1;
+      d_array_grid(icell,icol) += 1.0;
+    } else {   // SELECT
+      int icol = d_reaction2col(reaction);
+      if (icol >= 0) d_array_grid(icell,icol) += 1.0;
+    }
+  }
+
+ private:
+  DAT::tdual_float_1d k_vector_grid;
+  DAT::t_float_1d d_vector_grid;
+  DAT::tdual_float_2d_lr k_array_grid;
+  DAT::t_float_2d_lr d_array_grid;
+
+  DAT::t_int_1d d_reaction2col;   // reaction -> column map for SELECT mode
+
+  t_cinfo_1d d_cinfo;
+  DAT::t_int_2d d_s2g;
+};
+
+}
+
+#endif
+#endif
+
+/* ERROR/WARNING messages:
+
+*/
diff --git a/src/KOKKOS/update_kokkos.cpp b/src/KOKKOS/update_kokkos.cpp
index 5c79a5902..014373d8f 100644
--- a/src/KOKKOS/update_kokkos.cpp
+++ b/src/KOKKOS/update_kokkos.cpp
@@ -2157,8 +2157,8 @@ void UpdateKokkos::tally_set(bigint ntimestep)
   for (i = nslist_react_surf; i < KOKKOS_MAX_SLIST; i++)
     slist_active_react_surf_copy[i].copy(&tmp_compute_react_surf_kk);
 
-  if (ngas_tally)
-    error->all(FLERR,"Kokkos does not (yet) support tallying gas/gas collisions or reactions");
+  // gas/gas tally computes are validated and set up by CollideVSSKokkos,
+  //   which invokes their on-device gas_tally_kk() from the collision kernel
 }
 
 /* ---------------------------------------------------------------------- */
diff --git a/src/compute_gas_collision_grid.h b/src/compute_gas_collision_grid.h
index 4272f93e9..34e2a5d18 100644
--- a/src/compute_gas_collision_grid.h
+++ b/src/compute_gas_collision_grid.h
@@ -29,6 +29,7 @@ namespace SPARTA_NS {
 class ComputeGasCollisionGrid : public Compute {
  public:
   ComputeGasCollisionGrid(class SPARTA *, int, char **);
+  ComputeGasCollisionGrid(class SPARTA* sparta) : Compute(sparta) {} // needed for Kokkos
   ~ComputeGasCollisionGrid();
   void init();
   void compute_per_grid();
diff --git a/src/compute_gas_reaction_grid.h b/src/compute_gas_reaction_grid.h
index 5cb5a8f59..77c8b2bb4 100644
--- a/src/compute_gas_reaction_grid.h
+++ b/src/compute_gas_reaction_grid.h
@@ -29,6 +29,7 @@ namespace SPARTA_NS {
 class ComputeGasReactionGrid : public Compute {
  public:
   ComputeGasReactionGrid(class SPARTA *, int, char **);
+  ComputeGasReactionGrid(class SPARTA* sparta) : Compute(sparta) {} // needed for Kokkos
   ~ComputeGasReactionGrid();
   void init();
   void compute_per_grid();

From 10d749b0be6c96d8dc817e366ea387e0379dadd5 Mon Sep 17 00:00:00 2001
From: Claude 
Date: Tue, 23 Jun 2026 05:13:45 +0000
Subject: [PATCH 22/30] KOKKOS: add fix ave/surf/kk for surf-tally computes

Add a Kokkos-enabled fix ave/surf that accelerates the dominant use case:
time-averaging the per-surf output of surf-tally computes (compute surf,
compute react/surf). Previously fix ave/surf ran only on the host in a
Kokkos run, syncing each compute's per-surf tally device->host every sample.

Design (mirrors fix_ave_grid_kokkos's selective-acceleration philosophy):
 - The frequent per-sample work runs on device: each value reads a column of
   its compute's per-local-surf device tally and adds it into a device
   accumulator d_acc, with no per-sample device->host sync. Collide/move
   already produce the compute tally on device (post_surf_tally); the new
   query_tally_surf_kokkos() accessor exposes it without a host round-trip.
   The accumulation parallelizes over local surfs (one row per work item),
   so it needs no atomics.
 - The infrequent per-output work stays on host and mirrors the CPU base
   class exactly: surf->collate_* merges per-local-surf tallies to owned
   surfs (MPI), then normalize by sample count and apply the group mask.
 - The non-tally path (fix/variable/custom inputs, count_tally == 0) is
   delegated entirely to the host base class.

Also fixes a latent bug: FixAveSurf::~FixAveSurf lacked the copymode guard
that FixAveGrid has, so when Kokkos copies the fix functor for a kernel and
destroys the copy, the shared which/argindex/value2index/ids arrays were
freed out from under the real fix. Added the guard and made the base members
protected so the Kokkos subclass can reach them.

Verified bit-for-bit identical to the non-Kokkos path (Serial build with
SPARTA_KOKKOS_EXACT, 1 thread) for both the vector path (ave one) and the
multi-column array path (ave running), via surf dumps of the averaged output
over 500 steps of examples/adjust_temp/in.circle.constant. The 4-thread
OpenMP build runs cleanly. Marks fix ave/surf (k) in the command list.

Co-Authored-By: Claude Opus 4.8 
Claude-Session: https://claude.ai/code/session_01Q6nQVuxEiTUqSzDoDppCUM
Co-authored-by: stanmoore1 
---
 doc/Section_commands.html              |   2 +-
 doc/Section_commands.txt               |   2 +-
 src/KOKKOS/compute_react_surf_kokkos.h |   3 +
 src/KOKKOS/compute_surf_kokkos.h       |   3 +
 src/KOKKOS/fix_ave_surf_kokkos.cpp     | 265 +++++++++++++++++++++++++
 src/KOKKOS/fix_ave_surf_kokkos.h       |  66 ++++++
 src/fix_ave_surf.cpp                   |   2 +
 src/fix_ave_surf.h                     |   2 +-
 8 files changed, 342 insertions(+), 3 deletions(-)
 create mode 100644 src/KOKKOS/fix_ave_surf_kokkos.cpp
 create mode 100644 src/KOKKOS/fix_ave_surf_kokkos.h

diff --git a/doc/Section_commands.html b/doc/Section_commands.html
index 546759339..2375dc97e 100644
--- a/doc/Section_commands.html
+++ b/doc/Section_commands.html
@@ -343,7 +343,7 @@ 
      Fix styles
 
      
 
diff --git a/doc/Section_commands.txt b/doc/Section_commands.txt
index 44ef3e9bf..200fa442e 100644
--- a/doc/Section_commands.txt
+++ b/doc/Section_commands.txt
@@ -400,7 +400,7 @@ This is indicated by additional letters in parenthesis: k = KOKKOS.
 "ave/grid (k)"_fix_ave_grid.html,
 "ave/histo (k)"_fix_ave_histo.html,
 "ave/histo/weight (k)"_fix_ave_histo.html,
-"ave/surf"_fix_ave_surf.html,
+"ave/surf (k)"_fix_ave_surf.html,
 "ave/time"_fix_ave_time.html,
 "balance (k)"_fix_balance.html,
 "custom"_fix_custom.html,
diff --git a/src/KOKKOS/compute_react_surf_kokkos.h b/src/KOKKOS/compute_react_surf_kokkos.h
index 91a9a38b2..fa96316fd 100644
--- a/src/KOKKOS/compute_react_surf_kokkos.h
+++ b/src/KOKKOS/compute_react_surf_kokkos.h
@@ -38,6 +38,9 @@ class ComputeReactSurfKokkos : public ComputeReactSurf {
   void pre_surf_tally();
   void post_surf_tally();
 
+  // expose the per-local-surf device tally array for fix ave/surf/kk
+  void query_tally_surf_kokkos(DAT::t_float_2d_lr &d_array) { d_array = d_array_surf_tally; }
+
 /* ----------------------------------------------------------------------
    tally a surface reaction for particle colliding with surf element isurf
    mirrors ComputeReactSurf::surf_tally(); per-surf tally compressed to host
diff --git a/src/KOKKOS/compute_surf_kokkos.h b/src/KOKKOS/compute_surf_kokkos.h
index 3893ba59c..c9e811e17 100644
--- a/src/KOKKOS/compute_surf_kokkos.h
+++ b/src/KOKKOS/compute_surf_kokkos.h
@@ -45,6 +45,9 @@ class ComputeSurfKokkos : public ComputeSurf {
   void pre_surf_tally();
   void post_surf_tally();
 
+  // expose the per-local-surf device tally array for fix ave/surf/kk
+  void query_tally_surf_kokkos(DAT::t_float_2d_lr &d_array) { d_array = d_array_surf_tally; }
+
 enum{NUM,NUMWT,NFLUX,NFLUXIN,MFLUX,MFLUXIN,FX,FY,FZ,TX,TY,TZ,
   PRESS,XPRESS,YPRESS,ZPRESS,XSHEAR,YSHEAR,ZSHEAR,KE,EROT,EVIB,ECHEM,ETOT};
 
diff --git a/src/KOKKOS/fix_ave_surf_kokkos.cpp b/src/KOKKOS/fix_ave_surf_kokkos.cpp
new file mode 100644
index 000000000..0d3403e6f
--- /dev/null
+++ b/src/KOKKOS/fix_ave_surf_kokkos.cpp
@@ -0,0 +1,265 @@
+/* ----------------------------------------------------------------------
+   SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer
+   http://sparta.github.io
+   Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov
+   Sandia National Laboratories
+
+   Copyright (2014) Sandia Corporation.  Under the terms of Contract
+   DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains
+   certain rights in this software.  This software is distributed under
+   the GNU General Public License.
+
+   See the README file in the top-level SPARTA directory.
+------------------------------------------------------------------------- */
+
+#include "spatype.h"
+#include "string.h"
+#include "fix_ave_surf_kokkos.h"
+#include "surf.h"
+#include "domain.h"
+#include "update.h"
+#include "modify.h"
+#include "compute.h"
+#include "compute_surf_kokkos.h"
+#include "compute_react_surf_kokkos.h"
+#include "memory_kokkos.h"
+#include "error.h"
+#include "sparta_masks.h"
+
+using namespace SPARTA_NS;
+
+enum{COMPUTE,FIX,VARIABLE,CUSTOM};       // must match fix_ave_surf.cpp
+enum{ONE,RUNNING};                       // must match fix_ave_surf.cpp
+
+#define INVOKED_PER_SURF 32              // must match fix_ave_surf.cpp
+
+/* ---------------------------------------------------------------------- */
+
+FixAveSurfKokkos::FixAveSurfKokkos(SPARTA *sparta, int narg, char **arg) :
+  FixAveSurf(sparta, narg, arg)
+{
+  kokkos_flag = 1;
+  execution_space = Device;
+  datamask_read = EMPTY_MASK;
+  datamask_modify = EMPTY_MASK;
+
+  // only the all-tally path (averaging surf-tally computes) is accelerated
+  //   on device.  count_tally is 0 or nvalues, enforced by the base ctor.
+  // the non-tally path (fix/variable/custom inputs) runs on the host base class
+
+  kokkosable = (count_tally && count_tally == nvalues);
+
+  nstally = 0;
+  tally2surf_all = NULL;
+  acc_local_vec = NULL;
+  acc_local = NULL;
+}
+
+/* ---------------------------------------------------------------------- */
+
+FixAveSurfKokkos::~FixAveSurfKokkos()
+{
+  if (copymode) return;
+
+  memory->destroy(tally2surf_all);
+  memory->destroy(acc_local_vec);
+  memory->destroy(acc_local);
+}
+
+/* ---------------------------------------------------------------------- */
+
+void FixAveSurfKokkos::init()
+{
+  FixAveSurf::init();
+}
+
+/* ----------------------------------------------------------------------
+   allocate per-local-surf device accumulator and host collate buffers
+   build tally2surf_all mapping each local surf row to its surf ID
+------------------------------------------------------------------------- */
+
+void FixAveSurfKokkos::reallocate()
+{
+  int n = surf->nlocal + surf->nghost;
+  if (n == nstally && d_acc.extent(0)) return;
+  nstally = n;
+
+  d_acc = DAT::t_float_2d_lr("ave/surf:acc",nstally,nvalues);
+
+  memory->destroy(tally2surf_all);
+  memory->destroy(acc_local_vec);
+  memory->destroy(acc_local);
+  acc_local_vec = NULL;
+  acc_local = NULL;
+  memory->create(tally2surf_all,nstally,"ave/surf:tally2surf_all");
+  if (nvalues == 1) memory->create(acc_local_vec,nstally,"ave/surf:acc_local_vec");
+  else memory->create(acc_local,nstally,nvalues,"ave/surf:acc_local");
+
+  // surf ID of each local surf row, used by the host collate at output
+  // matches the per-local-surf row order of the Kokkos surf-tally computes
+
+  if (domain->dimension == 2) {
+    Surf::Line *lines = surf->lines;
+    for (int i = 0; i < nstally; i++) tally2surf_all[i] = lines[i].id;
+  } else {
+    Surf::Tri *tris = surf->tris;
+    for (int i = 0; i < nstally; i++) tally2surf_all[i] = tris[i].id;
+  }
+}
+
+/* ----------------------------------------------------------------------
+   only does something if nvalid = current timestep
+------------------------------------------------------------------------- */
+
+void FixAveSurfKokkos::setup()
+{
+  if (kokkosable) reallocate();
+  end_of_step();
+}
+
+/* ---------------------------------------------------------------------- */
+
+void FixAveSurfKokkos::end_of_step()
+{
+  int i,m,n;
+
+  // non-tally path runs entirely on the host base class
+
+  if (!kokkosable) {
+    FixAveSurf::end_of_step();
+    return;
+  }
+
+  // skip if not step which requires doing something
+
+  bigint ntimestep = update->ntimestep;
+  if (ntimestep != nvalid) return;
+
+  if (nstally != surf->nlocal + surf->nghost) reallocate();
+
+  // first sample of an averaging interval:
+  //   zero the per-interval device tally accumulator (== clearing the host hash)
+  //   zero the owned-surf accumulators if ave = ONE
+
+  if (irepeat == 0) {
+    Kokkos::deep_copy(d_acc,0.0);
+    if (ave == ONE) {
+      if (nvalues == 1)
+        for (i = 0; i < nown; i++) accvec[i] = 0.0;
+      else
+        for (i = 0; i < nown; i++)
+          for (m = 0; m < nvalues; m++) accarray[i][m] = 0.0;
+    }
+  }
+
+  // accumulate this sample's compute tallies into d_acc on device
+  // each value m reads a column of its compute's per-local-surf device tally
+  // compute/fix/variable may invoke computes, so wrap with clear/add
+
+  modify->clearstep_compute();
+
+  copymode = 1;
+  for (m = 0; m < nvalues; m++) {
+    n = value2index[m];
+    Compute *compute = modify->compute[n];
+
+    if (!compute->kokkos_flag)
+      error->all(FLERR,"Cannot (yet) use non-Kokkos computes with fix ave/surf/kk");
+
+    if (!(compute->invoked_flag & INVOKED_PER_SURF)) {
+      compute->compute_per_surf();
+      compute->invoked_flag |= INVOKED_PER_SURF;
+    }
+
+    // grab the compute's per-local-surf device tally array
+
+    if (strcmp(compute->style,"surf") == 0)
+      ((ComputeSurfKokkos*) compute)->query_tally_surf_kokkos(d_tally);
+    else if (strcmp(compute->style,"react/surf") == 0)
+      ((ComputeReactSurfKokkos*) compute)->query_tally_surf_kokkos(d_tally);
+    else
+      error->all(FLERR,"Fix ave/surf/kk requires Kokkos compute surf or compute react/surf");
+
+    acc_m = m;
+    acc_col = (argindex[m] == 0) ? 0 : argindex[m] - 1;
+    Kokkos::parallel_for(Kokkos::RangePolicy(0,nstally),*this);
+  }
+  copymode = 0;
+
+  // done if irepeat < nrepeat, else reset irepeat and nvalid
+
+  nsample++;
+  irepeat++;
+  if (irepeat < nrepeat) {
+    nvalid += nevery;
+    modify->addstep_compute(nvalid);
+    return;
+  }
+
+  irepeat = 0;
+  nvalid = ntimestep+per_surf_freq - (nrepeat-1)*nevery;
+  modify->addstep_compute(nvalid);
+
+  // copy the device tally accumulator to the host
+
+  auto h_acc = Kokkos::create_mirror_view(d_acc);
+  Kokkos::deep_copy(h_acc,d_acc);
+
+  // merge per-local-surf tallies to owned surfs via surf->collate (host MPI)
+  // then add the collated interval sum to the owned-surf accumulators
+
+  if (nvalues == 1) {
+    for (i = 0; i < nstally; i++) acc_local_vec[i] = h_acc(i,0);
+    surf->collate_vector(nstally,tally2surf_all,acc_local_vec,1,bufvec);
+    for (i = 0; i < nown; i++) accvec[i] += bufvec[i];
+  } else {
+    for (i = 0; i < nstally; i++)
+      for (m = 0; m < nvalues; m++) acc_local[i][m] = h_acc(i,m);
+    surf->collate_array(nstally,nvalues,tally2surf_all,acc_local,bufarray);
+    for (i = 0; i < nown; i++)
+      for (m = 0; m < nvalues; m++) accarray[i][m] += bufarray[i][m];
+  }
+
+  // normalize the accumulators for output, just by # of samples
+
+  if (ave == ONE) {
+    if (nvalues == 1)
+      for (i = 0; i < nown; i++) vector_surf[i] /= nsample;
+    else
+      for (i = 0; i < nown; i++)
+        for (m = 0; m < nvalues; m++) array_surf[i][m] /= nsample;
+  } else {
+    if (nvalues == 1)
+      for (i = 0; i < nown; i++) vector_surf[i] = accvec[i]/nsample;
+    else
+      for (i = 0; i < nown; i++)
+        for (m = 0; m < nvalues; m++) array_surf[i][m] = accarray[i][m]/nsample;
+  }
+
+  // set values for surfs not in group to zero
+
+  if (groupbit != 1) {
+    if (nvalues == 1) {
+      for (i = 0; i < nown; i++)
+        if (!(masks[i] & groupbit)) vector_surf[i] = 0.0;
+    } else {
+      for (i = 0; i < nown; i++)
+        if (!(masks[i] & groupbit))
+          for (m = 0; m < nvalues; m++) array_surf[i][m] = 0.0;
+    }
+  }
+
+  // reset nsample if ave = ONE
+
+  if (ave == ONE) nsample = 0;
+}
+
+/* ----------------------------------------------------------------------
+   add one value's per-local-surf compute tally column into d_acc
+------------------------------------------------------------------------- */
+
+KOKKOS_INLINE_FUNCTION
+void FixAveSurfKokkos::operator()(TagFixAveSurf_Add_tally, const int &i) const
+{
+  d_acc(i,acc_m) += d_tally(i,acc_col);
+}
diff --git a/src/KOKKOS/fix_ave_surf_kokkos.h b/src/KOKKOS/fix_ave_surf_kokkos.h
new file mode 100644
index 000000000..6881ce9f5
--- /dev/null
+++ b/src/KOKKOS/fix_ave_surf_kokkos.h
@@ -0,0 +1,66 @@
+/* ----------------------------------------------------------------------
+   SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer
+   http://sparta.github.io
+   Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov
+   Sandia National Laboratories
+
+   Copyright (2014) Sandia Corporation.  Under the terms of Contract
+   DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains
+   certain rights in this software.  This software is distributed under
+   the GNU General Public License.
+
+   See the README file in the top-level SPARTA directory.
+------------------------------------------------------------------------- */
+
+#ifdef FIX_CLASS
+
+FixStyle(ave/surf/kk,FixAveSurfKokkos)
+
+#else
+
+#ifndef SPARTA_FIX_AVE_SURF_KOKKOS_H
+#define SPARTA_FIX_AVE_SURF_KOKKOS_H
+
+#include "fix_ave_surf.h"
+#include "kokkos_type.h"
+
+namespace SPARTA_NS {
+
+struct TagFixAveSurf_Add_tally{};
+
+class FixAveSurfKokkos : public FixAveSurf {
+ public:
+  FixAveSurfKokkos(class SPARTA *, int, char **);
+  ~FixAveSurfKokkos();
+  void init();
+  void setup();
+  void end_of_step();
+
+  KOKKOS_INLINE_FUNCTION
+  void operator()(TagFixAveSurf_Add_tally, const int&) const;
+
+ private:
+  int kokkosable;        // 1 if the all-tally path runs on device
+                         // 0 if delegating entirely to the host base class
+  int nstally;           // # of local surf rows in a compute tally
+                         //   = surf->nlocal + surf->nghost
+  int acc_m,acc_col;     // value index / compute-tally column for current kernel
+
+  DAT::t_float_2d_lr d_acc;     // per-interval tally accumulator [nstally][nvalues]
+  DAT::t_float_2d_lr d_tally;   // current compute device tally (set per value)
+
+  surfint *tally2surf_all;      // surfID of each local surf row (host)
+  double *acc_local_vec;        // host copy of d_acc for collate (nvalues == 1)
+  double **acc_local;           // host copy of d_acc for collate (nvalues > 1)
+
+  void reallocate();
+};
+
+}
+
+#endif
+#endif
+
+/* ERROR/WARNING messages:
+
+*/
diff --git a/src/fix_ave_surf.cpp b/src/fix_ave_surf.cpp
index d7640c26f..602c3017b 100644
--- a/src/fix_ave_surf.cpp
+++ b/src/fix_ave_surf.cpp
@@ -291,6 +291,8 @@ FixAveSurf::FixAveSurf(SPARTA *sparta, int narg, char **arg) :
 
 FixAveSurf::~FixAveSurf()
 {
+  if (copymode) return;
+
   delete [] which;
   delete [] argindex;
   delete [] value2index;
diff --git a/src/fix_ave_surf.h b/src/fix_ave_surf.h
index 01a746116..3402b855d 100644
--- a/src/fix_ave_surf.h
+++ b/src/fix_ave_surf.h
@@ -36,7 +36,7 @@ class FixAveSurf : public Fix {
   void end_of_step();
   double memory_usage();
 
- private:
+ protected:
   int groupbit;
   int nvalues,maxvalues;
   int nrepeat,irepeat,nsample,ave;

From 2cacfb1a9ce601ca43a3df4ae239ceed281d9623 Mon Sep 17 00:00:00 2001
From: Claude 
Date: Tue, 23 Jun 2026 12:51:22 +0000
Subject: [PATCH 23/30] KOKKOS: support non-reacting multigroup ambipolar
 collisions in collide vss/kk

Add a Kokkos NTC collision path for ambipolar collisions with more than one
mixture group (ngroup > 1), mirroring the non-Kokkos
Collide::collisions_group_ambipolar algorithm. Previously the vss/kk style
errored out for multigroup ambipolar collisions.

As with the plain multigroup port, scope is limited to the non-reacting case
(react == NULL), so group membership and the per-cell electron list are static
within a timestep and the result is bit-for-bit identical to the non-Kokkos
version. Reacting multigroup ambipolar still raises a clear "not (yet)
supported" error.

Implementation (new TagCollideCollisionsGroupAmbipolar kernel and
collisions_group_ambipolar launcher):
 - per cell, builds group-contiguous lists of real particles in d_glist and a
   separate electron list d_elist (one electron per ambipolar ion, copied from
   the ion with its velambi velocity), with the electron species placed in its
   own group (egroup, required and checked at init)
 - pre-computes attempt counts per group pair into d_nattempt_pair, skipping
   electron/electron pairs and drawing RN for every other pair to preserve the
   non-Kokkos collision RNG ordering
 - keeps the electron group on the J side of each pair (matching the non-Kokkos
   gpair igroup/jgroup flip) so collision velocity assignment is identical
 - after collisions, copies the (scattered) electron velocities back into
   velambi, exactly as the non-Kokkos version does

Verified bit-for-bit identical to the non-Kokkos path (Serial build with
SPARTA_KOKKOS_EXACT, 1 thread) over 300 steps of a two-group thermal plasma
(heavy species + ambipolar electrons); a non-ambipolar control run confirms
the electron-group path is actually exercised. The 4-thread OpenMP build runs
cleanly. Adds examples/ambi/in.ambi.group.

Co-Authored-By: Claude Opus 4.8 
Claude-Session: https://claude.ai/code/session_01Q6nQVuxEiTUqSzDoDppCUM
Co-authored-by: stanmoore1 
---
 examples/ambi/in.ambi.group       |  47 +++++
 src/KOKKOS/collide_vss_kokkos.cpp | 293 +++++++++++++++++++++++++++++-
 src/KOKKOS/collide_vss_kokkos.h   |  13 ++
 3 files changed, 344 insertions(+), 9 deletions(-)
 create mode 100644 examples/ambi/in.ambi.group

diff --git a/examples/ambi/in.ambi.group b/examples/ambi/in.ambi.group
new file mode 100644
index 000000000..e69d6369f
--- /dev/null
+++ b/examples/ambi/in.ambi.group
@@ -0,0 +1,47 @@
+################################################################################
+# thermal plasma in a 2d box, non-reacting multigroup ambipolar collisions
+#
+# Exercises the ambipolar approximation with more than one collision group:
+# all heavy species (neutrals + ions) form one group and the ambipolar
+# electron species "e" is its own group, as required by collide ambipolar.
+#
+# Note:
+#  - The "comm/sort" option to the "global" command is used to match MPI runs.
+#  - The "twopass" option is used to match Kokkos runs.
+# The "comm/sort" and "twopass" options should not be used for production runs.
+################################################################################
+
+seed                12345
+dimension           2
+boundary            rr rr p
+global              gridcut 0.01 comm/sort yes
+create_box          -2.0 2.0 -2.0 2.0 -0.5 0.5
+create_grid         20 20 1
+balance_grid        rcb cell
+
+global              fnum 2.6404E16
+global              nrho 2.6404e20
+
+species             air.species N2 O2 N O NO N2+ O2+ N+ O+ NO+ e
+
+# collide mixture: all species, two groups
+# the ambipolar electron species e must be in a group by itself
+
+mixture             gas N2 O2 N O NO N2+ O2+ N+ O+ NO+ vstream 0 0 0 temp 5000.0 group heavy
+mixture             gas e group electron
+mixture             gas N2 frac 0.6
+mixture             gas N2+ frac 0.4
+
+fix                 ambi ambipolar e N+ N2+ NO+ O+ O2+
+
+collide             vss gas air.vss
+collide_modify      ambipolar yes
+
+create_particles    gas n 10000 twopass
+
+compute             temp temp
+stats               50
+stats_style         step np nattempt ncoll c_temp
+
+timestep            1.0e-8
+run                 300
diff --git a/src/KOKKOS/collide_vss_kokkos.cpp b/src/KOKKOS/collide_vss_kokkos.cpp
index 14bd975d6..ca48ce93b 100644
--- a/src/KOKKOS/collide_vss_kokkos.cpp
+++ b/src/KOKKOS/collide_vss_kokkos.cpp
@@ -307,7 +307,7 @@ void CollideVSSKokkos::init()
 
   if (ambiflag && mixture->ngroup > 1) {
     int *species2group = mixture->species2group;
-    int egroup = species2group[ambispecies];
+    egroup = species2group[ambispecies];
     if (mixture->groupsize[egroup] != 1)
       error->all(FLERR,"Multigroup ambipolar collisions require "
                  "electrons be their own group");
@@ -454,20 +454,26 @@ void CollideVSSKokkos::collisions()
     }
 
   // multiple groups
-  // Kokkos currently supports only non-reacting, non-ambipolar,
-  //   non-near-neighbor group collisions
+  // Kokkos currently supports only non-reacting, non-near-neighbor
+  //   group collisions (with or without the ambipolar approximation)
 
   } else {
     if (react)
       error->all(FLERR,"Kokkos does not (yet) support reacting group collisions");
-    if (ambiflag)
-      error->all(FLERR,"Kokkos does not (yet) support multigroup ambipolar collisions");
     if (nearcp)
       error->all(FLERR,"Kokkos does not (yet) support near-neighbor group collisions");
-    if (!ngas_tally) {
-      collisions_group<0,0>(reduce);
-    } else if (ngas_tally) {
-      collisions_group<0,1>(reduce);
+    if (!ambiflag) {
+      if (!ngas_tally) {
+        collisions_group<0,0>(reduce);
+      } else if (ngas_tally) {
+        collisions_group<0,1>(reduce);
+      }
+    } else if (ambiflag) {
+      if (!ngas_tally) {
+        collisions_group_ambipolar<0>(reduce);
+      } else if (ngas_tally) {
+        collisions_group_ambipolar<1>(reduce);
+      }
     }
   }
 
@@ -1141,6 +1147,275 @@ void CollideVSSKokkos::operator()(TagCollideCollisionsGroup< NEARCP, GASTALLY, A
   rand_pool.free_state(rand_gen);
 }
 
+/* ----------------------------------------------------------------------
+   NTC algorithm for multiple groups with ambipolar approximation
+   Kokkos version supports only the non-reacting case, so group membership
+     and the electron list are static within the timestep and no particles
+     are created or destroyed
+------------------------------------------------------------------------- */
+
+template < int GASTALLY >
+void CollideVSSKokkos::collisions_group_ambipolar(COLLIDE_REDUCE &reduce)
+{
+  if (ngroups > MAXGROUP)
+    error->all(FLERR,"Too many collision groups for Kokkos group collisions");
+
+  // ambipolar vectors
+
+  this->sync(Device,ALL_MASK);
+
+  ParticleKokkos* particle_kk = (ParticleKokkos*) particle;
+  particle_kk->sync(Device,PARTICLE_MASK|SPECIES_MASK|CUSTOM_MASK);
+  d_particles = particle_kk->k_particles.view_device();
+  d_species = particle_kk->k_species.view_device();
+  d_ewhich = particle_kk->k_ewhich.view_device();
+  auto h_ewhich = particle_kk->k_ewhich.view_host();
+  k_eivec = particle_kk->k_eivec;
+  k_eiarray = particle_kk->k_eiarray;
+  k_edarray = particle_kk->k_edarray;
+  d_ionambi = k_eivec.view_host()[h_ewhich[index_ionambi]].k_view.view_device();
+  d_velambi = k_edarray.view_host()[h_ewhich[index_velambi]].k_view.view_device();
+
+  GridKokkos* grid_kk = (GridKokkos*) grid;
+  grid_kk->sync(Device,CINFO_MASK);
+  d_plist = grid_kk->d_plist;
+
+  // allocate per-cell group scratch arrays (see collisions_group)
+
+  if (int(d_glist.extent(0)) < nglocal ||
+      int(d_glist.extent(1)) < int(d_plist.extent(1)))
+    MemKK::realloc_kokkos(d_glist,"collide:glist",nglocal,d_plist.extent(1));
+  if (int(d_nattempt_pair.extent(0)) < nglocal)
+    MemKK::realloc_kokkos(d_nattempt_pair,"collide:nattempt_pair",nglocal,ngroups,ngroups);
+
+  // per-cell electron list; non-reacting so nelectron <= cell particle count
+
+  maxcellcount = particle_kk->get_maxcellcount();
+  if (int(d_elist.extent(0)) < nglocal || int(d_elist.extent(1)) < maxcellcount) {
+    d_elist = t_particle_2d(); // reduce memory use by deallocating first
+    d_elist = t_particle_2d(Kokkos::view_alloc("collide:elist",Kokkos::WithoutInitializing),nglocal,maxcellcount);
+  }
+
+  copymode = 1;
+
+  // no particles are created or destroyed for non-reacting group collisions
+
+  ndelete = 0;
+
+  h_error_flag() = 0;
+  Kokkos::deep_copy(d_scalars,h_scalars);
+
+  grid_kk_copy.copy(grid_kk);
+
+  if (sparta->kokkos->atomic_reduction) {
+    if (sparta->kokkos->need_atomics)
+      Kokkos::parallel_for(Kokkos::RangePolicy >(0,nglocal),*this);
+    else
+      Kokkos::parallel_for(Kokkos::RangePolicy >(0,nglocal),*this);
+  } else
+    Kokkos::parallel_reduce(Kokkos::RangePolicy >(0,nglocal),*this,reduce);
+
+  Kokkos::deep_copy(h_scalars,d_scalars);
+
+  copymode = 0;
+
+  if (h_error_flag() == 1)
+    error->one(FLERR,"Collision cell volume is zero");
+  else if (h_error_flag() == 2)
+    error->one(FLERR,"Collisions in cell did not conserve electron count");
+
+  this->modified(Device,ALL_MASK);
+  particle_kk->modify(Device,PARTICLE_MASK|CUSTOM_MASK);
+
+  d_particles = t_particle_1d(); // destroy reference to reduce memory use
+  d_plist = {};
+}
+
+template < int GASTALLY, int ATOMIC_REDUCTION >
+KOKKOS_INLINE_FUNCTION
+void CollideVSSKokkos::operator()(TagCollideCollisionsGroupAmbipolar< GASTALLY, ATOMIC_REDUCTION >, const int &icell) const {
+  COLLIDE_REDUCE reduce;
+  this->template operator()< GASTALLY, ATOMIC_REDUCTION >(TagCollideCollisionsGroupAmbipolar< GASTALLY, ATOMIC_REDUCTION >(), icell, reduce);
+}
+
+template < int GASTALLY, int ATOMIC_REDUCTION >
+KOKKOS_INLINE_FUNCTION
+void CollideVSSKokkos::operator()(TagCollideCollisionsGroupAmbipolar< GASTALLY, ATOMIC_REDUCTION >, const int &icell, COLLIDE_REDUCE &reduce) const {
+
+  int np = grid_kk_copy.obj.d_cellcount[icell];
+  if (np <= 1) return;
+
+  const double volume = grid_kk_copy.obj.k_cinfo.view_device()[icell].volume / grid_kk_copy.obj.k_cinfo.view_device()[icell].weight;
+  if (volume == 0.0) d_error_flag() = 1;
+
+  // build per-group particle lists for this cell, plus the electron list
+  // gcount[g] = particle count in group g, with the electron count for egroup
+  // gstart[g] = offset of group g's real particles within d_glist(icell,*)
+  //   (the electron group egroup has no real particles, so it adds no entries)
+  // electrons (one per ambipolar ion) are created in d_elist in plist order
+
+  int gcount[MAXGROUP];
+  int gstart[MAXGROUP];
+  int gcursor[MAXGROUP];
+
+  for (int g = 0; g < ngroups; g++) gcount[g] = 0;
+
+  int nelectron = 0;
+  for (int n = 0; n < np; n++) {
+    const int ip = d_plist(icell,n);
+    const int isp = d_particles[ip].ispecies;
+    gcount[d_species2group[isp]]++;
+    if (d_ionambi[ip]) nelectron++;
+  }
+  gcount[egroup] = nelectron;
+
+  int offset = 0;
+  for (int g = 0; g < ngroups; g++) {
+    gstart[g] = offset;
+    gcursor[g] = offset;
+    if (g != egroup) offset += gcount[g];
+  }
+
+  int e = 0;
+  for (int n = 0; n < np; n++) {
+    const int ip = d_plist(icell,n);
+    const int isp = d_particles[ip].ispecies;
+    const int g = d_species2group[isp];
+    d_glist(icell,gcursor[g]++) = n;
+    if (d_ionambi[ip]) {
+      Particle::OnePart* p = &d_particles[ip];
+      Particle::OnePart* ep = &d_elist(icell,e);
+      *ep = *p;
+      ep->v[0] = d_velambi(ip,0);
+      ep->v[1] = d_velambi(ip,1);
+      ep->v[2] = d_velambi(ip,2);
+      ep->ispecies = ambispecies;
+      e++;
+    }
+  }
+
+  struct State precoln;       // state before collision
+  struct State postcoln;      // state after collision
+
+  rand_type rand_gen = rand_pool.get_state();
+
+  // pre-compute # of attempts for each pair of groups
+  // skip electron/electron pairs (no e/e collisions in the ambipolar model)
+  // draw RN for every other pair to match non-Kokkos collision ordering
+
+  for (int ig = 0; ig < ngroups; ig++)
+    for (int jg = ig; jg < ngroups; jg++) {
+      if (ig == egroup && jg == egroup) {
+        d_nattempt_pair(icell,ig,jg) = 0;
+        continue;
+      }
+      const double attempt =
+        attempt_collision_kokkos(icell,ig,jg,gcount[ig],gcount[jg],volume,rand_gen);
+      const int nattempt = static_cast (attempt);
+      d_nattempt_pair(icell,ig,jg) = nattempt;
+      if (nattempt) {
+        if (ATOMIC_REDUCTION == 1)
+          Kokkos::atomic_add(&d_nattempt_one(),nattempt);
+        else if (ATOMIC_REDUCTION == 0)
+          d_nattempt_one() += nattempt;
+        else
+          reduce.nattempt_one += nattempt;
+      }
+    }
+
+  // perform collisions for each pair of groups
+  // electron group is always the J side, so ipart is never an electron
+  //   (matches the non-Kokkos gpair igroup/jgroup flip)
+
+  for (int ig = 0; ig < ngroups; ig++)
+    for (int jg = ig; jg < ngroups; jg++) {
+      if (ig == egroup && jg == egroup) continue;
+      const int nattempt = d_nattempt_pair(icell,ig,jg);
+      if (!nattempt) continue;
+
+      int aig,ajg;
+      if (ig == egroup) { aig = jg; ajg = ig; }
+      else { aig = ig; ajg = jg; }
+
+      const int ni = gcount[aig];
+      const int nj = gcount[ajg];
+      if (ni == 0 || nj == 0) continue;
+      if (aig == ajg && ni == 1) continue;
+
+      for (int iattempt = 0; iattempt < nattempt; iattempt++) {
+        int i = ni * rand_gen.drand();
+        int j = nj * rand_gen.drand();
+        if (aig == ajg)
+          while (i == j) j = nj * rand_gen.drand();
+
+        Particle::OnePart* ipart =
+          &d_particles[d_plist(icell,d_glist(icell,gstart[aig]+i))];
+        Particle::OnePart* jpart;
+        if (ajg == egroup) jpart = &d_elist(icell,j);
+        else jpart = &d_particles[d_plist(icell,d_glist(icell,gstart[ajg]+j))];
+
+        // test if collision actually occurs
+
+        if (!test_collision_kokkos(icell,aig,ajg,ipart,jpart,precoln,rand_gen)) continue;
+
+        // perform collision (non-reacting: no chemistry, no 3rd particle)
+        // if GASTALLY: save iorig/jorig for tally
+
+        Particle::OnePart iorig,jorig;
+        if (GASTALLY) {
+          iorig = *ipart;
+          jorig = *jpart;
+        }
+
+        Particle::OnePart* kpart = NULL;
+        Particle::OnePart* recomb_part3 = NULL;
+        int recomb_species = -1;
+        double recomb_density = 0.0;
+        int index_kpart = 0;
+
+        setup_collision_kokkos(ipart,jpart,precoln,postcoln);
+        const int reactflag = perform_collision_kokkos(ipart,jpart,kpart,precoln,postcoln,rand_gen,
+                                 recomb_part3,recomb_species,recomb_density,index_kpart);
+
+        if (ATOMIC_REDUCTION == 1)
+          Kokkos::atomic_inc(&d_ncollide_one());
+        else if (ATOMIC_REDUCTION == 0)
+          d_ncollide_one()++;
+        else
+          reduce.ncollide_one++;
+
+        if (GASTALLY) {
+          for (int m = 0; m < nglist_collision; m++)
+            glist_collision_copy[m].obj.template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart);
+          for (int m = 0; m < nglist_reaction; m++)
+            glist_reaction_copy[m].obj.template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart);
+        }
+      }
+    }
+
+  // recombine ambipolar ions with their matching electrons
+  //   by copying the (possibly scattered) electron velocity back into velambi
+  // electrons were created in plist order, so the Nth ion gets the Nth electron
+
+  int melectron = 0;
+  for (int n = 0; n < np; n++) {
+    const int i = d_plist(icell,n);
+    if (d_ionambi[i]) {
+      if (melectron < nelectron) {
+        Particle::OnePart* ep = &d_elist(icell,melectron);
+        d_velambi(i,0) = ep->v[0];
+        d_velambi(i,1) = ep->v[1];
+        d_velambi(i,2) = ep->v[2];
+      }
+      melectron++;
+    }
+  }
+  if (melectron != nelectron)
+    d_error_flag() = 2;
+
+  rand_pool.free_state(rand_gen);
+}
+
 /* ----------------------------------------------------------------------
    NTC algorithm for a single group with ambipolar approximation
 ------------------------------------------------------------------------- */
diff --git a/src/KOKKOS/collide_vss_kokkos.h b/src/KOKKOS/collide_vss_kokkos.h
index ec62b333f..df1b3bb0e 100644
--- a/src/KOKKOS/collide_vss_kokkos.h
+++ b/src/KOKKOS/collide_vss_kokkos.h
@@ -69,6 +69,9 @@ struct TagCollideCollisionsOneAmbipolar{};
 template < int NEARCP, int GASTALLY, int ATOMIC_REDUCTION >
 struct TagCollideCollisionsGroup{};
 
+template < int GASTALLY, int ATOMIC_REDUCTION >
+struct TagCollideCollisionsGroupAmbipolar{};
+
 class CollideVSSKokkos : public CollideVSS {
  public:
   typedef COLLIDE_REDUCE value_type;
@@ -135,6 +138,14 @@ class CollideVSSKokkos : public CollideVSS {
   KOKKOS_INLINE_FUNCTION
   void operator()(TagCollideCollisionsGroup< NEARCP, GASTALLY, ATOMIC_REDUCTION >, const int&, COLLIDE_REDUCE&) const;
 
+  template < int GASTALLY, int ATOMIC_REDUCTION >
+  KOKKOS_INLINE_FUNCTION
+  void operator()(TagCollideCollisionsGroupAmbipolar< GASTALLY, ATOMIC_REDUCTION >, const int&) const;
+
+  template < int GASTALLY, int ATOMIC_REDUCTION >
+  KOKKOS_INLINE_FUNCTION
+  void operator()(TagCollideCollisionsGroupAmbipolar< GASTALLY, ATOMIC_REDUCTION >, const int&, COLLIDE_REDUCE&) const;
+
   typedef Kokkos::
     DualView tdual_params_2d;
   typedef tdual_params_2d::t_dev t_params_2d;
@@ -243,6 +254,8 @@ class CollideVSSKokkos : public CollideVSS {
   template < int NEARCP, int GASTALLY > void collisions_one(COLLIDE_REDUCE&);
   template < int GASTALLY > void collisions_one_ambipolar(COLLIDE_REDUCE&);
   template < int NEARCP, int GASTALLY > void collisions_group(COLLIDE_REDUCE&);
+  template < int GASTALLY > void collisions_group_ambipolar(COLLIDE_REDUCE&);
+  int egroup;        // mixture group containing the ambipolar electron species
 
   // VSS specific
 

From 3bd339cf75bcf4ae4f600f6b6a4ac2b5c04e5710 Mon Sep 17 00:00:00 2001
From: Claude 
Date: Thu, 25 Jun 2026 01:57:11 +0000
Subject: [PATCH 24/30] KOKKOS: review fixes for the gas-tally / multigroup
 collision ports

Address issues found in code review:

 - (major) Make compute gas/collision/grid/kk and gas/reaction/grid/kk inherit
   KokkosBase and expose their per-grid output through KokkosBase::d_vector_grid
   / d_array_grid (with a no-op compute_per_grid_kokkos). They set kokkos_flag=1
   but previously did not inherit KokkosBase, so feeding one into fix ave/grid/kk
   passed its kokkos_flag guard and then null-dereferenced the
   dynamic_cast result. Now fix ave/grid/kk can average these
   per-grid gas tallies; verified bit-for-bit identical to the non-Kokkos path.

 - (minor) collide vss/kk: reallocate d_nattempt_pair when ngroups grows, not
   just when nglocal grows (it is sized ngroups x ngroups); previously a later
   run with more collision groups could index it out of bounds.

 - initialize the egroup member (-1) and the SPARTA-only-ctor members of the gas
   computes; correct the gas-tally instance-limit error text; add a static_assert
   coupling the VAL_4 KKCopy initializers to KOKKOS_MAX_GLIST.

All existing bit-exact checks (group collisions, multigroup ambipolar, per-grid
gas tallies, fix ave/surf) still pass on the Serial+EXACT build, and the OpenMP
build compiles and runs cleanly.

Co-Authored-By: Claude Opus 4.8 
Claude-Session: https://claude.ai/code/session_01Q6nQVuxEiTUqSzDoDppCUM
Co-authored-by: stanmoore1 
---
 src/KOKKOS/collide_vss_kokkos.cpp                | 15 +++++++++++----
 src/KOKKOS/compute_gas_collision_grid_kokkos.cpp |  2 ++
 src/KOKKOS/compute_gas_collision_grid_kokkos.h   |  6 ++++--
 src/KOKKOS/compute_gas_reaction_grid_kokkos.cpp  |  4 ++++
 src/KOKKOS/compute_gas_reaction_grid_kokkos.h    |  8 +++++---
 5 files changed, 26 insertions(+), 9 deletions(-)

diff --git a/src/KOKKOS/collide_vss_kokkos.cpp b/src/KOKKOS/collide_vss_kokkos.cpp
index ca48ce93b..5d4b2296d 100644
--- a/src/KOKKOS/collide_vss_kokkos.cpp
+++ b/src/KOKKOS/collide_vss_kokkos.cpp
@@ -41,6 +41,10 @@ using namespace MathConst;
 #define VAL_2(X) VAL_1(X), VAL_1(X)
 #define VAL_4(X) VAL_2(X), VAL_2(X)
 
+// the glist KKCopy arrays below are brace-initialized with VAL_4 (4 elements)
+static_assert(KOKKOS_MAX_GLIST == 4,
+              "VAL_4 initializer lists assume KOKKOS_MAX_GLIST == 4");
+
 enum{NONE,DISCRETE,SMOOTH};            // several files
 enum{CONSTANT,VARIABLE};
 
@@ -75,6 +79,7 @@ CollideVSSKokkos::CollideVSSKokkos(SPARTA *sparta, int narg, char **arg) :
   kokkos_flag = 1;
   react_style = 0;
   nglist_collision = nglist_reaction = 0;
+  egroup = -1;
 
   // use 1D view for scalars to reduce GPU memory operations
 
@@ -536,7 +541,7 @@ void CollideVSSKokkos::setup_gas_tally()
       if (!ckk)
         error->all(FLERR,"Must use Kokkos-enabled compute gas/collision/grid with Kokkos");
       if (nglist_collision >= KOKKOS_MAX_GLIST)
-        error->all(FLERR,"Kokkos currently only supports two instances of compute gas/collision/grid");
+        error->all(FLERR,"Kokkos supports at most KOKKOS_MAX_GLIST instances of compute gas/collision/grid");
       ckk->pre_gas_tally();
       glist_collision_copy[nglist_collision].copy(ckk);
       nglist_collision++;
@@ -546,7 +551,7 @@ void CollideVSSKokkos::setup_gas_tally()
       if (!ckk)
         error->all(FLERR,"Must use Kokkos-enabled compute gas/reaction/grid with Kokkos");
       if (nglist_reaction >= KOKKOS_MAX_GLIST)
-        error->all(FLERR,"Kokkos currently only supports two instances of compute gas/reaction/grid");
+        error->all(FLERR,"Kokkos supports at most KOKKOS_MAX_GLIST instances of compute gas/reaction/grid");
       ckk->pre_gas_tally();
       glist_reaction_copy[nglist_reaction].copy(ckk);
       nglist_reaction++;
@@ -976,7 +981,8 @@ void CollideVSSKokkos::collisions_group(COLLIDE_REDUCE &reduce)
   if (int(d_glist.extent(0)) < nglocal ||
       int(d_glist.extent(1)) < int(d_plist.extent(1)))
     MemKK::realloc_kokkos(d_glist,"collide:glist",nglocal,d_plist.extent(1));
-  if (int(d_nattempt_pair.extent(0)) < nglocal)
+  if (int(d_nattempt_pair.extent(0)) < nglocal ||
+      int(d_nattempt_pair.extent(1)) < ngroups)
     MemKK::realloc_kokkos(d_nattempt_pair,"collide:nattempt_pair",nglocal,ngroups,ngroups);
 
   copymode = 1;
@@ -1185,7 +1191,8 @@ void CollideVSSKokkos::collisions_group_ambipolar(COLLIDE_REDUCE &reduce)
   if (int(d_glist.extent(0)) < nglocal ||
       int(d_glist.extent(1)) < int(d_plist.extent(1)))
     MemKK::realloc_kokkos(d_glist,"collide:glist",nglocal,d_plist.extent(1));
-  if (int(d_nattempt_pair.extent(0)) < nglocal)
+  if (int(d_nattempt_pair.extent(0)) < nglocal ||
+      int(d_nattempt_pair.extent(1)) < ngroups)
     MemKK::realloc_kokkos(d_nattempt_pair,"collide:nattempt_pair",nglocal,ngroups,ngroups);
 
   // per-cell electron list; non-reacting so nelectron <= cell particle count
diff --git a/src/KOKKOS/compute_gas_collision_grid_kokkos.cpp b/src/KOKKOS/compute_gas_collision_grid_kokkos.cpp
index 9df6b374a..9af8d22db 100644
--- a/src/KOKKOS/compute_gas_collision_grid_kokkos.cpp
+++ b/src/KOKKOS/compute_gas_collision_grid_kokkos.cpp
@@ -36,6 +36,8 @@ ComputeGasCollisionGridKokkos::ComputeGasCollisionGridKokkos(SPARTA *sparta) :
 {
   copy = 1;
   uncopy = 0;
+  vector_grid = NULL;
+  nglocal = 0;
 }
 
 /* ---------------------------------------------------------------------- */
diff --git a/src/KOKKOS/compute_gas_collision_grid_kokkos.h b/src/KOKKOS/compute_gas_collision_grid_kokkos.h
index d0cbd9769..ce3fc1d71 100644
--- a/src/KOKKOS/compute_gas_collision_grid_kokkos.h
+++ b/src/KOKKOS/compute_gas_collision_grid_kokkos.h
@@ -22,16 +22,18 @@ ComputeStyle(gas/collision/grid/kk,ComputeGasCollisionGridKokkos)
 #define SPARTA_COMPUTE_GAS_COLLISION_GRID_KOKKOS_H
 
 #include "compute_gas_collision_grid.h"
+#include "kokkos_base.h"
 #include "kokkos_type.h"
 #include "particle.h"
 
 namespace SPARTA_NS {
 
-class ComputeGasCollisionGridKokkos : public ComputeGasCollisionGrid {
+class ComputeGasCollisionGridKokkos : public ComputeGasCollisionGrid, public KokkosBase {
  public:
   ComputeGasCollisionGridKokkos(class SPARTA *, int, char **);
   ComputeGasCollisionGridKokkos(class SPARTA *);
   ~ComputeGasCollisionGridKokkos();
+  void compute_per_grid_kokkos() {}   // tallying happens in Collide, not here
   void clear();
   void pre_gas_tally();
   void post_gas_tally();
@@ -72,7 +74,7 @@ class ComputeGasCollisionGridKokkos : public ComputeGasCollisionGrid {
 
  private:
   DAT::tdual_float_1d k_vector_grid;
-  DAT::t_float_1d d_vector_grid;
+  // d_vector_grid is inherited from KokkosBase (read by fix ave/grid/kk)
 
   t_cinfo_1d d_cinfo;
   DAT::t_int_2d d_s2g;
diff --git a/src/KOKKOS/compute_gas_reaction_grid_kokkos.cpp b/src/KOKKOS/compute_gas_reaction_grid_kokkos.cpp
index 4ddf8bcb4..f325eba3e 100644
--- a/src/KOKKOS/compute_gas_reaction_grid_kokkos.cpp
+++ b/src/KOKKOS/compute_gas_reaction_grid_kokkos.cpp
@@ -37,6 +37,10 @@ ComputeGasReactionGridKokkos::ComputeGasReactionGridKokkos(SPARTA *sparta) :
 {
   copy = 1;
   uncopy = 0;
+  vector_grid = NULL;
+  array_grid = NULL;
+  ncol = 0;
+  nglocal = 0;
 }
 
 /* ---------------------------------------------------------------------- */
diff --git a/src/KOKKOS/compute_gas_reaction_grid_kokkos.h b/src/KOKKOS/compute_gas_reaction_grid_kokkos.h
index f0d6313df..d0d3b9ffd 100644
--- a/src/KOKKOS/compute_gas_reaction_grid_kokkos.h
+++ b/src/KOKKOS/compute_gas_reaction_grid_kokkos.h
@@ -22,12 +22,13 @@ ComputeStyle(gas/reaction/grid/kk,ComputeGasReactionGridKokkos)
 #define SPARTA_COMPUTE_GAS_REACTION_GRID_KOKKOS_H
 
 #include "compute_gas_reaction_grid.h"
+#include "kokkos_base.h"
 #include "kokkos_type.h"
 #include "particle.h"
 
 namespace SPARTA_NS {
 
-class ComputeGasReactionGridKokkos : public ComputeGasReactionGrid {
+class ComputeGasReactionGridKokkos : public ComputeGasReactionGrid, public KokkosBase {
  public:
   enum{ALL,EVERY,SELECT};        // must match compute_gas_reaction_grid.cpp
 
@@ -35,6 +36,7 @@ class ComputeGasReactionGridKokkos : public ComputeGasReactionGrid {
   ComputeGasReactionGridKokkos(class SPARTA *);
   ~ComputeGasReactionGridKokkos();
   void init();
+  void compute_per_grid_kokkos() {}   // tallying happens in Collide, not here
   void clear();
   void pre_gas_tally();
   void post_gas_tally();
@@ -84,9 +86,9 @@ class ComputeGasReactionGridKokkos : public ComputeGasReactionGrid {
 
  private:
   DAT::tdual_float_1d k_vector_grid;
-  DAT::t_float_1d d_vector_grid;
   DAT::tdual_float_2d_lr k_array_grid;
-  DAT::t_float_2d_lr d_array_grid;
+  // d_vector_grid and d_array_grid are inherited from KokkosBase
+  //   (read by fix ave/grid/kk)
 
   DAT::t_int_1d d_reaction2col;   // reaction -> column map for SELECT mode
 

From bf10c8a9485c55f7a5abb01f03dd16f879173a1a Mon Sep 17 00:00:00 2001
From: Claude 
Date: Mon, 29 Jun 2026 21:41:35 +0000
Subject: [PATCH 25/30] Drop Kokkos fix ave/surf port (correctness regression)

Remove the fix ave/surf/kk port and revert the associated compute surf
device-tally accessors back to master. The device-direct read path in
fix ave/surf/kk bypassed compute surf's host tallyinfo()/post_process_surf()
pipeline while still marking the compute invoked, so a downstream surf-style
variable (e.g. the torque computation in examples/torque) read stale host
tally state and produced zero output.

Unlike the per-grid case, compute surf's output is a compressed/remapped
tally fronted by a stateful host collate stage, which the device fast path
cannot safely skip without porting post_process_surf to device (out of scope).

Reverts:
- src/KOKKOS/fix_ave_surf_kokkos.{h,cpp} (deleted)
- src/fix_ave_surf.{h,cpp} restored to master
- src/KOKKOS/compute_surf_kokkos.h, compute_react_surf_kokkos.h
  device-tally accessors removed
- doc ave/surf (k) entry reverted

The remaining Kokkos ports in this branch (non-reacting group collisions,
per-grid gas tallies, non-reacting multigroup ambipolar) are unaffected and
remain bit-exact under SPARTA_KOKKOS_EXACT.

Co-authored-by: stanmoore1 
Co-Authored-By: Claude Opus 4.8 
Claude-Session: https://claude.ai/code/session_01Q6nQVuxEiTUqSzDoDppCUM
---
 doc/Section_commands.html              |   2 +-
 doc/Section_commands.txt               |   2 +-
 src/KOKKOS/compute_react_surf_kokkos.h |   3 -
 src/KOKKOS/compute_surf_kokkos.h       |   3 -
 src/KOKKOS/fix_ave_surf_kokkos.cpp     | 265 -------------------------
 src/KOKKOS/fix_ave_surf_kokkos.h       |  66 ------
 src/fix_ave_surf.cpp                   |   2 -
 src/fix_ave_surf.h                     |   2 +-
 8 files changed, 3 insertions(+), 342 deletions(-)
 delete mode 100644 src/KOKKOS/fix_ave_surf_kokkos.cpp
 delete mode 100644 src/KOKKOS/fix_ave_surf_kokkos.h

diff --git a/doc/Section_commands.html b/doc/Section_commands.html
index 9a4bc9c50..b9ab96bca 100644
--- a/doc/Section_commands.html
+++ b/doc/Section_commands.html
@@ -343,7 +343,7 @@ 
      Fix styles
 
      
 
      
-
+
      
 
      ablateadapt (k)ambipolar (k)ave/grid (k)ave/histo (k)ave/histo/weight (k)
      ave/surf (k)ave/timebalance (k)controllercustomdt/reset (k)
      ave/surfave/timebalance (k)controllercustomdt/reset (k)
      
 
      emit/face (k)emit/face/fileemit/surffield/gridfield/particlegrid/check (k)
      
 
      haltmove/surf (k)printsurf/temptemp/global/rescaletemp/rescale (k)
      
 
      vibmode (k)
diff --git a/doc/Section_commands.txt b/doc/Section_commands.txt
index 84b5bb183..a0c8a42b7 100644
--- a/doc/Section_commands.txt
+++ b/doc/Section_commands.txt
@@ -400,7 +400,7 @@ This is indicated by additional letters in parenthesis: k = KOKKOS.
 "ave/grid (k)"_fix_ave_grid.html,
 "ave/histo (k)"_fix_ave_histo.html,
 "ave/histo/weight (k)"_fix_ave_histo.html,
-"ave/surf (k)"_fix_ave_surf.html,
+"ave/surf"_fix_ave_surf.html,
 "ave/time"_fix_ave_time.html,
 "balance (k)"_fix_balance.html,
 "controller"_fix_controller.html,
diff --git a/src/KOKKOS/compute_react_surf_kokkos.h b/src/KOKKOS/compute_react_surf_kokkos.h
index fa96316fd..91a9a38b2 100644
--- a/src/KOKKOS/compute_react_surf_kokkos.h
+++ b/src/KOKKOS/compute_react_surf_kokkos.h
@@ -38,9 +38,6 @@ class ComputeReactSurfKokkos : public ComputeReactSurf {
   void pre_surf_tally();
   void post_surf_tally();
 
-  // expose the per-local-surf device tally array for fix ave/surf/kk
-  void query_tally_surf_kokkos(DAT::t_float_2d_lr &d_array) { d_array = d_array_surf_tally; }
-
 /* ----------------------------------------------------------------------
    tally a surface reaction for particle colliding with surf element isurf
    mirrors ComputeReactSurf::surf_tally(); per-surf tally compressed to host
diff --git a/src/KOKKOS/compute_surf_kokkos.h b/src/KOKKOS/compute_surf_kokkos.h
index c9e811e17..3893ba59c 100644
--- a/src/KOKKOS/compute_surf_kokkos.h
+++ b/src/KOKKOS/compute_surf_kokkos.h
@@ -45,9 +45,6 @@ class ComputeSurfKokkos : public ComputeSurf {
   void pre_surf_tally();
   void post_surf_tally();
 
-  // expose the per-local-surf device tally array for fix ave/surf/kk
-  void query_tally_surf_kokkos(DAT::t_float_2d_lr &d_array) { d_array = d_array_surf_tally; }
-
 enum{NUM,NUMWT,NFLUX,NFLUXIN,MFLUX,MFLUXIN,FX,FY,FZ,TX,TY,TZ,
   PRESS,XPRESS,YPRESS,ZPRESS,XSHEAR,YSHEAR,ZSHEAR,KE,EROT,EVIB,ECHEM,ETOT};
 
diff --git a/src/KOKKOS/fix_ave_surf_kokkos.cpp b/src/KOKKOS/fix_ave_surf_kokkos.cpp
deleted file mode 100644
index 0d3403e6f..000000000
--- a/src/KOKKOS/fix_ave_surf_kokkos.cpp
+++ /dev/null
@@ -1,265 +0,0 @@
-/* ----------------------------------------------------------------------
-   SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer
-   http://sparta.github.io
-   Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov
-   Sandia National Laboratories
-
-   Copyright (2014) Sandia Corporation.  Under the terms of Contract
-   DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains
-   certain rights in this software.  This software is distributed under
-   the GNU General Public License.
-
-   See the README file in the top-level SPARTA directory.
-------------------------------------------------------------------------- */
-
-#include "spatype.h"
-#include "string.h"
-#include "fix_ave_surf_kokkos.h"
-#include "surf.h"
-#include "domain.h"
-#include "update.h"
-#include "modify.h"
-#include "compute.h"
-#include "compute_surf_kokkos.h"
-#include "compute_react_surf_kokkos.h"
-#include "memory_kokkos.h"
-#include "error.h"
-#include "sparta_masks.h"
-
-using namespace SPARTA_NS;
-
-enum{COMPUTE,FIX,VARIABLE,CUSTOM};       // must match fix_ave_surf.cpp
-enum{ONE,RUNNING};                       // must match fix_ave_surf.cpp
-
-#define INVOKED_PER_SURF 32              // must match fix_ave_surf.cpp
-
-/* ---------------------------------------------------------------------- */
-
-FixAveSurfKokkos::FixAveSurfKokkos(SPARTA *sparta, int narg, char **arg) :
-  FixAveSurf(sparta, narg, arg)
-{
-  kokkos_flag = 1;
-  execution_space = Device;
-  datamask_read = EMPTY_MASK;
-  datamask_modify = EMPTY_MASK;
-
-  // only the all-tally path (averaging surf-tally computes) is accelerated
-  //   on device.  count_tally is 0 or nvalues, enforced by the base ctor.
-  // the non-tally path (fix/variable/custom inputs) runs on the host base class
-
-  kokkosable = (count_tally && count_tally == nvalues);
-
-  nstally = 0;
-  tally2surf_all = NULL;
-  acc_local_vec = NULL;
-  acc_local = NULL;
-}
-
-/* ---------------------------------------------------------------------- */
-
-FixAveSurfKokkos::~FixAveSurfKokkos()
-{
-  if (copymode) return;
-
-  memory->destroy(tally2surf_all);
-  memory->destroy(acc_local_vec);
-  memory->destroy(acc_local);
-}
-
-/* ---------------------------------------------------------------------- */
-
-void FixAveSurfKokkos::init()
-{
-  FixAveSurf::init();
-}
-
-/* ----------------------------------------------------------------------
-   allocate per-local-surf device accumulator and host collate buffers
-   build tally2surf_all mapping each local surf row to its surf ID
-------------------------------------------------------------------------- */
-
-void FixAveSurfKokkos::reallocate()
-{
-  int n = surf->nlocal + surf->nghost;
-  if (n == nstally && d_acc.extent(0)) return;
-  nstally = n;
-
-  d_acc = DAT::t_float_2d_lr("ave/surf:acc",nstally,nvalues);
-
-  memory->destroy(tally2surf_all);
-  memory->destroy(acc_local_vec);
-  memory->destroy(acc_local);
-  acc_local_vec = NULL;
-  acc_local = NULL;
-  memory->create(tally2surf_all,nstally,"ave/surf:tally2surf_all");
-  if (nvalues == 1) memory->create(acc_local_vec,nstally,"ave/surf:acc_local_vec");
-  else memory->create(acc_local,nstally,nvalues,"ave/surf:acc_local");
-
-  // surf ID of each local surf row, used by the host collate at output
-  // matches the per-local-surf row order of the Kokkos surf-tally computes
-
-  if (domain->dimension == 2) {
-    Surf::Line *lines = surf->lines;
-    for (int i = 0; i < nstally; i++) tally2surf_all[i] = lines[i].id;
-  } else {
-    Surf::Tri *tris = surf->tris;
-    for (int i = 0; i < nstally; i++) tally2surf_all[i] = tris[i].id;
-  }
-}
-
-/* ----------------------------------------------------------------------
-   only does something if nvalid = current timestep
-------------------------------------------------------------------------- */
-
-void FixAveSurfKokkos::setup()
-{
-  if (kokkosable) reallocate();
-  end_of_step();
-}
-
-/* ---------------------------------------------------------------------- */
-
-void FixAveSurfKokkos::end_of_step()
-{
-  int i,m,n;
-
-  // non-tally path runs entirely on the host base class
-
-  if (!kokkosable) {
-    FixAveSurf::end_of_step();
-    return;
-  }
-
-  // skip if not step which requires doing something
-
-  bigint ntimestep = update->ntimestep;
-  if (ntimestep != nvalid) return;
-
-  if (nstally != surf->nlocal + surf->nghost) reallocate();
-
-  // first sample of an averaging interval:
-  //   zero the per-interval device tally accumulator (== clearing the host hash)
-  //   zero the owned-surf accumulators if ave = ONE
-
-  if (irepeat == 0) {
-    Kokkos::deep_copy(d_acc,0.0);
-    if (ave == ONE) {
-      if (nvalues == 1)
-        for (i = 0; i < nown; i++) accvec[i] = 0.0;
-      else
-        for (i = 0; i < nown; i++)
-          for (m = 0; m < nvalues; m++) accarray[i][m] = 0.0;
-    }
-  }
-
-  // accumulate this sample's compute tallies into d_acc on device
-  // each value m reads a column of its compute's per-local-surf device tally
-  // compute/fix/variable may invoke computes, so wrap with clear/add
-
-  modify->clearstep_compute();
-
-  copymode = 1;
-  for (m = 0; m < nvalues; m++) {
-    n = value2index[m];
-    Compute *compute = modify->compute[n];
-
-    if (!compute->kokkos_flag)
-      error->all(FLERR,"Cannot (yet) use non-Kokkos computes with fix ave/surf/kk");
-
-    if (!(compute->invoked_flag & INVOKED_PER_SURF)) {
-      compute->compute_per_surf();
-      compute->invoked_flag |= INVOKED_PER_SURF;
-    }
-
-    // grab the compute's per-local-surf device tally array
-
-    if (strcmp(compute->style,"surf") == 0)
-      ((ComputeSurfKokkos*) compute)->query_tally_surf_kokkos(d_tally);
-    else if (strcmp(compute->style,"react/surf") == 0)
-      ((ComputeReactSurfKokkos*) compute)->query_tally_surf_kokkos(d_tally);
-    else
-      error->all(FLERR,"Fix ave/surf/kk requires Kokkos compute surf or compute react/surf");
-
-    acc_m = m;
-    acc_col = (argindex[m] == 0) ? 0 : argindex[m] - 1;
-    Kokkos::parallel_for(Kokkos::RangePolicy(0,nstally),*this);
-  }
-  copymode = 0;
-
-  // done if irepeat < nrepeat, else reset irepeat and nvalid
-
-  nsample++;
-  irepeat++;
-  if (irepeat < nrepeat) {
-    nvalid += nevery;
-    modify->addstep_compute(nvalid);
-    return;
-  }
-
-  irepeat = 0;
-  nvalid = ntimestep+per_surf_freq - (nrepeat-1)*nevery;
-  modify->addstep_compute(nvalid);
-
-  // copy the device tally accumulator to the host
-
-  auto h_acc = Kokkos::create_mirror_view(d_acc);
-  Kokkos::deep_copy(h_acc,d_acc);
-
-  // merge per-local-surf tallies to owned surfs via surf->collate (host MPI)
-  // then add the collated interval sum to the owned-surf accumulators
-
-  if (nvalues == 1) {
-    for (i = 0; i < nstally; i++) acc_local_vec[i] = h_acc(i,0);
-    surf->collate_vector(nstally,tally2surf_all,acc_local_vec,1,bufvec);
-    for (i = 0; i < nown; i++) accvec[i] += bufvec[i];
-  } else {
-    for (i = 0; i < nstally; i++)
-      for (m = 0; m < nvalues; m++) acc_local[i][m] = h_acc(i,m);
-    surf->collate_array(nstally,nvalues,tally2surf_all,acc_local,bufarray);
-    for (i = 0; i < nown; i++)
-      for (m = 0; m < nvalues; m++) accarray[i][m] += bufarray[i][m];
-  }
-
-  // normalize the accumulators for output, just by # of samples
-
-  if (ave == ONE) {
-    if (nvalues == 1)
-      for (i = 0; i < nown; i++) vector_surf[i] /= nsample;
-    else
-      for (i = 0; i < nown; i++)
-        for (m = 0; m < nvalues; m++) array_surf[i][m] /= nsample;
-  } else {
-    if (nvalues == 1)
-      for (i = 0; i < nown; i++) vector_surf[i] = accvec[i]/nsample;
-    else
-      for (i = 0; i < nown; i++)
-        for (m = 0; m < nvalues; m++) array_surf[i][m] = accarray[i][m]/nsample;
-  }
-
-  // set values for surfs not in group to zero
-
-  if (groupbit != 1) {
-    if (nvalues == 1) {
-      for (i = 0; i < nown; i++)
-        if (!(masks[i] & groupbit)) vector_surf[i] = 0.0;
-    } else {
-      for (i = 0; i < nown; i++)
-        if (!(masks[i] & groupbit))
-          for (m = 0; m < nvalues; m++) array_surf[i][m] = 0.0;
-    }
-  }
-
-  // reset nsample if ave = ONE
-
-  if (ave == ONE) nsample = 0;
-}
-
-/* ----------------------------------------------------------------------
-   add one value's per-local-surf compute tally column into d_acc
-------------------------------------------------------------------------- */
-
-KOKKOS_INLINE_FUNCTION
-void FixAveSurfKokkos::operator()(TagFixAveSurf_Add_tally, const int &i) const
-{
-  d_acc(i,acc_m) += d_tally(i,acc_col);
-}
diff --git a/src/KOKKOS/fix_ave_surf_kokkos.h b/src/KOKKOS/fix_ave_surf_kokkos.h
deleted file mode 100644
index 6881ce9f5..000000000
--- a/src/KOKKOS/fix_ave_surf_kokkos.h
+++ /dev/null
@@ -1,66 +0,0 @@
-/* ----------------------------------------------------------------------
-   SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer
-   http://sparta.github.io
-   Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov
-   Sandia National Laboratories
-
-   Copyright (2014) Sandia Corporation.  Under the terms of Contract
-   DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains
-   certain rights in this software.  This software is distributed under
-   the GNU General Public License.
-
-   See the README file in the top-level SPARTA directory.
-------------------------------------------------------------------------- */
-
-#ifdef FIX_CLASS
-
-FixStyle(ave/surf/kk,FixAveSurfKokkos)
-
-#else
-
-#ifndef SPARTA_FIX_AVE_SURF_KOKKOS_H
-#define SPARTA_FIX_AVE_SURF_KOKKOS_H
-
-#include "fix_ave_surf.h"
-#include "kokkos_type.h"
-
-namespace SPARTA_NS {
-
-struct TagFixAveSurf_Add_tally{};
-
-class FixAveSurfKokkos : public FixAveSurf {
- public:
-  FixAveSurfKokkos(class SPARTA *, int, char **);
-  ~FixAveSurfKokkos();
-  void init();
-  void setup();
-  void end_of_step();
-
-  KOKKOS_INLINE_FUNCTION
-  void operator()(TagFixAveSurf_Add_tally, const int&) const;
-
- private:
-  int kokkosable;        // 1 if the all-tally path runs on device
-                         // 0 if delegating entirely to the host base class
-  int nstally;           // # of local surf rows in a compute tally
-                         //   = surf->nlocal + surf->nghost
-  int acc_m,acc_col;     // value index / compute-tally column for current kernel
-
-  DAT::t_float_2d_lr d_acc;     // per-interval tally accumulator [nstally][nvalues]
-  DAT::t_float_2d_lr d_tally;   // current compute device tally (set per value)
-
-  surfint *tally2surf_all;      // surfID of each local surf row (host)
-  double *acc_local_vec;        // host copy of d_acc for collate (nvalues == 1)
-  double **acc_local;           // host copy of d_acc for collate (nvalues > 1)
-
-  void reallocate();
-};
-
-}
-
-#endif
-#endif
-
-/* ERROR/WARNING messages:
-
-*/
diff --git a/src/fix_ave_surf.cpp b/src/fix_ave_surf.cpp
index 602c3017b..d7640c26f 100644
--- a/src/fix_ave_surf.cpp
+++ b/src/fix_ave_surf.cpp
@@ -291,8 +291,6 @@ FixAveSurf::FixAveSurf(SPARTA *sparta, int narg, char **arg) :
 
 FixAveSurf::~FixAveSurf()
 {
-  if (copymode) return;
-
   delete [] which;
   delete [] argindex;
   delete [] value2index;
diff --git a/src/fix_ave_surf.h b/src/fix_ave_surf.h
index 3402b855d..01a746116 100644
--- a/src/fix_ave_surf.h
+++ b/src/fix_ave_surf.h
@@ -36,7 +36,7 @@ class FixAveSurf : public Fix {
   void end_of_step();
   double memory_usage();
 
- protected:
+ private:
   int groupbit;
   int nvalues,maxvalues;
   int nrepeat,irepeat,nsample,ave;

From 86629c5fc209c116e965d65affff05ae4d6a6c69 Mon Sep 17 00:00:00 2001
From: Claude 
Date: Mon, 29 Jun 2026 21:50:31 +0000
Subject: [PATCH 26/30] Enable KOKKOS regression tests for ported features

Now that this branch ports the surf_collide adiabatic/cll/td/impulsive
models, surf_react adsorb, and the gas-phase collide features, the KOKKOS
exclusion list (used when SPARTA_KOKKOS_EXACT runs the regression suite with
"-k on -sf kk") no longer needs to skip those inputs. Remove from the
SPARTA_KOKKOS_EXACT skip list the 18 inputs that now run bit-for-bit
identical to the non-KOKKOS gold logs:

  in.ablation.2d
  in.beam.{adiabatic,cll,impulsive,td}
  in.circle.{adiabatic,cll,impulsive,td}
  in.beam.face.{gs,gs_ps,ps}
  in.beam.surf.{gs,gs_ps,ps}
  in.circle.{gs,gs_ps,ps}

Kept excluded (still not KOKKOS-supported):
  in.ablation.3d   - implicit-surf collision hits zero cell volume under KOKKOS
  in.bfield        - external field fix not KOKKOS-enabled
  in.bfield.grid   - external field fix not KOKKOS-enabled

Also add committed CPU gold logs for the new feature examples
(in.collide.group, in.chem.gastally, in.ambi.group) so the KOKKOS job
compares each "-sf kk" run against the non-KOKKOS reference rather than
auto-generating a (trivially passing) KOKKOS-vs-KOKKOS log.

All enabled inputs verified bit-for-bit (Serial backend, SPARTA_KOKKOS_EXACT,
1 thread, np=1) against the non-KOKKOS reference.

Co-authored-by: stanmoore1 
Co-Authored-By: Claude Opus 4.8 
Claude-Session: https://claude.ai/code/session_01Q6nQVuxEiTUqSzDoDppCUM
---
 cmake/common/set/sparta_cmake_defaults.cmake  |  23 +---
 examples/ambi/log.29Jun26.mpi_1.ambi.group    | 121 ++++++++++++++++
 examples/chem/log.29Jun26.mpi_1.chem.gastally | 129 ++++++++++++++++++
 .../collide/log.29Jun26.mpi_1.collide.group   | 125 +++++++++++++++++
 4 files changed, 377 insertions(+), 21 deletions(-)
 create mode 100644 examples/ambi/log.29Jun26.mpi_1.ambi.group
 create mode 100644 examples/chem/log.29Jun26.mpi_1.chem.gastally
 create mode 100644 examples/collide/log.29Jun26.mpi_1.collide.group

diff --git a/cmake/common/set/sparta_cmake_defaults.cmake b/cmake/common/set/sparta_cmake_defaults.cmake
index de2cc2e52..09efdf18a 100644
--- a/cmake/common/set/sparta_cmake_defaults.cmake
+++ b/cmake/common/set/sparta_cmake_defaults.cmake
@@ -73,28 +73,9 @@ if(SPARTA_ENABLE_TESTING)
   # the non-KOKKOS configurations.
   if(SPARTA_KOKKOS_EXACT)
     list(APPEND SPARTA_DISABLED_TESTS
-        # fix ave/grid for grid/surf inputs not yet supported in KOKKOS
-        "in.ablation.2d"
+        # implicit-surface ablation in 3D errors under KOKKOS (zero collision
+        # cell volume); the 2D case runs bit-for-bit and is enabled
         "in.ablation.3d"
-        # surf_collide adiabatic/cll/td/impulsive styles not KOKKOS-enabled
-        "in.beam.adiabatic"
-        "in.beam.cll"
-        "in.beam.impulsive"
-        "in.beam.td"
-        "in.circle.adiabatic"
-        "in.circle.cll"
-        "in.circle.impulsive"
-        "in.circle.td"
-        # surf_react gs/ps styles use a non-KOKKOS-enabled surf_collide method
-        "in.beam.face.gs"
-        "in.beam.face.gs_ps"
-        "in.beam.face.ps"
-        "in.beam.surf.gs"
-        "in.beam.surf.gs_ps"
-        "in.beam.surf.ps"
-        "in.circle.gs"
-        "in.circle.gs_ps"
-        "in.circle.ps"
         # external field fix not KOKKOS-enabled
         "in.bfield"
         "in.bfield.grid"
diff --git a/examples/ambi/log.29Jun26.mpi_1.ambi.group b/examples/ambi/log.29Jun26.mpi_1.ambi.group
new file mode 100644
index 000000000..e5fedfb56
--- /dev/null
+++ b/examples/ambi/log.29Jun26.mpi_1.ambi.group
@@ -0,0 +1,121 @@
+SPARTA (24 Sep 2025)
+Running on 1 MPI task(s)
+################################################################################
+# thermal plasma in a 2d box, non-reacting multigroup ambipolar collisions
+#
+# Exercises the ambipolar approximation with more than one collision group:
+# all heavy species (neutrals + ions) form one group and the ambipolar
+# electron species "e" is its own group, as required by collide ambipolar.
+#
+# Note:
+#  - The "comm/sort" option to the "global" command is used to match MPI runs.
+#  - The "twopass" option is used to match Kokkos runs.
+# The "comm/sort" and "twopass" options should not be used for production runs.
+################################################################################
+
+seed                12345
+dimension           2
+boundary            rr rr p
+global              gridcut 0.01 comm/sort yes
+create_box          -2.0 2.0 -2.0 2.0 -0.5 0.5
+Created orthogonal box = (-2 -2 -0.5) to (2 2 0.5)
+create_grid         20 20 1
+Created 400 child grid cells
+  CPU time = 0.000936031 secs
+  create/ghost percent = 93.9124 6.08762
+balance_grid        rcb cell
+Balance grid migrated 0 cells
+  CPU time = 5.50747e-05 secs
+  reassign/sort/migrate/ghost percent = 43.7229 0 21.645 34.632
+
+global              fnum 2.6404E16
+global              nrho 2.6404e20
+
+species             air.species N2 O2 N O NO N2+ O2+ N+ O+ NO+ e
+
+# collide mixture: all species, two groups
+# the ambipolar electron species e must be in a group by itself
+
+mixture             gas N2 O2 N O NO N2+ O2+ N+ O+ NO+ vstream 0 0 0 temp 5000.0 group heavy
+mixture             gas e group electron
+mixture             gas N2 frac 0.6
+mixture             gas N2+ frac 0.4
+
+fix                 ambi ambipolar e N+ N2+ NO+ O+ O2+
+
+collide             vss gas air.vss
+collide_modify      ambipolar yes
+
+create_particles    gas n 10000 twopass
+Created 10000 particles
+  CPU time = 0.00320506 secs
+
+compute             temp temp
+stats               50
+stats_style         step np nattempt ncoll c_temp
+
+timestep            1.0e-8
+run                 300
+Memory usage per proc in Mbytes:
+  particles (ave,min,max) = 2 2 2
+  grid      (ave,min,max) = 1.51379 1.51379 1.51379
+  surf      (ave,min,max) = 0 0 0
+  total     (ave,min,max) = 3.51379 3.51379 3.51379
+Step Np Natt Ncoll c_temp 
+       0    10000        0        0    4975.0406 
+      50    10000       77       34    4975.0911 
+     100    10000      108       38    4975.0667 
+     150    10000      134       57    4975.0591 
+     200    10000       37       16    4974.9255 
+     250    10000       70       28    4976.2863 
+     300    10000       50       18    4974.4373 
+Loop time of 0.0635989 on 1 procs for 300 steps with 10000 particles
+Performance: 4717.065 timesteps/s, 47.171 Mparticle-step/s
+
+MPI task timing breakdown:
+Section |  min time  |  avg time  |  max time  |%varavg| %total
+---------------------------------------------------------------
+Move    | 0.015433   | 0.015433   | 0.015433   |   0.0 | 24.27
+Coll    | 0.042727   | 0.042727   | 0.042727   |   0.0 | 67.18
+Sort    | 0.0051994  | 0.0051994  | 0.0051994  |   0.0 |  8.18
+Comm    | 5.1737e-05 | 5.1737e-05 | 5.1737e-05 |   0.0 |  0.08
+Modify  | 0          | 0          | 0          |   0.0 |  0.00
+Output  | 0.00013208 | 0.00013208 | 0.00013208 |   0.0 |  0.21
+Other   |            | 5.627e-05  |            |       |  0.09
+
+Particle moves    = 3000000 (3M)
+Cells touched     = 3000293 (3M)
+Particle comms    = 0 (0K)
+Boundary collides = 20 (0.02K)
+Boundary exits    = 0 (0K)
+SurfColl checks   = 0 (0K)
+SurfColl occurs   = 0 (0K)
+Surf reactions    = 0 (0K)
+Collide attempts  = 20534 (20.5K)
+Collide occurs    = 8182 (8.18K)
+Reactions         = 0 (0K)
+Particles stuck   = 0
+Axisymm bad moves = 0
+
+Particle-moves/CPUsec/proc: 4.71706e+07
+Particle-moves/step: 10000
+Cell-touches/particle/step: 1.0001
+Particle comm iterations/step: 1
+Particle fraction communicated: 0
+Particle fraction colliding with boundary: 6.66667e-06
+Particle fraction exiting boundary: 0
+Surface-checks/particle/step: 0
+Surface-collisions/particle/step: 0
+Surf-reactions/particle/step: 0
+Collision-attempts/particle/step: 0.00684467
+Collisions/particle/step: 0.00272733
+Reactions/particle/step: 0
+
+Particles: 10000 ave 10000 max 10000 min
+Histogram: 1 0 0 0 0 0 0 0 0 0
+Cells:      400 ave 400 max 400 min
+Histogram: 1 0 0 0 0 0 0 0 0 0
+GhostCell: 0 ave 0 max 0 min
+Histogram: 1 0 0 0 0 0 0 0 0 0
+EmptyCell: 0 ave 0 max 0 min
+Histogram: 1 0 0 0 0 0 0 0 0 0
diff --git a/examples/chem/log.29Jun26.mpi_1.chem.gastally b/examples/chem/log.29Jun26.mpi_1.chem.gastally
new file mode 100644
index 000000000..a3a347843
--- /dev/null
+++ b/examples/chem/log.29Jun26.mpi_1.chem.gastally
@@ -0,0 +1,129 @@
+SPARTA (24 Sep 2025)
+Running on 1 MPI task(s)
+################################################################################
+# thermal gas in a 3d box with collisions and reactions
+# tally per-grid-cell gas collisions and reactions
+#
+# Demonstrates/verifies compute gas/collision/grid and compute gas/reaction/grid
+# (the latter in all/every/select modes).
+#
+# Note:
+#  - The "comm/sort" option to the "global" command is used to match MPI runs.
+#  - The "twopass" option is used to match Kokkos runs.
+# The "comm/sort" and "twopass" options should not be used for production runs.
+################################################################################
+
+seed	    	    12345
+dimension   	    3
+global              gridcut 1.0e-5 comm/sort yes
+boundary	    rr rr rr
+create_box  	    0 0.0001 0 0.0001 0 0.0001
+Created orthogonal box = (0 0 0) to (0.0001 0.0001 0.0001)
+create_grid 	    10 10 10
+Created 1000 child grid cells
+  CPU time = 0.000930071 secs
+  create/ghost percent = 81.9533 18.0467
+balance_grid        rcb part
+Balance grid migrated 0 cells
+  CPU time = 0.000136852 secs
+  reassign/sort/migrate/ghost percent = 48.2578 0.696864 13.0662 37.9791
+
+species		    air.species N2 N
+mixture		    air N2 N vstream 0.0 0.0 0.0 temp 20000.0
+mixture             air N2 frac 1.0
+mixture             air N frac 0.0
+
+global              nrho 7.07043E22
+global              fnum 7.07043E5
+
+collide		    vss air air.vss
+react               tce air.tce
+
+create_particles    air n 10000 twopass
+Created 10000 particles
+  CPU time = 0.00198483 secs
+
+# per-grid-cell tallies of gas collisions and reactions
+
+compute             cc gas/collision/grid all all
+compute             cr gas/reaction/grid all all all
+compute             cre gas/reaction/grid all all every
+
+# sums over all cells: c_sumcc == ncoll-nreact and c_sumcr == nreact each step
+
+compute             sumcc reduce sum c_cc
+compute             sumcr reduce sum c_cr
+
+stats		    100
+compute             temp temp
+stats_style	    step np nattempt ncoll nreact c_temp c_sumcc c_sumcr
+
+timestep 	    7.00E-9
+run 		    500
+Memory usage per proc in Mbytes:
+  particles (ave,min,max) = 1.5625 1.5625 1.5625
+  grid      (ave,min,max) = 1.51379 1.51379 1.51379
+  surf      (ave,min,max) = 0 0 0
+  total     (ave,min,max) = 3.43488 3.43488 3.43488
+Step Np Natt Ncoll Nreact c_temp c_sumcc c_sumcr 
+       0    10000        0        0        0    19907.187            0            0 
+     100    10190     1000      262        3    18764.989          259            3 
+     200    10335     1011      269        2    18038.695          267            2 
+     300    10484     1079      255        4    17426.475          251            4 
+     400    10627     1070      238        1    16707.966          237            1 
+     500    10760     1124      289        3    16036.998          286            3 
+Loop time of 0.779421 on 1 procs for 500 steps with 10760 particles
+Performance: 641.502 timesteps/s, 6.903 Mparticle-step/s
+
+MPI task timing breakdown:
+Section |  min time  |  avg time  |  max time  |%varavg| %total
+---------------------------------------------------------------
+Move    | 0.68919    | 0.68919    | 0.68919    |   0.0 | 88.42
+Coll    | 0.079147   | 0.079147   | 0.079147   |   0.0 | 10.15
+Sort    | 0.010417   | 0.010417   | 0.010417   |   0.0 |  1.34
+Comm    | 0.00019526 | 0.00019526 | 0.00019526 |   0.0 |  0.03
+Modify  | 0          | 0          | 0          |   0.0 |  0.00
+Output  | 0.0002346  | 0.0002346  | 0.0002346  |   0.0 |  0.03
+Other   |            | 0.0002329  |            |       |  0.03
+
+Particle moves    = 5201491 (5.2M)
+Cells touched     = 23735594 (23.7M)
+Particle comms    = 0 (0K)
+Boundary collides = 2059037 (2.06M)
+Boundary exits    = 0 (0K)
+SurfColl checks   = 0 (0K)
+SurfColl occurs   = 0 (0K)
+Surf reactions    = 0 (0K)
+Collide attempts  = 512014 (0.512M)
+Collide occurs    = 128199 (0.128M)
+Reactions         = 760 (0.76K)
+Particles stuck   = 0
+Axisymm bad moves = 0
+
+Particle-moves/CPUsec/proc: 6.67353e+06
+Particle-moves/step: 10403
+Cell-touches/particle/step: 4.56323
+Particle comm iterations/step: 1
+Particle fraction communicated: 0
+Particle fraction colliding with boundary: 0.395855
+Particle fraction exiting boundary: 0
+Surface-checks/particle/step: 0
+Surface-collisions/particle/step: 0
+Surf-reactions/particle/step: 0
+Collision-attempts/particle/step: 0.098436
+Collisions/particle/step: 0.0246466
+Reactions/particle/step: 0.000146112
+
+Gas reaction tallies:
+  style tce #-of-reactions 45
+  reaction N2 + N2 --> N + N + N2: 578
+  reaction N2 + N --> N + N + N: 182
+
+Particles: 10760 ave 10760 max 10760 min
+Histogram: 1 0 0 0 0 0 0 0 0 0
+Cells:      1000 ave 1000 max 1000 min
+Histogram: 1 0 0 0 0 0 0 0 0 0
+GhostCell: 0 ave 0 max 0 min
+Histogram: 1 0 0 0 0 0 0 0 0 0
+EmptyCell: 0 ave 0 max 0 min
+Histogram: 1 0 0 0 0 0 0 0 0 0
diff --git a/examples/collide/log.29Jun26.mpi_1.collide.group b/examples/collide/log.29Jun26.mpi_1.collide.group
new file mode 100644
index 000000000..86a004a07
--- /dev/null
+++ b/examples/collide/log.29Jun26.mpi_1.collide.group
@@ -0,0 +1,125 @@
+SPARTA (24 Sep 2025)
+Running on 1 MPI task(s)
+################################################################################
+# thermal gas in a 3d box with collisions, multiple collision groups
+# particles reflect off global box boundaries
+#
+# Demonstrates/verifies non-reacting multigroup (ngroup > 1) collisions.
+# The species are split into two collision groups: "heavy" and "light".
+#
+# Note:
+#  - The "comm/sort" option to the "global" command is used to match MPI runs.
+#  - The "twopass" option is used to match Kokkos runs.
+# The "comm/sort" and "twopass" options should not be used for production runs.
+################################################################################
+
+seed	    	    12345
+dimension   	    3
+global              gridcut 1.0e-5 comm/sort yes
+
+boundary	    rr rr rr
+
+create_box  	    0 0.0001 0 0.0001 0 0.0001
+Created orthogonal box = (0 0 0) to (0.0001 0.0001 0.0001)
+create_grid 	    10 10 10
+Created 1000 child grid cells
+  CPU time = 0.00102401 secs
+  create/ghost percent = 85.4482 14.5518
+
+balance_grid        rcb part
+Balance grid migrated 0 cells
+  CPU time = 0.000148058 secs
+  reassign/sort/migrate/ghost percent = 47.343 0.644122 12.8824 39.1304
+
+species		    6SpeciesAir.species N2 O2 NO N O Ar
+
+mixture		    air O2 N2 O N vstream 0.0 0.0 0.0 temp 273.1
+mixture		    air O2 frac 0.21 group heavy
+mixture		    air N2 frac 0.78 group heavy
+mixture		    air NO group heavy
+mixture		    air Ar frac 0.009 group heavy
+mixture		    air O group light
+mixture		    air N group light
+
+global              nrho 7.07043E22
+global              fnum 7.07043E6
+
+collide		    vss air 6SpeciesAirII.vss
+
+create_particles    air n 10000 twopass
+Created 10000 particles
+  CPU time = 0.00219703 secs
+
+stats		    100
+compute             temp temp
+stats_style	    step cpu np nattempt ncoll c_temp
+
+timestep 	    7.00E-9
+run 		    1000
+Memory usage per proc in Mbytes:
+  particles (ave,min,max) = 1.5625 1.5625 1.5625
+  grid      (ave,min,max) = 1.51379 1.51379 1.51379
+  surf      (ave,min,max) = 0 0 0
+  total     (ave,min,max) = 3.07629 3.07629 3.07629
+Step CPU Np Natt Ncoll c_temp 
+       0            0    10000        0        0    277.59158 
+     100  0.068479061    10000     1314      705    273.84816 
+     200   0.13391995    10000     1368      732    276.91389 
+     300   0.18817091    10000     1387      712    277.42376 
+     400   0.23841596    10000     1441      732    276.07365 
+     500   0.28786993    10000     1465      751    275.21438 
+     600   0.33715987    10000     1410      717    275.57577 
+     700   0.38843393    10000     1456      698    275.57243 
+     800   0.43906307    10000     1456      677    275.21249 
+     900   0.49090099    10000     1459      729    276.30512 
+    1000   0.54217601    10000     1513      741    277.13434 
+Loop time of 0.542186 on 1 procs for 1000 steps with 10000 particles
+Performance: 1844.385 timesteps/s, 18.444 Mparticle-step/s
+
+MPI task timing breakdown:
+Section |  min time  |  avg time  |  max time  |%varavg| %total
+---------------------------------------------------------------
+Move    | 0.2215     | 0.2215     | 0.2215     |   0.0 | 40.85
+Coll    | 0.29954    | 0.29954    | 0.29954    |   0.0 | 55.25
+Sort    | 0.020196   | 0.020196   | 0.020196   |   0.0 |  3.72
+Comm    | 0.00033998 | 0.00033998 | 0.00033998 |   0.0 |  0.06
+Modify  | 0          | 0          | 0          |   0.0 |  0.00
+Output  | 0.00033808 | 0.00033808 | 0.00033808 |   0.0 |  0.06
+Other   |            | 0.0002744  |            |       |  0.05
+
+Particle moves    = 10000000 (10M)
+Cells touched     = 14254616 (14.3M)
+Particle comms    = 0 (0K)
+Boundary collides = 472688 (0.473M)
+Boundary exits    = 0 (0K)
+SurfColl checks   = 0 (0K)
+SurfColl occurs   = 0 (0K)
+Surf reactions    = 0 (0K)
+Collide attempts  = 1408271 (1.41M)
+Collide occurs    = 718301 (0.718M)
+Reactions         = 0 (0K)
+Particles stuck   = 0
+Axisymm bad moves = 0
+
+Particle-moves/CPUsec/proc: 1.84439e+07
+Particle-moves/step: 10000
+Cell-touches/particle/step: 1.42546
+Particle comm iterations/step: 1
+Particle fraction communicated: 0
+Particle fraction colliding with boundary: 0.0472688
+Particle fraction exiting boundary: 0
+Surface-checks/particle/step: 0
+Surface-collisions/particle/step: 0
+Surf-reactions/particle/step: 0
+Collision-attempts/particle/step: 0.140827
+Collisions/particle/step: 0.0718301
+Reactions/particle/step: 0
+
+Particles: 10000 ave 10000 max 10000 min
+Histogram: 1 0 0 0 0 0 0 0 0 0
+Cells:      1000 ave 1000 max 1000 min
+Histogram: 1 0 0 0 0 0 0 0 0 0
+GhostCell: 0 ave 0 max 0 min
+Histogram: 1 0 0 0 0 0 0 0 0 0
+EmptyCell: 0 ave 0 max 0 min
+Histogram: 1 0 0 0 0 0 0 0 0 0

From fb792ea07234cbe888fad0955ce5d8ab74122800 Mon Sep 17 00:00:00 2001
From: Claude 
Date: Tue, 7 Jul 2026 23:02:31 +0000
Subject: [PATCH 27/30] Remove platform-specific gold logs for new KOKKOS
 examples

The committed gold logs for in.collide.group, in.chem.gastally, and
in.ambi.group were blessed on a dev machine whose libm/FP behavior differs
from the CI runner (ubuntu-22.04). Over a multi-hundred-step stochastic
collision run, last-bit differences in exp/sqrt/pow accumulate until an
integer collision count flips, so a bit-exact (1e-7) comparison against an
off-platform gold log fails in upstream CI. This is cross-OS numerical drift,
not a correctness bug: on any single machine the KOKKOS ("-k on -sf kk") and
non-KOKKOS runs of these inputs are bit-for-bit identical under
SPARTA_KOKKOS_EXACT.

Drop the gold logs so the regression harness auto-generates the reference on
the CI platform per run (self-consistent, same OS), matching the many other
examples that ship without a committed log. The KOKKOS-vs-non-KOKKOS
bit-exactness of these inputs remains verified locally.

Co-authored-by: stanmoore1 
Co-Authored-By: Claude Opus 4.8 
Claude-Session: https://claude.ai/code/session_01Q6nQVuxEiTUqSzDoDppCUM
---
 examples/ambi/log.29Jun26.mpi_1.ambi.group    | 121 ----------------
 examples/chem/log.29Jun26.mpi_1.chem.gastally | 129 ------------------
 .../collide/log.29Jun26.mpi_1.collide.group   | 125 -----------------
 3 files changed, 375 deletions(-)
 delete mode 100644 examples/ambi/log.29Jun26.mpi_1.ambi.group
 delete mode 100644 examples/chem/log.29Jun26.mpi_1.chem.gastally
 delete mode 100644 examples/collide/log.29Jun26.mpi_1.collide.group

diff --git a/examples/ambi/log.29Jun26.mpi_1.ambi.group b/examples/ambi/log.29Jun26.mpi_1.ambi.group
deleted file mode 100644
index e5fedfb56..000000000
--- a/examples/ambi/log.29Jun26.mpi_1.ambi.group
+++ /dev/null
@@ -1,121 +0,0 @@
-SPARTA (24 Sep 2025)
-Running on 1 MPI task(s)
-################################################################################
-# thermal plasma in a 2d box, non-reacting multigroup ambipolar collisions
-#
-# Exercises the ambipolar approximation with more than one collision group:
-# all heavy species (neutrals + ions) form one group and the ambipolar
-# electron species "e" is its own group, as required by collide ambipolar.
-#
-# Note:
-#  - The "comm/sort" option to the "global" command is used to match MPI runs.
-#  - The "twopass" option is used to match Kokkos runs.
-# The "comm/sort" and "twopass" options should not be used for production runs.
-################################################################################
-
-seed                12345
-dimension           2
-boundary            rr rr p
-global              gridcut 0.01 comm/sort yes
-create_box          -2.0 2.0 -2.0 2.0 -0.5 0.5
-Created orthogonal box = (-2 -2 -0.5) to (2 2 0.5)
-create_grid         20 20 1
-Created 400 child grid cells
-  CPU time = 0.000936031 secs
-  create/ghost percent = 93.9124 6.08762
-balance_grid        rcb cell
-Balance grid migrated 0 cells
-  CPU time = 5.50747e-05 secs
-  reassign/sort/migrate/ghost percent = 43.7229 0 21.645 34.632
-
-global              fnum 2.6404E16
-global              nrho 2.6404e20
-
-species             air.species N2 O2 N O NO N2+ O2+ N+ O+ NO+ e
-
-# collide mixture: all species, two groups
-# the ambipolar electron species e must be in a group by itself
-
-mixture             gas N2 O2 N O NO N2+ O2+ N+ O+ NO+ vstream 0 0 0 temp 5000.0 group heavy
-mixture             gas e group electron
-mixture             gas N2 frac 0.6
-mixture             gas N2+ frac 0.4
-
-fix                 ambi ambipolar e N+ N2+ NO+ O+ O2+
-
-collide             vss gas air.vss
-collide_modify      ambipolar yes
-
-create_particles    gas n 10000 twopass
-Created 10000 particles
-  CPU time = 0.00320506 secs
-
-compute             temp temp
-stats               50
-stats_style         step np nattempt ncoll c_temp
-
-timestep            1.0e-8
-run                 300
-Memory usage per proc in Mbytes:
-  particles (ave,min,max) = 2 2 2
-  grid      (ave,min,max) = 1.51379 1.51379 1.51379
-  surf      (ave,min,max) = 0 0 0
-  total     (ave,min,max) = 3.51379 3.51379 3.51379
-Step Np Natt Ncoll c_temp 
-       0    10000        0        0    4975.0406 
-      50    10000       77       34    4975.0911 
-     100    10000      108       38    4975.0667 
-     150    10000      134       57    4975.0591 
-     200    10000       37       16    4974.9255 
-     250    10000       70       28    4976.2863 
-     300    10000       50       18    4974.4373 
-Loop time of 0.0635989 on 1 procs for 300 steps with 10000 particles
-Performance: 4717.065 timesteps/s, 47.171 Mparticle-step/s
-
-MPI task timing breakdown:
-Section |  min time  |  avg time  |  max time  |%varavg| %total
----------------------------------------------------------------
-Move    | 0.015433   | 0.015433   | 0.015433   |   0.0 | 24.27
-Coll    | 0.042727   | 0.042727   | 0.042727   |   0.0 | 67.18
-Sort    | 0.0051994  | 0.0051994  | 0.0051994  |   0.0 |  8.18
-Comm    | 5.1737e-05 | 5.1737e-05 | 5.1737e-05 |   0.0 |  0.08
-Modify  | 0          | 0          | 0          |   0.0 |  0.00
-Output  | 0.00013208 | 0.00013208 | 0.00013208 |   0.0 |  0.21
-Other   |            | 5.627e-05  |            |       |  0.09
-
-Particle moves    = 3000000 (3M)
-Cells touched     = 3000293 (3M)
-Particle comms    = 0 (0K)
-Boundary collides = 20 (0.02K)
-Boundary exits    = 0 (0K)
-SurfColl checks   = 0 (0K)
-SurfColl occurs   = 0 (0K)
-Surf reactions    = 0 (0K)
-Collide attempts  = 20534 (20.5K)
-Collide occurs    = 8182 (8.18K)
-Reactions         = 0 (0K)
-Particles stuck   = 0
-Axisymm bad moves = 0
-
-Particle-moves/CPUsec/proc: 4.71706e+07
-Particle-moves/step: 10000
-Cell-touches/particle/step: 1.0001
-Particle comm iterations/step: 1
-Particle fraction communicated: 0
-Particle fraction colliding with boundary: 6.66667e-06
-Particle fraction exiting boundary: 0
-Surface-checks/particle/step: 0
-Surface-collisions/particle/step: 0
-Surf-reactions/particle/step: 0
-Collision-attempts/particle/step: 0.00684467
-Collisions/particle/step: 0.00272733
-Reactions/particle/step: 0
-
-Particles: 10000 ave 10000 max 10000 min
-Histogram: 1 0 0 0 0 0 0 0 0 0
-Cells:      400 ave 400 max 400 min
-Histogram: 1 0 0 0 0 0 0 0 0 0
-GhostCell: 0 ave 0 max 0 min
-Histogram: 1 0 0 0 0 0 0 0 0 0
-EmptyCell: 0 ave 0 max 0 min
-Histogram: 1 0 0 0 0 0 0 0 0 0
diff --git a/examples/chem/log.29Jun26.mpi_1.chem.gastally b/examples/chem/log.29Jun26.mpi_1.chem.gastally
deleted file mode 100644
index a3a347843..000000000
--- a/examples/chem/log.29Jun26.mpi_1.chem.gastally
+++ /dev/null
@@ -1,129 +0,0 @@
-SPARTA (24 Sep 2025)
-Running on 1 MPI task(s)
-################################################################################
-# thermal gas in a 3d box with collisions and reactions
-# tally per-grid-cell gas collisions and reactions
-#
-# Demonstrates/verifies compute gas/collision/grid and compute gas/reaction/grid
-# (the latter in all/every/select modes).
-#
-# Note:
-#  - The "comm/sort" option to the "global" command is used to match MPI runs.
-#  - The "twopass" option is used to match Kokkos runs.
-# The "comm/sort" and "twopass" options should not be used for production runs.
-################################################################################
-
-seed	    	    12345
-dimension   	    3
-global              gridcut 1.0e-5 comm/sort yes
-boundary	    rr rr rr
-create_box  	    0 0.0001 0 0.0001 0 0.0001
-Created orthogonal box = (0 0 0) to (0.0001 0.0001 0.0001)
-create_grid 	    10 10 10
-Created 1000 child grid cells
-  CPU time = 0.000930071 secs
-  create/ghost percent = 81.9533 18.0467
-balance_grid        rcb part
-Balance grid migrated 0 cells
-  CPU time = 0.000136852 secs
-  reassign/sort/migrate/ghost percent = 48.2578 0.696864 13.0662 37.9791
-
-species		    air.species N2 N
-mixture		    air N2 N vstream 0.0 0.0 0.0 temp 20000.0
-mixture             air N2 frac 1.0
-mixture             air N frac 0.0
-
-global              nrho 7.07043E22
-global              fnum 7.07043E5
-
-collide		    vss air air.vss
-react               tce air.tce
-
-create_particles    air n 10000 twopass
-Created 10000 particles
-  CPU time = 0.00198483 secs
-
-# per-grid-cell tallies of gas collisions and reactions
-
-compute             cc gas/collision/grid all all
-compute             cr gas/reaction/grid all all all
-compute             cre gas/reaction/grid all all every
-
-# sums over all cells: c_sumcc == ncoll-nreact and c_sumcr == nreact each step
-
-compute             sumcc reduce sum c_cc
-compute             sumcr reduce sum c_cr
-
-stats		    100
-compute             temp temp
-stats_style	    step np nattempt ncoll nreact c_temp c_sumcc c_sumcr
-
-timestep 	    7.00E-9
-run 		    500
-Memory usage per proc in Mbytes:
-  particles (ave,min,max) = 1.5625 1.5625 1.5625
-  grid      (ave,min,max) = 1.51379 1.51379 1.51379
-  surf      (ave,min,max) = 0 0 0
-  total     (ave,min,max) = 3.43488 3.43488 3.43488
-Step Np Natt Ncoll Nreact c_temp c_sumcc c_sumcr 
-       0    10000        0        0        0    19907.187            0            0 
-     100    10190     1000      262        3    18764.989          259            3 
-     200    10335     1011      269        2    18038.695          267            2 
-     300    10484     1079      255        4    17426.475          251            4 
-     400    10627     1070      238        1    16707.966          237            1 
-     500    10760     1124      289        3    16036.998          286            3 
-Loop time of 0.779421 on 1 procs for 500 steps with 10760 particles
-Performance: 641.502 timesteps/s, 6.903 Mparticle-step/s
-
-MPI task timing breakdown:
-Section |  min time  |  avg time  |  max time  |%varavg| %total
----------------------------------------------------------------
-Move    | 0.68919    | 0.68919    | 0.68919    |   0.0 | 88.42
-Coll    | 0.079147   | 0.079147   | 0.079147   |   0.0 | 10.15
-Sort    | 0.010417   | 0.010417   | 0.010417   |   0.0 |  1.34
-Comm    | 0.00019526 | 0.00019526 | 0.00019526 |   0.0 |  0.03
-Modify  | 0          | 0          | 0          |   0.0 |  0.00
-Output  | 0.0002346  | 0.0002346  | 0.0002346  |   0.0 |  0.03
-Other   |            | 0.0002329  |            |       |  0.03
-
-Particle moves    = 5201491 (5.2M)
-Cells touched     = 23735594 (23.7M)
-Particle comms    = 0 (0K)
-Boundary collides = 2059037 (2.06M)
-Boundary exits    = 0 (0K)
-SurfColl checks   = 0 (0K)
-SurfColl occurs   = 0 (0K)
-Surf reactions    = 0 (0K)
-Collide attempts  = 512014 (0.512M)
-Collide occurs    = 128199 (0.128M)
-Reactions         = 760 (0.76K)
-Particles stuck   = 0
-Axisymm bad moves = 0
-
-Particle-moves/CPUsec/proc: 6.67353e+06
-Particle-moves/step: 10403
-Cell-touches/particle/step: 4.56323
-Particle comm iterations/step: 1
-Particle fraction communicated: 0
-Particle fraction colliding with boundary: 0.395855
-Particle fraction exiting boundary: 0
-Surface-checks/particle/step: 0
-Surface-collisions/particle/step: 0
-Surf-reactions/particle/step: 0
-Collision-attempts/particle/step: 0.098436
-Collisions/particle/step: 0.0246466
-Reactions/particle/step: 0.000146112
-
-Gas reaction tallies:
-  style tce #-of-reactions 45
-  reaction N2 + N2 --> N + N + N2: 578
-  reaction N2 + N --> N + N + N: 182
-
-Particles: 10760 ave 10760 max 10760 min
-Histogram: 1 0 0 0 0 0 0 0 0 0
-Cells:      1000 ave 1000 max 1000 min
-Histogram: 1 0 0 0 0 0 0 0 0 0
-GhostCell: 0 ave 0 max 0 min
-Histogram: 1 0 0 0 0 0 0 0 0 0
-EmptyCell: 0 ave 0 max 0 min
-Histogram: 1 0 0 0 0 0 0 0 0 0
diff --git a/examples/collide/log.29Jun26.mpi_1.collide.group b/examples/collide/log.29Jun26.mpi_1.collide.group
deleted file mode 100644
index 86a004a07..000000000
--- a/examples/collide/log.29Jun26.mpi_1.collide.group
+++ /dev/null
@@ -1,125 +0,0 @@
-SPARTA (24 Sep 2025)
-Running on 1 MPI task(s)
-################################################################################
-# thermal gas in a 3d box with collisions, multiple collision groups
-# particles reflect off global box boundaries
-#
-# Demonstrates/verifies non-reacting multigroup (ngroup > 1) collisions.
-# The species are split into two collision groups: "heavy" and "light".
-#
-# Note:
-#  - The "comm/sort" option to the "global" command is used to match MPI runs.
-#  - The "twopass" option is used to match Kokkos runs.
-# The "comm/sort" and "twopass" options should not be used for production runs.
-################################################################################
-
-seed	    	    12345
-dimension   	    3
-global              gridcut 1.0e-5 comm/sort yes
-
-boundary	    rr rr rr
-
-create_box  	    0 0.0001 0 0.0001 0 0.0001
-Created orthogonal box = (0 0 0) to (0.0001 0.0001 0.0001)
-create_grid 	    10 10 10
-Created 1000 child grid cells
-  CPU time = 0.00102401 secs
-  create/ghost percent = 85.4482 14.5518
-
-balance_grid        rcb part
-Balance grid migrated 0 cells
-  CPU time = 0.000148058 secs
-  reassign/sort/migrate/ghost percent = 47.343 0.644122 12.8824 39.1304
-
-species		    6SpeciesAir.species N2 O2 NO N O Ar
-
-mixture		    air O2 N2 O N vstream 0.0 0.0 0.0 temp 273.1
-mixture		    air O2 frac 0.21 group heavy
-mixture		    air N2 frac 0.78 group heavy
-mixture		    air NO group heavy
-mixture		    air Ar frac 0.009 group heavy
-mixture		    air O group light
-mixture		    air N group light
-
-global              nrho 7.07043E22
-global              fnum 7.07043E6
-
-collide		    vss air 6SpeciesAirII.vss
-
-create_particles    air n 10000 twopass
-Created 10000 particles
-  CPU time = 0.00219703 secs
-
-stats		    100
-compute             temp temp
-stats_style	    step cpu np nattempt ncoll c_temp
-
-timestep 	    7.00E-9
-run 		    1000
-Memory usage per proc in Mbytes:
-  particles (ave,min,max) = 1.5625 1.5625 1.5625
-  grid      (ave,min,max) = 1.51379 1.51379 1.51379
-  surf      (ave,min,max) = 0 0 0
-  total     (ave,min,max) = 3.07629 3.07629 3.07629
-Step CPU Np Natt Ncoll c_temp 
-       0            0    10000        0        0    277.59158 
-     100  0.068479061    10000     1314      705    273.84816 
-     200   0.13391995    10000     1368      732    276.91389 
-     300   0.18817091    10000     1387      712    277.42376 
-     400   0.23841596    10000     1441      732    276.07365 
-     500   0.28786993    10000     1465      751    275.21438 
-     600   0.33715987    10000     1410      717    275.57577 
-     700   0.38843393    10000     1456      698    275.57243 
-     800   0.43906307    10000     1456      677    275.21249 
-     900   0.49090099    10000     1459      729    276.30512 
-    1000   0.54217601    10000     1513      741    277.13434 
-Loop time of 0.542186 on 1 procs for 1000 steps with 10000 particles
-Performance: 1844.385 timesteps/s, 18.444 Mparticle-step/s
-
-MPI task timing breakdown:
-Section |  min time  |  avg time  |  max time  |%varavg| %total
----------------------------------------------------------------
-Move    | 0.2215     | 0.2215     | 0.2215     |   0.0 | 40.85
-Coll    | 0.29954    | 0.29954    | 0.29954    |   0.0 | 55.25
-Sort    | 0.020196   | 0.020196   | 0.020196   |   0.0 |  3.72
-Comm    | 0.00033998 | 0.00033998 | 0.00033998 |   0.0 |  0.06
-Modify  | 0          | 0          | 0          |   0.0 |  0.00
-Output  | 0.00033808 | 0.00033808 | 0.00033808 |   0.0 |  0.06
-Other   |            | 0.0002744  |            |       |  0.05
-
-Particle moves    = 10000000 (10M)
-Cells touched     = 14254616 (14.3M)
-Particle comms    = 0 (0K)
-Boundary collides = 472688 (0.473M)
-Boundary exits    = 0 (0K)
-SurfColl checks   = 0 (0K)
-SurfColl occurs   = 0 (0K)
-Surf reactions    = 0 (0K)
-Collide attempts  = 1408271 (1.41M)
-Collide occurs    = 718301 (0.718M)
-Reactions         = 0 (0K)
-Particles stuck   = 0
-Axisymm bad moves = 0
-
-Particle-moves/CPUsec/proc: 1.84439e+07
-Particle-moves/step: 10000
-Cell-touches/particle/step: 1.42546
-Particle comm iterations/step: 1
-Particle fraction communicated: 0
-Particle fraction colliding with boundary: 0.0472688
-Particle fraction exiting boundary: 0
-Surface-checks/particle/step: 0
-Surface-collisions/particle/step: 0
-Surf-reactions/particle/step: 0
-Collision-attempts/particle/step: 0.140827
-Collisions/particle/step: 0.0718301
-Reactions/particle/step: 0
-
-Particles: 10000 ave 10000 max 10000 min
-Histogram: 1 0 0 0 0 0 0 0 0 0
-Cells:      1000 ave 1000 max 1000 min
-Histogram: 1 0 0 0 0 0 0 0 0 0
-GhostCell: 0 ave 0 max 0 min
-Histogram: 1 0 0 0 0 0 0 0 0 0
-EmptyCell: 0 ave 0 max 0 min
-Histogram: 1 0 0 0 0 0 0 0 0 0

From 6b4143f129d1c660b21ef73638642cd7ba351945 Mon Sep 17 00:00:00 2001
From: Claude 
Date: Wed, 8 Jul 2026 13:29:43 +0000
Subject: [PATCH 28/30] Rebless in.collide.group gold logs from CI platform

Add committed gold logs for the group-collision example, generated on the
CI runner (ubuntu-22.04, SPARTA 24 Sep 2025) at 1 and 4 MPI ranks. These
replace the dev-machine logs removed earlier: a gold log is only bit-exact
on the OS it was blessed on, so the reference must come from the CI platform.

With these in place the regression harness performs a real bit-exact
comparison (KOKKOS "-k on -sf kk" and non-KOKKOS runs vs the committed
reference) rather than auto-generating a self-consistent log.

Co-authored-by: stanmoore1 
Co-Authored-By: Claude Opus 4.8 
Claude-Session: https://claude.ai/code/session_01Q6nQVuxEiTUqSzDoDppCUM
---
 .../collide/log.24Sep25.mpi_1.collide.group   | 125 +++++++++++++++++
 .../collide/log.24Sep25.mpi_4.collide.group   | 126 ++++++++++++++++++
 2 files changed, 251 insertions(+)
 create mode 100644 examples/collide/log.24Sep25.mpi_1.collide.group
 create mode 100644 examples/collide/log.24Sep25.mpi_4.collide.group

diff --git a/examples/collide/log.24Sep25.mpi_1.collide.group b/examples/collide/log.24Sep25.mpi_1.collide.group
new file mode 100644
index 000000000..05905d5a1
--- /dev/null
+++ b/examples/collide/log.24Sep25.mpi_1.collide.group
@@ -0,0 +1,125 @@
+SPARTA (24 Sep 2025)
+Running on 1 MPI task(s)
+################################################################################
+# thermal gas in a 3d box with collisions, multiple collision groups
+# particles reflect off global box boundaries
+#
+# Demonstrates/verifies non-reacting multigroup (ngroup > 1) collisions.
+# The species are split into two collision groups: "heavy" and "light".
+#
+# Note:
+#  - The "comm/sort" option to the "global" command is used to match MPI runs.
+#  - The "twopass" option is used to match Kokkos runs.
+# The "comm/sort" and "twopass" options should not be used for production runs.
+################################################################################
+
+seed	    	    12345
+dimension   	    3
+global              gridcut 1.0e-5 comm/sort yes
+
+boundary	    rr rr rr
+
+create_box  	    0 0.0001 0 0.0001 0 0.0001
+Created orthogonal box = (0 0 0) to (0.0001 0.0001 0.0001)
+create_grid 	    10 10 10
+Created 1000 child grid cells
+  CPU time = 0.00142212 secs
+  create/ghost percent = 83.9976 16.0024
+
+balance_grid        rcb part
+Balance grid migrated 0 cells
+  CPU time = 0.000282915 secs
+  reassign/sort/migrate/ghost percent = 58.3313 0.385982 11.6957 29.587
+
+species		    6SpeciesAir.species N2 O2 NO N O Ar
+
+mixture		    air O2 N2 O N vstream 0.0 0.0 0.0 temp 273.1
+mixture		    air O2 frac 0.21 group heavy
+mixture		    air N2 frac 0.78 group heavy
+mixture		    air NO group heavy
+mixture		    air Ar frac 0.009 group heavy
+mixture		    air O group light
+mixture		    air N group light
+
+global              nrho 7.07043E22
+global              fnum 7.07043E6
+
+collide		    vss air 6SpeciesAirII.vss
+
+create_particles    air n 10000 twopass
+Created 10000 particles
+  CPU time = 0.00303553 secs
+
+stats		    100
+compute             temp temp
+stats_style	    step cpu np nattempt ncoll c_temp
+
+timestep 	    7.00E-9
+run 		    1000
+Memory usage per proc in Mbytes:
+  particles (ave,min,max) = 1.5625 1.5625 1.5625
+  grid      (ave,min,max) = 1.51379 1.51379 1.51379
+  surf      (ave,min,max) = 0 0 0
+  total     (ave,min,max) = 3.07629 3.07629 3.07629
+Step CPU Np Natt Ncoll c_temp 
+       0            0    10000        0        0    277.59158 
+     100  0.074204402    10000     1314      705    273.84816 
+     200   0.15109194    10000     1368      732    276.91389 
+     300    0.2302166    10000     1387      712    277.42376 
+     400    0.3073353    10000     1441      732    276.07365 
+     500    0.3850255    10000     1465      751    275.21438 
+     600   0.46709235    10000     1410      717    275.57577 
+     700   0.54452659    10000     1456      698    275.57243 
+     800   0.62213841    10000     1456      677    275.21249 
+     900   0.70012387    10000     1486      756    277.62023 
+    1000   0.77825893    10000     1464      692    277.53426 
+Loop time of 0.778362 on 1 procs for 1000 steps with 10000 particles
+Performance: 1284.750 timesteps/s, 12.847 Mparticle-step/s
+
+MPI task timing breakdown:
+Section |  min time  |  avg time  |  max time  |%varavg| %total
+---------------------------------------------------------------
+Move    | 0.28888    | 0.28888    | 0.28888    |   0.0 | 37.11
+Coll    | 0.4618     | 0.4618     | 0.4618     |   0.0 | 59.33
+Sort    | 0.025941   | 0.025941   | 0.025941   |   0.0 |  3.33
+Comm    | 0.00036157 | 0.00036157 | 0.00036157 |   0.0 |  0.05
+Modify  | 0          | 0          | 0          |   0.0 |  0.00
+Output  | 0.00094166 | 0.00094166 | 0.00094166 |   0.0 |  0.12
+Other   |            | 0.0004362  |            |       |  0.06
+
+Particle moves    = 10000000 (10M)
+Cells touched     = 14254925 (14.3M)
+Particle comms    = 0 (0K)
+Boundary collides = 472768 (0.473M)
+Boundary exits    = 0 (0K)
+SurfColl checks   = 0 (0K)
+SurfColl occurs   = 0 (0K)
+Surf reactions    = 0 (0K)
+Collide attempts  = 1408068 (1.41M)
+Collide occurs    = 718153 (0.718M)
+Reactions         = 0 (0K)
+Particles stuck   = 0
+Axisymm bad moves = 0
+
+Particle-moves/CPUsec/proc: 1.28475e+07
+Particle-moves/step: 10000
+Cell-touches/particle/step: 1.42549
+Particle comm iterations/step: 1
+Particle fraction communicated: 0
+Particle fraction colliding with boundary: 0.0472768
+Particle fraction exiting boundary: 0
+Surface-checks/particle/step: 0
+Surface-collisions/particle/step: 0
+Surf-reactions/particle/step: 0
+Collision-attempts/particle/step: 0.140807
+Collisions/particle/step: 0.0718153
+Reactions/particle/step: 0
+
+Particles: 10000 ave 10000 max 10000 min
+Histogram: 1 0 0 0 0 0 0 0 0 0
+Cells:      1000 ave 1000 max 1000 min
+Histogram: 1 0 0 0 0 0 0 0 0 0
+GhostCell: 0 ave 0 max 0 min
+Histogram: 1 0 0 0 0 0 0 0 0 0
+EmptyCell: 0 ave 0 max 0 min
+Histogram: 1 0 0 0 0 0 0 0 0 0
diff --git a/examples/collide/log.24Sep25.mpi_4.collide.group b/examples/collide/log.24Sep25.mpi_4.collide.group
new file mode 100644
index 000000000..0fee37ae8
--- /dev/null
+++ b/examples/collide/log.24Sep25.mpi_4.collide.group
@@ -0,0 +1,126 @@
+SPARTA (24 Sep 2025)
+Running on 4 MPI task(s)
+################################################################################
+# thermal gas in a 3d box with collisions, multiple collision groups
+# particles reflect off global box boundaries
+#
+# Demonstrates/verifies non-reacting multigroup (ngroup > 1) collisions.
+# The species are split into two collision groups: "heavy" and "light".
+#
+# Note:
+#  - The "comm/sort" option to the "global" command is used to match MPI runs.
+#  - The "twopass" option is used to match Kokkos runs.
+# The "comm/sort" and "twopass" options should not be used for production runs.
+################################################################################
+
+seed	    	    12345
+dimension   	    3
+global              gridcut 1.0e-5 comm/sort yes
+
+boundary	    rr rr rr
+
+create_box  	    0 0.0001 0 0.0001 0 0.0001
+Created orthogonal box = (0 0 0) to (0.0001 0.0001 0.0001)
+create_grid 	    10 10 10
+WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/runner/work/sparta/sparta/src/grid.cpp:473)
+Created 1000 child grid cells
+  CPU time = 0.00256409 secs
+  create/ghost percent = 96.1988 3.80119
+
+balance_grid        rcb part
+Balance grid migrated 740 cells
+  CPU time = 0.0015163 secs
+  reassign/sort/migrate/ghost percent = 45.2081 0.572645 18.8967 35.3225
+
+species		    6SpeciesAir.species N2 O2 NO N O Ar
+
+mixture		    air O2 N2 O N vstream 0.0 0.0 0.0 temp 273.1
+mixture		    air O2 frac 0.21 group heavy
+mixture		    air N2 frac 0.78 group heavy
+mixture		    air NO group heavy
+mixture		    air Ar frac 0.009 group heavy
+mixture		    air O group light
+mixture		    air N group light
+
+global              nrho 7.07043E22
+global              fnum 7.07043E6
+
+collide		    vss air 6SpeciesAirII.vss
+
+create_particles    air n 10000 twopass
+Created 10000 particles
+  CPU time = 0.00346542 secs
+
+stats		    100
+compute             temp temp
+stats_style	    step cpu np nattempt ncoll c_temp
+
+timestep 	    7.00E-9
+run 		    1000
+Memory usage per proc in Mbytes:
+  particles (ave,min,max) = 1.5625 1.5625 1.5625
+  grid      (ave,min,max) = 1.51379 1.51379 1.51379
+  surf      (ave,min,max) = 0 0 0
+  total     (ave,min,max) = 3.07629 3.07629 3.07629
+Step CPU Np Natt Ncoll c_temp 
+       0            0    10000        0        0    273.34821 
+     100  0.028047249    10000     1329      716    276.86384 
+     200  0.050504899    10000     1338      677    274.71485 
+     300  0.073207879    10000     1394      712      274.004 
+     400  0.095735905    10000     1424      732    273.64174 
+     500    0.1181215    10000     1408      733    272.42923 
+     600   0.14066922    10000     1464      734    273.94597 
+     700   0.16327536    10000     1457      692    276.09022 
+     800    0.1860353    10000     1502      737    274.82414 
+     900   0.20903339    10000     1474      707    274.63488 
+    1000   0.23218167    10000     1486      697    275.00172 
+Loop time of 0.23223 on 4 procs for 1000 steps with 10000 particles
+Performance: 4306.081 timesteps/s, 43.061 Mparticle-step/s
+
+MPI task timing breakdown:
+Section |  min time  |  avg time  |  max time  |%varavg| %total
+---------------------------------------------------------------
+Move    | 0.076994   | 0.077297   | 0.077817   |   0.1 | 33.28
+Coll    | 0.10799    | 0.10932    | 0.11021    |   0.3 | 47.07
+Sort    | 0.0082983  | 0.0083786  | 0.0084676  |   0.1 |  3.61
+Comm    | 0.020273   | 0.02226    | 0.027417   |   2.0 |  9.59
+Modify  | 0          | 0          | 0          |   0.0 |  0.00
+Output  | 0.00034317 | 0.00047312 | 0.00079935 |   0.0 |  0.20
+Other   |            | 0.0145     |            |       |  6.24
+
+Particle moves    = 10000000 (10M)
+Cells touched     = 14241042 (14.2M)
+Particle comms    = 311728 (0.312M)
+Boundary collides = 471432 (0.471M)
+Boundary exits    = 0 (0K)
+SurfColl checks   = 0 (0K)
+SurfColl occurs   = 0 (0K)
+Surf reactions    = 0 (0K)
+Collide attempts  = 1403898 (1.4M)
+Collide occurs    = 715997 (0.716M)
+Reactions         = 0 (0K)
+Particles stuck   = 0
+Axisymm bad moves = 0
+
+Particle-moves/CPUsec/proc: 1.07652e+07
+Particle-moves/step: 10000
+Cell-touches/particle/step: 1.4241
+Particle comm iterations/step: 1
+Particle fraction communicated: 0.0311728
+Particle fraction colliding with boundary: 0.0471432
+Particle fraction exiting boundary: 0
+Surface-checks/particle/step: 0
+Surface-collisions/particle/step: 0
+Surf-reactions/particle/step: 0
+Collision-attempts/particle/step: 0.14039
+Collisions/particle/step: 0.0715997
+Reactions/particle/step: 0
+
+Particles: 2500 ave 2586 max 2332 min
+Histogram: 1 0 0 0 0 0 0 1 0 2
+Cells:      250 ave 250 max 250 min
+Histogram: 4 0 0 0 0 0 0 0 0 0
+GhostCell: 172.5 ave 240 max 110 min
+Histogram: 1 0 0 0 2 0 0 0 0 1
+EmptyCell: 62.5 ave 130 max 0 min
+Histogram: 1 0 0 0 2 0 0 0 0 1

From 6623a553250a9a85fb13b22306c7f9ae3f4c2e0c Mon Sep 17 00:00:00 2001
From: Claude 
Date: Wed, 8 Jul 2026 13:35:54 +0000
Subject: [PATCH 29/30] Rebless in.ambi.group and in.chem.gastally gold logs
 from CI platform

Add committed gold logs for the multigroup-ambipolar and per-grid gas-tally
examples, generated on the CI runner (ubuntu-22.04, SPARTA 24 Sep 2025) at
1 and 4 MPI ranks. Same rationale as in.collide.group: a gold log is only
bit-exact on the OS it was blessed on, so the reference must come from the
CI platform. With these in place all three new KOKKOS examples are held to a
real bit-exact comparison (KOKKOS "-k on -sf kk" and non-KOKKOS runs vs the
committed reference).

Co-authored-by: stanmoore1 
Co-Authored-By: Claude Opus 4.8 
Claude-Session: https://claude.ai/code/session_01Q6nQVuxEiTUqSzDoDppCUM
---
 examples/ambi/log.24Sep25.mpi_1.ambi.group    | 121 ++++++++++++++++
 examples/ambi/log.24Sep25.mpi_4.ambi.group    | 122 ++++++++++++++++
 examples/chem/log.24Sep25.mpi_1.chem.gastally | 129 +++++++++++++++++
 examples/chem/log.24Sep25.mpi_4.chem.gastally | 130 ++++++++++++++++++
 4 files changed, 502 insertions(+)
 create mode 100644 examples/ambi/log.24Sep25.mpi_1.ambi.group
 create mode 100644 examples/ambi/log.24Sep25.mpi_4.ambi.group
 create mode 100644 examples/chem/log.24Sep25.mpi_1.chem.gastally
 create mode 100644 examples/chem/log.24Sep25.mpi_4.chem.gastally

diff --git a/examples/ambi/log.24Sep25.mpi_1.ambi.group b/examples/ambi/log.24Sep25.mpi_1.ambi.group
new file mode 100644
index 000000000..e39e4d4b8
--- /dev/null
+++ b/examples/ambi/log.24Sep25.mpi_1.ambi.group
@@ -0,0 +1,121 @@
+SPARTA (24 Sep 2025)
+Running on 1 MPI task(s)
+################################################################################
+# thermal plasma in a 2d box, non-reacting multigroup ambipolar collisions
+#
+# Exercises the ambipolar approximation with more than one collision group:
+# all heavy species (neutrals + ions) form one group and the ambipolar
+# electron species "e" is its own group, as required by collide ambipolar.
+#
+# Note:
+#  - The "comm/sort" option to the "global" command is used to match MPI runs.
+#  - The "twopass" option is used to match Kokkos runs.
+# The "comm/sort" and "twopass" options should not be used for production runs.
+################################################################################
+
+seed                12345
+dimension           2
+boundary            rr rr p
+global              gridcut 0.01 comm/sort yes
+create_box          -2.0 2.0 -2.0 2.0 -0.5 0.5
+Created orthogonal box = (-2 -2 -0.5) to (2 2 0.5)
+create_grid         20 20 1
+Created 400 child grid cells
+  CPU time = 0.00135691 secs
+  create/ghost percent = 90.3865 9.61347
+balance_grid        rcb cell
+Balance grid migrated 0 cells
+  CPU time = 0.000476826 secs
+  reassign/sort/migrate/ghost percent = 82.2793 0.611334 6.06154 11.0478
+
+global              fnum 2.6404E16
+global              nrho 2.6404e20
+
+species             air.species N2 O2 N O NO N2+ O2+ N+ O+ NO+ e
+
+# collide mixture: all species, two groups
+# the ambipolar electron species e must be in a group by itself
+
+mixture             gas N2 O2 N O NO N2+ O2+ N+ O+ NO+ vstream 0 0 0 temp 5000.0 group heavy
+mixture             gas e group electron
+mixture             gas N2 frac 0.6
+mixture             gas N2+ frac 0.4
+
+fix                 ambi ambipolar e N+ N2+ NO+ O+ O2+
+
+collide             vss gas air.vss
+collide_modify      ambipolar yes
+
+create_particles    gas n 10000 twopass
+Created 10000 particles
+  CPU time = 0.00470918 secs
+
+compute             temp temp
+stats               50
+stats_style         step np nattempt ncoll c_temp
+
+timestep            1.0e-8
+run                 300
+Memory usage per proc in Mbytes:
+  particles (ave,min,max) = 2 2 2
+  grid      (ave,min,max) = 1.51379 1.51379 1.51379
+  surf      (ave,min,max) = 0 0 0
+  total     (ave,min,max) = 3.51379 3.51379 3.51379
+Step Np Natt Ncoll c_temp 
+       0    10000        0        0    4975.0406 
+      50    10000       77       34    4975.0911 
+     100    10000      108       38    4975.0667 
+     150    10000      134       57    4975.0591 
+     200    10000       37       16    4974.9255 
+     250    10000       70       28    4976.2863 
+     300    10000       50       18    4974.4373 
+Loop time of 0.0799685 on 1 procs for 300 steps with 10000 particles
+Performance: 3751.478 timesteps/s, 37.515 Mparticle-step/s
+
+MPI task timing breakdown:
+Section |  min time  |  avg time  |  max time  |%varavg| %total
+---------------------------------------------------------------
+Move    | 0.025585   | 0.025585   | 0.025585   |   0.0 | 31.99
+Coll    | 0.042859   | 0.042859   | 0.042859   |   0.0 | 53.59
+Sort    | 0.010504   | 0.010504   | 0.010504   |   0.0 | 13.13
+Comm    | 8.2923e-05 | 8.2923e-05 | 8.2923e-05 |   0.0 |  0.10
+Modify  | 0          | 0          | 0          |   0.0 |  0.00
+Output  | 0.00084199 | 0.00084199 | 0.00084199 |   0.0 |  1.05
+Other   |            | 9.562e-05  |            |       |  0.12
+
+Particle moves    = 3000000 (3M)
+Cells touched     = 3000293 (3M)
+Particle comms    = 0 (0K)
+Boundary collides = 20 (0.02K)
+Boundary exits    = 0 (0K)
+SurfColl checks   = 0 (0K)
+SurfColl occurs   = 0 (0K)
+Surf reactions    = 0 (0K)
+Collide attempts  = 20534 (20.5K)
+Collide occurs    = 8182 (8.18K)
+Reactions         = 0 (0K)
+Particles stuck   = 0
+Axisymm bad moves = 0
+
+Particle-moves/CPUsec/proc: 3.75148e+07
+Particle-moves/step: 10000
+Cell-touches/particle/step: 1.0001
+Particle comm iterations/step: 1
+Particle fraction communicated: 0
+Particle fraction colliding with boundary: 6.66667e-06
+Particle fraction exiting boundary: 0
+Surface-checks/particle/step: 0
+Surface-collisions/particle/step: 0
+Surf-reactions/particle/step: 0
+Collision-attempts/particle/step: 0.00684467
+Collisions/particle/step: 0.00272733
+Reactions/particle/step: 0
+
+Particles: 10000 ave 10000 max 10000 min
+Histogram: 1 0 0 0 0 0 0 0 0 0
+Cells:      400 ave 400 max 400 min
+Histogram: 1 0 0 0 0 0 0 0 0 0
+GhostCell: 0 ave 0 max 0 min
+Histogram: 1 0 0 0 0 0 0 0 0 0
+EmptyCell: 0 ave 0 max 0 min
+Histogram: 1 0 0 0 0 0 0 0 0 0
diff --git a/examples/ambi/log.24Sep25.mpi_4.ambi.group b/examples/ambi/log.24Sep25.mpi_4.ambi.group
new file mode 100644
index 000000000..328c2c59d
--- /dev/null
+++ b/examples/ambi/log.24Sep25.mpi_4.ambi.group
@@ -0,0 +1,122 @@
+SPARTA (24 Sep 2025)
+Running on 4 MPI task(s)
+################################################################################
+# thermal plasma in a 2d box, non-reacting multigroup ambipolar collisions
+#
+# Exercises the ambipolar approximation with more than one collision group:
+# all heavy species (neutrals + ions) form one group and the ambipolar
+# electron species "e" is its own group, as required by collide ambipolar.
+#
+# Note:
+#  - The "comm/sort" option to the "global" command is used to match MPI runs.
+#  - The "twopass" option is used to match Kokkos runs.
+# The "comm/sort" and "twopass" options should not be used for production runs.
+################################################################################
+
+seed                12345
+dimension           2
+boundary            rr rr p
+global              gridcut 0.01 comm/sort yes
+create_box          -2.0 2.0 -2.0 2.0 -0.5 0.5
+Created orthogonal box = (-2 -2 -0.5) to (2 2 0.5)
+create_grid         20 20 1
+WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/runner/work/sparta/sparta/src/grid.cpp:473)
+Created 400 child grid cells
+  CPU time = 0.00264936 secs
+  create/ghost percent = 94.9017 5.09833
+balance_grid        rcb cell
+Balance grid migrated 280 cells
+  CPU time = 0.00113465 secs
+  reassign/sort/migrate/ghost percent = 46.3118 0.712375 19.4597 33.5161
+
+global              fnum 2.6404E16
+global              nrho 2.6404e20
+
+species             air.species N2 O2 N O NO N2+ O2+ N+ O+ NO+ e
+
+# collide mixture: all species, two groups
+# the ambipolar electron species e must be in a group by itself
+
+mixture             gas N2 O2 N O NO N2+ O2+ N+ O+ NO+ vstream 0 0 0 temp 5000.0 group heavy
+mixture             gas e group electron
+mixture             gas N2 frac 0.6
+mixture             gas N2+ frac 0.4
+
+fix                 ambi ambipolar e N+ N2+ NO+ O+ O2+
+
+collide             vss gas air.vss
+collide_modify      ambipolar yes
+
+create_particles    gas n 10000 twopass
+Created 10000 particles
+  CPU time = 0.00489833 secs
+
+compute             temp temp
+stats               50
+stats_style         step np nattempt ncoll c_temp
+
+timestep            1.0e-8
+run                 300
+Memory usage per proc in Mbytes:
+  particles (ave,min,max) = 2 2 2
+  grid      (ave,min,max) = 1.51379 1.51379 1.51379
+  surf      (ave,min,max) = 0 0 0
+  total     (ave,min,max) = 3.51379 3.51379 3.51379
+Step Np Natt Ncoll c_temp 
+       0    10000        0        0    4961.4899 
+      50    10000       62       18    4961.3574 
+     100    10000       96       42    4961.4647 
+     150    10000      129       56    4961.2506 
+     200    10000       40       21    4961.1044 
+     250    10000       64       24    4961.2342 
+     300    10000       44       17    4962.6572 
+Loop time of 0.027287 on 4 procs for 300 steps with 10000 particles
+Performance: 10994.232 timesteps/s, 109.942 Mparticle-step/s
+
+MPI task timing breakdown:
+Section |  min time  |  avg time  |  max time  |%varavg| %total
+---------------------------------------------------------------
+Move    | 0.0081502  | 0.0082497  | 0.0084222  |   0.1 | 30.23
+Coll    | 0.012323   | 0.012456   | 0.012749   |   0.2 | 45.65
+Sort    | 0.00287    | 0.0028725  | 0.0028778  |   0.0 | 10.53
+Comm    | 0.0019475  | 0.0020446  | 0.0021515  |   0.2 |  7.49
+Modify  | 0          | 0          | 0          |   0.0 |  0.00
+Output  | 0.00013547 | 0.00016465 | 0.00023228 |   0.0 |  0.60
+Other   |            | 0.0015     |            |       |  5.50
+
+Particle moves    = 3000000 (3M)
+Cells touched     = 3000288 (3M)
+Particle comms    = 15 (0.015K)
+Boundary collides = 20 (0.02K)
+Boundary exits    = 0 (0K)
+SurfColl checks   = 0 (0K)
+SurfColl occurs   = 0 (0K)
+Surf reactions    = 0 (0K)
+Collide attempts  = 20225 (20.2K)
+Collide occurs    = 8077 (8.08K)
+Reactions         = 0 (0K)
+Particles stuck   = 0
+Axisymm bad moves = 0
+
+Particle-moves/CPUsec/proc: 2.74856e+07
+Particle-moves/step: 10000
+Cell-touches/particle/step: 1.0001
+Particle comm iterations/step: 1
+Particle fraction communicated: 5e-06
+Particle fraction colliding with boundary: 6.66667e-06
+Particle fraction exiting boundary: 0
+Surface-checks/particle/step: 0
+Surface-collisions/particle/step: 0
+Surf-reactions/particle/step: 0
+Collision-attempts/particle/step: 0.00674167
+Collisions/particle/step: 0.00269233
+Reactions/particle/step: 0
+
+Particles: 2500 ave 2501 max 2499 min
+Histogram: 1 0 0 0 0 2 0 0 0 1
+Cells:      100 ave 100 max 100 min
+Histogram: 4 0 0 0 0 0 0 0 0 0
+GhostCell: 21 ave 21 max 21 min
+Histogram: 4 0 0 0 0 0 0 0 0 0
+EmptyCell: 0 ave 0 max 0 min
+Histogram: 4 0 0 0 0 0 0 0 0 0
diff --git a/examples/chem/log.24Sep25.mpi_1.chem.gastally b/examples/chem/log.24Sep25.mpi_1.chem.gastally
new file mode 100644
index 000000000..182561784
--- /dev/null
+++ b/examples/chem/log.24Sep25.mpi_1.chem.gastally
@@ -0,0 +1,129 @@
+SPARTA (24 Sep 2025)
+Running on 1 MPI task(s)
+################################################################################
+# thermal gas in a 3d box with collisions and reactions
+# tally per-grid-cell gas collisions and reactions
+#
+# Demonstrates/verifies compute gas/collision/grid and compute gas/reaction/grid
+# (the latter in all/every/select modes).
+#
+# Note:
+#  - The "comm/sort" option to the "global" command is used to match MPI runs.
+#  - The "twopass" option is used to match Kokkos runs.
+# The "comm/sort" and "twopass" options should not be used for production runs.
+################################################################################
+
+seed	    	    12345
+dimension   	    3
+global              gridcut 1.0e-5 comm/sort yes
+boundary	    rr rr rr
+create_box  	    0 0.0001 0 0.0001 0 0.0001
+Created orthogonal box = (0 0 0) to (0.0001 0.0001 0.0001)
+create_grid 	    10 10 10
+Created 1000 child grid cells
+  CPU time = 0.00157439 secs
+  create/ghost percent = 84.3119 15.6881
+balance_grid        rcb part
+Balance grid migrated 0 cells
+  CPU time = 0.000282464 secs
+  reassign/sort/migrate/ghost percent = 57.7153 0.322519 11.4595 30.5026
+
+species		    air.species N2 N
+mixture		    air N2 N vstream 0.0 0.0 0.0 temp 20000.0
+mixture             air N2 frac 1.0
+mixture             air N frac 0.0
+
+global              nrho 7.07043E22
+global              fnum 7.07043E5
+
+collide		    vss air air.vss
+react               tce air.tce
+
+create_particles    air n 10000 twopass
+Created 10000 particles
+  CPU time = 0.00309303 secs
+
+# per-grid-cell tallies of gas collisions and reactions
+
+compute             cc gas/collision/grid all all
+compute             cr gas/reaction/grid all all all
+compute             cre gas/reaction/grid all all every
+
+# sums over all cells: c_sumcc == ncoll-nreact and c_sumcr == nreact each step
+
+compute             sumcc reduce sum c_cc
+compute             sumcr reduce sum c_cr
+
+stats		    100
+compute             temp temp
+stats_style	    step np nattempt ncoll nreact c_temp c_sumcc c_sumcr
+
+timestep 	    7.00E-9
+run 		    500
+Memory usage per proc in Mbytes:
+  particles (ave,min,max) = 1.5625 1.5625 1.5625
+  grid      (ave,min,max) = 1.51379 1.51379 1.51379
+  surf      (ave,min,max) = 0 0 0
+  total     (ave,min,max) = 3.43488 3.43488 3.43488
+Step Np Natt Ncoll Nreact c_temp c_sumcc c_sumcr 
+       0    10000        0        0        0    19907.187            0            0 
+     100    10190     1000      262        3    18764.989          259            3 
+     200    10335     1011      269        2    18038.695          267            2 
+     300    10484     1079      255        4    17426.475          251            4 
+     400    10627     1070      238        1    16707.966          237            1 
+     500    10760     1124      289        3    16036.998          286            3 
+Loop time of 0.917186 on 1 procs for 500 steps with 10760 particles
+Performance: 545.145 timesteps/s, 5.866 Mparticle-step/s
+
+MPI task timing breakdown:
+Section |  min time  |  avg time  |  max time  |%varavg| %total
+---------------------------------------------------------------
+Move    | 0.78531    | 0.78531    | 0.78531    |   0.0 | 85.62
+Coll    | 0.11393    | 0.11393    | 0.11393    |   0.0 | 12.42
+Sort    | 0.016552   | 0.016552   | 0.016552   |   0.0 |  1.80
+Comm    | 0.00035723 | 0.00035723 | 0.00035723 |   0.0 |  0.04
+Modify  | 0          | 0          | 0          |   0.0 |  0.00
+Output  | 0.00053396 | 0.00053396 | 0.00053396 |   0.0 |  0.06
+Other   |            | 0.0005075  |            |       |  0.06
+
+Particle moves    = 5201491 (5.2M)
+Cells touched     = 23735594 (23.7M)
+Particle comms    = 0 (0K)
+Boundary collides = 2059037 (2.06M)
+Boundary exits    = 0 (0K)
+SurfColl checks   = 0 (0K)
+SurfColl occurs   = 0 (0K)
+Surf reactions    = 0 (0K)
+Collide attempts  = 512014 (0.512M)
+Collide occurs    = 128199 (0.128M)
+Reactions         = 760 (0.76K)
+Particles stuck   = 0
+Axisymm bad moves = 0
+
+Particle-moves/CPUsec/proc: 5.67114e+06
+Particle-moves/step: 10403
+Cell-touches/particle/step: 4.56323
+Particle comm iterations/step: 1
+Particle fraction communicated: 0
+Particle fraction colliding with boundary: 0.395855
+Particle fraction exiting boundary: 0
+Surface-checks/particle/step: 0
+Surface-collisions/particle/step: 0
+Surf-reactions/particle/step: 0
+Collision-attempts/particle/step: 0.098436
+Collisions/particle/step: 0.0246466
+Reactions/particle/step: 0.000146112
+
+Gas reaction tallies:
+  style tce #-of-reactions 45
+  reaction N2 + N2 --> N + N + N2: 578
+  reaction N2 + N --> N + N + N: 182
+
+Particles: 10760 ave 10760 max 10760 min
+Histogram: 1 0 0 0 0 0 0 0 0 0
+Cells:      1000 ave 1000 max 1000 min
+Histogram: 1 0 0 0 0 0 0 0 0 0
+GhostCell: 0 ave 0 max 0 min
+Histogram: 1 0 0 0 0 0 0 0 0 0
+EmptyCell: 0 ave 0 max 0 min
+Histogram: 1 0 0 0 0 0 0 0 0 0
diff --git a/examples/chem/log.24Sep25.mpi_4.chem.gastally b/examples/chem/log.24Sep25.mpi_4.chem.gastally
new file mode 100644
index 000000000..3d2175281
--- /dev/null
+++ b/examples/chem/log.24Sep25.mpi_4.chem.gastally
@@ -0,0 +1,130 @@
+SPARTA (24 Sep 2025)
+Running on 4 MPI task(s)
+################################################################################
+# thermal gas in a 3d box with collisions and reactions
+# tally per-grid-cell gas collisions and reactions
+#
+# Demonstrates/verifies compute gas/collision/grid and compute gas/reaction/grid
+# (the latter in all/every/select modes).
+#
+# Note:
+#  - The "comm/sort" option to the "global" command is used to match MPI runs.
+#  - The "twopass" option is used to match Kokkos runs.
+# The "comm/sort" and "twopass" options should not be used for production runs.
+################################################################################
+
+seed	    	    12345
+dimension   	    3
+global              gridcut 1.0e-5 comm/sort yes
+boundary	    rr rr rr
+create_box  	    0 0.0001 0 0.0001 0 0.0001
+Created orthogonal box = (0 0 0) to (0.0001 0.0001 0.0001)
+create_grid 	    10 10 10
+WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/runner/work/sparta/sparta/src/grid.cpp:473)
+Created 1000 child grid cells
+  CPU time = 0.00205981 secs
+  create/ghost percent = 94.305 5.69498
+balance_grid        rcb part
+Balance grid migrated 740 cells
+  CPU time = 0.00114597 secs
+  reassign/sort/migrate/ghost percent = 49.0399 0.678202 20.0193 30.2626
+
+species		    air.species N2 N
+mixture		    air N2 N vstream 0.0 0.0 0.0 temp 20000.0
+mixture             air N2 frac 1.0
+mixture             air N frac 0.0
+
+global              nrho 7.07043E22
+global              fnum 7.07043E5
+
+collide		    vss air air.vss
+react               tce air.tce
+
+create_particles    air n 10000 twopass
+Created 10000 particles
+  CPU time = 0.00177956 secs
+
+# per-grid-cell tallies of gas collisions and reactions
+
+compute             cc gas/collision/grid all all
+compute             cr gas/reaction/grid all all all
+compute             cre gas/reaction/grid all all every
+
+# sums over all cells: c_sumcc == ncoll-nreact and c_sumcr == nreact each step
+
+compute             sumcc reduce sum c_cc
+compute             sumcr reduce sum c_cr
+
+stats		    100
+compute             temp temp
+stats_style	    step np nattempt ncoll nreact c_temp c_sumcc c_sumcr
+
+timestep 	    7.00E-9
+run 		    500
+Memory usage per proc in Mbytes:
+  particles (ave,min,max) = 1.5625 1.5625 1.5625
+  grid      (ave,min,max) = 1.51379 1.51379 1.51379
+  surf      (ave,min,max) = 0 0 0
+  total     (ave,min,max) = 3.16594 3.16594 3.16594
+Step Np Natt Ncoll Nreact c_temp c_sumcc c_sumcr 
+       0    10000        0        0        0    19847.392            0            0 
+     100    10187      998      269        0    18979.069          269            0 
+     200    10362     1020      239        2     18220.89          237            2 
+     300    10529     1067      259        2    17415.447          257            2 
+     400    10658     1079      260        0    16700.368          260            0 
+     500    10778     1097      246        1    15978.454          245            1 
+Loop time of 0.292439 on 4 procs for 500 steps with 10778 particles
+Performance: 1709.759 timesteps/s, 18.428 Mparticle-step/s
+
+MPI task timing breakdown:
+Section |  min time  |  avg time  |  max time  |%varavg| %total
+---------------------------------------------------------------
+Move    | 0.20483    | 0.20772    | 0.21009    |   0.4 | 71.03
+Coll    | 0.026371   | 0.02658    | 0.02671    |   0.1 |  9.09
+Sort    | 0.0047512  | 0.004797   | 0.0048304  |   0.0 |  1.64
+Comm    | 0.040441   | 0.041421   | 0.041872   |   0.3 | 14.16
+Modify  | 0          | 0          | 0          |   0.0 |  0.00
+Output  | 0.00024169 | 0.00036943 | 0.00074573 |   0.0 |  0.13
+Other   |            | 0.01155    |            |       |  3.95
+
+Particle moves    = 5212700 (5.21M)
+Cells touched     = 24150571 (24.2M)
+Particle comms    = 1312904 (1.31M)
+Boundary collides = 2072594 (2.07M)
+Boundary exits    = 0 (0K)
+SurfColl checks   = 0 (0K)
+SurfColl occurs   = 0 (0K)
+Surf reactions    = 0 (0K)
+Collide attempts  = 514360 (0.514M)
+Collide occurs    = 129535 (0.13M)
+Reactions         = 778 (0.778K)
+Particles stuck   = 0
+Axisymm bad moves = 0
+
+Particle-moves/CPUsec/proc: 4.45623e+06
+Particle-moves/step: 10425.4
+Cell-touches/particle/step: 4.63303
+Particle comm iterations/step: 2.994
+Particle fraction communicated: 0.251866
+Particle fraction colliding with boundary: 0.397605
+Particle fraction exiting boundary: 0
+Surface-checks/particle/step: 0
+Surface-collisions/particle/step: 0
+Surf-reactions/particle/step: 0
+Collision-attempts/particle/step: 0.0986744
+Collisions/particle/step: 0.0248499
+Reactions/particle/step: 0.000149251
+
+Gas reaction tallies:
+  style tce #-of-reactions 45
+  reaction N2 + N2 --> N + N + N2: 577
+  reaction N2 + N --> N + N + N: 201
+
+Particles: 2694.5 ave 2767 max 2647 min
+Histogram: 1 1 0 0 1 0 0 0 0 1
+Cells:      250 ave 250 max 250 min
+Histogram: 4 0 0 0 0 0 0 0 0 0
+GhostCell: 172.5 ave 240 max 110 min
+Histogram: 1 0 0 0 2 0 0 0 0 1
+EmptyCell: 62.5 ave 130 max 0 min
+Histogram: 1 0 0 0 2 0 0 0 0 1

From 86940ed050462715449132a06877e58d04e103e3 Mon Sep 17 00:00:00 2001
From: Claude 
Date: Wed, 8 Jul 2026 14:53:54 +0000
Subject: [PATCH 30/30] Fix KOKKOS implicit-surface ablation; enable
 in.ablation.3d

Running in.ablation.3d under KOKKOS (SPARTA_KOKKOS_EXACT, "-k on -sf kk")
errored at the first post-ablation step with "Collision cell volume is
zero". Two distinct KOKKOS-only bugs in the mid-run implicit-surface
ablation path were responsible; the first masked the second.

1. Stale device per-cell surf graphs after ablation.
   fix ablate regenerates the implicit surfaces on the host every Nevery
   steps, but the device per-cell surf lists (d_csurfs/d_csplits/d_csubs)
   are only built by wrap_kokkos_graphs() in UpdateKokkos::setup(), i.e.
   once per run. After ablation the surf move used stale per-cell surf
   lists, so particles penetrated newly-solid cells and were left in
   fully-inside (zero-volume) cells, tripping the collision volume check.
   Fix: add Grid::changed, set by notify_changed(), and have the KOKKOS run
   loop resync the device grid/surf graphs (mirroring setup()) after any
   end-of-step fix that changes grid/surf topology. The resync happens
   between moves, where grid_kk_copy is not live, so it is KKCopy-safe.

2. Per-surf isurf/grid tally arrays not regrown on surf-count change.
   ComputeISurfGridKokkos sizes its isurf-indexed per-surf tally arrays in
   init_normflux(), called from reallocate(), which early-returns when
   grid->nlocal is unchanged. Ablation changes the surf count without
   changing the cell count, so when the surf count grew the device tally
   kernel and host tallyinfo() wrote/read out of bounds, corrupting the
   heap (manifesting as a later double-free/segfault). Fix: override
   ComputeISurfGridKokkos::reallocate() to re-run init_normflux() (which
   regrows the arrays and recomputes normflux) when the surf count changes.
   The sibling react/isurf/grid compute already self-heals in clear().

With both fixes in.ablation.3d runs to completion and is bit-for-bit
identical to the non-KOKKOS reference over 100 steps (thermo + f_ablate),
matching the committed gold logs. Remove it from the SPARTA_KOKKOS_EXACT
skip list. in.ablation.2d and the full KOKKOS regression suite are
unaffected.

Co-authored-by: stanmoore1 
Co-Authored-By: Claude Opus 4.8 
Claude-Session: https://claude.ai/code/session_01Q6nQVuxEiTUqSzDoDppCUM
---
 cmake/common/set/sparta_cmake_defaults.cmake |  3 ---
 src/KOKKOS/compute_isurf_grid_kokkos.cpp     | 19 +++++++++++++++++
 src/KOKKOS/compute_isurf_grid_kokkos.h       |  3 +++
 src/KOKKOS/update_kokkos.cpp                 | 22 ++++++++++++++++++++
 src/grid.cpp                                 |  3 +++
 src/grid.h                                   |  3 +++
 6 files changed, 50 insertions(+), 3 deletions(-)

diff --git a/cmake/common/set/sparta_cmake_defaults.cmake b/cmake/common/set/sparta_cmake_defaults.cmake
index 09efdf18a..79dbb70a2 100644
--- a/cmake/common/set/sparta_cmake_defaults.cmake
+++ b/cmake/common/set/sparta_cmake_defaults.cmake
@@ -73,9 +73,6 @@ if(SPARTA_ENABLE_TESTING)
   # the non-KOKKOS configurations.
   if(SPARTA_KOKKOS_EXACT)
     list(APPEND SPARTA_DISABLED_TESTS
-        # implicit-surface ablation in 3D errors under KOKKOS (zero collision
-        # cell volume); the 2D case runs bit-for-bit and is enabled
-        "in.ablation.3d"
         # external field fix not KOKKOS-enabled
         "in.bfield"
         "in.bfield.grid"
diff --git a/src/KOKKOS/compute_isurf_grid_kokkos.cpp b/src/KOKKOS/compute_isurf_grid_kokkos.cpp
index a325b5aa7..97efc1717 100644
--- a/src/KOKKOS/compute_isurf_grid_kokkos.cpp
+++ b/src/KOKKOS/compute_isurf_grid_kokkos.cpp
@@ -33,6 +33,8 @@ ComputeISurfGridKokkos::ComputeISurfGridKokkos(SPARTA *sparta, int narg, char **
 {
   kokkos_flag = 1;
 
+  nsurf_tally_alloc = -1;
+
   // hash is allocated/used only on the host; not needed for device tally
 
   d_which = DAT::t_int_1d("isurf/grid:which",nvalue);
@@ -92,6 +94,23 @@ void ComputeISurfGridKokkos::init_normflux()
 
   memoryKK->grow_kokkos(k_array_surf_tally,array_surf_tally,nsurf,ntotal,"isurf/grid:array_surf_tally");
   d_array_surf_tally = k_array_surf_tally.view_device();
+
+  nsurf_tally_alloc = nsurf;
+}
+
+/* ----------------------------------------------------------------------
+   reallocate per-cell and per-surf arrays after the grid/surfs change
+   the per-surf tally arrays are indexed by isurf, so they must track the
+   surf count; ablation regenerates surfs without changing grid->nlocal, so
+   the base reallocate() (keyed on grid->nlocal) can leave them stale
+------------------------------------------------------------------------- */
+
+void ComputeISurfGridKokkos::reallocate()
+{
+  ComputeISurfGrid::reallocate();
+
+  int nsurf = surf->nlocal + surf->nghost;
+  if (nsurf != nsurf_tally_alloc) init_normflux();
 }
 
 /* ---------------------------------------------------------------------- */
diff --git a/src/KOKKOS/compute_isurf_grid_kokkos.h b/src/KOKKOS/compute_isurf_grid_kokkos.h
index 35679bb22..1878ae49f 100644
--- a/src/KOKKOS/compute_isurf_grid_kokkos.h
+++ b/src/KOKKOS/compute_isurf_grid_kokkos.h
@@ -34,6 +34,7 @@ class ComputeISurfGridKokkos : public ComputeISurfGrid {
   ~ComputeISurfGridKokkos();
   void init();
   void init_normflux();
+  void reallocate();
   void clear();
   int tallyinfo(surfint *&);
   void pre_surf_tally();
@@ -278,6 +279,8 @@ void surf_tally_kk(double /*dtremain*/, int isurf, int /*icell*/, int /*reaction
   t_line_1d d_lines;
   t_tri_1d d_tris;
 
+  int nsurf_tally_alloc;   // # of surfs the per-surf tally arrays are sized for
+
   void grow_tally();
 };
 
diff --git a/src/KOKKOS/update_kokkos.cpp b/src/KOKKOS/update_kokkos.cpp
index 014373d8f..98e512c3f 100644
--- a/src/KOKKOS/update_kokkos.cpp
+++ b/src/KOKKOS/update_kokkos.cpp
@@ -329,6 +329,10 @@ void UpdateKokkos::setup()
   }
   hash_kk = grid_kk->hash_kk;
 
+  // device grid/surf graphs are now current; clear any pending change flag so
+  // the run loop does not do a redundant resync on the first step
+  grid->changed = 0;
+
   Update::setup(); // must come after prewrap since computes are called by setup()
 
   // For MPI debugging
@@ -431,6 +435,24 @@ void UpdateKokkos::run(int nsteps)
       timer->stamp(TIME_MODIFY);
     }
 
+    // if an end-of-step fix changed the grid/surf topology (e.g. fix ablate
+    // regenerated implicit surfaces), the host grid is now authoritative but
+    // the device per-cell surf graphs (d_csurfs/d_csplits/d_csubs) are stale.
+    // Resync them to the device before the next move, mirroring setup().
+    // Safe here: grid_kk_copy from this step's move is no longer in use and is
+    // refreshed at the start of the next move.
+
+    if (grid->changed) {
+      GridKokkos* grid_kk = (GridKokkos*) grid;
+      grid_kk->modify(Host,ALL_MASK);
+      grid_kk->update_hash();
+      if (surf->exist) {
+        ((SurfKokkos*)surf)->modify(Host,ALL_MASK);
+        grid_kk->wrap_kokkos_graphs();
+      }
+      grid->changed = 0;
+    }
+
     // all output
 
     if (ntimestep == output->next) {
diff --git a/src/grid.cpp b/src/grid.cpp
index 849384c84..e1a1d1a0c 100644
--- a/src/grid.cpp
+++ b/src/grid.cpp
@@ -76,6 +76,7 @@ int corners[6][4] = {{0,2,4,6}, {1,3,5,7}, {0,1,4,5}, {2,3,6,7},
 Grid::Grid(SPARTA *sparta) : Pointers(sparta)
 {
   exist = exist_ghost = clumped = 0;
+  changed = 0;
   MPI_Comm_rank(world,&me);
 
   gnames = (char **) memory->smalloc(MAXGROUP*sizeof(char *),"grid:gnames");
@@ -367,6 +368,8 @@ void Grid::add_sub_cell(int icell, int ownflag)
 
 void Grid::notify_changed()
 {
+  changed = 1;
+
   if (modify->n_pergrid) modify->grid_changed();
 
   Compute **compute = modify->compute;
diff --git a/src/grid.h b/src/grid.h
index 2f40084fb..914106e62 100644
--- a/src/grid.h
+++ b/src/grid.h
@@ -27,6 +27,9 @@ class Grid : protected Pointers {
  public:
   int exist;            // 1 if grid is defined
   int exist_ghost;      // 1 if ghost cells exist
+  int changed;          // set by notify_changed() when grid/surf topology changes
+                        // (e.g. ablation, adaptation); consumed by KOKKOS to
+                        // resync per-cell surf graphs to device mid-run
   int clumped;          // 1 if grid ownership is clumped, due to RCB
                         // if not, some operations are not allowed