fix(analyzer): Summary storage improvement

core/opentaint-dataflow-core/opentaint-dataflow/src/main/kotlin/org/opentaint/dataflow/ap/ifds/access/automata/MethodAutomataAccessPathSubscription.kt

@@ -151,8 +151,8 @@ class MethodAutomataAccessPathSubscription : CommonAPSub<AccessGraph, AccessGrap
 
         override fun createStorage(idx: Int): Storage<AccessGraph, AccessGraph> = FactStorage(idx)
 
-        override fun relevantStorageIndices(summaryInitialFact: AccessGraph): BitSet =
-            graphIndex.localizeIndexedGraphContainsAllGraph(summaryInitialFact)
+        override fun forEachRelevantStorageIndex(summaryInitialFact: AccessGraph, body: (Int) -> Unit) =
+            graphIndex.localizeIndexedGraphContainsAllGraph(summaryInitialFact).forEach(body)
     }
 }
 

core/opentaint-dataflow-core/opentaint-dataflow/src/main/kotlin/org/opentaint/dataflow/ap/ifds/access/cactus/MethodCactusAccessPathSubscription.kt

@@ -97,8 +97,9 @@ private class NDSubStorage(private val cactusManager: CactusApManager, callerEp:
             current?.filterStartsWith(summaryInitialFact)?.let { dst.add(it) }
         }
     }
-    override fun relevantStorageIndices(summaryInitialFact: AccessPathWithCycles.AccessNode?): BitSet =
-        BitSet().also { it.set(0, maxIdx + 1) }
+    override fun forEachRelevantStorageIndex(summaryInitialFact: AccessPathWithCycles.AccessNode?, body: (Int) -> Unit) {
+        for (idx in 0..maxIdx) body(idx)
+    }
 }
 
 private class ZeroEdgeSubBuilder(override val cactusManager: CactusApManager) : CommonZeroEdgeSubBuilder<AccessCactus.AccessNode>(), CactusFinalApAccess

core/opentaint-dataflow-core/opentaint-dataflow/src/main/kotlin/org/opentaint/dataflow/ap/ifds/access/common/ndf2f/DefaultNDF2FSubStorage.kt

@@ -18,7 +18,14 @@ abstract class DefaultNDF2FSubStorage<IAP, FAP : Any> :
         fun collect(dst: MutableList<FAP>, summaryInitialFact: IAP)
     }
 
-    abstract fun relevantStorageIndices(summaryInitialFact: IAP): BitSet
+    /**
+     * Feeds [body] every storage index whose stored fact may match [summaryInitialFact], ascending.
+     *
+     * This used to hand back a `BitSet`. The tree implementation's index is sparse - 87.75 % of its
+     * nodes hold a single item - so materialising one would have cost more than the set it
+     * describes, the backing `long[]` being sized by the largest index rather than by the count.
+     */
+    abstract fun forEachRelevantStorageIndex(summaryInitialFact: IAP, body: (Int) -> Unit)
 
     override fun add(
         callerInitial: Set<InitialFactAp>,
@@ -35,7 +42,7 @@ abstract class DefaultNDF2FSubStorage<IAP, FAP : Any> :
         summaryInitialFact: IAP,
         emptyDeltaRequired: Boolean,
     ) {
-        relevantStorageIndices(summaryInitialFact).forEach { storageIdx ->
+        forEachRelevantStorageIndex(summaryInitialFact) { storageIdx ->
             val callerInitialAp = initialApStorage[storageIdx]
             val callerExitAp = exitApStorage[storageIdx]
             collectToListWithPostProcess(

core/opentaint-dataflow-core/opentaint-dataflow/src/main/kotlin/org/opentaint/dataflow/ap/ifds/access/tree/AccessBasedStorage.kt

@@ -4,6 +4,7 @@ import it.unimi.dsi.fastutil.ints.IntArrayList
 import org.opentaint.dataflow.ap.ifds.access.tree.AccessPath.AccessNode.Companion.createNodeFromAccessors
 import org.opentaint.dataflow.ap.ifds.access.util.AccessorIdx
 import org.opentaint.dataflow.ap.ifds.access.util.AccessorInterner.Companion.FINAL_ACCESSOR_IDX
+import org.opentaint.dataflow.util.ConcurrentReadSafeInt2ObjectMap
 import org.opentaint.dataflow.util.forEachEntry
 import org.opentaint.dataflow.util.forEachInt
 import org.opentaint.dataflow.util.getOrCreateNullable
@@ -12,7 +13,33 @@ import org.opentaint.dataflow.util.int2ObjectMap
 abstract class AccessBasedStorage<S : AccessBasedStorage<S>>(
     val manager: TreeApManager
 ) {
-    private val children = int2ObjectMap<S?>()
+    /**
+     * Allocated on the first child, not on construction.
+     *
+     * A premise trie is mostly leaves: in a ThingsBoard heap dump 85.5 % of the 1,652,208
+     * `IF2FFStorage` nodes had no children at all and 94.2 % had at most one, yet every node
+     * carried a full [ConcurrentReadSafeInt2ObjectMap] - the map plus its two 17-slot tables is
+     * ~256 B, and those empty maps alone were 345 MiB of a 9.7 GiB live heap.
+     *
+     * `@Volatile` is load-bearing, not decoration. The field used to be a `val`, so final-field
+     * semantics published the map's internals for free; a plain `var` assigned after construction
+     * would let a reader see a non-null map whose `key`/`value` tables are not yet visible, which
+     * the map's own seqlock cannot repair - it guards mutations of a published map, not the
+     * publication of the map itself. The volatile write here is the release that pairs with the
+     * reader's acquiring load, and it happens once per node.
+     */
+    @Volatile
+    private var children: ConcurrentReadSafeInt2ObjectMap<S?>? = null
+
+    /** Double-checked under the monitor, so racing writers cannot each install a table. */
+    private fun childrenForWrite(): ConcurrentReadSafeInt2ObjectMap<S?> {
+        children?.let { return it }
+
+        synchronized(this) {
+            children?.let { return it }
+            return int2ObjectMap<S?>().also { children = it }
+        }
+    }
 
     abstract fun createStorage(): S
 
@@ -57,7 +84,7 @@ abstract class AccessBasedStorage<S : AccessBasedStorage<S>>(
         nodes.add(this as S)
 
         if (pattern.isFinal) {
-            children.get(FINAL_ACCESSOR_IDX)?.let { nodes.add(it) }
+            children?.get(FINAL_ACCESSOR_IDX)?.let { nodes.add(it) }
         }
 
         pattern.forEachAccessor { accessor, accessorPattern ->
@@ -70,7 +97,7 @@ abstract class AccessBasedStorage<S : AccessBasedStorage<S>>(
         accessor: AccessorIdx,
         nodes: MutableList<S>
     ) {
-        children.get(accessor)?.collectNodesContains(pattern, nodes)
+        children?.get(accessor)?.collectNodesContains(pattern, nodes)
     }
 
     fun allNodes(): Sequence<S> {
@@ -82,7 +109,7 @@ abstract class AccessBasedStorage<S : AccessBasedStorage<S>>(
             @Suppress("UNCHECKED_CAST")
             storages.add(storage as S)
 
-            storage.children.forEachEntry { _, s ->
+            storage.children?.forEachEntry { _, s ->
                 if (s == null) return@forEachEntry
                 unprocessedStorages.add(s)
             }
@@ -106,7 +133,7 @@ abstract class AccessBasedStorage<S : AccessBasedStorage<S>>(
             @Suppress("UNCHECKED_CAST")
             body(accessors, storage as S)
 
-            storage.children.forEachEntry { accessor, s ->
+            storage.children?.forEachEntry { accessor, s ->
                 if (s == null) return@forEachEntry
 
                 val childrenAccessors = accessors.clone()
@@ -118,6 +145,7 @@ abstract class AccessBasedStorage<S : AccessBasedStorage<S>>(
     }
 
     fun removeChildren(predicate: (AccessorIdx, S) -> Boolean) {
+        val children = this.children ?: return
         val accessorsToRemove = IntArrayList()
 
         children.forEachEntry { accessor, s ->
@@ -136,10 +164,10 @@ abstract class AccessBasedStorage<S : AccessBasedStorage<S>>(
     }
 
     open fun getOrCreateChild(accessor: AccessorIdx): S =
-        children.getOrCreateNullable(accessor) { createStorage() }
+        childrenForWrite().getOrCreateNullable(accessor) { createStorage() }
 
     open fun findChild(accessor: AccessorIdx): S? =
-        children.get(accessor)
+        children?.get(accessor)
 
     override fun toString(): String = buildString {
         print(this, prefix = "")
@@ -149,7 +177,7 @@ abstract class AccessBasedStorage<S : AccessBasedStorage<S>>(
 
     fun print(builder: StringBuilder, prefix: String) {
         builder.appendLine("$prefix${printStorageNode()}")
-        children.forEachEntry { accessorIdx, s ->
+        children?.forEachEntry { accessorIdx, s ->
             if (s == null) return@forEachEntry
             val accessor = with(manager) { accessorIdx.accessor }
             builder.appendLine("$prefix$accessor ->")

core/opentaint-dataflow-core/opentaint-dataflow/src/main/kotlin/org/opentaint/dataflow/ap/ifds/access/tree/MethodEdgesFinalTreeApSet.kt

@@ -8,24 +8,25 @@ import org.opentaint.dataflow.ap.ifds.access.tree.AccessTree.AccessNode as Acces
 
 class MethodEdgesFinalTreeApSet(
     methodInitialStatement: CommonInst,
-    private val maxInstIdx: Int,
+    @Suppress("UNUSED_PARAMETER") maxInstIdx: Int,
     private val languageManager: LanguageManager,
     override val apManager: TreeApManager,
 ) : CommonZ2FSet<AccessTreeNode>(methodInitialStatement), TreeFinalApAccess {
     override fun createApStorage(): ApStorage<AccessTreeNode> =
-        ZeroInitialFactEdges(maxInstIdx, languageManager, apManager)
+        ZeroInitialFactEdges(languageManager, apManager)
 
     private class ZeroInitialFactEdges(
-        maxInstIdx: Int,
         private val languageManager: LanguageManager,
         manager: TreeApManager,
-    ): TreeSetWithCompression(maxInstIdx, manager), ApStorage<AccessTreeNode> {
+    ): TreeSetWithCompression(COLUMNS, manager), ApStorage<AccessTreeNode> {
         override fun addEdge(statement: CommonInst, accessPath: AccessTreeNode): AccessTreeNode? {
             val factSetIdx = instructionStorageIdx(statement, languageManager)
-            val factSet = edges[factSetIdx]
+            val row = rowsForWrite(factSetIdx)
+            val offset = offsetOf(row, factSetIdx)
+            val factSet = row.values[offset] as AccessTreeNode?
 
             if (factSet == null) {
-                edges[factSetIdx] = internIfRequired(accessPath)
+                row.values[offset] = internIfRequired(accessPath)
                 return accessPath
             }
 
@@ -34,13 +35,21 @@ class MethodEdgesFinalTreeApSet(
                 return null
             }
 
-            edges[factSetIdx] = internIfRequired(mergedFacts)
-            intern(factSetIdx)
+            val storedFacts = internIfRequired(mergedFacts)
+            row.values[offset] = storedFacts
+            intern(storedFacts)
             return mergedFacts
         }
 
         override fun collectApAtStatement(statement: CommonInst, dst: MutableList<AccessTreeNode>) {
-            dst += edges[instructionStorageIdx(statement, languageManager)] ?: return
+            val row = rows() ?: return
+            val offset = offsetOf(row, instructionStorageIdx(statement, languageManager))
+            if (offset < 0) return
+            dst += row.values[offset] as AccessTreeNode? ?: return
+        }
+
+        private companion object {
+            private const val COLUMNS = 1
         }
     }
 }

core/opentaint-dataflow-core/opentaint-dataflow/src/main/kotlin/org/opentaint/dataflow/ap/ifds/access/tree/MethodEdgesInitialToFinalTreeApSet.kt

@@ -10,7 +10,7 @@ import org.opentaint.ir.api.common.cfg.CommonInst
 
 class MethodEdgesInitialToFinalTreeApSet(
     methodInitialStatement: CommonInst,
-    private val maxInstIdx: Int,
+    @Suppress("UNUSED_PARAMETER") maxInstIdx: Int,
     private val languageManager: LanguageManager,
     override val apManager: TreeApManager,
 ) : CommonF2FSet<AccessPath.AccessNode?, AccessTree.AccessNode>(methodInitialStatement),
@@ -22,7 +22,7 @@ class MethodEdgesInitialToFinalTreeApSet(
     override fun mostAbstractPattern(base: AccessPathBase): AccessPath.AccessNode? = null
 
     private inner class TaintedFactAccessEdgeStorage : ApStorage<AccessPath.AccessNode?, AccessTree.AccessNode> {
-        private val sameInitialAccessEdges = IF2FFStorage(maxInstIdx, languageManager, apManager)
+        private val sameInitialAccessEdges = IF2FFStorage(languageManager, apManager)
 
         override fun add(
             statement: CommonInst,
@@ -60,60 +60,78 @@ class MethodEdgesInitialToFinalTreeApSet(
     }
 
     private class IF2FFStorage(
-        val maxInstIdx: Int,
         private val languageManager: LanguageManager,
         manager: TreeApManager,
     ) : AccessBasedStorage<IF2FFStorage>(manager) {
-        val current = EdgeNonUniverseExclusionMergingStorage(maxInstIdx, languageManager, manager)
+        val current = EdgeNonUniverseExclusionMergingStorage(languageManager, manager)
 
         override fun createStorage(): IF2FFStorage =
-            IF2FFStorage(maxInstIdx, languageManager, manager)
+            IF2FFStorage(languageManager, manager)
 
         override fun printStorageNode(): String = current.toString()
     }
 
+    /**
+     * Two columns per instruction: the merged access tree and the merged exclusion set. They are
+     * written and cleared together - measured on a ThingsBoard dump, the two former arrays had
+     * byte-identical non-null distributions - so one sparse table carries both, and a reader that
+     * finds a row is guaranteed the pair belongs to the same instruction.
+     */
     private class EdgeNonUniverseExclusionMergingStorage(
-        maxInstIdx: Int,
         private val languageManager: LanguageManager,
         manager: TreeApManager,
-    ): TreeSetWithCompression(maxInstIdx, manager) {
-        private val exclusions = arrayOfNulls<ExclusionSet>(MethodAnalyzerEdges.instructionStorageSize(maxInstIdx))
+    ): TreeSetWithCompression(COLUMNS, manager) {
 
         fun add(
             statement: CommonInst,
             accessWithExclusion: AccessWithExclusion<AccessTree.AccessNode>
         ): AccessWithExclusion<AccessTree.AccessNode>? {
             val edgeSetIdx = MethodAnalyzerEdges.instructionStorageIdx(statement, languageManager)
-            val currentExclusion = exclusions[edgeSetIdx]
+            val row = rowsForWrite(edgeSetIdx)
+            val offset = offsetOf(row, edgeSetIdx)
+
+            val currentExclusion = row.values[offset + EXCLUSION] as ExclusionSet?
 
             if (currentExclusion == null) {
-                exclusions[edgeSetIdx] = accessWithExclusion.exclusion
-                edges[edgeSetIdx] = internIfRequired(accessWithExclusion.access)
+                // Access first: a reader takes a non-null exclusion as the sign that the row is populated.
+                row.values[offset + ACCESS] = internIfRequired(accessWithExclusion.access)
+                row.values[offset + EXCLUSION] = accessWithExclusion.exclusion
                 return accessWithExclusion
             }
 
             val mergedExclusion = currentExclusion.union(accessWithExclusion.exclusion)
-            exclusions[edgeSetIdx] = mergedExclusion
+            row.values[offset + EXCLUSION] = mergedExclusion
 
-            val currentAccess = edges[edgeSetIdx]!!
+            val currentAccess = row.values[offset + ACCESS] as AccessTree.AccessNode
             val mergedAccess = currentAccess.mergeAdd(accessWithExclusion.access)
             if (mergedAccess === currentAccess) {
                 if (mergedExclusion === currentExclusion) return null
 
                 return AccessWithExclusion(mergedAccess, mergedExclusion)
             }
 
-            edges[edgeSetIdx] = internIfRequired(mergedAccess)
-            intern(edgeSetIdx)
+            val storedAccess = internIfRequired(mergedAccess)
+            row.values[offset + ACCESS] = storedAccess
+            intern(storedAccess)
 
             return AccessWithExclusion(mergedAccess, mergedExclusion)
         }
 
         fun allApAtStatement(dst: MutableList<AccessWithExclusion<AccessTree.AccessNode>>, statement: CommonInst) {
             val edgeSetIdx = MethodAnalyzerEdges.instructionStorageIdx(statement, languageManager)
-            val currentExclusion = exclusions[edgeSetIdx] ?: return
-            val access = edges[edgeSetIdx] ?: return
+            val row = rows() ?: return
+            val offset = offsetOf(row, edgeSetIdx)
+            if (offset < 0) return
+
+            val currentExclusion = row.values[offset + EXCLUSION] as ExclusionSet? ?: return
+            val access = row.values[offset + ACCESS] as AccessTree.AccessNode? ?: return
             dst += AccessWithExclusion(access, currentExclusion)
         }
+
+        companion object {
+            private const val ACCESS = 0
+            private const val EXCLUSION = 1
+            private const val COLUMNS = 2
+        }
     }
 }