From a488ef2e0024df30123d44a03f893f0c03fca613 Mon Sep 17 00:00:00 2001 From: zefutoff Date: Thu, 8 Oct 2026 23:38:58 +0200 Subject: [PATCH] test: add runtime and memory stress coverage --- .github/workflows/ci.yml | 3 + CHANGELOG.md | 1 + Makefile | 9 +- ROADMAP.md | 2 +- docs/testing.md | 39 ++ engine-grammalecte/build.gradle.kts | 8 + .../engine/GrammalecteQuickJsEngine.kt | 28 ++ .../engine/EngineRuntimeStressTest.kt | 357 ++++++++++++++++++ 8 files changed, 445 insertions(+), 2 deletions(-) create mode 100644 engine-grammalecte/src/test/kotlin/fr/grammalecteandroid/engine/EngineRuntimeStressTest.kt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3e4e5f2..f5487dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,6 +43,9 @@ jobs: - name: Run performance regression checks run: make performance-check + - name: Run runtime stress checks + run: make stress-check + - name: Build debug APK run: make assemble diff --git a/CHANGELOG.md b/CHANGELOG.md index e39ae44..ff68392 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Added - Real-engine performance regression tests covering QuickJS runtime reuse and tolerant cold-start, sentence and paragraph timing budgets. +- Runtime stress coverage for prolonged Grammalecte analysis, concurrent engine calls, repeated QuickJS lifecycle cycles and retained QuickJS memory growth. - Initial modular Android project foundation. - Pure Kotlin `GrammarEngine` abstraction for correction results. - Native Android `SpellCheckerService` integration. diff --git a/Makefile b/Makefile index 13bb1f1..928e1a2 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ GRADLE := ./gradlew -.PHONY: bootstrap vendor check performance-check test lint assemble instrumented refresh-verification-metadata clean +.PHONY: bootstrap vendor check performance-check stress-check test lint assemble instrumented refresh-verification-metadata clean bootstrap: ./tools/bootstrap-gradle-wrapper.sh @@ -22,6 +22,13 @@ performance-check: -PgrammalectePerf=1 \ --rerun-tasks +stress-check: + $(GRADLE) --no-daemon \ + :engine-grammalecte:testDebugUnitTest \ + --tests '*EngineRuntimeStressTest*' \ + -PgrammalecteStress=1 \ + --rerun-tasks + test: $(GRADLE) --no-daemon :core:test :engine-grammalecte:test :spellchecker:test diff --git a/ROADMAP.md b/ROADMAP.md index 33d6547..42c1c73 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -95,7 +95,7 @@ Exit criterion: correction remains usable in common editable applications even w - [x] Review all user-facing French strings - [x] Add compatibility documentation - [x] Add performance regression tests -- [ ] Add memory/runtime stress tests +- [x] Add memory/runtime stress tests ## Phase 5 — Public distribution diff --git a/docs/testing.md b/docs/testing.md index c9dc60d..68aba2b 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -246,3 +246,42 @@ profiling concern. Performance regression tests are intended to detect substantial regressions, such as accidental runtime reinitialization, repeated asset loading or major analysis slowdowns, rather than small timing variations. + +## Runtime and memory stress tests + +`EngineRuntimeStressTest` exercises the real embedded QuickJS/Grammalecte +runtime under sustained use. + +The stress suite covers: + +- a long-lived engine performing hundreds of grammar analyses and repeated + spelling checks; +- retained QuickJS memory before and after sustained analysis, measured after + explicit QuickJS garbage collection; +- concurrent bursts of grammar and spelling requests sharing one serialized + runtime; +- repeated engine creation, use and closure cycles. + +Run the suite with: + + make stress-check + +The stress tests are opt-in through `-PgrammalecteStress=1` and are executed +by CI through the dedicated `stress-check` target. + +The initial development measurement after 400 long-lived analyses reported: + +- QuickJS memory before stress: 30.00 MiB; +- QuickJS memory after stress: 30.00 MiB; +- retained-memory growth: 0.00 MiB; +- QuickJS allocated memory after stress: 40.62 MiB; +- objects after stress: 91,502; +- strings after stress: 36,620. + +The retained-memory regression budget is 8 MiB. This intentionally allows +normal runtime and host variation while detecting substantial accumulation +across repeated corrections. + +These measurements concern the embedded QuickJS runtime used by the JVM test +environment. They protect engine lifecycle and memory behavior but are not a +claim about total Android process memory or device-specific RSS. diff --git a/engine-grammalecte/build.gradle.kts b/engine-grammalecte/build.gradle.kts index 2087366..996585d 100644 --- a/engine-grammalecte/build.gradle.kts +++ b/engine-grammalecte/build.gradle.kts @@ -64,6 +64,14 @@ tasks.withType().configureEach { .orElse("0") .get(), ) + + systemProperty( + "grammalecte.stress", + providers + .gradleProperty("grammalecteStress") + .orElse("0") + .get(), + ) } dependencies { diff --git a/engine-grammalecte/src/main/kotlin/fr/grammalecteandroid/engine/GrammalecteQuickJsEngine.kt b/engine-grammalecte/src/main/kotlin/fr/grammalecteandroid/engine/GrammalecteQuickJsEngine.kt index c1f68f3..731968c 100644 --- a/engine-grammalecte/src/main/kotlin/fr/grammalecteandroid/engine/GrammalecteQuickJsEngine.kt +++ b/engine-grammalecte/src/main/kotlin/fr/grammalecteandroid/engine/GrammalecteQuickJsEngine.kt @@ -13,6 +13,13 @@ import kotlinx.coroutines.runBlocking import org.json.JSONArray import org.json.JSONObject +internal data class GrammalecteRuntimeMemorySnapshot( + val memoryUsedBytes: Long, + val mallocBytes: Long, + val objectCount: Long, + val stringCount: Long, +) + class GrammalecteQuickJsEngine internal constructor( private val assetLoader: AssetTextLoader, private val rulePreferences: GrammalecteRulePreferences? = null, @@ -598,6 +605,27 @@ class GrammalecteQuickJsEngine internal constructor( } } + internal fun collectRuntimeGarbageAndSnapshot(): GrammalecteRuntimeMemorySnapshot = + synchronized(lock) { + val js = + runtime + ?: createRuntime().also { + runtime = it + } + + js.gc() + + val usage = + js.memoryUsage + + GrammalecteRuntimeMemorySnapshot( + memoryUsedBytes = usage.memoryUsedSize, + mallocBytes = usage.mallocSize, + objectCount = usage.objCount, + stringCount = usage.strCount, + ) + } + override fun close() { synchronized(lock) { runtime?.close() diff --git a/engine-grammalecte/src/test/kotlin/fr/grammalecteandroid/engine/EngineRuntimeStressTest.kt b/engine-grammalecte/src/test/kotlin/fr/grammalecteandroid/engine/EngineRuntimeStressTest.kt new file mode 100644 index 0000000..e0b2764 --- /dev/null +++ b/engine-grammalecte/src/test/kotlin/fr/grammalecteandroid/engine/EngineRuntimeStressTest.kt @@ -0,0 +1,357 @@ +package fr.grammalecteandroid.engine + +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Assume.assumeTrue +import org.junit.Test +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Path +import java.util.concurrent.CountDownLatch +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit + +class EngineRuntimeStressTest { + @Test + fun longLivedRuntimeMemoryRemainsBounded() { + requireStressEnabled() + + val engine = + newEngine() + + try { + repeat(WARMUP_ROUNDS) { + CORPUS.forEach { text -> + assertTrue( + engine + .check( + text = text, + localeTag = "fr-FR", + ).isNotEmpty(), + ) + } + + assertSpellingCorrection(engine) + } + + val before = + engine.collectRuntimeGarbageAndSnapshot() + + repeat(LONG_LIVED_ITERATIONS) { index -> + val text = + CORPUS[ + index % + CORPUS.size, + ] + + assertTrue( + "Expected issues during stress iteration $index", + engine + .check( + text = text, + localeTag = "fr-FR", + ).isNotEmpty(), + ) + + if ( + index % + WORD_CHECK_INTERVAL == + 0 + ) { + assertSpellingCorrection(engine) + } + } + + val after = + engine.collectRuntimeGarbageAndSnapshot() + + val growth = + ( + after.memoryUsedBytes - + before.memoryUsedBytes + ).coerceAtLeast(0) + + println() + println("GRAMMALECTE RUNTIME MEMORY") + println("--------------------------") + println( + "Before : ${formatBytes(before.memoryUsedBytes)}", + ) + println( + "After : ${formatBytes(after.memoryUsedBytes)}", + ) + println( + "Growth : ${formatBytes(growth)}", + ) + println( + "Malloc : ${formatBytes(after.mallocBytes)}", + ) + println( + "Objects : ${after.objectCount}", + ) + println( + "Strings : ${after.stringCount}", + ) + println() + + assertTrue( + "QuickJS retained-memory growth " + + "${formatBytes(growth)} exceeds " + + "${formatBytes(MAX_RETAINED_MEMORY_GROWTH_BYTES)}", + growth <= + MAX_RETAINED_MEMORY_GROWTH_BYTES, + ) + } finally { + engine.close() + } + } + + @Test + fun concurrentBurstRemainsCorrect() { + requireStressEnabled() + + val engine = + newEngine() + + val executor = + Executors.newFixedThreadPool( + CONCURRENT_THREADS, + ) + + val startGate = + CountDownLatch(1) + + try { + assertSpellingCorrection(engine) + + val futures = + List(CONCURRENT_THREADS) { worker -> + executor.submit { + startGate.await() + + repeat( + CONCURRENT_ITERATIONS_PER_THREAD, + ) { iteration -> + if ( + ( + worker + + iteration + ) % 2 == + 0 + ) { + val text = + CORPUS[ + ( + worker + + iteration + ) % + CORPUS.size, + ] + + if ( + engine + .check( + text = text, + localeTag = "fr-FR", + ).isEmpty() + ) { + return@submit false + } + } else { + val check = + engine.checkWord( + word = "magazin", + localeTag = "fr-FR", + suggestionLimit = 8, + ) + + if ( + check.valid || + "magasin" !in + check.suggestions + ) { + return@submit false + } + } + } + + true + } + } + + startGate.countDown() + + futures.forEachIndexed { index, future -> + assertTrue( + "Concurrent stress worker $index failed", + future.get( + CONCURRENT_TIMEOUT_SECONDS, + TimeUnit.SECONDS, + ), + ) + } + } finally { + startGate.countDown() + + executor.shutdown() + + if ( + !executor.awaitTermination( + CONCURRENT_TIMEOUT_SECONDS, + TimeUnit.SECONDS, + ) + ) { + executor.shutdownNow() + } + + engine.close() + } + } + + @Test + fun repeatedRuntimeCreateUseCloseCyclesRemainStable() { + requireStressEnabled() + + repeat(RUNTIME_LIFECYCLE_CYCLES) { cycle -> + val engine = + newEngine() + + try { + assertSpellingCorrection(engine) + + val issues = + engine.check( + text = + CORPUS[ + cycle % + CORPUS.size, + ], + localeTag = "fr-FR", + ) + + assertTrue( + "Runtime lifecycle cycle $cycle returned no issues", + issues.isNotEmpty(), + ) + } finally { + engine.close() + } + } + } + + private fun assertSpellingCorrection(engine: GrammalecteQuickJsEngine) { + val check = + engine.checkWord( + word = "magazin", + localeTag = "fr-FR", + suggestionLimit = 8, + ) + + assertFalse( + "Expected « magazin » to remain invalid", + check.valid, + ) + + assertTrue( + "Expected « magasin » among suggestions", + "magasin" in check.suggestions, + ) + } + + private fun newEngine(): GrammalecteQuickJsEngine = + GrammalecteQuickJsEngine( + FileAssetTextLoader( + findAssetRoot(), + ), + ) + + private fun requireStressEnabled() { + assumeTrue( + "Set -PgrammalecteStress=1 to run runtime stress tests", + System.getProperty("grammalecte.stress") == + "1", + ) + } + + private fun formatBytes(bytes: Long): String = + "%.2f MiB".format( + java.util.Locale.ROOT, + bytes.toDouble() / + BYTES_PER_MEBIBYTE, + ) + + private fun findAssetRoot(): Path { + val candidates = + listOf( + Path.of("src/main/assets"), + Path.of( + "engine-grammalecte/src/main/assets", + ), + ).map { + it + .toAbsolutePath() + .normalize() + } + + return candidates.firstOrNull { + Files.isDirectory(it) + } ?: error( + "Unable to locate engine assets. Tried: $candidates", + ) + } + + private class FileAssetTextLoader( + private val root: Path, + ) : AssetTextLoader { + override fun readText(path: String): String { + val normalizedPath = + AssetPathNormalizer.normalize(path) + + val file = + root + .resolve(normalizedPath) + .normalize() + + check(file.startsWith(root)) { + "Asset path escapes test root: $path" + } + + return Files + .newBufferedReader( + file, + StandardCharsets.UTF_8, + ).use { + it.readText() + } + } + } + + private companion object { + const val WARMUP_ROUNDS = 3 + const val LONG_LIVED_ITERATIONS = 400 + const val WORD_CHECK_INTERVAL = 10 + + const val CONCURRENT_THREADS = 8 + const val CONCURRENT_ITERATIONS_PER_THREAD = 20 + const val CONCURRENT_TIMEOUT_SECONDS = 120L + + const val RUNTIME_LIFECYCLE_CYCLES = 10 + + const val MAX_RETAINED_MEMORY_GROWTH_BYTES = + 8L * 1024L * 1024L + + const val BYTES_PER_MEBIBYTE = + 1024.0 * 1024.0 + + val CORPUS = + listOf( + "Je suis aller au magazin hier et " + + "j'ai acheter deux pomme.", + "Les enfant joue dans le jardin " + + "pendant que nous somme arrivé en retard.", + "Elle c'est rendu compte de son erreur.", + "Il faut que tu fasse attention car " + + "ces une bonne idée.", + ) + } +}