diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ef95982..3e4e5f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,6 +40,9 @@ jobs: - name: Run project checks run: make check + - name: Run performance regression checks + run: make performance-check + - name: Build debug APK run: make assemble diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f625d9..e39ae44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Added +- Real-engine performance regression tests covering QuickJS runtime reuse and tolerant cold-start, sentence and paragraph timing budgets. - Initial modular Android project foundation. - Pure Kotlin `GrammarEngine` abstraction for correction results. - Native Android `SpellCheckerService` integration. diff --git a/Makefile b/Makefile index a4a77fd..13bb1f1 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ GRADLE := ./gradlew -.PHONY: bootstrap vendor check test lint assemble instrumented refresh-verification-metadata clean +.PHONY: bootstrap vendor check performance-check test lint assemble instrumented refresh-verification-metadata clean bootstrap: ./tools/bootstrap-gradle-wrapper.sh @@ -15,6 +15,13 @@ check: node tools/test-js-bridge.mjs $(GRADLE) --no-daemon ktlintCheck :core:test :engine-grammalecte:test :spellchecker:test :app:lintDebug +performance-check: + $(GRADLE) --no-daemon \ + :engine-grammalecte:testDebugUnitTest \ + --tests '*EnginePerformanceRegressionTest*' \ + -PgrammalectePerf=1 \ + --rerun-tasks + test: $(GRADLE) --no-daemon :core:test :engine-grammalecte:test :spellchecker:test diff --git a/ROADMAP.md b/ROADMAP.md index 70a6863..33d6547 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -94,7 +94,7 @@ Exit criterion: correction remains usable in common editable applications even w - [x] Improve accessibility of the application UI - [x] Review all user-facing French strings - [x] Add compatibility documentation -- [ ] Add performance regression tests +- [x] Add performance regression tests - [ ] Add memory/runtime stress tests ## Phase 5 — Public distribution diff --git a/docs/testing.md b/docs/testing.md index 9365dfb..c9dc60d 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -198,9 +198,12 @@ Use short synthetic French sentences designed to isolate one behavior. ## Performance tests -An opt-in JVM characterization test measures the real embedded QuickJS/Grammalecte engine without adding unstable timing requirements to normal CI. +Performance coverage uses two complementary JVM tests against the real +embedded QuickJS/Grammalecte engine. -Run it with: +`EnginePerformanceCharacterizationTest` remains an opt-in measurement tool. +It reports cold-start, sentence and paragraph timings without enforcing +thresholds: ./gradlew \ :engine-grammalecte:testDebugUnitTest \ @@ -209,20 +212,37 @@ Run it with: --rerun-tasks \ --no-daemon -The initial development baseline measured: +`EnginePerformanceRegressionTest` adds regression protection. -- cold first correction: approximately 808 ms; -- sentence median: approximately 13.5 ms; -- sentence p95: approximately 15.6 ms; -- paragraph median: approximately 30.9 ms; -- paragraph p95: approximately 35.7 ms. +Its deterministic test verifies that repeated analyses reuse the initialized +QuickJS runtime without reloading the embedded Grammalecte assets. This part +runs with the normal engine test suite. -These measurements characterize the JVM development environment and are not Android-device performance guarantees. +Its timing test is enabled by `-PgrammalectePerf=1` and is run in CI through: -Future performance work should additionally cover: + make performance-check -- cold initialization on representative physical Android devices; -- memory use after repeated corrections; -- rapid repeated IME analyses. +The timing budgets are intentionally wider than the development baseline so +that normal host load does not make CI flaky: -Performance tests should primarily detect regressions rather than enforce unrealistic device-independent timing thresholds. +- cold first correction: at most 3000 ms; +- sentence median: at most 75 ms; +- sentence p95: at most 200 ms; +- paragraph median: at most 150 ms; +- paragraph p95: at most 400 ms. + +A three-run development baseline measured: + +- cold first correction: 809.54–828.45 ms; +- sentence median: 13.48–13.84 ms; +- sentence p95: 14.94–16.45 ms; +- paragraph median: 31.89–32.23 ms; +- paragraph p95: 32.85–34.35 ms. + +These JVM measurements are regression guards, not Android-device performance +guarantees. Physical-device latency remains a separate compatibility and +profiling concern. + +Performance regression tests are intended to detect substantial regressions, +such as accidental runtime reinitialization, repeated asset loading or major +analysis slowdowns, rather than small timing variations. diff --git a/engine-grammalecte/src/test/kotlin/fr/grammalecteandroid/engine/EnginePerformanceRegressionTest.kt b/engine-grammalecte/src/test/kotlin/fr/grammalecteandroid/engine/EnginePerformanceRegressionTest.kt new file mode 100644 index 0000000..3aadcd3 --- /dev/null +++ b/engine-grammalecte/src/test/kotlin/fr/grammalecteandroid/engine/EnginePerformanceRegressionTest.kt @@ -0,0 +1,366 @@ +package fr.grammalecteandroid.engine + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Assume.assumeTrue +import org.junit.Test +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Path +import kotlin.math.ceil +import kotlin.system.measureNanoTime + +class EnginePerformanceRegressionTest { + @Test + fun repeatedAnalysesReuseInitializedRuntimeAssets() { + val loader = + CountingFileAssetTextLoader( + findAssetRoot(), + ) + + val engine = + GrammalecteQuickJsEngine( + loader, + ) + + try { + assertTrue( + engine + .check( + text = SENTENCE, + localeTag = "fr-FR", + ).isNotEmpty(), + ) + + assertTrue( + engine + .check( + text = PARAGRAPH, + localeTag = "fr-FR", + ).isNotEmpty(), + ) + + val readsAfterWarmup = + loader.readCount + + assertTrue( + "Expected engine initialization to load assets", + readsAfterWarmup > 0, + ) + + repeat(REUSE_ITERATIONS) { + engine.check( + text = SENTENCE, + localeTag = "fr-FR", + ) + + engine.check( + text = PARAGRAPH, + localeTag = "fr-FR", + ) + } + + assertEquals( + "Repeated analyses must reuse the initialized " + + "QuickJS runtime without reloading assets", + readsAfterWarmup, + loader.readCount, + ) + } finally { + engine.close() + } + } + + @Test + fun realEngineStaysWithinRegressionBudgets() { + assumeTrue( + "Set -PgrammalectePerf=1 to run timing budgets", + System.getProperty("grammalecte.perf") == "1", + ) + + val assetRoot = + findAssetRoot() + + val coldFirstCheckMillis = + measureMillis { + val engine = + GrammalecteQuickJsEngine( + FileAssetTextLoader(assetRoot), + ) + + try { + engine.checkWord( + word = "magazin", + localeTag = "fr-FR", + suggestionLimit = 8, + ) + } finally { + engine.close() + } + } + + val engine = + GrammalecteQuickJsEngine( + FileAssetTextLoader(assetRoot), + ) + + try { + engine.checkWord( + word = "magazin", + localeTag = "fr-FR", + suggestionLimit = 8, + ) + + repeat(WARMUP_ITERATIONS) { + engine.check( + text = SENTENCE, + localeTag = "fr-FR", + ) + } + + engine.check( + text = PARAGRAPH, + localeTag = "fr-FR", + ) + + val sentenceSamples = + measureSamples(SENTENCE_ITERATIONS) { + engine.check( + text = SENTENCE, + localeTag = "fr-FR", + ) + } + + val paragraphSamples = + measureSamples(PARAGRAPH_ITERATIONS) { + engine.check( + text = PARAGRAPH, + localeTag = "fr-FR", + ) + } + + val sentenceMedian = + percentile( + sentenceSamples, + 50, + ) + + val sentenceP95 = + percentile( + sentenceSamples, + 95, + ) + + val paragraphMedian = + percentile( + paragraphSamples, + 50, + ) + + val paragraphP95 = + percentile( + paragraphSamples, + 95, + ) + + println() + println("GRAMMALECTE PERFORMANCE REGRESSION") + println("----------------------------------") + println( + "Cold first check : ${format(coldFirstCheckMillis)} ms", + ) + println( + "Sentence median : ${format(sentenceMedian)} ms", + ) + println( + "Sentence p95 : ${format(sentenceP95)} ms", + ) + println( + "Paragraph median : ${format(paragraphMedian)} ms", + ) + println( + "Paragraph p95 : ${format(paragraphP95)} ms", + ) + println() + + assertWithinBudget( + label = "Cold first check", + measuredMillis = coldFirstCheckMillis, + budgetMillis = COLD_FIRST_CHECK_BUDGET_MILLIS, + ) + + assertWithinBudget( + label = "Sentence median", + measuredMillis = sentenceMedian, + budgetMillis = SENTENCE_MEDIAN_BUDGET_MILLIS, + ) + + assertWithinBudget( + label = "Sentence p95", + measuredMillis = sentenceP95, + budgetMillis = SENTENCE_P95_BUDGET_MILLIS, + ) + + assertWithinBudget( + label = "Paragraph median", + measuredMillis = paragraphMedian, + budgetMillis = PARAGRAPH_MEDIAN_BUDGET_MILLIS, + ) + + assertWithinBudget( + label = "Paragraph p95", + measuredMillis = paragraphP95, + budgetMillis = PARAGRAPH_P95_BUDGET_MILLIS, + ) + } finally { + engine.close() + } + } + + private fun assertWithinBudget( + label: String, + measuredMillis: Double, + budgetMillis: Double, + ) { + assertTrue( + "$label regression: ${format(measuredMillis)} ms " + + "exceeds ${format(budgetMillis)} ms budget", + measuredMillis <= budgetMillis, + ) + } + + private fun measureSamples( + iterations: Int, + block: () -> Unit, + ): List = + List(iterations) { + measureMillis(block) + } + + private fun measureMillis(block: () -> Unit): Double = + measureNanoTime(block) / + NANOS_PER_MILLISECOND + + private fun percentile( + samples: List, + percentile: Int, + ): Double { + val sorted = + samples.sorted() + + val index = + ( + ceil( + percentile / 100.0 * + sorted.size, + ).toInt() - 1 + ).coerceIn( + 0, + sorted.lastIndex, + ) + + return sorted[index] + } + + private fun format(value: Double): String = + "%.2f".format( + java.util.Locale.ROOT, + value, + ) + + private fun findAssetRoot(): Path { + val candidates = + listOf( + Path.of("src/main/assets"), + Path.of( + "engine-grammalecte/src/main/assets", + ), + ).map { + it.toAbsolutePath().normalize() + } + + return candidates.firstOrNull { + Files.isDirectory(it) + } ?: error( + "Unable to locate engine assets. Tried: $candidates", + ) + } + + private open class FileAssetTextLoader( + private val root: Path, + ) : AssetTextLoader { + override fun readText(path: String): String { + val normalizedPath = + AssetPathNormalizer.normalize(path) + + val file = + root + .resolve(normalizedPath) + .normalize() + + check(file.startsWith(root)) { + "Asset path escapes test root: $path" + } + + return Files + .newBufferedReader( + file, + StandardCharsets.UTF_8, + ).use { + it.readText() + } + } + } + + private class CountingFileAssetTextLoader( + root: Path, + ) : FileAssetTextLoader(root) { + var readCount: Int = 0 + private set + + override fun readText(path: String): String { + readCount += 1 + + return super.readText(path) + } + } + + private companion object { + const val REUSE_ITERATIONS = 10 + const val WARMUP_ITERATIONS = 3 + const val SENTENCE_ITERATIONS = 20 + const val PARAGRAPH_ITERATIONS = 12 + + const val NANOS_PER_MILLISECOND = + 1_000_000.0 + + const val COLD_FIRST_CHECK_BUDGET_MILLIS = + 3_000.0 + + const val SENTENCE_MEDIAN_BUDGET_MILLIS = + 75.0 + + const val SENTENCE_P95_BUDGET_MILLIS = + 200.0 + + const val PARAGRAPH_MEDIAN_BUDGET_MILLIS = + 150.0 + + const val PARAGRAPH_P95_BUDGET_MILLIS = + 400.0 + + const val SENTENCE = + "Je suis aller au magazin hier et " + + "j'ai acheter deux pomme." + + const val PARAGRAPH = + "Je suis aller au magazin hier et " + + "j'ai acheter deux pomme. " + + "Les enfant joue dans le jardin pendant que " + + "nous somme arrivé en retard. " + + "Il faut que tu fasse attention car ces une " + + "bonne idée de vérifier le texte. " + + "Je voudrai savoir si vous êtes disponible " + + "demain matin." + } +}