From e068ae3d6e5e730b4edf23c2659c9b914da3a0ac Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Sat, 15 Aug 2026 18:37:36 +0200 Subject: [PATCH 01/16] refactor(evaluation): preserve optional quality presence in Room --- .../daniele21/localllm/evaluation/room/EvaluationRoomEntities.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomEntities.kt b/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomEntities.kt index 8d58d4951..066796286 100644 --- a/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomEntities.kt +++ b/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomEntities.kt @@ -22,6 +22,7 @@ data class EvaluationRunEntity( @Embedded(prefix = "identity_") val identity: EvaluationRunIdentityEntity?, val state: String, @Embedded(prefix = "progress_") val progress: EvaluationProgressEntity, + @ColumnInfo(name = "quality_present") val qualityPresent: Boolean, @ColumnInfo(name = "quality_aggregate_score") val qualityAggregateScore: Double?, @Embedded(prefix = "reliability_") val reliability: EvaluationReliabilityEntity?, @ColumnInfo(name = "started_at_epoch_ms") val startedAtEpochMs: Long, From c783121c2020377e0dc139e4b9ca1ee6eea288c1 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Sat, 15 Aug 2026 18:37:46 +0200 Subject: [PATCH 02/16] build(evaluation): enable Room repository generation --- evaluation/room-store/build.gradle.kts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/evaluation/room-store/build.gradle.kts b/evaluation/room-store/build.gradle.kts index a547029f5..1b0230a49 100644 --- a/evaluation/room-store/build.gradle.kts +++ b/evaluation/room-store/build.gradle.kts @@ -9,6 +9,11 @@ android { defaultConfig { minSdk = libs.versions.minSdk.get().toInt() + javaCompileOptions { + annotationProcessorOptions { + argument("room.schemaLocation", "$projectDir/schemas") + } + } } compileOptions { @@ -25,7 +30,9 @@ android { } dependencies { - implementation(project(":evaluation:contracts")) + api(project(":evaluation:contracts")) implementation(libs.room.runtime) + implementation(libs.kotlinx.coroutines.android) + annotationProcessor(libs.room.compiler) testImplementation(libs.junit4) } From 290e54df67d9d2822a665ce94f8026d443247d9a Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Sat, 15 Aug 2026 18:38:00 +0200 Subject: [PATCH 03/16] feat(evaluation): add transactional Room run snapshot --- .../evaluation/room/EvaluationStoredRun.java | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 evaluation/room-store/src/main/java/io/github/daniele21/localllm/evaluation/room/EvaluationStoredRun.java diff --git a/evaluation/room-store/src/main/java/io/github/daniele21/localllm/evaluation/room/EvaluationStoredRun.java b/evaluation/room-store/src/main/java/io/github/daniele21/localllm/evaluation/room/EvaluationStoredRun.java new file mode 100644 index 000000000..f80d1e0a7 --- /dev/null +++ b/evaluation/room-store/src/main/java/io/github/daniele21/localllm/evaluation/room/EvaluationStoredRun.java @@ -0,0 +1,24 @@ +package io.github.daniele21.localllm.evaluation.room; + +import java.util.List; + +public final class EvaluationStoredRun { + public final EvaluationRunEntity run; + public final List samples; + public final List categoryScores; + public final List caseResults; + public final List evaluatorParameters; + + public EvaluationStoredRun( + EvaluationRunEntity run, + List samples, + List categoryScores, + List caseResults, + List evaluatorParameters) { + this.run = run; + this.samples = samples; + this.categoryScores = categoryScores; + this.caseResults = caseResults; + this.evaluatorParameters = evaluatorParameters; + } +} From 305fd6789134c0fe3e7dcb400368ae48d55def03 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Sat, 15 Aug 2026 18:38:17 +0200 Subject: [PATCH 04/16] feat(evaluation): add evaluation Room DAO --- .../evaluation/room/EvaluationDao.java | 132 ++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 evaluation/room-store/src/main/java/io/github/daniele21/localllm/evaluation/room/EvaluationDao.java diff --git a/evaluation/room-store/src/main/java/io/github/daniele21/localllm/evaluation/room/EvaluationDao.java b/evaluation/room-store/src/main/java/io/github/daniele21/localllm/evaluation/room/EvaluationDao.java new file mode 100644 index 000000000..c823aca3a --- /dev/null +++ b/evaluation/room-store/src/main/java/io/github/daniele21/localllm/evaluation/room/EvaluationDao.java @@ -0,0 +1,132 @@ +package io.github.daniele21.localllm.evaluation.room; + +import androidx.annotation.Nullable; +import androidx.room.Dao; +import androidx.room.Insert; +import androidx.room.OnConflictStrategy; +import androidx.room.Query; +import androidx.room.RawQuery; +import androidx.room.Transaction; +import androidx.room.Update; +import androidx.sqlite.db.SupportSQLiteQuery; +import java.util.List; + +@Dao +public interface EvaluationDao { + @Insert(onConflict = OnConflictStrategy.ABORT) + void insertRun(EvaluationRunEntity run); + + @Update + int updateRun(EvaluationRunEntity run); + + @Insert(onConflict = OnConflictStrategy.ABORT) + void insertSamples(List samples); + + @Insert(onConflict = OnConflictStrategy.REPLACE) + void insertCategoryScores(List scores); + + @Insert(onConflict = OnConflictStrategy.REPLACE) + void upsertCaseResult(EvaluationCaseResultEntity result); + + @Insert(onConflict = OnConflictStrategy.REPLACE) + void insertEvaluatorParameters(List parameters); + + @Nullable + @Query("SELECT * FROM evaluation_runs WHERE run_id = :runId LIMIT 1") + EvaluationRunEntity findRun(String runId); + + @Query("SELECT * FROM evaluation_sample_cases WHERE run_id = :runId ORDER BY ordinal ASC") + List sampleCases(String runId); + + @Query("SELECT * FROM evaluation_category_scores WHERE run_id = :runId ORDER BY category_id ASC") + List categoryScores(String runId); + + @Query( + "SELECT r.* FROM evaluation_case_results r " + + "JOIN evaluation_sample_cases s ON s.run_id = r.run_id AND s.case_id = r.case_id " + + "WHERE r.run_id = :runId ORDER BY s.ordinal ASC") + List caseResults(String runId); + + @Query( + "SELECT * FROM evaluation_evaluator_parameters WHERE run_id = :runId " + + "ORDER BY case_id ASC, parameter_key ASC") + List evaluatorParameters(String runId); + + @Query( + "SELECT COUNT(*) FROM evaluation_sample_cases " + + "WHERE run_id = :runId AND case_id = :caseId") + int sampleCaseCount(String runId, String caseId); + + @RawQuery + List queryRuns(SupportSQLiteQuery query); + + @Query( + "SELECT * FROM evaluation_runs WHERE state IN ('COMPLETED','CANCELLED','FAILED') " + + "ORDER BY started_at_epoch_ms DESC, run_id ASC") + List terminalRunsNewestFirst(); + + @Query("DELETE FROM evaluation_category_scores WHERE run_id = :runId") + void deleteCategoryScores(String runId); + + @Query( + "DELETE FROM evaluation_evaluator_parameters " + + "WHERE run_id = :runId AND case_id = :caseId") + void deleteEvaluatorParameters(String runId, String caseId); + + @Query("DELETE FROM evaluation_runs WHERE run_id = :runId") + int deleteRunRow(String runId); + + @Query("DELETE FROM evaluation_runs WHERE run_id IN (:runIds)") + int deleteRunRows(List runIds); + + @Transaction + default void createRunGraph( + EvaluationRunEntity run, + List samples, + List scores) { + insertRun(run); + insertSamples(samples); + if (!scores.isEmpty()) { + insertCategoryScores(scores); + } + } + + @Transaction + default void updateRunGraph( + EvaluationRunEntity run, + List scores) { + if (updateRun(run) != 1) { + throw new IllegalStateException("Evaluation run does not exist"); + } + deleteCategoryScores(run.getRunId()); + if (!scores.isEmpty()) { + insertCategoryScores(scores); + } + } + + @Transaction + default void upsertCaseResultGraph( + EvaluationCaseResultEntity result, + List parameters) { + upsertCaseResult(result); + deleteEvaluatorParameters(result.getRunId(), result.getCaseId()); + if (!parameters.isEmpty()) { + insertEvaluatorParameters(parameters); + } + } + + @Nullable + @Transaction + default EvaluationStoredRun loadStoredRun(String runId) { + EvaluationRunEntity run = findRun(runId); + if (run == null) { + return null; + } + return new EvaluationStoredRun( + run, + sampleCases(runId), + categoryScores(runId), + caseResults(runId), + evaluatorParameters(runId)); + } +} From 7461cb370b3c3b19be435b9d4f00582acd9f258d Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Sat, 15 Aug 2026 18:38:28 +0200 Subject: [PATCH 05/16] feat(evaluation): add evaluation Room database --- .../evaluation/room/EvaluationDatabase.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 evaluation/room-store/src/main/java/io/github/daniele21/localllm/evaluation/room/EvaluationDatabase.java diff --git a/evaluation/room-store/src/main/java/io/github/daniele21/localllm/evaluation/room/EvaluationDatabase.java b/evaluation/room-store/src/main/java/io/github/daniele21/localllm/evaluation/room/EvaluationDatabase.java new file mode 100644 index 000000000..a70735821 --- /dev/null +++ b/evaluation/room-store/src/main/java/io/github/daniele21/localllm/evaluation/room/EvaluationDatabase.java @@ -0,0 +1,18 @@ +package io.github.daniele21.localllm.evaluation.room; + +import androidx.room.Database; +import androidx.room.RoomDatabase; + +@Database( + entities = { + EvaluationRunEntity.class, + EvaluationSampleCaseEntity.class, + EvaluationCategoryScoreEntity.class, + EvaluationCaseResultEntity.class, + EvaluationEvaluatorParameterEntity.class + }, + version = 1, + exportSchema = true) +public abstract class EvaluationDatabase extends RoomDatabase { + public abstract EvaluationDao evaluationDao(); +} From faa3448b77bf65dda578821256e9f8458a7e83c6 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Sat, 15 Aug 2026 18:39:05 +0200 Subject: [PATCH 06/16] refactor(evaluation): preserve category score order in Room --- .../localllm/evaluation/room/EvaluationRoomEntities.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomEntities.kt b/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomEntities.kt index 066796286..2a6d0322c 100644 --- a/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomEntities.kt +++ b/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomEntities.kt @@ -138,10 +138,11 @@ data class EvaluationSampleCaseEntity( onDelete = ForeignKey.CASCADE, ), ], - indices = [Index(value = ["run_id"])], + indices = [Index(value = ["run_id"]), Index(value = ["run_id", "ordinal"], unique = true)], ) data class EvaluationCategoryScoreEntity( @ColumnInfo(name = "run_id") val runId: String, + val ordinal: Int, @ColumnInfo(name = "category_id") val categoryId: String, val score: Double, @ColumnInfo(name = "scored_case_count") val scoredCaseCount: Int, From 1d502a42b23e33a7e63d35dae216f864b38b5317 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Sat, 15 Aug 2026 18:39:28 +0200 Subject: [PATCH 07/16] fix(evaluation): load category scores in persisted order --- .../daniele21/localllm/evaluation/room/EvaluationDao.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evaluation/room-store/src/main/java/io/github/daniele21/localllm/evaluation/room/EvaluationDao.java b/evaluation/room-store/src/main/java/io/github/daniele21/localllm/evaluation/room/EvaluationDao.java index c823aca3a..50f420923 100644 --- a/evaluation/room-store/src/main/java/io/github/daniele21/localllm/evaluation/room/EvaluationDao.java +++ b/evaluation/room-store/src/main/java/io/github/daniele21/localllm/evaluation/room/EvaluationDao.java @@ -38,7 +38,7 @@ public interface EvaluationDao { @Query("SELECT * FROM evaluation_sample_cases WHERE run_id = :runId ORDER BY ordinal ASC") List sampleCases(String runId); - @Query("SELECT * FROM evaluation_category_scores WHERE run_id = :runId ORDER BY category_id ASC") + @Query("SELECT * FROM evaluation_category_scores WHERE run_id = :runId ORDER BY ordinal ASC") List categoryScores(String runId); @Query( From ee5612210d07e1f9d3a109a8ad94e4a35e9c9471 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Sat, 15 Aug 2026 18:40:10 +0200 Subject: [PATCH 08/16] feat(evaluation): map privacy-safe Room graph to domain --- .../evaluation/room/EvaluationRoomMapper.kt | 428 ++++++++++++++++++ 1 file changed, 428 insertions(+) create mode 100644 evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomMapper.kt diff --git a/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomMapper.kt b/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomMapper.kt new file mode 100644 index 000000000..1ee59c0f2 --- /dev/null +++ b/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomMapper.kt @@ -0,0 +1,428 @@ +package io.github.daniele21.localllm.evaluation.room + +import io.github.daniele21.localllm.contracts.ChatTemplateSource +import io.github.daniele21.localllm.contracts.InferencePresetId +import io.github.daniele21.localllm.contracts.InferencePresetRef +import io.github.daniele21.localllm.contracts.ModelDigest +import io.github.daniele21.localllm.contracts.RequestId +import io.github.daniele21.localllm.contracts.SeedPolicyType +import io.github.daniele21.localllm.contracts.ThinkingMode +import io.github.daniele21.localllm.evaluation.CaseExecutionSemanticsDigest +import io.github.daniele21.localllm.evaluation.EvaluationCaseId +import io.github.daniele21.localllm.evaluation.EvaluationCaseMetrics +import io.github.daniele21.localllm.evaluation.EvaluationCaseResult +import io.github.daniele21.localllm.evaluation.EvaluationCaseStatus +import io.github.daniele21.localllm.evaluation.EvaluationCategoryId +import io.github.daniele21.localllm.evaluation.EvaluationCategoryScore +import io.github.daniele21.localllm.evaluation.EvaluationDatasetDigest +import io.github.daniele21.localllm.evaluation.EvaluationDatasetId +import io.github.daniele21.localllm.evaluation.EvaluationDatasetIdentity +import io.github.daniele21.localllm.evaluation.EvaluationDatasetVersion +import io.github.daniele21.localllm.evaluation.EvaluationExecutionProfileId +import io.github.daniele21.localllm.evaluation.EvaluationExecutionProfileRef +import io.github.daniele21.localllm.evaluation.EvaluationFailure +import io.github.daniele21.localllm.evaluation.EvaluationFailureCode +import io.github.daniele21.localllm.evaluation.EvaluationFailureStage +import io.github.daniele21.localllm.evaluation.EvaluationModelIdentity +import io.github.daniele21.localllm.evaluation.EvaluationModelLoadPolicy +import io.github.daniele21.localllm.evaluation.EvaluationOutcome +import io.github.daniele21.localllm.evaluation.EvaluationProgress +import io.github.daniele21.localllm.evaluation.EvaluationQualitySummary +import io.github.daniele21.localllm.evaluation.EvaluationReliabilitySummary +import io.github.daniele21.localllm.evaluation.EvaluationRunConfig +import io.github.daniele21.localllm.evaluation.EvaluationRunId +import io.github.daniele21.localllm.evaluation.EvaluationRunIdentity +import io.github.daniele21.localllm.evaluation.EvaluationRunSummary +import io.github.daniele21.localllm.evaluation.EvaluationRunState +import io.github.daniele21.localllm.evaluation.EvaluationRuntimeEnvironmentIdentity +import io.github.daniele21.localllm.evaluation.EvaluationSemanticExecution +import io.github.daniele21.localllm.evaluation.EvaluationSemanticExecutionIdentity +import io.github.daniele21.localllm.evaluation.EvaluationWarmupPolicy +import io.github.daniele21.localllm.evaluation.EvaluatorOutcomeCode +import io.github.daniele21.localllm.evaluation.EvaluatorSetDigest +import io.github.daniele21.localllm.evaluation.EvaluatorSpec +import io.github.daniele21.localllm.evaluation.EvaluatorType +import io.github.daniele21.localllm.evaluation.EvaluatorVersion +import io.github.daniele21.localllm.evaluation.NormalizedScore +import io.github.daniele21.localllm.evaluation.PersistedEvaluationRun +import io.github.daniele21.localllm.evaluation.SamplingPolicyId +import io.github.daniele21.localllm.evaluation.SamplingPolicyRef +import io.github.daniele21.localllm.evaluation.SamplingSelection + +@Suppress("TooManyFunctions") +internal object EvaluationRoomMapper { + fun runEntity(summary: EvaluationRunSummary): EvaluationRunEntity = EvaluationRunEntity( + runId = summary.runId.value, + config = configEntity(summary.config), + identity = summary.identity?.let(::identityEntity), + state = summary.state.name, + progress = progressEntity(summary.progress), + qualityPresent = summary.quality != null, + qualityAggregateScore = summary.quality?.aggregateScore?.value, + reliability = summary.reliability?.let(::reliabilityEntity), + startedAtEpochMs = summary.startedAtEpochMs, + completedAtEpochMs = summary.completedAtEpochMs, + failure = summary.failure?.let(::failureEntity), + ) + + fun sampleEntities(summary: EvaluationRunSummary): List = + summary.config.sampling.orderedCaseIds.mapIndexed { ordinal, caseId -> + EvaluationSampleCaseEntity(summary.runId.value, ordinal, caseId.value) + } + + fun categoryScoreEntities(summary: EvaluationRunSummary): List = + summary.quality?.categoryScores.orEmpty().mapIndexed { ordinal, score -> + EvaluationCategoryScoreEntity( + runId = summary.runId.value, + ordinal = ordinal, + categoryId = score.categoryId.value, + score = score.score.value, + scoredCaseCount = score.scoredCaseCount, + weight = score.weight, + ) + } + + fun caseResultEntity(runId: EvaluationRunId, result: EvaluationCaseResult): EvaluationCaseResultEntity = + EvaluationCaseResultEntity( + runId = runId.value, + caseId = result.caseId.value, + categoryId = result.categoryId.value, + evaluatorType = result.evaluator.type.name, + evaluatorVersion = result.evaluator.version.value, + status = result.status.name, + outcomeScore = result.outcome?.score?.value, + outcomeCode = result.outcome?.code?.name, + requestId = result.requestId?.value, + timeToFirstTokenMs = result.metrics.timeToFirstTokenMs, + totalMs = result.metrics.totalMs, + prefillMs = result.metrics.prefillMs, + decodeMs = result.metrics.decodeMs, + inputTokens = result.metrics.inputTokens, + outputTokens = result.metrics.outputTokens, + decodeTokensPerSecond = result.metrics.decodeTokensPerSecond, + processPssBytes = result.metrics.processPssBytes, + availableMemoryBytes = result.metrics.availableMemoryBytes, + thermalStatus = result.metrics.thermalStatus, + failure = result.failure?.let(::failureEntity), + ) + + fun evaluatorParameterEntities(runId: EvaluationRunId, result: EvaluationCaseResult): List = + result.evaluator.parameters.toSortedMap().map { (key, value) -> + EvaluationEvaluatorParameterEntity(runId.value, result.caseId.value, key, value) + } + + fun summary(stored: EvaluationStoredRun): EvaluationRunSummary { + val config = config(stored.run.config, stored.samples) + return summary(stored.run, config, stored.categoryScores) + } + + fun persistedRun(stored: EvaluationStoredRun): PersistedEvaluationRun { + val config = config(stored.run.config, stored.samples) + val summary = summary(stored.run, config, stored.categoryScores) + val parametersByCase = stored.evaluatorParameters.groupBy { it.caseId } + val results = stored.caseResults.map { entity -> + caseResult(entity, parametersByCase[entity.caseId].orEmpty()) + } + return PersistedEvaluationRun(summary, results) + } + + private fun configEntity(config: EvaluationRunConfig): EvaluationRunConfigEntity = EvaluationRunConfigEntity( + modelDigest = config.model.artifactDigest.sha256, + modelProfileId = config.model.modelProfileId, + modelTier = config.model.tier, + modelQuantization = config.model.quantization, + datasetId = config.dataset.id.value, + datasetVersion = config.dataset.version.value, + datasetDigest = config.dataset.digest.sha256, + sampleSetDigest = config.sampling.digest.sha256, + samplingPolicyId = config.sampling.policy.id.value, + samplingPolicyVersion = config.sampling.policy.version, + samplingSeed = config.sampling.seed, + executionProfileId = config.executionProfile.id.value, + executionProfileVersion = config.executionProfile.version, + loadPolicy = config.loadPolicy.name, + warmupPolicy = config.warmupPolicy.name, + caseTimeoutMs = config.caseTimeoutMs, + ) + + private fun identityEntity(identity: EvaluationRunIdentity): EvaluationRunIdentityEntity { + val semantic = identity.semanticExecution.execution + val runtime = identity.runtimeEnvironment + return EvaluationRunIdentityEntity( + evaluatorSetDigest = identity.evaluatorSetDigest.sha256, + semanticExecutionFingerprint = identity.semanticExecution.fingerprint.sha256, + runFingerprint = identity.fingerprint.sha256, + semantic = EvaluationSemanticExecutionEntity( + semanticsVersion = semantic.semanticsVersion, + backendRevision = semantic.backendRevision, + contextSize = semantic.contextSize, + presetId = semantic.preset?.id?.value, + presetVersion = semantic.preset?.version, + thinkingMode = semantic.thinkingMode.name, + temperature = semantic.temperature, + topP = semantic.topP, + topK = semantic.topK, + minP = semantic.minP, + presencePenalty = semantic.presencePenalty, + repeatPenalty = semantic.repeatPenalty, + repeatLastN = semantic.repeatLastN, + seedPolicy = semantic.seedPolicy.name, + effectiveSeed = semantic.effectiveSeed, + maxOutputTokens = semantic.maxOutputTokens, + chatTemplateId = semantic.chatTemplateId, + chatTemplateSource = semantic.chatTemplateSource.name, + systemPromptVersion = semantic.systemPromptVersion, + caseExecutionSemanticsDigest = semantic.caseExecutionSemanticsDigest.sha256, + ), + runtime = EvaluationRuntimeEnvironmentEntity( + deviceClass = runtime.deviceClass, + androidApiLevel = runtime.androidApiLevel, + abi = runtime.abi, + backendRevision = runtime.backendRevision, + harnessBuildIdentity = runtime.harnessBuildIdentity, + runtimeTuningProfileId = runtime.runtimeTuningProfileId, + runtimeTuningProfileVersion = runtime.runtimeTuningProfileVersion, + loadPolicy = runtime.loadPolicy.name, + warmupPolicy = runtime.warmupPolicy.name, + ), + ) + } + + private fun progressEntity(progress: EvaluationProgress) = EvaluationProgressEntity( + totalCases = progress.totalCases, + attemptedCases = progress.attemptedCases, + completedCases = progress.completedCases, + currentCaseId = progress.currentCaseId?.value, + ) + + private fun reliabilityEntity(summary: EvaluationReliabilitySummary) = EvaluationReliabilityEntity( + totalCases = summary.totalCases, + completedAndScored = summary.completedAndScored, + incorrectButValid = summary.incorrectButValid, + invalidOutput = summary.invalidOutput, + timeout = summary.timeout, + runtimeFailure = summary.runtimeFailure, + cancelled = summary.cancelled, + skipped = summary.skipped, + ) + + private fun failureEntity(failure: EvaluationFailure) = EvaluationFailureEntity( + stage = failure.stage.name, + code = failure.code.name, + caseId = failure.caseId?.value, + retryable = failure.retryable, + ) + + private fun config(entity: EvaluationRunConfigEntity, samples: List): EvaluationRunConfig { + require(samples.map { it.ordinal } == samples.indices.toList()) { "Persisted evaluation sample ordinals must be contiguous" } + val dataset = EvaluationDatasetIdentity( + id = EvaluationDatasetId(entity.datasetId), + version = EvaluationDatasetVersion(entity.datasetVersion), + digest = EvaluationDatasetDigest(entity.datasetDigest), + ) + val sampling = SamplingSelection.create( + dataset = dataset, + policy = SamplingPolicyRef(SamplingPolicyId(entity.samplingPolicyId), entity.samplingPolicyVersion), + seed = entity.samplingSeed, + orderedCaseIds = samples.map { EvaluationCaseId(it.caseId) }, + ) + require(sampling.digest.sha256 == entity.sampleSetDigest) { "Persisted evaluation sample digest does not match sample rows" } + return EvaluationRunConfig( + runId = EvaluationRunId(samples.firstOrNull()?.runId ?: error("Persisted evaluation run has no sample rows")), + model = EvaluationModelIdentity( + artifactDigest = ModelDigest(entity.modelDigest), + modelProfileId = entity.modelProfileId, + tier = entity.modelTier, + quantization = entity.modelQuantization, + ), + dataset = dataset, + sampling = sampling, + executionProfile = EvaluationExecutionProfileRef( + EvaluationExecutionProfileId(entity.executionProfileId), + entity.executionProfileVersion, + ), + loadPolicy = EvaluationModelLoadPolicy.valueOf(entity.loadPolicy), + warmupPolicy = EvaluationWarmupPolicy.valueOf(entity.warmupPolicy), + caseTimeoutMs = entity.caseTimeoutMs, + ) + } + + private fun summary( + entity: EvaluationRunEntity, + config: EvaluationRunConfig, + categories: List, + ): EvaluationRunSummary { + require(config.runId.value == entity.runId) { "Persisted evaluation run ID does not match sample rows" } + val quality = quality(entity, categories) + return EvaluationRunSummary( + runId = EvaluationRunId(entity.runId), + config = config, + identity = entity.identity?.let { identity(it, config) }, + state = EvaluationRunState.valueOf(entity.state), + progress = EvaluationProgress( + totalCases = entity.progress.totalCases, + attemptedCases = entity.progress.attemptedCases, + completedCases = entity.progress.completedCases, + currentCaseId = entity.progress.currentCaseId?.let(::EvaluationCaseId), + ), + quality = quality, + reliability = entity.reliability?.let(::reliability), + startedAtEpochMs = entity.startedAtEpochMs, + completedAtEpochMs = entity.completedAtEpochMs, + failure = entity.failure?.let(::failure), + ) + } + + private fun quality( + entity: EvaluationRunEntity, + categories: List, + ): EvaluationQualitySummary? { + if (!entity.qualityPresent) { + require(entity.qualityAggregateScore == null && categories.isEmpty()) { + "Persisted quality rows exist while quality is absent" + } + return null + } + require(categories.map { it.ordinal } == categories.indices.toList()) { + "Persisted category score ordinals must be contiguous" + } + return EvaluationQualitySummary( + aggregateScore = entity.qualityAggregateScore?.let(::NormalizedScore), + categoryScores = categories.map { row -> + EvaluationCategoryScore( + categoryId = EvaluationCategoryId(row.categoryId), + score = NormalizedScore(row.score), + scoredCaseCount = row.scoredCaseCount, + weight = row.weight, + ) + }, + ) + } + + private fun identity(entity: EvaluationRunIdentityEntity, config: EvaluationRunConfig): EvaluationRunIdentity { + val semanticExecution = semanticExecution(entity.semantic, config.executionProfile) + val semanticIdentity = EvaluationSemanticExecutionIdentity.create(semanticExecution) + require(semanticIdentity.fingerprint.sha256 == entity.semanticExecutionFingerprint) { + "Persisted semantic execution fingerprint does not match semantic fields" + } + val runtime = EvaluationRuntimeEnvironmentIdentity( + deviceClass = entity.runtime.deviceClass, + androidApiLevel = entity.runtime.androidApiLevel, + abi = entity.runtime.abi, + backendRevision = entity.runtime.backendRevision, + harnessBuildIdentity = entity.runtime.harnessBuildIdentity, + runtimeTuningProfileId = entity.runtime.runtimeTuningProfileId, + runtimeTuningProfileVersion = entity.runtime.runtimeTuningProfileVersion, + loadPolicy = EvaluationModelLoadPolicy.valueOf(entity.runtime.loadPolicy), + warmupPolicy = EvaluationWarmupPolicy.valueOf(entity.runtime.warmupPolicy), + ) + val identity = EvaluationRunIdentity.create( + model = config.model, + dataset = config.dataset, + sampleSetDigest = config.sampling.digest, + samplingPolicy = config.sampling.policy, + samplingSeed = config.sampling.seed, + evaluatorSetDigest = EvaluatorSetDigest(entity.evaluatorSetDigest), + semanticExecution = semanticIdentity, + runtimeEnvironment = runtime, + ) + require(identity.fingerprint.sha256 == entity.runFingerprint) { + "Persisted evaluation run fingerprint does not match identity fields" + } + return identity + } + + private fun semanticExecution( + entity: EvaluationSemanticExecutionEntity, + profile: EvaluationExecutionProfileRef, + ): EvaluationSemanticExecution { + val preset = when { + entity.presetId == null && entity.presetVersion == null -> null + entity.presetId != null && entity.presetVersion != null -> + InferencePresetRef(InferencePresetId(entity.presetId), entity.presetVersion) + else -> error("Persisted inference preset identity is incomplete") + } + return EvaluationSemanticExecution( + semanticsVersion = entity.semanticsVersion, + profile = profile, + backendRevision = entity.backendRevision, + contextSize = entity.contextSize, + preset = preset, + thinkingMode = ThinkingMode.valueOf(entity.thinkingMode), + temperature = entity.temperature, + topP = entity.topP, + topK = entity.topK, + minP = entity.minP, + presencePenalty = entity.presencePenalty, + repeatPenalty = entity.repeatPenalty, + repeatLastN = entity.repeatLastN, + seedPolicy = SeedPolicyType.valueOf(entity.seedPolicy), + effectiveSeed = entity.effectiveSeed, + maxOutputTokens = entity.maxOutputTokens, + chatTemplateId = entity.chatTemplateId, + chatTemplateSource = ChatTemplateSource.valueOf(entity.chatTemplateSource), + systemPromptVersion = entity.systemPromptVersion, + caseExecutionSemanticsDigest = CaseExecutionSemanticsDigest(entity.caseExecutionSemanticsDigest), + ) + } + + private fun reliability(entity: EvaluationReliabilityEntity) = EvaluationReliabilitySummary( + totalCases = entity.totalCases, + completedAndScored = entity.completedAndScored, + incorrectButValid = entity.incorrectButValid, + invalidOutput = entity.invalidOutput, + timeout = entity.timeout, + runtimeFailure = entity.runtimeFailure, + cancelled = entity.cancelled, + skipped = entity.skipped, + ) + + private fun failure(entity: EvaluationFailureEntity) = EvaluationFailure( + stage = EvaluationFailureStage.valueOf(entity.stage), + code = EvaluationFailureCode.valueOf(entity.code), + caseId = entity.caseId?.let(::EvaluationCaseId), + retryable = entity.retryable, + ) + + private fun caseResult( + entity: EvaluationCaseResultEntity, + parameters: List, + ): EvaluationCaseResult { + require(parameters.all { it.runId == entity.runId && it.caseId == entity.caseId }) { + "Persisted evaluator parameter identity does not match case result" + } + val outcome = when { + entity.outcomeScore == null && entity.outcomeCode == null -> null + entity.outcomeScore != null && entity.outcomeCode != null -> EvaluationOutcome( + score = NormalizedScore(entity.outcomeScore), + code = EvaluatorOutcomeCode.valueOf(entity.outcomeCode), + ) + else -> error("Persisted evaluator outcome is incomplete") + } + return EvaluationCaseResult( + caseId = EvaluationCaseId(entity.caseId), + categoryId = EvaluationCategoryId(entity.categoryId), + evaluator = EvaluatorSpec( + type = EvaluatorType.valueOf(entity.evaluatorType), + version = EvaluatorVersion(entity.evaluatorVersion), + parameters = parameters.associate { it.parameterKey to it.parameterValue }, + ), + status = EvaluationCaseStatus.valueOf(entity.status), + outcome = outcome, + requestId = entity.requestId?.let(::RequestId), + metrics = EvaluationCaseMetrics( + timeToFirstTokenMs = entity.timeToFirstTokenMs, + totalMs = entity.totalMs, + prefillMs = entity.prefillMs, + decodeMs = entity.decodeMs, + inputTokens = entity.inputTokens, + outputTokens = entity.outputTokens, + decodeTokensPerSecond = entity.decodeTokensPerSecond, + processPssBytes = entity.processPssBytes, + availableMemoryBytes = entity.availableMemoryBytes, + thermalStatus = entity.thermalStatus, + ), + failure = entity.failure?.let(::failure), + ) + } +} From c444fc33674946cbe167eae80595863b70a828c2 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Sat, 15 Aug 2026 18:40:39 +0200 Subject: [PATCH 09/16] feat(evaluation): expose bounded Room retention count --- .../daniele21/localllm/evaluation/room/EvaluationDao.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/evaluation/room-store/src/main/java/io/github/daniele21/localllm/evaluation/room/EvaluationDao.java b/evaluation/room-store/src/main/java/io/github/daniele21/localllm/evaluation/room/EvaluationDao.java index 50f420923..1d7f6c7e2 100644 --- a/evaluation/room-store/src/main/java/io/github/daniele21/localllm/evaluation/room/EvaluationDao.java +++ b/evaluation/room-store/src/main/java/io/github/daniele21/localllm/evaluation/room/EvaluationDao.java @@ -57,6 +57,9 @@ public interface EvaluationDao { + "WHERE run_id = :runId AND case_id = :caseId") int sampleCaseCount(String runId, String caseId); + @Query("SELECT COUNT(*) FROM evaluation_runs") + int runCount(); + @RawQuery List queryRuns(SupportSQLiteQuery query); From bea63efa6e020d8ec72dc5640a20ad88998ffd9c Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Sat, 15 Aug 2026 18:41:07 +0200 Subject: [PATCH 10/16] feat(evaluation): implement Room evaluation result repository --- .../room/RoomEvaluationResultRepository.kt | 194 ++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/RoomEvaluationResultRepository.kt diff --git a/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/RoomEvaluationResultRepository.kt b/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/RoomEvaluationResultRepository.kt new file mode 100644 index 000000000..df949df2e --- /dev/null +++ b/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/RoomEvaluationResultRepository.kt @@ -0,0 +1,194 @@ +package io.github.daniele21.localllm.evaluation.room + +import android.content.Context +import androidx.room.Room +import androidx.room.migration.Migration +import androidx.sqlite.db.SimpleSQLiteQuery +import io.github.daniele21.localllm.evaluation.EvaluationCaseResult +import io.github.daniele21.localllm.evaluation.EvaluationResultRepository +import io.github.daniele21.localllm.evaluation.EvaluationRetentionPolicy +import io.github.daniele21.localllm.evaluation.EvaluationRetentionResult +import io.github.daniele21.localllm.evaluation.EvaluationRunDeleteStatus +import io.github.daniele21.localllm.evaluation.EvaluationRunId +import io.github.daniele21.localllm.evaluation.EvaluationRunQuery +import io.github.daniele21.localllm.evaluation.EvaluationRunState +import io.github.daniele21.localllm.evaluation.EvaluationRunSummary +import io.github.daniele21.localllm.evaluation.PersistedEvaluationRun +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext + +class RoomEvaluationResultRepository internal constructor( + private val dao: EvaluationDao, + private val clock: () -> Long = System::currentTimeMillis, + private val closeDatabase: () -> Unit = {}, +) : EvaluationResultRepository, + AutoCloseable { + override suspend fun createRun(summary: EvaluationRunSummary) = withContext(Dispatchers.IO) { + require(dao.findRun(summary.runId.value) == null) { "Evaluation run already exists" } + dao.createRunGraph( + EvaluationRoomMapper.runEntity(summary), + EvaluationRoomMapper.sampleEntities(summary), + EvaluationRoomMapper.categoryScoreEntities(summary), + ) + } + + override suspend fun updateRunSummary(summary: EvaluationRunSummary) = withContext(Dispatchers.IO) { + val current = requireStoredRun(summary.runId) + val currentSummary = EvaluationRoomMapper.summary(current) + require(currentSummary.config == summary.config) { "Evaluation run config cannot change after creation" } + requireValidTransition(currentSummary.state, summary.state) + dao.updateRunGraph( + EvaluationRoomMapper.runEntity(summary), + EvaluationRoomMapper.categoryScoreEntities(summary), + ) + } + + override suspend fun appendCaseResult(runId: EvaluationRunId, result: EvaluationCaseResult) = withContext(Dispatchers.IO) { + val stored = requireStoredRun(runId) + val summary = EvaluationRoomMapper.summary(stored) + require(!summary.state.isTerminal()) { "Cannot append case result to terminal evaluation run" } + require(result.caseId in summary.config.sampling.orderedCaseIds) { + "Evaluation case result must belong to run sample set" + } + require(dao.sampleCaseCount(runId.value, result.caseId.value) == 1) { + "Evaluation case result must reference exactly one sampled case" + } + dao.upsertCaseResultGraph( + EvaluationRoomMapper.caseResultEntity(runId, result), + EvaluationRoomMapper.evaluatorParameterEntities(runId, result), + ) + } + + override suspend fun getRun(runId: EvaluationRunId): PersistedEvaluationRun? = withContext(Dispatchers.IO) { + dao.loadStoredRun(runId.value)?.let(EvaluationRoomMapper::persistedRun) + } + + override suspend fun queryRuns(query: EvaluationRunQuery): List = withContext(Dispatchers.IO) { + dao.queryRuns(query.toSqlQuery()).map { entity -> + val stored = requireNotNull(dao.loadStoredRun(entity.runId)) { + "Evaluation run disappeared while reading history" + } + EvaluationRoomMapper.summary(stored) + } + } + + override suspend fun deleteRun(runId: EvaluationRunId): EvaluationRunDeleteStatus = withContext(Dispatchers.IO) { + val run = dao.findRun(runId.value) ?: return@withContext EvaluationRunDeleteStatus.NOT_FOUND + if (!EvaluationRunState.valueOf(run.state).isTerminal()) { + EvaluationRunDeleteStatus.ACTIVE_RUN + } else { + check(dao.deleteRunRow(runId.value) == 1) { "Evaluation run disappeared while deleting" } + EvaluationRunDeleteStatus.DELETED + } + } + + override suspend fun applyRetention(policy: EvaluationRetentionPolicy): EvaluationRetentionResult = withContext(Dispatchers.IO) { + val now = clock() + val terminal = dao.terminalRunsNewestFirst() + val keepByCount = terminal.take(policy.maxTerminalRuns).map { it.runId }.toSet() + val expiredByAge = policy.maxAgeMs?.let { maxAgeMs -> + terminal.filter { now - it.startedAtEpochMs >= maxAgeMs }.map { it.runId }.toSet() + }.orEmpty() + val deleteIds = terminal.asSequence() + .map { it.runId } + .filter { it !in keepByCount || it in expiredByAge } + .toList() + if (deleteIds.isNotEmpty()) { + dao.deleteRunRows(deleteIds) + } + EvaluationRetentionResult( + deletedRunIds = deleteIds.map(::EvaluationRunId), + retainedRunCount = dao.runCount(), + ) + } + + override fun close() { + closeDatabase() + } + + private fun requireStoredRun(runId: EvaluationRunId): EvaluationStoredRun = requireNotNull(dao.loadStoredRun(runId.value)) { + "Evaluation run does not exist" + } + + private fun requireValidTransition(from: EvaluationRunState, to: EvaluationRunState) { + require(from == to || to in ALLOWED_TRANSITIONS.getValue(from)) { + "Invalid evaluation run transition: $from -> $to" + } + } + + companion object { + const val DEFAULT_DATABASE_NAME: String = "local-llm-evaluation.db" + private val MIGRATIONS: Array = emptyArray() + + fun open( + context: Context, + databaseName: String = DEFAULT_DATABASE_NAME, + ): RoomEvaluationResultRepository { + require(databaseName.isNotBlank()) { "Evaluation database name must not be blank" } + val database = Room.databaseBuilder( + context.applicationContext, + EvaluationDatabase::class.java, + databaseName, + ).addMigrations(*MIGRATIONS).build() + return RoomEvaluationResultRepository( + dao = database.evaluationDao(), + closeDatabase = database::close, + ) + } + + private val ALLOWED_TRANSITIONS: Map> = mapOf( + EvaluationRunState.CREATED to setOf(EvaluationRunState.VALIDATING, EvaluationRunState.CANCELLING, EvaluationRunState.FAILED), + EvaluationRunState.VALIDATING to + setOf(EvaluationRunState.PREPARING_MODEL, EvaluationRunState.CANCELLING, EvaluationRunState.FAILED), + EvaluationRunState.PREPARING_MODEL to setOf( + EvaluationRunState.WARMING_UP, + EvaluationRunState.RUNNING, + EvaluationRunState.CANCELLING, + EvaluationRunState.FAILED, + ), + EvaluationRunState.WARMING_UP to setOf(EvaluationRunState.RUNNING, EvaluationRunState.CANCELLING, EvaluationRunState.FAILED), + EvaluationRunState.RUNNING to setOf(EvaluationRunState.AGGREGATING, EvaluationRunState.CANCELLING, EvaluationRunState.FAILED), + EvaluationRunState.AGGREGATING to setOf(EvaluationRunState.COMPLETED, EvaluationRunState.CANCELLING, EvaluationRunState.FAILED), + EvaluationRunState.CANCELLING to setOf(EvaluationRunState.CANCELLED, EvaluationRunState.FAILED), + EvaluationRunState.COMPLETED to emptySet(), + EvaluationRunState.CANCELLED to emptySet(), + EvaluationRunState.FAILED to emptySet(), + ) + } +} + +private fun EvaluationRunQuery.toSqlQuery(): SimpleSQLiteQuery { + val sql = StringBuilder("SELECT * FROM evaluation_runs WHERE 1=1") + val args = mutableListOf() + if (states.isNotEmpty()) { + val orderedStates = states.sortedBy { it.name } + sql.append(" AND state IN (") + sql.append(orderedStates.joinToString(",") { "?" }) + sql.append(')') + args += orderedStates.map { it.name } + } + datasetId?.let { + sql.append(" AND config_dataset_id = ?") + args += it.value + } + modelDigest?.let { + sql.append(" AND config_model_digest = ?") + args += it.sha256 + } + startedBeforeEpochMs?.let { + sql.append(" AND started_at_epoch_ms < ?") + args += it + } + sql.append(" ORDER BY started_at_epoch_ms DESC, run_id ASC LIMIT ?") + args += limit + return SimpleSQLiteQuery(sql.toString(), args.toTypedArray()) +} + +private fun EvaluationRunState.isTerminal(): Boolean = when (this) { + EvaluationRunState.COMPLETED, + EvaluationRunState.CANCELLED, + EvaluationRunState.FAILED, + -> true + + else -> false +} From fda6c18bb4526c3ef3f990b45c8390fc9612a13c Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Sat, 15 Aug 2026 18:41:32 +0200 Subject: [PATCH 11/16] test(evaluation): cover Room domain round trip --- .../room/EvaluationRoomMapperTest.kt | 125 ++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 evaluation/room-store/src/test/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomMapperTest.kt diff --git a/evaluation/room-store/src/test/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomMapperTest.kt b/evaluation/room-store/src/test/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomMapperTest.kt new file mode 100644 index 000000000..7dd2965a1 --- /dev/null +++ b/evaluation/room-store/src/test/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomMapperTest.kt @@ -0,0 +1,125 @@ +package io.github.daniele21.localllm.evaluation.room + +import io.github.daniele21.localllm.contracts.ModelDigest +import io.github.daniele21.localllm.contracts.RequestId +import io.github.daniele21.localllm.evaluation.EvaluationCaseId +import io.github.daniele21.localllm.evaluation.EvaluationCaseMetrics +import io.github.daniele21.localllm.evaluation.EvaluationCaseResult +import io.github.daniele21.localllm.evaluation.EvaluationCaseStatus +import io.github.daniele21.localllm.evaluation.EvaluationCategoryId +import io.github.daniele21.localllm.evaluation.EvaluationDatasetDigest +import io.github.daniele21.localllm.evaluation.EvaluationDatasetId +import io.github.daniele21.localllm.evaluation.EvaluationDatasetIdentity +import io.github.daniele21.localllm.evaluation.EvaluationDatasetVersion +import io.github.daniele21.localllm.evaluation.EvaluationExecutionProfileId +import io.github.daniele21.localllm.evaluation.EvaluationExecutionProfileRef +import io.github.daniele21.localllm.evaluation.EvaluationModelIdentity +import io.github.daniele21.localllm.evaluation.EvaluationModelLoadPolicy +import io.github.daniele21.localllm.evaluation.EvaluationOutcome +import io.github.daniele21.localllm.evaluation.EvaluationProgress +import io.github.daniele21.localllm.evaluation.EvaluationQualitySummary +import io.github.daniele21.localllm.evaluation.EvaluationRunConfig +import io.github.daniele21.localllm.evaluation.EvaluationRunId +import io.github.daniele21.localllm.evaluation.EvaluationRunState +import io.github.daniele21.localllm.evaluation.EvaluationRunSummary +import io.github.daniele21.localllm.evaluation.EvaluationWarmupPolicy +import io.github.daniele21.localllm.evaluation.EvaluatorOutcomeCode +import io.github.daniele21.localllm.evaluation.EvaluatorSpec +import io.github.daniele21.localllm.evaluation.EvaluatorType +import io.github.daniele21.localllm.evaluation.EvaluatorVersion +import io.github.daniele21.localllm.evaluation.NormalizedScore +import io.github.daniele21.localllm.evaluation.SamplingPolicyId +import io.github.daniele21.localllm.evaluation.SamplingPolicyRef +import io.github.daniele21.localllm.evaluation.SamplingSelection +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +class EvaluationRoomMapperTest { + @Test + fun `round trip preserves optional empty quality and sample order`() { + val summary = summary() + val stored = EvaluationStoredRun( + EvaluationRoomMapper.runEntity(summary), + EvaluationRoomMapper.sampleEntities(summary), + EvaluationRoomMapper.categoryScoreEntities(summary), + emptyList(), + emptyList(), + ) + + val restored = EvaluationRoomMapper.summary(stored) + + assertEquals(summary, restored) + assertTrue(stored.run.qualityPresent) + assertEquals(listOf("case-b", "case-a"), stored.samples.map { it.caseId }) + } + + @Test + fun `case result round trip preserves only privacy safe result fields`() { + val summary = summary() + val result = EvaluationCaseResult( + caseId = CASE_B, + categoryId = EvaluationCategoryId("general"), + evaluator = EvaluatorSpec( + EvaluatorType.EXACT_MATCH, + EvaluatorVersion(1), + mapOf("case" to "sensitive"), + ), + status = EvaluationCaseStatus.SCORED, + outcome = EvaluationOutcome(NormalizedScore(1.0), EvaluatorOutcomeCode.CORRECT), + requestId = RequestId("request-1"), + metrics = EvaluationCaseMetrics(totalMs = 42, outputTokens = 3), + ) + val stored = EvaluationStoredRun( + EvaluationRoomMapper.runEntity(summary), + EvaluationRoomMapper.sampleEntities(summary), + EvaluationRoomMapper.categoryScoreEntities(summary), + listOf(EvaluationRoomMapper.caseResultEntity(RUN_ID, result)), + EvaluationRoomMapper.evaluatorParameterEntities(RUN_ID, result), + ) + + val restored = EvaluationRoomMapper.persistedRun(stored) + + assertEquals(listOf(result), restored.caseResults) + assertEquals(summary, restored.summary) + } + + private fun summary(): EvaluationRunSummary = EvaluationRunSummary( + runId = RUN_ID, + config = CONFIG, + identity = null, + state = EvaluationRunState.CREATED, + progress = EvaluationProgress(totalCases = 2, attemptedCases = 0, completedCases = 0), + quality = EvaluationQualitySummary(aggregateScore = null, categoryScores = emptyList()), + reliability = null, + startedAtEpochMs = 100, + completedAtEpochMs = null, + failure = null, + ) + + private companion object { + val RUN_ID = EvaluationRunId("run-1") + val CASE_A = EvaluationCaseId("case-a") + val CASE_B = EvaluationCaseId("case-b") + val DATASET = EvaluationDatasetIdentity( + EvaluationDatasetId("fixture"), + EvaluationDatasetVersion("1"), + EvaluationDatasetDigest("1".repeat(64)), + ) + val CONFIG = EvaluationRunConfig( + runId = RUN_ID, + model = EvaluationModelIdentity(ModelDigest("a".repeat(64)), "model-profile", quantization = "Q4_K_M"), + dataset = DATASET, + sampling = SamplingSelection.create( + dataset = DATASET, + policy = SamplingPolicyRef(SamplingPolicyId("fixed"), 1), + seed = 7, + orderedCaseIds = listOf(CASE_B, CASE_A), + ), + executionProfile = EvaluationExecutionProfileRef(EvaluationExecutionProfileId("direct"), 1), + loadPolicy = EvaluationModelLoadPolicy.PRESERVE_CURRENT_RESIDENCY, + warmupPolicy = EvaluationWarmupPolicy.NONE, + caseTimeoutMs = 30_000, + ) + } +} From 21c6559c8cf7e43091769b17803bb232a0499f7a Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Sat, 15 Aug 2026 18:50:34 +0200 Subject: [PATCH 12/16] fix(evaluation): append Room query state arguments safely --- .../localllm/evaluation/room/RoomEvaluationResultRepository.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/RoomEvaluationResultRepository.kt b/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/RoomEvaluationResultRepository.kt index df949df2e..d8d1b7206 100644 --- a/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/RoomEvaluationResultRepository.kt +++ b/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/RoomEvaluationResultRepository.kt @@ -165,7 +165,7 @@ private fun EvaluationRunQuery.toSqlQuery(): SimpleSQLiteQuery { sql.append(" AND state IN (") sql.append(orderedStates.joinToString(",") { "?" }) sql.append(')') - args += orderedStates.map { it.name } + args.addAll(orderedStates.map { it.name }) } datasetId?.let { sql.append(" AND config_dataset_id = ?") From 50205e6fdcf24b3520cbc95bc786fcec9f4939bf Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Sat, 15 Aug 2026 20:22:06 +0200 Subject: [PATCH 13/16] style(evaluation): apply Room mapper Spotless format --- .../evaluation/room/EvaluationRoomMapper.kt | 47 +++++++++---------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomMapper.kt b/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomMapper.kt index 1ee59c0f2..ae9462296 100644 --- a/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomMapper.kt +++ b/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomMapper.kt @@ -32,8 +32,8 @@ import io.github.daniele21.localllm.evaluation.EvaluationReliabilitySummary import io.github.daniele21.localllm.evaluation.EvaluationRunConfig import io.github.daniele21.localllm.evaluation.EvaluationRunId import io.github.daniele21.localllm.evaluation.EvaluationRunIdentity -import io.github.daniele21.localllm.evaluation.EvaluationRunSummary import io.github.daniele21.localllm.evaluation.EvaluationRunState +import io.github.daniele21.localllm.evaluation.EvaluationRunSummary import io.github.daniele21.localllm.evaluation.EvaluationRuntimeEnvironmentIdentity import io.github.daniele21.localllm.evaluation.EvaluationSemanticExecution import io.github.daniele21.localllm.evaluation.EvaluationSemanticExecutionIdentity @@ -82,29 +82,28 @@ internal object EvaluationRoomMapper { ) } - fun caseResultEntity(runId: EvaluationRunId, result: EvaluationCaseResult): EvaluationCaseResultEntity = - EvaluationCaseResultEntity( - runId = runId.value, - caseId = result.caseId.value, - categoryId = result.categoryId.value, - evaluatorType = result.evaluator.type.name, - evaluatorVersion = result.evaluator.version.value, - status = result.status.name, - outcomeScore = result.outcome?.score?.value, - outcomeCode = result.outcome?.code?.name, - requestId = result.requestId?.value, - timeToFirstTokenMs = result.metrics.timeToFirstTokenMs, - totalMs = result.metrics.totalMs, - prefillMs = result.metrics.prefillMs, - decodeMs = result.metrics.decodeMs, - inputTokens = result.metrics.inputTokens, - outputTokens = result.metrics.outputTokens, - decodeTokensPerSecond = result.metrics.decodeTokensPerSecond, - processPssBytes = result.metrics.processPssBytes, - availableMemoryBytes = result.metrics.availableMemoryBytes, - thermalStatus = result.metrics.thermalStatus, - failure = result.failure?.let(::failureEntity), - ) + fun caseResultEntity(runId: EvaluationRunId, result: EvaluationCaseResult): EvaluationCaseResultEntity = EvaluationCaseResultEntity( + runId = runId.value, + caseId = result.caseId.value, + categoryId = result.categoryId.value, + evaluatorType = result.evaluator.type.name, + evaluatorVersion = result.evaluator.version.value, + status = result.status.name, + outcomeScore = result.outcome?.score?.value, + outcomeCode = result.outcome?.code?.name, + requestId = result.requestId?.value, + timeToFirstTokenMs = result.metrics.timeToFirstTokenMs, + totalMs = result.metrics.totalMs, + prefillMs = result.metrics.prefillMs, + decodeMs = result.metrics.decodeMs, + inputTokens = result.metrics.inputTokens, + outputTokens = result.metrics.outputTokens, + decodeTokensPerSecond = result.metrics.decodeTokensPerSecond, + processPssBytes = result.metrics.processPssBytes, + availableMemoryBytes = result.metrics.availableMemoryBytes, + thermalStatus = result.metrics.thermalStatus, + failure = result.failure?.let(::failureEntity), + ) fun evaluatorParameterEntities(runId: EvaluationRunId, result: EvaluationCaseResult): List = result.evaluator.parameters.toSortedMap().map { (key, value) -> From 0df061867cce810b46dea8e6a7a8415a3db1c6b1 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Sat, 15 Aug 2026 20:28:36 +0200 Subject: [PATCH 14/16] style(evaluation): apply repository Spotless format --- .../evaluation/room/RoomEvaluationResultRepository.kt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/RoomEvaluationResultRepository.kt b/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/RoomEvaluationResultRepository.kt index d8d1b7206..22e2583d3 100644 --- a/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/RoomEvaluationResultRepository.kt +++ b/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/RoomEvaluationResultRepository.kt @@ -120,10 +120,7 @@ class RoomEvaluationResultRepository internal constructor( const val DEFAULT_DATABASE_NAME: String = "local-llm-evaluation.db" private val MIGRATIONS: Array = emptyArray() - fun open( - context: Context, - databaseName: String = DEFAULT_DATABASE_NAME, - ): RoomEvaluationResultRepository { + fun open(context: Context, databaseName: String = DEFAULT_DATABASE_NAME): RoomEvaluationResultRepository { require(databaseName.isNotBlank()) { "Evaluation database name must not be blank" } val database = Room.databaseBuilder( context.applicationContext, From 545c34330246a6bf1361a136b03cafaf95d505b3 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Sat, 15 Aug 2026 20:29:24 +0200 Subject: [PATCH 15/16] style(evaluation): finish Room mapper Spotless format --- .../localllm/evaluation/room/EvaluationRoomMapper.kt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomMapper.kt b/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomMapper.kt index ae9462296..d5c37d70b 100644 --- a/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomMapper.kt +++ b/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomMapper.kt @@ -272,10 +272,7 @@ internal object EvaluationRoomMapper { ) } - private fun quality( - entity: EvaluationRunEntity, - categories: List, - ): EvaluationQualitySummary? { + private fun quality(entity: EvaluationRunEntity, categories: List): EvaluationQualitySummary? { if (!entity.qualityPresent) { require(entity.qualityAggregateScore == null && categories.isEmpty()) { "Persisted quality rows exist while quality is absent" @@ -337,8 +334,10 @@ internal object EvaluationRoomMapper { ): EvaluationSemanticExecution { val preset = when { entity.presetId == null && entity.presetVersion == null -> null + entity.presetId != null && entity.presetVersion != null -> InferencePresetRef(InferencePresetId(entity.presetId), entity.presetVersion) + else -> error("Persisted inference preset identity is incomplete") } return EvaluationSemanticExecution( @@ -392,10 +391,12 @@ internal object EvaluationRoomMapper { } val outcome = when { entity.outcomeScore == null && entity.outcomeCode == null -> null + entity.outcomeScore != null && entity.outcomeCode != null -> EvaluationOutcome( score = NormalizedScore(entity.outcomeScore), code = EvaluatorOutcomeCode.valueOf(entity.outcomeCode), ) + else -> error("Persisted evaluator outcome is incomplete") } return EvaluationCaseResult( From b3d327cdcd16fe55c68445c63e903abd18c47769 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Sat, 15 Aug 2026 20:34:53 +0200 Subject: [PATCH 16/16] refactor(evaluation): avoid migration spread allocation --- .../evaluation/room/RoomEvaluationResultRepository.kt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/RoomEvaluationResultRepository.kt b/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/RoomEvaluationResultRepository.kt index 22e2583d3..7391a8ebd 100644 --- a/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/RoomEvaluationResultRepository.kt +++ b/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/RoomEvaluationResultRepository.kt @@ -118,15 +118,17 @@ class RoomEvaluationResultRepository internal constructor( companion object { const val DEFAULT_DATABASE_NAME: String = "local-llm-evaluation.db" - private val MIGRATIONS: Array = emptyArray() + private val MIGRATIONS: List = emptyList() fun open(context: Context, databaseName: String = DEFAULT_DATABASE_NAME): RoomEvaluationResultRepository { require(databaseName.isNotBlank()) { "Evaluation database name must not be blank" } - val database = Room.databaseBuilder( + val builder = Room.databaseBuilder( context.applicationContext, EvaluationDatabase::class.java, databaseName, - ).addMigrations(*MIGRATIONS).build() + ) + MIGRATIONS.forEach { migration -> builder.addMigrations(migration) } + val database = builder.build() return RoomEvaluationResultRepository( dao = database.evaluationDao(), closeDatabase = database::close,