diff --git a/evaluation/room-store/build.gradle.kts b/evaluation/room-store/build.gradle.kts index a547029f5..1b0230a49 100644 --- a/evaluation/room-store/build.gradle.kts +++ b/evaluation/room-store/build.gradle.kts @@ -9,6 +9,11 @@ android { defaultConfig { minSdk = libs.versions.minSdk.get().toInt() + javaCompileOptions { + annotationProcessorOptions { + argument("room.schemaLocation", "$projectDir/schemas") + } + } } compileOptions { @@ -25,7 +30,9 @@ android { } dependencies { - implementation(project(":evaluation:contracts")) + api(project(":evaluation:contracts")) implementation(libs.room.runtime) + implementation(libs.kotlinx.coroutines.android) + annotationProcessor(libs.room.compiler) testImplementation(libs.junit4) } diff --git a/evaluation/room-store/src/main/java/io/github/daniele21/localllm/evaluation/room/EvaluationDao.java b/evaluation/room-store/src/main/java/io/github/daniele21/localllm/evaluation/room/EvaluationDao.java new file mode 100644 index 000000000..1d7f6c7e2 --- /dev/null +++ b/evaluation/room-store/src/main/java/io/github/daniele21/localllm/evaluation/room/EvaluationDao.java @@ -0,0 +1,135 @@ +package io.github.daniele21.localllm.evaluation.room; + +import androidx.annotation.Nullable; +import androidx.room.Dao; +import androidx.room.Insert; +import androidx.room.OnConflictStrategy; +import androidx.room.Query; +import androidx.room.RawQuery; +import androidx.room.Transaction; +import androidx.room.Update; +import androidx.sqlite.db.SupportSQLiteQuery; +import java.util.List; + +@Dao +public interface EvaluationDao { + @Insert(onConflict = OnConflictStrategy.ABORT) + void insertRun(EvaluationRunEntity run); + + @Update + int updateRun(EvaluationRunEntity run); + + @Insert(onConflict = OnConflictStrategy.ABORT) + void insertSamples(List samples); + + @Insert(onConflict = OnConflictStrategy.REPLACE) + void insertCategoryScores(List scores); + + @Insert(onConflict = OnConflictStrategy.REPLACE) + void upsertCaseResult(EvaluationCaseResultEntity result); + + @Insert(onConflict = OnConflictStrategy.REPLACE) + void insertEvaluatorParameters(List parameters); + + @Nullable + @Query("SELECT * FROM evaluation_runs WHERE run_id = :runId LIMIT 1") + EvaluationRunEntity findRun(String runId); + + @Query("SELECT * FROM evaluation_sample_cases WHERE run_id = :runId ORDER BY ordinal ASC") + List sampleCases(String runId); + + @Query("SELECT * FROM evaluation_category_scores WHERE run_id = :runId ORDER BY ordinal ASC") + List categoryScores(String runId); + + @Query( + "SELECT r.* FROM evaluation_case_results r " + + "JOIN evaluation_sample_cases s ON s.run_id = r.run_id AND s.case_id = r.case_id " + + "WHERE r.run_id = :runId ORDER BY s.ordinal ASC") + List caseResults(String runId); + + @Query( + "SELECT * FROM evaluation_evaluator_parameters WHERE run_id = :runId " + + "ORDER BY case_id ASC, parameter_key ASC") + List evaluatorParameters(String runId); + + @Query( + "SELECT COUNT(*) FROM evaluation_sample_cases " + + "WHERE run_id = :runId AND case_id = :caseId") + int sampleCaseCount(String runId, String caseId); + + @Query("SELECT COUNT(*) FROM evaluation_runs") + int runCount(); + + @RawQuery + List queryRuns(SupportSQLiteQuery query); + + @Query( + "SELECT * FROM evaluation_runs WHERE state IN ('COMPLETED','CANCELLED','FAILED') " + + "ORDER BY started_at_epoch_ms DESC, run_id ASC") + List terminalRunsNewestFirst(); + + @Query("DELETE FROM evaluation_category_scores WHERE run_id = :runId") + void deleteCategoryScores(String runId); + + @Query( + "DELETE FROM evaluation_evaluator_parameters " + + "WHERE run_id = :runId AND case_id = :caseId") + void deleteEvaluatorParameters(String runId, String caseId); + + @Query("DELETE FROM evaluation_runs WHERE run_id = :runId") + int deleteRunRow(String runId); + + @Query("DELETE FROM evaluation_runs WHERE run_id IN (:runIds)") + int deleteRunRows(List runIds); + + @Transaction + default void createRunGraph( + EvaluationRunEntity run, + List samples, + List scores) { + insertRun(run); + insertSamples(samples); + if (!scores.isEmpty()) { + insertCategoryScores(scores); + } + } + + @Transaction + default void updateRunGraph( + EvaluationRunEntity run, + List scores) { + if (updateRun(run) != 1) { + throw new IllegalStateException("Evaluation run does not exist"); + } + deleteCategoryScores(run.getRunId()); + if (!scores.isEmpty()) { + insertCategoryScores(scores); + } + } + + @Transaction + default void upsertCaseResultGraph( + EvaluationCaseResultEntity result, + List parameters) { + upsertCaseResult(result); + deleteEvaluatorParameters(result.getRunId(), result.getCaseId()); + if (!parameters.isEmpty()) { + insertEvaluatorParameters(parameters); + } + } + + @Nullable + @Transaction + default EvaluationStoredRun loadStoredRun(String runId) { + EvaluationRunEntity run = findRun(runId); + if (run == null) { + return null; + } + return new EvaluationStoredRun( + run, + sampleCases(runId), + categoryScores(runId), + caseResults(runId), + evaluatorParameters(runId)); + } +} diff --git a/evaluation/room-store/src/main/java/io/github/daniele21/localllm/evaluation/room/EvaluationDatabase.java b/evaluation/room-store/src/main/java/io/github/daniele21/localllm/evaluation/room/EvaluationDatabase.java new file mode 100644 index 000000000..a70735821 --- /dev/null +++ b/evaluation/room-store/src/main/java/io/github/daniele21/localllm/evaluation/room/EvaluationDatabase.java @@ -0,0 +1,18 @@ +package io.github.daniele21.localllm.evaluation.room; + +import androidx.room.Database; +import androidx.room.RoomDatabase; + +@Database( + entities = { + EvaluationRunEntity.class, + EvaluationSampleCaseEntity.class, + EvaluationCategoryScoreEntity.class, + EvaluationCaseResultEntity.class, + EvaluationEvaluatorParameterEntity.class + }, + version = 1, + exportSchema = true) +public abstract class EvaluationDatabase extends RoomDatabase { + public abstract EvaluationDao evaluationDao(); +} diff --git a/evaluation/room-store/src/main/java/io/github/daniele21/localllm/evaluation/room/EvaluationStoredRun.java b/evaluation/room-store/src/main/java/io/github/daniele21/localllm/evaluation/room/EvaluationStoredRun.java new file mode 100644 index 000000000..f80d1e0a7 --- /dev/null +++ b/evaluation/room-store/src/main/java/io/github/daniele21/localllm/evaluation/room/EvaluationStoredRun.java @@ -0,0 +1,24 @@ +package io.github.daniele21.localllm.evaluation.room; + +import java.util.List; + +public final class EvaluationStoredRun { + public final EvaluationRunEntity run; + public final List samples; + public final List categoryScores; + public final List caseResults; + public final List evaluatorParameters; + + public EvaluationStoredRun( + EvaluationRunEntity run, + List samples, + List categoryScores, + List caseResults, + List evaluatorParameters) { + this.run = run; + this.samples = samples; + this.categoryScores = categoryScores; + this.caseResults = caseResults; + this.evaluatorParameters = evaluatorParameters; + } +} diff --git a/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomEntities.kt b/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomEntities.kt index 8d58d4951..2a6d0322c 100644 --- a/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomEntities.kt +++ b/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomEntities.kt @@ -22,6 +22,7 @@ data class EvaluationRunEntity( @Embedded(prefix = "identity_") val identity: EvaluationRunIdentityEntity?, val state: String, @Embedded(prefix = "progress_") val progress: EvaluationProgressEntity, + @ColumnInfo(name = "quality_present") val qualityPresent: Boolean, @ColumnInfo(name = "quality_aggregate_score") val qualityAggregateScore: Double?, @Embedded(prefix = "reliability_") val reliability: EvaluationReliabilityEntity?, @ColumnInfo(name = "started_at_epoch_ms") val startedAtEpochMs: Long, @@ -137,10 +138,11 @@ data class EvaluationSampleCaseEntity( onDelete = ForeignKey.CASCADE, ), ], - indices = [Index(value = ["run_id"])], + indices = [Index(value = ["run_id"]), Index(value = ["run_id", "ordinal"], unique = true)], ) data class EvaluationCategoryScoreEntity( @ColumnInfo(name = "run_id") val runId: String, + val ordinal: Int, @ColumnInfo(name = "category_id") val categoryId: String, val score: Double, @ColumnInfo(name = "scored_case_count") val scoredCaseCount: Int, diff --git a/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomMapper.kt b/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomMapper.kt new file mode 100644 index 000000000..d5c37d70b --- /dev/null +++ b/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomMapper.kt @@ -0,0 +1,428 @@ +package io.github.daniele21.localllm.evaluation.room + +import io.github.daniele21.localllm.contracts.ChatTemplateSource +import io.github.daniele21.localllm.contracts.InferencePresetId +import io.github.daniele21.localllm.contracts.InferencePresetRef +import io.github.daniele21.localllm.contracts.ModelDigest +import io.github.daniele21.localllm.contracts.RequestId +import io.github.daniele21.localllm.contracts.SeedPolicyType +import io.github.daniele21.localllm.contracts.ThinkingMode +import io.github.daniele21.localllm.evaluation.CaseExecutionSemanticsDigest +import io.github.daniele21.localllm.evaluation.EvaluationCaseId +import io.github.daniele21.localllm.evaluation.EvaluationCaseMetrics +import io.github.daniele21.localllm.evaluation.EvaluationCaseResult +import io.github.daniele21.localllm.evaluation.EvaluationCaseStatus +import io.github.daniele21.localllm.evaluation.EvaluationCategoryId +import io.github.daniele21.localllm.evaluation.EvaluationCategoryScore +import io.github.daniele21.localllm.evaluation.EvaluationDatasetDigest +import io.github.daniele21.localllm.evaluation.EvaluationDatasetId +import io.github.daniele21.localllm.evaluation.EvaluationDatasetIdentity +import io.github.daniele21.localllm.evaluation.EvaluationDatasetVersion +import io.github.daniele21.localllm.evaluation.EvaluationExecutionProfileId +import io.github.daniele21.localllm.evaluation.EvaluationExecutionProfileRef +import io.github.daniele21.localllm.evaluation.EvaluationFailure +import io.github.daniele21.localllm.evaluation.EvaluationFailureCode +import io.github.daniele21.localllm.evaluation.EvaluationFailureStage +import io.github.daniele21.localllm.evaluation.EvaluationModelIdentity +import io.github.daniele21.localllm.evaluation.EvaluationModelLoadPolicy +import io.github.daniele21.localllm.evaluation.EvaluationOutcome +import io.github.daniele21.localllm.evaluation.EvaluationProgress +import io.github.daniele21.localllm.evaluation.EvaluationQualitySummary +import io.github.daniele21.localllm.evaluation.EvaluationReliabilitySummary +import io.github.daniele21.localllm.evaluation.EvaluationRunConfig +import io.github.daniele21.localllm.evaluation.EvaluationRunId +import io.github.daniele21.localllm.evaluation.EvaluationRunIdentity +import io.github.daniele21.localllm.evaluation.EvaluationRunState +import io.github.daniele21.localllm.evaluation.EvaluationRunSummary +import io.github.daniele21.localllm.evaluation.EvaluationRuntimeEnvironmentIdentity +import io.github.daniele21.localllm.evaluation.EvaluationSemanticExecution +import io.github.daniele21.localllm.evaluation.EvaluationSemanticExecutionIdentity +import io.github.daniele21.localllm.evaluation.EvaluationWarmupPolicy +import io.github.daniele21.localllm.evaluation.EvaluatorOutcomeCode +import io.github.daniele21.localllm.evaluation.EvaluatorSetDigest +import io.github.daniele21.localllm.evaluation.EvaluatorSpec +import io.github.daniele21.localllm.evaluation.EvaluatorType +import io.github.daniele21.localllm.evaluation.EvaluatorVersion +import io.github.daniele21.localllm.evaluation.NormalizedScore +import io.github.daniele21.localllm.evaluation.PersistedEvaluationRun +import io.github.daniele21.localllm.evaluation.SamplingPolicyId +import io.github.daniele21.localllm.evaluation.SamplingPolicyRef +import io.github.daniele21.localllm.evaluation.SamplingSelection + +@Suppress("TooManyFunctions") +internal object EvaluationRoomMapper { + fun runEntity(summary: EvaluationRunSummary): EvaluationRunEntity = EvaluationRunEntity( + runId = summary.runId.value, + config = configEntity(summary.config), + identity = summary.identity?.let(::identityEntity), + state = summary.state.name, + progress = progressEntity(summary.progress), + qualityPresent = summary.quality != null, + qualityAggregateScore = summary.quality?.aggregateScore?.value, + reliability = summary.reliability?.let(::reliabilityEntity), + startedAtEpochMs = summary.startedAtEpochMs, + completedAtEpochMs = summary.completedAtEpochMs, + failure = summary.failure?.let(::failureEntity), + ) + + fun sampleEntities(summary: EvaluationRunSummary): List = + summary.config.sampling.orderedCaseIds.mapIndexed { ordinal, caseId -> + EvaluationSampleCaseEntity(summary.runId.value, ordinal, caseId.value) + } + + fun categoryScoreEntities(summary: EvaluationRunSummary): List = + summary.quality?.categoryScores.orEmpty().mapIndexed { ordinal, score -> + EvaluationCategoryScoreEntity( + runId = summary.runId.value, + ordinal = ordinal, + categoryId = score.categoryId.value, + score = score.score.value, + scoredCaseCount = score.scoredCaseCount, + weight = score.weight, + ) + } + + fun caseResultEntity(runId: EvaluationRunId, result: EvaluationCaseResult): EvaluationCaseResultEntity = EvaluationCaseResultEntity( + runId = runId.value, + caseId = result.caseId.value, + categoryId = result.categoryId.value, + evaluatorType = result.evaluator.type.name, + evaluatorVersion = result.evaluator.version.value, + status = result.status.name, + outcomeScore = result.outcome?.score?.value, + outcomeCode = result.outcome?.code?.name, + requestId = result.requestId?.value, + timeToFirstTokenMs = result.metrics.timeToFirstTokenMs, + totalMs = result.metrics.totalMs, + prefillMs = result.metrics.prefillMs, + decodeMs = result.metrics.decodeMs, + inputTokens = result.metrics.inputTokens, + outputTokens = result.metrics.outputTokens, + decodeTokensPerSecond = result.metrics.decodeTokensPerSecond, + processPssBytes = result.metrics.processPssBytes, + availableMemoryBytes = result.metrics.availableMemoryBytes, + thermalStatus = result.metrics.thermalStatus, + failure = result.failure?.let(::failureEntity), + ) + + fun evaluatorParameterEntities(runId: EvaluationRunId, result: EvaluationCaseResult): List = + result.evaluator.parameters.toSortedMap().map { (key, value) -> + EvaluationEvaluatorParameterEntity(runId.value, result.caseId.value, key, value) + } + + fun summary(stored: EvaluationStoredRun): EvaluationRunSummary { + val config = config(stored.run.config, stored.samples) + return summary(stored.run, config, stored.categoryScores) + } + + fun persistedRun(stored: EvaluationStoredRun): PersistedEvaluationRun { + val config = config(stored.run.config, stored.samples) + val summary = summary(stored.run, config, stored.categoryScores) + val parametersByCase = stored.evaluatorParameters.groupBy { it.caseId } + val results = stored.caseResults.map { entity -> + caseResult(entity, parametersByCase[entity.caseId].orEmpty()) + } + return PersistedEvaluationRun(summary, results) + } + + private fun configEntity(config: EvaluationRunConfig): EvaluationRunConfigEntity = EvaluationRunConfigEntity( + modelDigest = config.model.artifactDigest.sha256, + modelProfileId = config.model.modelProfileId, + modelTier = config.model.tier, + modelQuantization = config.model.quantization, + datasetId = config.dataset.id.value, + datasetVersion = config.dataset.version.value, + datasetDigest = config.dataset.digest.sha256, + sampleSetDigest = config.sampling.digest.sha256, + samplingPolicyId = config.sampling.policy.id.value, + samplingPolicyVersion = config.sampling.policy.version, + samplingSeed = config.sampling.seed, + executionProfileId = config.executionProfile.id.value, + executionProfileVersion = config.executionProfile.version, + loadPolicy = config.loadPolicy.name, + warmupPolicy = config.warmupPolicy.name, + caseTimeoutMs = config.caseTimeoutMs, + ) + + private fun identityEntity(identity: EvaluationRunIdentity): EvaluationRunIdentityEntity { + val semantic = identity.semanticExecution.execution + val runtime = identity.runtimeEnvironment + return EvaluationRunIdentityEntity( + evaluatorSetDigest = identity.evaluatorSetDigest.sha256, + semanticExecutionFingerprint = identity.semanticExecution.fingerprint.sha256, + runFingerprint = identity.fingerprint.sha256, + semantic = EvaluationSemanticExecutionEntity( + semanticsVersion = semantic.semanticsVersion, + backendRevision = semantic.backendRevision, + contextSize = semantic.contextSize, + presetId = semantic.preset?.id?.value, + presetVersion = semantic.preset?.version, + thinkingMode = semantic.thinkingMode.name, + temperature = semantic.temperature, + topP = semantic.topP, + topK = semantic.topK, + minP = semantic.minP, + presencePenalty = semantic.presencePenalty, + repeatPenalty = semantic.repeatPenalty, + repeatLastN = semantic.repeatLastN, + seedPolicy = semantic.seedPolicy.name, + effectiveSeed = semantic.effectiveSeed, + maxOutputTokens = semantic.maxOutputTokens, + chatTemplateId = semantic.chatTemplateId, + chatTemplateSource = semantic.chatTemplateSource.name, + systemPromptVersion = semantic.systemPromptVersion, + caseExecutionSemanticsDigest = semantic.caseExecutionSemanticsDigest.sha256, + ), + runtime = EvaluationRuntimeEnvironmentEntity( + deviceClass = runtime.deviceClass, + androidApiLevel = runtime.androidApiLevel, + abi = runtime.abi, + backendRevision = runtime.backendRevision, + harnessBuildIdentity = runtime.harnessBuildIdentity, + runtimeTuningProfileId = runtime.runtimeTuningProfileId, + runtimeTuningProfileVersion = runtime.runtimeTuningProfileVersion, + loadPolicy = runtime.loadPolicy.name, + warmupPolicy = runtime.warmupPolicy.name, + ), + ) + } + + private fun progressEntity(progress: EvaluationProgress) = EvaluationProgressEntity( + totalCases = progress.totalCases, + attemptedCases = progress.attemptedCases, + completedCases = progress.completedCases, + currentCaseId = progress.currentCaseId?.value, + ) + + private fun reliabilityEntity(summary: EvaluationReliabilitySummary) = EvaluationReliabilityEntity( + totalCases = summary.totalCases, + completedAndScored = summary.completedAndScored, + incorrectButValid = summary.incorrectButValid, + invalidOutput = summary.invalidOutput, + timeout = summary.timeout, + runtimeFailure = summary.runtimeFailure, + cancelled = summary.cancelled, + skipped = summary.skipped, + ) + + private fun failureEntity(failure: EvaluationFailure) = EvaluationFailureEntity( + stage = failure.stage.name, + code = failure.code.name, + caseId = failure.caseId?.value, + retryable = failure.retryable, + ) + + private fun config(entity: EvaluationRunConfigEntity, samples: List): EvaluationRunConfig { + require(samples.map { it.ordinal } == samples.indices.toList()) { "Persisted evaluation sample ordinals must be contiguous" } + val dataset = EvaluationDatasetIdentity( + id = EvaluationDatasetId(entity.datasetId), + version = EvaluationDatasetVersion(entity.datasetVersion), + digest = EvaluationDatasetDigest(entity.datasetDigest), + ) + val sampling = SamplingSelection.create( + dataset = dataset, + policy = SamplingPolicyRef(SamplingPolicyId(entity.samplingPolicyId), entity.samplingPolicyVersion), + seed = entity.samplingSeed, + orderedCaseIds = samples.map { EvaluationCaseId(it.caseId) }, + ) + require(sampling.digest.sha256 == entity.sampleSetDigest) { "Persisted evaluation sample digest does not match sample rows" } + return EvaluationRunConfig( + runId = EvaluationRunId(samples.firstOrNull()?.runId ?: error("Persisted evaluation run has no sample rows")), + model = EvaluationModelIdentity( + artifactDigest = ModelDigest(entity.modelDigest), + modelProfileId = entity.modelProfileId, + tier = entity.modelTier, + quantization = entity.modelQuantization, + ), + dataset = dataset, + sampling = sampling, + executionProfile = EvaluationExecutionProfileRef( + EvaluationExecutionProfileId(entity.executionProfileId), + entity.executionProfileVersion, + ), + loadPolicy = EvaluationModelLoadPolicy.valueOf(entity.loadPolicy), + warmupPolicy = EvaluationWarmupPolicy.valueOf(entity.warmupPolicy), + caseTimeoutMs = entity.caseTimeoutMs, + ) + } + + private fun summary( + entity: EvaluationRunEntity, + config: EvaluationRunConfig, + categories: List, + ): EvaluationRunSummary { + require(config.runId.value == entity.runId) { "Persisted evaluation run ID does not match sample rows" } + val quality = quality(entity, categories) + return EvaluationRunSummary( + runId = EvaluationRunId(entity.runId), + config = config, + identity = entity.identity?.let { identity(it, config) }, + state = EvaluationRunState.valueOf(entity.state), + progress = EvaluationProgress( + totalCases = entity.progress.totalCases, + attemptedCases = entity.progress.attemptedCases, + completedCases = entity.progress.completedCases, + currentCaseId = entity.progress.currentCaseId?.let(::EvaluationCaseId), + ), + quality = quality, + reliability = entity.reliability?.let(::reliability), + startedAtEpochMs = entity.startedAtEpochMs, + completedAtEpochMs = entity.completedAtEpochMs, + failure = entity.failure?.let(::failure), + ) + } + + private fun quality(entity: EvaluationRunEntity, categories: List): EvaluationQualitySummary? { + if (!entity.qualityPresent) { + require(entity.qualityAggregateScore == null && categories.isEmpty()) { + "Persisted quality rows exist while quality is absent" + } + return null + } + require(categories.map { it.ordinal } == categories.indices.toList()) { + "Persisted category score ordinals must be contiguous" + } + return EvaluationQualitySummary( + aggregateScore = entity.qualityAggregateScore?.let(::NormalizedScore), + categoryScores = categories.map { row -> + EvaluationCategoryScore( + categoryId = EvaluationCategoryId(row.categoryId), + score = NormalizedScore(row.score), + scoredCaseCount = row.scoredCaseCount, + weight = row.weight, + ) + }, + ) + } + + private fun identity(entity: EvaluationRunIdentityEntity, config: EvaluationRunConfig): EvaluationRunIdentity { + val semanticExecution = semanticExecution(entity.semantic, config.executionProfile) + val semanticIdentity = EvaluationSemanticExecutionIdentity.create(semanticExecution) + require(semanticIdentity.fingerprint.sha256 == entity.semanticExecutionFingerprint) { + "Persisted semantic execution fingerprint does not match semantic fields" + } + val runtime = EvaluationRuntimeEnvironmentIdentity( + deviceClass = entity.runtime.deviceClass, + androidApiLevel = entity.runtime.androidApiLevel, + abi = entity.runtime.abi, + backendRevision = entity.runtime.backendRevision, + harnessBuildIdentity = entity.runtime.harnessBuildIdentity, + runtimeTuningProfileId = entity.runtime.runtimeTuningProfileId, + runtimeTuningProfileVersion = entity.runtime.runtimeTuningProfileVersion, + loadPolicy = EvaluationModelLoadPolicy.valueOf(entity.runtime.loadPolicy), + warmupPolicy = EvaluationWarmupPolicy.valueOf(entity.runtime.warmupPolicy), + ) + val identity = EvaluationRunIdentity.create( + model = config.model, + dataset = config.dataset, + sampleSetDigest = config.sampling.digest, + samplingPolicy = config.sampling.policy, + samplingSeed = config.sampling.seed, + evaluatorSetDigest = EvaluatorSetDigest(entity.evaluatorSetDigest), + semanticExecution = semanticIdentity, + runtimeEnvironment = runtime, + ) + require(identity.fingerprint.sha256 == entity.runFingerprint) { + "Persisted evaluation run fingerprint does not match identity fields" + } + return identity + } + + private fun semanticExecution( + entity: EvaluationSemanticExecutionEntity, + profile: EvaluationExecutionProfileRef, + ): EvaluationSemanticExecution { + val preset = when { + entity.presetId == null && entity.presetVersion == null -> null + + entity.presetId != null && entity.presetVersion != null -> + InferencePresetRef(InferencePresetId(entity.presetId), entity.presetVersion) + + else -> error("Persisted inference preset identity is incomplete") + } + return EvaluationSemanticExecution( + semanticsVersion = entity.semanticsVersion, + profile = profile, + backendRevision = entity.backendRevision, + contextSize = entity.contextSize, + preset = preset, + thinkingMode = ThinkingMode.valueOf(entity.thinkingMode), + temperature = entity.temperature, + topP = entity.topP, + topK = entity.topK, + minP = entity.minP, + presencePenalty = entity.presencePenalty, + repeatPenalty = entity.repeatPenalty, + repeatLastN = entity.repeatLastN, + seedPolicy = SeedPolicyType.valueOf(entity.seedPolicy), + effectiveSeed = entity.effectiveSeed, + maxOutputTokens = entity.maxOutputTokens, + chatTemplateId = entity.chatTemplateId, + chatTemplateSource = ChatTemplateSource.valueOf(entity.chatTemplateSource), + systemPromptVersion = entity.systemPromptVersion, + caseExecutionSemanticsDigest = CaseExecutionSemanticsDigest(entity.caseExecutionSemanticsDigest), + ) + } + + private fun reliability(entity: EvaluationReliabilityEntity) = EvaluationReliabilitySummary( + totalCases = entity.totalCases, + completedAndScored = entity.completedAndScored, + incorrectButValid = entity.incorrectButValid, + invalidOutput = entity.invalidOutput, + timeout = entity.timeout, + runtimeFailure = entity.runtimeFailure, + cancelled = entity.cancelled, + skipped = entity.skipped, + ) + + private fun failure(entity: EvaluationFailureEntity) = EvaluationFailure( + stage = EvaluationFailureStage.valueOf(entity.stage), + code = EvaluationFailureCode.valueOf(entity.code), + caseId = entity.caseId?.let(::EvaluationCaseId), + retryable = entity.retryable, + ) + + private fun caseResult( + entity: EvaluationCaseResultEntity, + parameters: List, + ): EvaluationCaseResult { + require(parameters.all { it.runId == entity.runId && it.caseId == entity.caseId }) { + "Persisted evaluator parameter identity does not match case result" + } + val outcome = when { + entity.outcomeScore == null && entity.outcomeCode == null -> null + + entity.outcomeScore != null && entity.outcomeCode != null -> EvaluationOutcome( + score = NormalizedScore(entity.outcomeScore), + code = EvaluatorOutcomeCode.valueOf(entity.outcomeCode), + ) + + else -> error("Persisted evaluator outcome is incomplete") + } + return EvaluationCaseResult( + caseId = EvaluationCaseId(entity.caseId), + categoryId = EvaluationCategoryId(entity.categoryId), + evaluator = EvaluatorSpec( + type = EvaluatorType.valueOf(entity.evaluatorType), + version = EvaluatorVersion(entity.evaluatorVersion), + parameters = parameters.associate { it.parameterKey to it.parameterValue }, + ), + status = EvaluationCaseStatus.valueOf(entity.status), + outcome = outcome, + requestId = entity.requestId?.let(::RequestId), + metrics = EvaluationCaseMetrics( + timeToFirstTokenMs = entity.timeToFirstTokenMs, + totalMs = entity.totalMs, + prefillMs = entity.prefillMs, + decodeMs = entity.decodeMs, + inputTokens = entity.inputTokens, + outputTokens = entity.outputTokens, + decodeTokensPerSecond = entity.decodeTokensPerSecond, + processPssBytes = entity.processPssBytes, + availableMemoryBytes = entity.availableMemoryBytes, + thermalStatus = entity.thermalStatus, + ), + failure = entity.failure?.let(::failure), + ) + } +} diff --git a/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/RoomEvaluationResultRepository.kt b/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/RoomEvaluationResultRepository.kt new file mode 100644 index 000000000..7391a8ebd --- /dev/null +++ b/evaluation/room-store/src/main/kotlin/io/github/daniele21/localllm/evaluation/room/RoomEvaluationResultRepository.kt @@ -0,0 +1,193 @@ +package io.github.daniele21.localllm.evaluation.room + +import android.content.Context +import androidx.room.Room +import androidx.room.migration.Migration +import androidx.sqlite.db.SimpleSQLiteQuery +import io.github.daniele21.localllm.evaluation.EvaluationCaseResult +import io.github.daniele21.localllm.evaluation.EvaluationResultRepository +import io.github.daniele21.localllm.evaluation.EvaluationRetentionPolicy +import io.github.daniele21.localllm.evaluation.EvaluationRetentionResult +import io.github.daniele21.localllm.evaluation.EvaluationRunDeleteStatus +import io.github.daniele21.localllm.evaluation.EvaluationRunId +import io.github.daniele21.localllm.evaluation.EvaluationRunQuery +import io.github.daniele21.localllm.evaluation.EvaluationRunState +import io.github.daniele21.localllm.evaluation.EvaluationRunSummary +import io.github.daniele21.localllm.evaluation.PersistedEvaluationRun +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext + +class RoomEvaluationResultRepository internal constructor( + private val dao: EvaluationDao, + private val clock: () -> Long = System::currentTimeMillis, + private val closeDatabase: () -> Unit = {}, +) : EvaluationResultRepository, + AutoCloseable { + override suspend fun createRun(summary: EvaluationRunSummary) = withContext(Dispatchers.IO) { + require(dao.findRun(summary.runId.value) == null) { "Evaluation run already exists" } + dao.createRunGraph( + EvaluationRoomMapper.runEntity(summary), + EvaluationRoomMapper.sampleEntities(summary), + EvaluationRoomMapper.categoryScoreEntities(summary), + ) + } + + override suspend fun updateRunSummary(summary: EvaluationRunSummary) = withContext(Dispatchers.IO) { + val current = requireStoredRun(summary.runId) + val currentSummary = EvaluationRoomMapper.summary(current) + require(currentSummary.config == summary.config) { "Evaluation run config cannot change after creation" } + requireValidTransition(currentSummary.state, summary.state) + dao.updateRunGraph( + EvaluationRoomMapper.runEntity(summary), + EvaluationRoomMapper.categoryScoreEntities(summary), + ) + } + + override suspend fun appendCaseResult(runId: EvaluationRunId, result: EvaluationCaseResult) = withContext(Dispatchers.IO) { + val stored = requireStoredRun(runId) + val summary = EvaluationRoomMapper.summary(stored) + require(!summary.state.isTerminal()) { "Cannot append case result to terminal evaluation run" } + require(result.caseId in summary.config.sampling.orderedCaseIds) { + "Evaluation case result must belong to run sample set" + } + require(dao.sampleCaseCount(runId.value, result.caseId.value) == 1) { + "Evaluation case result must reference exactly one sampled case" + } + dao.upsertCaseResultGraph( + EvaluationRoomMapper.caseResultEntity(runId, result), + EvaluationRoomMapper.evaluatorParameterEntities(runId, result), + ) + } + + override suspend fun getRun(runId: EvaluationRunId): PersistedEvaluationRun? = withContext(Dispatchers.IO) { + dao.loadStoredRun(runId.value)?.let(EvaluationRoomMapper::persistedRun) + } + + override suspend fun queryRuns(query: EvaluationRunQuery): List = withContext(Dispatchers.IO) { + dao.queryRuns(query.toSqlQuery()).map { entity -> + val stored = requireNotNull(dao.loadStoredRun(entity.runId)) { + "Evaluation run disappeared while reading history" + } + EvaluationRoomMapper.summary(stored) + } + } + + override suspend fun deleteRun(runId: EvaluationRunId): EvaluationRunDeleteStatus = withContext(Dispatchers.IO) { + val run = dao.findRun(runId.value) ?: return@withContext EvaluationRunDeleteStatus.NOT_FOUND + if (!EvaluationRunState.valueOf(run.state).isTerminal()) { + EvaluationRunDeleteStatus.ACTIVE_RUN + } else { + check(dao.deleteRunRow(runId.value) == 1) { "Evaluation run disappeared while deleting" } + EvaluationRunDeleteStatus.DELETED + } + } + + override suspend fun applyRetention(policy: EvaluationRetentionPolicy): EvaluationRetentionResult = withContext(Dispatchers.IO) { + val now = clock() + val terminal = dao.terminalRunsNewestFirst() + val keepByCount = terminal.take(policy.maxTerminalRuns).map { it.runId }.toSet() + val expiredByAge = policy.maxAgeMs?.let { maxAgeMs -> + terminal.filter { now - it.startedAtEpochMs >= maxAgeMs }.map { it.runId }.toSet() + }.orEmpty() + val deleteIds = terminal.asSequence() + .map { it.runId } + .filter { it !in keepByCount || it in expiredByAge } + .toList() + if (deleteIds.isNotEmpty()) { + dao.deleteRunRows(deleteIds) + } + EvaluationRetentionResult( + deletedRunIds = deleteIds.map(::EvaluationRunId), + retainedRunCount = dao.runCount(), + ) + } + + override fun close() { + closeDatabase() + } + + private fun requireStoredRun(runId: EvaluationRunId): EvaluationStoredRun = requireNotNull(dao.loadStoredRun(runId.value)) { + "Evaluation run does not exist" + } + + private fun requireValidTransition(from: EvaluationRunState, to: EvaluationRunState) { + require(from == to || to in ALLOWED_TRANSITIONS.getValue(from)) { + "Invalid evaluation run transition: $from -> $to" + } + } + + companion object { + const val DEFAULT_DATABASE_NAME: String = "local-llm-evaluation.db" + private val MIGRATIONS: List = emptyList() + + fun open(context: Context, databaseName: String = DEFAULT_DATABASE_NAME): RoomEvaluationResultRepository { + require(databaseName.isNotBlank()) { "Evaluation database name must not be blank" } + val builder = Room.databaseBuilder( + context.applicationContext, + EvaluationDatabase::class.java, + databaseName, + ) + MIGRATIONS.forEach { migration -> builder.addMigrations(migration) } + val database = builder.build() + return RoomEvaluationResultRepository( + dao = database.evaluationDao(), + closeDatabase = database::close, + ) + } + + private val ALLOWED_TRANSITIONS: Map> = mapOf( + EvaluationRunState.CREATED to setOf(EvaluationRunState.VALIDATING, EvaluationRunState.CANCELLING, EvaluationRunState.FAILED), + EvaluationRunState.VALIDATING to + setOf(EvaluationRunState.PREPARING_MODEL, EvaluationRunState.CANCELLING, EvaluationRunState.FAILED), + EvaluationRunState.PREPARING_MODEL to setOf( + EvaluationRunState.WARMING_UP, + EvaluationRunState.RUNNING, + EvaluationRunState.CANCELLING, + EvaluationRunState.FAILED, + ), + EvaluationRunState.WARMING_UP to setOf(EvaluationRunState.RUNNING, EvaluationRunState.CANCELLING, EvaluationRunState.FAILED), + EvaluationRunState.RUNNING to setOf(EvaluationRunState.AGGREGATING, EvaluationRunState.CANCELLING, EvaluationRunState.FAILED), + EvaluationRunState.AGGREGATING to setOf(EvaluationRunState.COMPLETED, EvaluationRunState.CANCELLING, EvaluationRunState.FAILED), + EvaluationRunState.CANCELLING to setOf(EvaluationRunState.CANCELLED, EvaluationRunState.FAILED), + EvaluationRunState.COMPLETED to emptySet(), + EvaluationRunState.CANCELLED to emptySet(), + EvaluationRunState.FAILED to emptySet(), + ) + } +} + +private fun EvaluationRunQuery.toSqlQuery(): SimpleSQLiteQuery { + val sql = StringBuilder("SELECT * FROM evaluation_runs WHERE 1=1") + val args = mutableListOf() + if (states.isNotEmpty()) { + val orderedStates = states.sortedBy { it.name } + sql.append(" AND state IN (") + sql.append(orderedStates.joinToString(",") { "?" }) + sql.append(')') + args.addAll(orderedStates.map { it.name }) + } + datasetId?.let { + sql.append(" AND config_dataset_id = ?") + args += it.value + } + modelDigest?.let { + sql.append(" AND config_model_digest = ?") + args += it.sha256 + } + startedBeforeEpochMs?.let { + sql.append(" AND started_at_epoch_ms < ?") + args += it + } + sql.append(" ORDER BY started_at_epoch_ms DESC, run_id ASC LIMIT ?") + args += limit + return SimpleSQLiteQuery(sql.toString(), args.toTypedArray()) +} + +private fun EvaluationRunState.isTerminal(): Boolean = when (this) { + EvaluationRunState.COMPLETED, + EvaluationRunState.CANCELLED, + EvaluationRunState.FAILED, + -> true + + else -> false +} diff --git a/evaluation/room-store/src/test/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomMapperTest.kt b/evaluation/room-store/src/test/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomMapperTest.kt new file mode 100644 index 000000000..7dd2965a1 --- /dev/null +++ b/evaluation/room-store/src/test/kotlin/io/github/daniele21/localllm/evaluation/room/EvaluationRoomMapperTest.kt @@ -0,0 +1,125 @@ +package io.github.daniele21.localllm.evaluation.room + +import io.github.daniele21.localllm.contracts.ModelDigest +import io.github.daniele21.localllm.contracts.RequestId +import io.github.daniele21.localllm.evaluation.EvaluationCaseId +import io.github.daniele21.localllm.evaluation.EvaluationCaseMetrics +import io.github.daniele21.localllm.evaluation.EvaluationCaseResult +import io.github.daniele21.localllm.evaluation.EvaluationCaseStatus +import io.github.daniele21.localllm.evaluation.EvaluationCategoryId +import io.github.daniele21.localllm.evaluation.EvaluationDatasetDigest +import io.github.daniele21.localllm.evaluation.EvaluationDatasetId +import io.github.daniele21.localllm.evaluation.EvaluationDatasetIdentity +import io.github.daniele21.localllm.evaluation.EvaluationDatasetVersion +import io.github.daniele21.localllm.evaluation.EvaluationExecutionProfileId +import io.github.daniele21.localllm.evaluation.EvaluationExecutionProfileRef +import io.github.daniele21.localllm.evaluation.EvaluationModelIdentity +import io.github.daniele21.localllm.evaluation.EvaluationModelLoadPolicy +import io.github.daniele21.localllm.evaluation.EvaluationOutcome +import io.github.daniele21.localllm.evaluation.EvaluationProgress +import io.github.daniele21.localllm.evaluation.EvaluationQualitySummary +import io.github.daniele21.localllm.evaluation.EvaluationRunConfig +import io.github.daniele21.localllm.evaluation.EvaluationRunId +import io.github.daniele21.localllm.evaluation.EvaluationRunState +import io.github.daniele21.localllm.evaluation.EvaluationRunSummary +import io.github.daniele21.localllm.evaluation.EvaluationWarmupPolicy +import io.github.daniele21.localllm.evaluation.EvaluatorOutcomeCode +import io.github.daniele21.localllm.evaluation.EvaluatorSpec +import io.github.daniele21.localllm.evaluation.EvaluatorType +import io.github.daniele21.localllm.evaluation.EvaluatorVersion +import io.github.daniele21.localllm.evaluation.NormalizedScore +import io.github.daniele21.localllm.evaluation.SamplingPolicyId +import io.github.daniele21.localllm.evaluation.SamplingPolicyRef +import io.github.daniele21.localllm.evaluation.SamplingSelection +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +class EvaluationRoomMapperTest { + @Test + fun `round trip preserves optional empty quality and sample order`() { + val summary = summary() + val stored = EvaluationStoredRun( + EvaluationRoomMapper.runEntity(summary), + EvaluationRoomMapper.sampleEntities(summary), + EvaluationRoomMapper.categoryScoreEntities(summary), + emptyList(), + emptyList(), + ) + + val restored = EvaluationRoomMapper.summary(stored) + + assertEquals(summary, restored) + assertTrue(stored.run.qualityPresent) + assertEquals(listOf("case-b", "case-a"), stored.samples.map { it.caseId }) + } + + @Test + fun `case result round trip preserves only privacy safe result fields`() { + val summary = summary() + val result = EvaluationCaseResult( + caseId = CASE_B, + categoryId = EvaluationCategoryId("general"), + evaluator = EvaluatorSpec( + EvaluatorType.EXACT_MATCH, + EvaluatorVersion(1), + mapOf("case" to "sensitive"), + ), + status = EvaluationCaseStatus.SCORED, + outcome = EvaluationOutcome(NormalizedScore(1.0), EvaluatorOutcomeCode.CORRECT), + requestId = RequestId("request-1"), + metrics = EvaluationCaseMetrics(totalMs = 42, outputTokens = 3), + ) + val stored = EvaluationStoredRun( + EvaluationRoomMapper.runEntity(summary), + EvaluationRoomMapper.sampleEntities(summary), + EvaluationRoomMapper.categoryScoreEntities(summary), + listOf(EvaluationRoomMapper.caseResultEntity(RUN_ID, result)), + EvaluationRoomMapper.evaluatorParameterEntities(RUN_ID, result), + ) + + val restored = EvaluationRoomMapper.persistedRun(stored) + + assertEquals(listOf(result), restored.caseResults) + assertEquals(summary, restored.summary) + } + + private fun summary(): EvaluationRunSummary = EvaluationRunSummary( + runId = RUN_ID, + config = CONFIG, + identity = null, + state = EvaluationRunState.CREATED, + progress = EvaluationProgress(totalCases = 2, attemptedCases = 0, completedCases = 0), + quality = EvaluationQualitySummary(aggregateScore = null, categoryScores = emptyList()), + reliability = null, + startedAtEpochMs = 100, + completedAtEpochMs = null, + failure = null, + ) + + private companion object { + val RUN_ID = EvaluationRunId("run-1") + val CASE_A = EvaluationCaseId("case-a") + val CASE_B = EvaluationCaseId("case-b") + val DATASET = EvaluationDatasetIdentity( + EvaluationDatasetId("fixture"), + EvaluationDatasetVersion("1"), + EvaluationDatasetDigest("1".repeat(64)), + ) + val CONFIG = EvaluationRunConfig( + runId = RUN_ID, + model = EvaluationModelIdentity(ModelDigest("a".repeat(64)), "model-profile", quantization = "Q4_K_M"), + dataset = DATASET, + sampling = SamplingSelection.create( + dataset = DATASET, + policy = SamplingPolicyRef(SamplingPolicyId("fixed"), 1), + seed = 7, + orderedCaseIds = listOf(CASE_B, CASE_A), + ), + executionProfile = EvaluationExecutionProfileRef(EvaluationExecutionProfileId("direct"), 1), + loadPolicy = EvaluationModelLoadPolicy.PRESERVE_CURRENT_RESIDENCY, + warmupPolicy = EvaluationWarmupPolicy.NONE, + caseTimeoutMs = 30_000, + ) + } +}