+,
+ #[arg(long)]
+ signature: Option,
+ #[arg(long, default_value_t = 0)]
+ workers: u32,
+ #[arg(long)]
+ json: bool,
+ },
+ Identify {
+ #[arg(long)]
+ workspace: PathBuf,
+ #[arg(long)]
+ run: String,
+ #[arg(long)]
+ siegfried: Option,
+ #[arg(long)]
+ signature: Option,
+ #[arg(long, default_value_t = 2_048)]
+ batch_size: u32,
+ #[arg(long, default_value_t = 0)]
+ workers: u32,
+ #[arg(long, default_value_t = 900)]
+ timeout_seconds: u64,
+ #[arg(long, default_value_t = true)]
+ resume: bool,
+ #[arg(long)]
+ allow_migration: bool,
+ },
+ Summary {
+ #[arg(long)]
+ workspace: PathBuf,
+ #[arg(long)]
+ run: String,
+ #[arg(long)]
+ json: bool,
+ },
+ List {
+ #[arg(long)]
+ workspace: PathBuf,
+ #[arg(long)]
+ run: String,
+ #[arg(long)]
+ state: Option,
+ #[arg(long)]
+ puid: Option,
+ #[arg(long)]
+ mismatch_only: bool,
+ #[arg(long)]
+ search: Option,
+ #[arg(long, default_value_t = 100)]
+ limit: u32,
+ #[arg(long)]
+ json: bool,
+ },
+}
+
+#[derive(Debug)]
+struct IdentifyFormatsArgs {
+ workspace: PathBuf,
+ run: String,
+ siegfried: Option,
+ signature: Option,
+ batch_size: u32,
+ workers: u32,
+ timeout_seconds: u64,
+ resume: bool,
+ allow_migration: bool,
+}
+
+#[derive(Debug)]
+struct ListFormatsArgs {
+ workspace: PathBuf,
+ run: String,
+ state: Option,
+ puid: Option,
+ mismatch_only: bool,
+ search: Option,
+ limit: u32,
+ json: bool,
+}
+
#[derive(Debug, Subcommand)]
enum ExportCommand {
SanitizedSummary {
@@ -238,6 +328,7 @@ fn run(arguments: Arguments) -> ProfilerResult<()> {
Command::Workspace { command } => run_workspace_command(command),
Command::Runs { command } => run_runs_command(command),
Command::Findings { command } => run_findings_command(command),
+ Command::Formats { command } => run_formats_command(command),
Command::Export { command } => run_export_command(command),
}
}
@@ -519,6 +610,189 @@ fn run_finding_note(
)?)
}
+fn run_formats_command(command: FormatsCommand) -> ProfilerResult<()> {
+ match command {
+ FormatsCommand::Probe {
+ siegfried,
+ signature,
+ workers,
+ json,
+ } => run_format_probe(siegfried, signature, workers, json),
+ FormatsCommand::Identify {
+ workspace,
+ run,
+ siegfried,
+ signature,
+ batch_size,
+ workers,
+ timeout_seconds,
+ resume,
+ allow_migration,
+ } => run_format_identification(IdentifyFormatsArgs {
+ workspace,
+ run,
+ siegfried,
+ signature,
+ batch_size,
+ workers,
+ timeout_seconds,
+ resume,
+ allow_migration,
+ }),
+ FormatsCommand::Summary {
+ workspace,
+ run,
+ json,
+ } => run_format_summary(&workspace, &run, json),
+ FormatsCommand::List {
+ workspace,
+ run,
+ state,
+ puid,
+ mismatch_only,
+ search,
+ limit,
+ json,
+ } => run_format_list(ListFormatsArgs {
+ workspace,
+ run,
+ state,
+ puid,
+ mismatch_only,
+ search,
+ limit,
+ json,
+ }),
+ }
+}
+
+fn run_format_probe(
+ siegfried: Option,
+ signature: Option,
+ workers: u32,
+ json: bool,
+) -> ProfilerResult<()> {
+ let identity = ExactFormatEngine.probe_tool(siegfried, signature, workers)?;
+ if json {
+ print_json(&identity)
+ } else {
+ println!("Tool: {} {}", identity.tool_name, identity.tool_version);
+ println!("Executable SHA-256: {}", identity.executable_sha256);
+ println!("Signature: {}", identity.signature_path);
+ println!("PRONOM signature version: {}", identity.signature_version);
+ Ok(())
+ }
+}
+
+fn run_format_identification(args: IdentifyFormatsArgs) -> ProfilerResult<()> {
+ let session = open_review_workspace(&args.workspace, args.allow_migration)?;
+ let context = session.context();
+ let archive_root = single_source_root(&context.source_roots)?;
+
+ print_json(&ExactFormatEngine.identify(
+ &FormatIdentificationRequest {
+ baseline_run_id: args.run,
+ workspace_root: context.root_path,
+ profiler_database: context.profiler_database,
+ archive_root,
+ siegfried_path: args.siegfried,
+ signature_path: args.signature,
+ options: FormatOptions {
+ batch_size: args.batch_size,
+ workers: args.workers,
+ timeout_seconds: args.timeout_seconds,
+ resume: args.resume,
+ },
+ },
+ &JsonLineProgress,
+ )?)
+}
+
+fn run_format_summary(workspace: &Path, run: &str, json: bool) -> ProfilerResult<()> {
+ let session = open_read_only_workspace(workspace)?;
+ let summary = ExactFormatEngine.summary(&session.context().profiler_database, run)?;
+
+ if json {
+ print_json(&summary)
+ } else {
+ println!("Baseline run: {}", summary.baseline_run_id);
+ println!(
+ "Latest format run: {}",
+ summary.latest_format_run_id.as_deref().unwrap_or("none")
+ );
+ println!(
+ "State: {}",
+ summary.latest_run_state.as_deref().unwrap_or("not_started")
+ );
+ println!("Identified: {}", summary.identified);
+ println!("Unknown: {}", summary.unknown);
+ println!("Ambiguous: {}", summary.ambiguous);
+ println!("Tool errors: {}", summary.tool_errors);
+ println!("Extension mismatches: {}", summary.extension_mismatches);
+ Ok(())
+ }
+}
+
+fn run_format_list(args: ListFormatsArgs) -> ProfilerResult<()> {
+ let session = open_read_only_workspace(&args.workspace)?;
+ let page = ExactFormatEngine.page(
+ &session.context().profiler_database,
+ &FormatPageRequest {
+ baseline_run_id: args.run,
+ filters: FormatFilters {
+ search: args.search,
+ state: args.state.as_deref().map(parse_format_state).transpose()?,
+ puid: args.puid,
+ mismatch_only: args.mismatch_only,
+ },
+ after_sha256: None,
+ limit: args.limit,
+ },
+ )?;
+
+ if args.json {
+ print_json(&page)
+ } else {
+ for item in page.items {
+ println!(
+ "{} {} {} {}",
+ item.state,
+ item.primary_identifier.as_deref().unwrap_or("—"),
+ item.primary_format_name.as_deref().unwrap_or("—"),
+ item.sha256
+ );
+ }
+ Ok(())
+ }
+}
+
+fn parse_format_state(value: &str) -> ProfilerResult {
+ match value.trim().to_ascii_lowercase().as_str() {
+ "uninspected" => Ok(FormatState::Uninspected),
+ "identified" => Ok(FormatState::Identified),
+ "unknown" => Ok(FormatState::Unknown),
+ "ambiguous" => Ok(FormatState::Ambiguous),
+ "empty" => Ok(FormatState::Empty),
+ "skipped_unavailable" => Ok(FormatState::SkippedUnavailable),
+ "tool_error" => Ok(FormatState::ToolError),
+ _ => Err(profiler_core::ProfilerError::InvalidArgument(
+ "format state is not supported".into(),
+ )),
+ }
+}
+
+fn single_source_root(roots: &[PathBuf]) -> ProfilerResult {
+ match roots {
+ [root] => Ok(root.clone()),
+ [] => Err(profiler_core::ProfilerError::IncompatibleSource(
+ "workspace does not contain a MailVault source root".into(),
+ )),
+ _ => Err(profiler_core::ProfilerError::IncompatibleSource(
+ "workspace contains multiple source roots; select a single collection workspace".into(),
+ )),
+ }
+}
+
fn run_export_command(command: ExportCommand) -> ProfilerResult<()> {
match command {
ExportCommand::SanitizedSummary {
diff --git a/apps/desktop/package.json b/apps/desktop/package.json
index 2b3bfa1..d2c8b76 100644
--- a/apps/desktop/package.json
+++ b/apps/desktop/package.json
@@ -1,7 +1,7 @@
{
"name": "@firexcore/mailvault-profiler-desktop",
"private": true,
- "version": "0.1.0-alpha.3",
+ "version": "0.1.0-alpha.4",
"type": "module",
"scripts": {
"dev": "vite --port 1420 --strictPort",
diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs
index 3be6488..03940ea 100644
--- a/apps/desktop/src-tauri/src/lib.rs
+++ b/apps/desktop/src-tauri/src/lib.rs
@@ -3,13 +3,15 @@ use std::{path::PathBuf, str::FromStr, sync::Mutex};
use profiler_adapter_mailvault::MailVaultAdapter;
use profiler_core::{
ActiveRunContext, CollectionAdapter, ContentObjectDetail, ErrorReport, FindingCategory,
- FindingDetail, FindingReviewHistory, FindingsPage, FindingsPageRequest, InventoryFilters,
- InventoryPage, InventoryPageRequest, OpenWorkspaceResult, PreflightReport, ProfilerError,
- ProfilerResult, ProgressEvent, ProgressSink, ReviewActorKind, ReviewStatus, RunCatalogEntry,
- SnapshotOptions, SnapshotRequest, SnapshotResult, WorkspaceInspection, WorkspaceOpenMode,
+ FindingDetail, FindingReviewHistory, FindingsPage, FindingsPageRequest, FormatFilters,
+ FormatIdentificationRequest, FormatIdentificationResult, FormatOptions, FormatPage,
+ FormatPageRequest, FormatSummary, FormatToolIdentity, InventoryFilters, InventoryPage,
+ InventoryPageRequest, OpenWorkspaceResult, PreflightReport, ProfilerError, ProfilerResult,
+ ProgressEvent, ProgressSink, ReviewActorKind, ReviewStatus, RunCatalogEntry, SnapshotOptions,
+ SnapshotRequest, SnapshotResult, WorkspaceInspection, WorkspaceOpenMode,
};
use profiler_engine::{
- ProfileEngine, ProfileOptions, ProfileRequest, ProfileResult,
+ ExactFormatEngine, ProfileEngine, ProfileOptions, ProfileRequest, ProfileResult,
workspace::{
WorkspaceContext, WorkspaceSession, add_review_note, clear_review_status,
export_sanitized_run, finding_detail, findings_page as query_findings_page,
@@ -17,7 +19,7 @@ use profiler_engine::{
},
};
use serde::Deserialize;
-use tauri::{State, ipc::Channel};
+use tauri::{AppHandle, Manager, State, ipc::Channel};
use uuid::Uuid;
#[derive(Debug, Deserialize)]
@@ -32,6 +34,27 @@ struct FindingsPageCommandRequest {
limit: u32,
}
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct FormatPageCommandRequest {
+ search: Option,
+ state: Option,
+ puid: Option,
+ mismatch_only: bool,
+ after_sha256: Option,
+ limit: u32,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct FormatIdentifyCommandRequest {
+ siegfried_path: Option,
+ batch_size: u32,
+ workers: u32,
+ timeout_seconds: u64,
+ resume: bool,
+}
+
struct ChannelProgressSink {
channel: Channel,
}
@@ -442,6 +465,144 @@ async fn add_finding_review_note(
.map_err(|error| error.report())
}
+fn bundled_siegfried_paths(app: &AppHandle) -> ProfilerResult<(Option, Option)> {
+ let resource_root = app.path().resource_dir().map_err(|error| {
+ ProfilerError::Internal(format!("resolving application resource directory: {error}"))
+ })?;
+ let directory = resource_root.join("tools").join("siegfried");
+ let executable = directory.join(if cfg!(windows) { "sf.exe" } else { "sf" });
+ let signature = directory.join("default.sig");
+ Ok((
+ executable.is_file().then_some(executable),
+ signature.is_file().then_some(signature),
+ ))
+}
+
+#[allow(clippy::needless_pass_by_value)]
+#[tauri::command]
+async fn probe_format_tool(
+ siegfried_path: Option,
+ app: AppHandle,
+ state: State<'_, DesktopState>,
+) -> Result {
+ let _ = state.context().map_err(|error| error.report())?;
+ let (bundled_executable, bundled_signature) =
+ bundled_siegfried_paths(&app).map_err(|error| error.report())?;
+ tauri::async_runtime::spawn_blocking(move || {
+ ExactFormatEngine.probe_tool(
+ siegfried_path.map(PathBuf::from).or(bundled_executable),
+ bundled_signature,
+ 0,
+ )
+ })
+ .await
+ .map_err(|error| {
+ ProfilerError::Internal(format!("format-tool probe failed: {error}")).report()
+ })?
+ .map_err(|error| error.report())
+}
+
+#[allow(clippy::needless_pass_by_value)]
+#[tauri::command]
+async fn identify_formats(
+ request: FormatIdentifyCommandRequest,
+ on_event: Channel,
+ app: AppHandle,
+ state: State<'_, DesktopState>,
+) -> Result {
+ let context = state.context().map_err(|error| error.report())?;
+ let active = state.active_run().map_err(|error| error.report())?;
+ context.open_writer().map_err(|error| error.report())?;
+ let archive_root = match context.source_roots.as_slice() {
+ [root] => root.clone(),
+ [] => {
+ return Err(ProfilerError::IncompatibleSource(
+ "workspace has no MailVault source root".into(),
+ )
+ .report());
+ }
+ _ => {
+ return Err(ProfilerError::IncompatibleSource(
+ "workspace contains multiple source roots".into(),
+ )
+ .report());
+ }
+ };
+ let (bundled_executable, bundled_signature) =
+ bundled_siegfried_paths(&app).map_err(|error| error.report())?;
+ let format_request = FormatIdentificationRequest {
+ baseline_run_id: active.run.run_id,
+ workspace_root: context.root_path,
+ profiler_database: context.profiler_database,
+ archive_root,
+ siegfried_path: request
+ .siegfried_path
+ .map(PathBuf::from)
+ .or(bundled_executable),
+ signature_path: bundled_signature,
+ options: FormatOptions {
+ batch_size: request.batch_size,
+ workers: request.workers,
+ timeout_seconds: request.timeout_seconds,
+ resume: request.resume,
+ },
+ };
+ tauri::async_runtime::spawn_blocking(move || {
+ ExactFormatEngine.identify(&format_request, &ChannelProgressSink { channel: on_event })
+ })
+ .await
+ .map_err(|error| {
+ ProfilerError::Internal(format!("format-identification worker failed: {error}")).report()
+ })?
+ .map_err(|error| error.report())
+}
+
+#[allow(clippy::needless_pass_by_value)]
+#[tauri::command]
+async fn format_summary(state: State<'_, DesktopState>) -> Result {
+ let context = state.context().map_err(|error| error.report())?;
+ let active = state.active_run().map_err(|error| error.report())?;
+ tauri::async_runtime::spawn_blocking(move || {
+ ExactFormatEngine.summary(&context.profiler_database, &active.run.run_id)
+ })
+ .await
+ .map_err(|error| {
+ ProfilerError::Internal(format!("format-summary worker failed: {error}")).report()
+ })?
+ .map_err(|error| error.report())
+}
+
+#[allow(clippy::needless_pass_by_value)]
+#[tauri::command]
+async fn format_page(
+ request: FormatPageCommandRequest,
+ state: State<'_, DesktopState>,
+) -> Result {
+ let context = state.context().map_err(|error| error.report())?;
+ let active = state.active_run().map_err(|error| error.report())?;
+ tauri::async_runtime::spawn_blocking(move || {
+ ExactFormatEngine.page(
+ &context.profiler_database,
+ &FormatPageRequest {
+ baseline_run_id: active.run.run_id,
+ filters: FormatFilters {
+ search: request.search,
+ state: request.state,
+ puid: request.puid,
+ mismatch_only: request.mismatch_only,
+ },
+ after_sha256: request.after_sha256,
+ limit: request.limit,
+ },
+ )
+ })
+ .await
+ .map_err(|error| {
+ ProfilerError::Internal(format!("format-page worker failed: {error}")).report()
+ })?
+ .map_err(|error| error.report())
+}
+
#[tauri::command]
async fn export_sanitized_summary(
run_id: String,
@@ -490,6 +651,10 @@ pub fn run() {
set_finding_review_status,
clear_finding_review_status,
add_finding_review_note,
+ probe_format_tool,
+ identify_formats,
+ format_summary,
+ format_page,
export_sanitized_summary
])
.run(tauri::generate_context!())
diff --git a/apps/desktop/src-tauri/tauri.conf.json b/apps/desktop/src-tauri/tauri.conf.json
index 322e720..210d89d 100644
--- a/apps/desktop/src-tauri/tauri.conf.json
+++ b/apps/desktop/src-tauri/tauri.conf.json
@@ -1,7 +1,7 @@
{
"$schema": "https://schema.tauri.app/config/2",
"productName": "MailVault Collection Profiler",
- "version": "0.1.0-alpha.3",
+ "version": "0.1.0-alpha.4",
"identifier": "com.firexcore.mailvault-profiler",
"build": {
"beforeDevCommand": "npm run dev",
@@ -40,8 +40,14 @@
],
"windows": {
"wix": {
- "version": "0.1.0.3"
+ "version": "0.1.0.4"
}
+ },
+ "resources": {
+ "../../../tools/siegfried/windows-x86_64/sf.exe": "tools/siegfried/sf.exe",
+ "../../../tools/siegfried/windows-x86_64/default.sig": "tools/siegfried/default.sig",
+ "../../../tools/siegfried/windows-x86_64/tool-manifest.json": "tools/siegfried/tool-manifest.json",
+ "../../../THIRD_PARTY_NOTICES.md": "THIRD_PARTY_NOTICES.md"
}
}
}
diff --git a/apps/desktop/src/App.tsx b/apps/desktop/src/App.tsx
index ff18192..5accc4e 100644
--- a/apps/desktop/src/App.tsx
+++ b/apps/desktop/src/App.tsx
@@ -10,16 +10,21 @@ import {
getCurrentWorkspace,
getFindingDetail,
getFindingsPage,
+ getFormatPage,
+ getFormatSummary,
getInventoryPage,
getWorkspaceRuns,
+ identifyFormats,
inspectWorkspace,
normalizeError,
openExistingRun,
openWorkspace,
preflightArchive,
+ probeFormatTool,
profileCollection,
setFindingReviewStatus,
} from './api';
+import FormatsView from './FormatsView';
import type {
ActiveRunContext,
AvailabilityState,
@@ -29,6 +34,10 @@ import type {
FindingDetail,
FindingView,
FindingsPage,
+ FormatPage,
+ FormatState,
+ FormatSummary,
+ FormatToolIdentity,
InventoryFilters,
InventoryObjectRow,
InventoryPage,
@@ -53,7 +62,7 @@ const dateTime = new Intl.DateTimeFormat('en-US', {
});
const PAGE_SIZE = 100;
-type View = 'setup' | 'runs' | 'inventory' | 'findings';
+type View = 'setup' | 'runs' | 'inventory' | 'formats' | 'findings';
type BusyState =
| 'preflight'
| 'profile'
@@ -61,6 +70,9 @@ type BusyState =
| 'runs'
| 'inventory'
| 'findings'
+ | 'formats'
+ | 'format_tool'
+ | 'format_run'
| 'detail'
| 'review'
| 'export'
@@ -534,6 +546,17 @@ function App() {
const [findingsVersion, setFindingsVersion] = useState(0);
const [findingDetail, setFindingDetail] = useState(null);
+ const [formatSummary, setFormatSummary] = useState(null);
+ const [formatPage, setFormatPage] = useState(null);
+ const [formatTool, setFormatTool] = useState(null);
+ const [formatSearch, setFormatSearch] = useState('');
+ const [formatState, setFormatState] = useState('');
+ const [formatPuid, setFormatPuid] = useState('');
+ const [formatMismatchOnly, setFormatMismatchOnly] = useState(false);
+ const [appliedFormatFilters, setAppliedFormatFilters] = useState({ search: '', state: '' as FormatState | '', puid: '', mismatchOnly: false });
+ const [formatAfter, setFormatAfter] = useState();
+ const [formatVersion, setFormatVersion] = useState(0);
+
const canReview = workspace?.descriptor.accessMode === 'read_write' && workspace.descriptor.reviewIntegrityValid;
async function runPreflight() {
@@ -590,7 +613,7 @@ function App() {
}
function resetRunViews() {
- setInventoryPage(null); setInventoryFilters({}); setAppliedInventoryFilters({}); setInventoryAfter(undefined); setInventoryHistory([]); setInventoryVersion((value) => value + 1); setFindingsPage(null); setFindingAfter(undefined); setFindingDetail(null); setObjectDetail(null); setFindingsVersion((value) => value + 1);
+ setInventoryPage(null); setInventoryFilters({}); setAppliedInventoryFilters({}); setInventoryAfter(undefined); setInventoryHistory([]); setInventoryVersion((value) => value + 1); setFindingsPage(null); setFindingAfter(undefined); setFindingDetail(null); setObjectDetail(null); setFindingsVersion((value) => value + 1); setFormatSummary(null); setFormatPage(null); setFormatTool(null); setFormatAfter(undefined); setFormatVersion((value) => value + 1);
}
const loadInventory = useCallback(async () => {
@@ -605,8 +628,46 @@ function App() {
try { setFindingsPage(await getFindingsPage(appliedFindingFilters.code || null, appliedFindingFilters.severity || null, appliedFindingFilters.reviewStatus || null, appliedFindingFilters.category, appliedFindingFilters.search || null, findingAfter ?? null, PAGE_SIZE)); } catch (caught) { setError(normalizeError(caught)); } finally { setBusy(null); }
}, [activeRun, appliedFindingFilters, findingAfter]);
+ const loadFormats = useCallback(async () => {
+ if (!activeRun) return;
+ setBusy('formats'); setError(null);
+ try {
+ const [summary, page] = await Promise.all([
+ getFormatSummary(),
+ getFormatPage(
+ appliedFormatFilters.search || null,
+ appliedFormatFilters.state || null,
+ appliedFormatFilters.puid || null,
+ appliedFormatFilters.mismatchOnly,
+ formatAfter ?? null,
+ PAGE_SIZE,
+ ),
+ ]);
+ setFormatSummary(summary);
+ setFormatPage(page);
+ if (summary.tool) setFormatTool(summary.tool);
+ } catch (caught) { setError(normalizeError(caught)); } finally { setBusy(null); }
+ }, [activeRun, appliedFormatFilters, formatAfter]);
+
+ async function probeFormats() {
+ setBusy('format_tool'); setError(null);
+ try { setFormatTool(await probeFormatTool()); } catch (caught) { setError(normalizeError(caught)); } finally { setBusy(null); }
+ }
+
+ async function runFormats() {
+ if (!activeRun || !canReview) return;
+ setBusy('format_run'); setError(null); setProgress(null);
+ try {
+ await identifyFormats({ batchSize: 2048, workers: 0, timeoutSeconds: 900, resume: true }, setProgress);
+ setFormatVersion((value) => value + 1);
+ setFormatAfter(undefined);
+ await loadFormats();
+ } catch (caught) { setError(normalizeError(caught)); } finally { setBusy(null); }
+ }
+
useEffect(() => { if (view === 'inventory' && activeRun) void loadInventory(); }, [view, activeRun, inventoryVersion, loadInventory]);
useEffect(() => { if (view === 'findings' && activeRun) void loadFindings(); }, [view, activeRun, findingsVersion, loadFindings]);
+ useEffect(() => { if (view === 'formats' && activeRun) void loadFormats(); }, [view, activeRun, formatVersion, loadFormats]);
async function openObject(row: InventoryObjectRow) {
setBusy('detail'); setError(null);
@@ -641,7 +702,8 @@ function App() {
{ id: 'setup' as const, index: '01', label: 'Start', enabled: true },
{ id: 'runs' as const, index: '02', label: 'Workspace runs', enabled: workspace !== null },
{ id: 'inventory' as const, index: '03', label: 'Physical inventory', enabled: activeRun !== null },
- { id: 'findings' as const, index: '04', label: 'Findings review', enabled: activeRun !== null },
+ { id: 'formats' as const, index: '04', label: 'Exact formats', enabled: activeRun !== null },
+ { id: 'findings' as const, index: '05', label: 'Findings review', enabled: activeRun !== null },
], [workspace, activeRun]);
return (
@@ -659,6 +721,7 @@ function App() {
{view === 'setup' ? void runPreflight()} onProfile={() => void createProfile()} onInspectWorkspace={() => void inspectExistingWorkspace()} onOpenWorkspace={(readOnly, allowMigration) => void openExistingWorkspace(readOnly, allowMigration)} /> : null}
{view === 'runs' && workspace ? void activateRun(runId)} /> : null}
{view === 'inventory' && activeRun ? 0} onFilters={setInventoryFilters} onSearch={() => { setAppliedInventoryFilters({ ...inventoryFilters }); setInventoryAfter(undefined); setInventoryHistory([]); setInventoryVersion((value) => value + 1); }} onNext={() => { if (!inventoryPage?.nextAfterSha256) return; setInventoryHistory((history) => [...history, inventoryAfter]); setInventoryAfter(inventoryPage.nextAfterSha256 ?? undefined); }} onPrevious={() => { setInventoryHistory((history) => { const previous = history.at(-1); setInventoryAfter(previous); return history.slice(0, -1); }); }} onSelect={(row) => void openObject(row)} /> : null}
+ {view === 'formats' && activeRun ? { setAppliedFormatFilters({ search: formatSearch, state: formatState, puid: formatPuid, mismatchOnly: formatMismatchOnly }); setFormatAfter(undefined); setFormatVersion((value) => value + 1); }} onProbe={() => void probeFormats()} onRun={() => void runFormats()} onNext={() => { if (formatPage?.nextAfterSha256) setFormatAfter(formatPage.nextAfterSha256); }} /> : null}
{view === 'findings' && activeRun ? { setFindingCategory(category); setAppliedFindingFilters((filters) => ({ ...filters, category })); setFindingAfter(undefined); setFindingsVersion((value) => value + 1); }} onSeverity={setFindingSeverity} onCode={setFindingCode} onReviewStatus={setFindingReviewStatusFilter} onSearch={setFindingSearch} onReload={() => { setAppliedFindingFilters({ category: findingCategory, severity: findingSeverity, code: findingCode, reviewStatus: findingReviewStatus, search: findingSearch }); setFindingAfter(undefined); setFindingsVersion((value) => value + 1); }} onNext={() => { if (findingsPage?.nextAfterId) setFindingAfter(findingsPage.nextAfterId); }} onSelect={(finding) => void openFinding(finding)} onExport={(extension) => void exportRun(extension)} /> : null}
{objectDetail ? setObjectDetail(null)} /> : null}
diff --git a/apps/desktop/src/FormatsView.tsx b/apps/desktop/src/FormatsView.tsx
new file mode 100644
index 0000000..cecfb64
--- /dev/null
+++ b/apps/desktop/src/FormatsView.tsx
@@ -0,0 +1,206 @@
+import type {
+ FormatObjectRow,
+ FormatPage,
+ FormatState,
+ FormatSummary,
+ FormatToolIdentity,
+ ProgressEvent,
+} from './types';
+
+const number = new Intl.NumberFormat('en-US');
+const decimal = new Intl.NumberFormat('en-US', { maximumFractionDigits: 2 });
+
+function formatBytes(value: number): string {
+ if (value < 1024) return `${number.format(value)} B`;
+ const units = ['KiB', 'MiB', 'GiB', 'TiB'] as const;
+ let size = value / 1024;
+ let unit: string = units[0];
+ for (let index = 1; index < units.length && size >= 1024; index += 1) {
+ size /= 1024;
+ unit = units[index] ?? unit;
+ }
+ return `${decimal.format(size)} ${unit}`;
+}
+
+function titleCase(value: string): string {
+ return value.replaceAll('_', ' ').replace(/\b\w/g, (letter) => letter.toUpperCase());
+}
+
+function shortHash(value: string): string {
+ return value.length > 16 ? `${value.slice(0, 12)}…${value.slice(-4)}` : value;
+}
+
+function Metric({ label, value, tone }: { label: string; value: number; tone?: string }) {
+ return (
+
+ {label}
+ {number.format(value)}
+
+ );
+}
+
+function FormatRow({ item }: { item: FormatObjectRow }) {
+ const extensionLabel = item.extensionMismatch
+ ? 'Mismatch'
+ : item.extensionChecked
+ ? 'No mismatch'
+ : 'Not checked';
+ const extensionClass = item.extensionMismatch
+ ? 'mismatch-yes'
+ : item.extensionChecked
+ ? 'mismatch-no'
+ : 'mismatch-unchecked';
+ return (
+
+
+
+
+
+
+
+
+
+ {(running || progress || summary?.latestFormatRunId) ? (
+
+
+ ) : null}
+
+
+
+
+ );
+}
diff --git a/apps/desktop/src/api.ts b/apps/desktop/src/api.ts
index 7789def..e11de0d 100644
--- a/apps/desktop/src/api.ts
+++ b/apps/desktop/src/api.ts
@@ -8,6 +8,11 @@ import type {
FindingDetail,
FindingReviewHistory,
FindingsPage,
+ FormatIdentifyRequest,
+ FormatPage,
+ FormatState,
+ FormatSummary,
+ FormatToolIdentity,
InventoryFilters,
InventoryPage,
OpenWorkspaceResult,
@@ -89,6 +94,38 @@ export async function createSourceSnapshot(
return invoke('create_source_snapshot', { root, workspace, onEvent });
}
+export async function probeFormatTool(
+ siegfriedPath: string | null = null,
+): Promise {
+ return invoke('probe_format_tool', { siegfriedPath });
+}
+
+export async function identifyFormats(
+ request: FormatIdentifyRequest,
+ onProgress: (event: ProgressEvent) => void,
+): Promise {
+ const onEvent = new Channel();
+ onEvent.onmessage = onProgress;
+ return invoke('identify_formats', { request, onEvent });
+}
+
+export async function getFormatSummary(): Promise {
+ return invoke('format_summary');
+}
+
+export async function getFormatPage(
+ search: string | null,
+ state: FormatState | null,
+ puid: string | null,
+ mismatchOnly: boolean,
+ afterSha256: string | null,
+ limit: number,
+): Promise {
+ return invoke('format_page', {
+ request: { search, state, puid, mismatchOnly, afterSha256, limit },
+ });
+}
+
export function normalizeError(error: unknown): ErrorReport {
if (typeof error === 'object' && error !== null && 'message' in error) {
const candidate = error as Partial;
diff --git a/apps/desktop/src/styles.css b/apps/desktop/src/styles.css
index dbeb77e..9259843 100644
--- a/apps/desktop/src/styles.css
+++ b/apps/desktop/src/styles.css
@@ -1872,3 +1872,288 @@ button:disabled {
.pipeline-metrics { grid-template-columns: repeat(3, minmax(0, 1fr)); }
.findings-overview { grid-template-columns: repeat(3, minmax(0, 1fr)); }
}
+
+/* Exact format identification */
+.format-hero {
+ display: grid;
+ grid-template-columns: minmax(0, 1.2fr) minmax(360px, 0.8fr);
+ gap: 22px;
+ padding: 26px;
+ border: 1px solid #1a3e49;
+ border-radius: 18px;
+ background:
+ radial-gradient(circle at 90% 0, rgba(43, 212, 170, 0.12), transparent 38%),
+ linear-gradient(135deg, rgba(14, 40, 50, 0.95), rgba(7, 23, 32, 0.96));
+}
+
+.format-hero h2 {
+ margin: 7px 0 10px;
+ font-size: 31px;
+ letter-spacing: -0.045em;
+}
+
+.format-hero p {
+ max-width: 740px;
+ color: #87a1aa;
+ line-height: 1.65;
+ font-size: 13px;
+}
+
+.tool-card {
+ display: grid;
+ gap: 12px;
+ padding: 18px;
+ border: 1px solid #24515a;
+ border-radius: 14px;
+ background: rgba(5, 19, 27, 0.74);
+}
+
+.tool-card > div:not(.button-row) {
+ display: grid;
+ grid-template-columns: 120px minmax(0, 1fr);
+ gap: 12px;
+ align-items: center;
+}
+
+.tool-card span,
+.tool-card small {
+ color: #718d97;
+ font-size: 10px;
+ text-transform: uppercase;
+ letter-spacing: 0.08em;
+}
+
+.tool-card strong,
+.tool-card code {
+ overflow: hidden;
+ text-overflow: ellipsis;
+ white-space: nowrap;
+ font-size: 12px;
+}
+
+.format-write-warning {
+ color: #e9b765 !important;
+ text-transform: none !important;
+ letter-spacing: 0 !important;
+}
+
+.format-metrics-grid {
+ display: grid;
+ grid-template-columns: repeat(8, minmax(0, 1fr));
+ gap: 9px;
+ margin: 13px 0;
+}
+
+.format-metric {
+ min-width: 0;
+ padding: 14px;
+ border: 1px solid #173641;
+ border-radius: 12px;
+ background: #081923;
+}
+
+.format-metric span {
+ display: block;
+ min-height: 25px;
+ color: #6c8994;
+ font-size: 9px;
+ line-height: 1.3;
+ text-transform: uppercase;
+ letter-spacing: 0.06em;
+}
+
+.format-metric strong {
+ display: block;
+ margin-top: 7px;
+ font-size: 20px;
+ letter-spacing: -0.04em;
+}
+
+.metric-good strong { color: #47ddb9; }
+.metric-warn strong { color: #e7b35d; }
+.metric-error strong { color: #e7798a; }
+
+.format-progress-panel {
+ margin-bottom: 13px;
+ padding: 18px;
+ border: 1px solid #1c4b47;
+ border-radius: 14px;
+ background: linear-gradient(90deg, rgba(25, 106, 87, 0.14), rgba(7, 24, 32, 0.85));
+}
+
+.format-progress-heading {
+ display: flex;
+ justify-content: space-between;
+ align-items: end;
+ margin-bottom: 12px;
+}
+
+.format-progress-heading h3 {
+ margin: 5px 0 0;
+}
+
+.format-progress-heading > strong {
+ color: #52ddbd;
+ font-size: 23px;
+}
+
+.format-progress-panel dl {
+ display: grid;
+ grid-template-columns: repeat(4, 1fr);
+ gap: 14px;
+ margin: 13px 0 0;
+}
+
+.format-progress-panel dl div {
+ display: grid;
+ gap: 4px;
+}
+
+.format-progress-panel dt {
+ color: #65828d;
+ font-size: 9px;
+ text-transform: uppercase;
+}
+
+.format-progress-panel dd {
+ margin: 0;
+ font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
+ font-size: 11px;
+}
+
+.format-browser {
+ border: 1px solid #173641;
+ border-radius: 14px;
+ background: #071821;
+ overflow: hidden;
+}
+
+.format-filterbar {
+ display: grid;
+ grid-template-columns: minmax(240px, 1fr) 170px 170px auto auto;
+ gap: 9px;
+ padding: 14px;
+ border-bottom: 1px solid #173641;
+}
+
+.format-filterbar input,
+.format-filterbar select {
+ min-width: 0;
+ height: 40px;
+ padding: 0 11px;
+ border: 1px solid #254653;
+ border-radius: 9px;
+ color: #d7e5ea;
+ background: #091d27;
+ outline: none;
+}
+
+.format-filterbar input:focus,
+.format-filterbar select:focus {
+ border-color: #37c9a8;
+ box-shadow: 0 0 0 3px rgba(55, 201, 168, 0.08);
+}
+
+.checkbox-filter {
+ display: flex;
+ align-items: center;
+ gap: 8px;
+ padding: 0 8px;
+ color: #86a0aa;
+ font-size: 11px;
+ white-space: nowrap;
+}
+
+.checkbox-filter input { accent-color: #32cfa9; }
+
+.format-table-head,
+.format-row {
+ display: grid;
+ grid-template-columns: minmax(280px, 1.35fr) 140px minmax(240px, 1fr) 110px 100px;
+ gap: 14px;
+ align-items: center;
+}
+
+.format-table-head {
+ min-height: 38px;
+ padding: 0 16px;
+ color: #587680;
+ background: #0b202a;
+ font-size: 9px;
+ letter-spacing: 0.08em;
+ text-transform: uppercase;
+}
+
+.format-row {
+ width: 100%;
+ min-height: 62px;
+ padding: 10px 16px;
+ border: 0;
+ border-top: 1px solid #122f39;
+ color: inherit;
+ background: transparent;
+ text-align: left;
+}
+
+.format-row:hover { background: rgba(37, 105, 105, 0.08); }
+
+.format-file,
+.format-identity {
+ min-width: 0;
+ display: grid;
+ gap: 4px;
+}
+
+.format-file strong,
+.format-identity strong {
+ overflow: hidden;
+ text-overflow: ellipsis;
+ white-space: nowrap;
+ font-size: 12px;
+}
+
+.format-file span,
+.format-identity span {
+ overflow: hidden;
+ text-overflow: ellipsis;
+ white-space: nowrap;
+ color: #67848f;
+ font-size: 10px;
+}
+
+.format-row code {
+ color: #74d9c0;
+ font-size: 10px;
+}
+
+.mismatch-yes,
+.mismatch-no {
+ font-size: 10px;
+ font-weight: 650;
+}
+
+.mismatch-yes { color: #e9b15d; }
+.mismatch-no { color: #62847e; }
+
+.state-identified { border-color: #287c69; color: #50d7b5; background: rgba(38, 142, 116, 0.13); }
+.state-unknown,
+.state-ambiguous { border-color: #755d32; color: #e4b66a; background: rgba(139, 100, 38, 0.14); }
+.state-tool_error { border-color: #763d4b; color: #eb8495; background: rgba(141, 50, 69, 0.14); }
+.state-empty,
+.state-skipped_unavailable,
+.state-uninspected { border-color: #405762; color: #8ca0a8; background: rgba(83, 105, 113, 0.12); }
+
+.table-empty {
+ padding: 42px;
+ color: #6b8791;
+ text-align: center;
+ font-size: 12px;
+}
+
+@media (max-width: 1380px) {
+ .format-metrics-grid { grid-template-columns: repeat(4, minmax(0, 1fr)); }
+ .format-filterbar { grid-template-columns: minmax(240px, 1fr) 150px 150px; }
+ .checkbox-filter { grid-column: span 1; }
+}
+
+.mismatch-unchecked { color: #78909c; }
diff --git a/apps/desktop/src/types.ts b/apps/desktop/src/types.ts
index 49f23ee..03cfacf 100644
--- a/apps/desktop/src/types.ts
+++ b/apps/desktop/src/types.ts
@@ -421,3 +421,83 @@ export interface FindingDetail {
object: ContentObjectDetail | null;
review: FindingReviewHistory;
}
+
+export type FormatState =
+ | 'uninspected'
+ | 'identified'
+ | 'unknown'
+ | 'ambiguous'
+ | 'empty'
+ | 'skipped_unavailable'
+ | 'tool_error';
+
+export interface FormatIdentifierIdentity {
+ name: string;
+ details: string;
+}
+
+export interface FormatToolIdentity {
+ toolName: string;
+ toolVersion: string;
+ executablePath: string;
+ executableSha256: string;
+ signaturePath: string;
+ signatureSha256: string | null;
+ signatureVersion: string;
+ signatureCreated: string | null;
+ identifiers: FormatIdentifierIdentity[];
+ probedAt: string;
+}
+
+export interface FormatSummary {
+ baselineRunId: string;
+ latestFormatRunId: string | null;
+ latestRunState: string | null;
+ totalObjects: number;
+ eligibleObjects: number;
+ completedObjects: number;
+ totalBytes: number;
+ completedBytes: number;
+ identified: number;
+ unknown: number;
+ ambiguous: number;
+ empty: number;
+ skippedUnavailable: number;
+ toolErrors: number;
+ extensionMismatches: number;
+ distinctPuids: number;
+ tool: FormatToolIdentity | null;
+ startedAt: string | null;
+ finishedAt: string | null;
+}
+
+export interface FormatObjectRow {
+ contentObjectId: string;
+ sha256: string;
+ primaryFilename: string;
+ expectedSizeBytes: number;
+ sourceMimeType: string;
+ state: FormatState;
+ primaryIdentifier: string | null;
+ primaryFormatName: string | null;
+ primaryFormatVersion: string | null;
+ primaryMimeType: string | null;
+ matchCount: number;
+ extensionChecked: boolean;
+ extensionMismatch: boolean;
+}
+
+export interface FormatPage {
+ items: FormatObjectRow[];
+ totalFiltered: number;
+ nextAfterSha256: string | null;
+ hasMore: boolean;
+}
+
+export interface FormatIdentifyRequest {
+ siegfriedPath?: string | null;
+ batchSize: number;
+ workers: number;
+ timeoutSeconds: number;
+ resume: boolean;
+}
diff --git a/crates/profiler-core/src/error.rs b/crates/profiler-core/src/error.rs
index b897e5d..66c997c 100644
--- a/crates/profiler-core/src/error.rs
+++ b/crates/profiler-core/src/error.rs
@@ -37,6 +37,12 @@ pub enum ErrorCode {
ReviewHistoryIntegrityFailure,
ReviewWriteNotAllowed,
SanitizedExportFailed,
+ FormatToolNotFound,
+ FormatToolIncompatible,
+ FormatRunNotFound,
+ FormatRunFailed,
+ FormatRunAlreadyActive,
+ FormatOutputInvalid,
Internal,
}
diff --git a/crates/profiler-core/src/format.rs b/crates/profiler-core/src/format.rs
new file mode 100644
index 0000000..aca771a
--- /dev/null
+++ b/crates/profiler-core/src/format.rs
@@ -0,0 +1,243 @@
+use std::path::PathBuf;
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum FormatState {
+ Uninspected,
+ Identified,
+ Unknown,
+ Ambiguous,
+ Empty,
+ SkippedUnavailable,
+ ToolError,
+}
+
+impl FormatState {
+ pub const fn as_str(self) -> &'static str {
+ match self {
+ Self::Uninspected => "uninspected",
+ Self::Identified => "identified",
+ Self::Unknown => "unknown",
+ Self::Ambiguous => "ambiguous",
+ Self::Empty => "empty",
+ Self::SkippedUnavailable => "skipped_unavailable",
+ Self::ToolError => "tool_error",
+ }
+ }
+}
+
+impl std::fmt::Display for FormatState {
+ fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ formatter.write_str(self.as_str())
+ }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct FormatToolIdentity {
+ pub tool_name: String,
+ pub tool_version: String,
+ pub executable_path: String,
+ pub executable_sha256: String,
+ pub signature_path: String,
+ pub signature_sha256: Option,
+ pub signature_version: String,
+ pub signature_created: Option,
+ pub identifiers: Vec,
+ pub probed_at: String,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct FormatIdentifierIdentity {
+ pub name: String,
+ pub details: String,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct FormatOptions {
+ pub batch_size: u32,
+ pub workers: u32,
+ pub timeout_seconds: u64,
+ pub resume: bool,
+}
+
+impl Default for FormatOptions {
+ fn default() -> Self {
+ Self {
+ batch_size: 2_048,
+ workers: 0,
+ timeout_seconds: 900,
+ resume: true,
+ }
+ }
+}
+
+#[derive(Debug, Clone)]
+pub struct FormatIdentificationRequest {
+ pub baseline_run_id: String,
+ pub workspace_root: PathBuf,
+ pub profiler_database: PathBuf,
+ pub archive_root: PathBuf,
+ pub siegfried_path: Option,
+ pub signature_path: Option,
+ pub options: FormatOptions,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct FormatWorkItem {
+ pub content_object_id: String,
+ pub sha256: String,
+ pub expected_size_bytes: u64,
+ pub source_mime_type: String,
+ pub canonical_path_display: String,
+ pub availability_state: String,
+ pub preferred_extension: Option,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct FormatRunRegistration {
+ pub format_run_id: String,
+ pub collection_id: String,
+ pub total_objects: u64,
+ pub eligible_objects: u64,
+ pub total_bytes: u64,
+ pub resume_after_sha256: Option,
+ pub completed_objects: u64,
+ pub completed_bytes: u64,
+ pub checkpoint_sequence: u64,
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct FormatRunStartRequest<'a> {
+ pub baseline_run_id: &'a str,
+ pub tool: &'a FormatToolIdentity,
+ pub configuration_fingerprint: &'a str,
+ pub batch_size: u32,
+ pub worker_count: u32,
+ pub timeout_seconds: u64,
+ pub resume: bool,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct FormatMatch {
+ pub namespace: String,
+ pub identifier: String,
+ pub format_name: String,
+ pub format_version: String,
+ pub mime_type: String,
+ pub format_class: Option,
+ pub basis: String,
+ pub warning: String,
+ pub is_primary: bool,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct FormatObservation {
+ pub content_object_id: String,
+ pub sha256: String,
+ pub state: FormatState,
+ pub source_mime_type: String,
+ pub preferred_extension: Option,
+ pub staging_mode: String,
+ pub primary_identifier: Option,
+ pub primary_format_name: Option,
+ pub primary_format_version: Option,
+ pub primary_mime_type: Option,
+ pub match_count: u64,
+ pub extension_checked: bool,
+ pub extension_mismatch: bool,
+ pub error_code: Option,
+ pub error_message: Option,
+ pub matches: Vec,
+ pub observed_at: String,
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct FormatSummary {
+ pub baseline_run_id: String,
+ pub latest_format_run_id: Option,
+ pub latest_run_state: Option,
+ pub total_objects: u64,
+ pub eligible_objects: u64,
+ pub completed_objects: u64,
+ pub total_bytes: u64,
+ pub completed_bytes: u64,
+ pub identified: u64,
+ pub unknown: u64,
+ pub ambiguous: u64,
+ pub empty: u64,
+ pub skipped_unavailable: u64,
+ pub tool_errors: u64,
+ pub extension_mismatches: u64,
+ pub distinct_puids: u64,
+ pub tool: Option,
+ pub started_at: Option,
+ pub finished_at: Option,
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct FormatFilters {
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub search: Option,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub state: Option,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub puid: Option,
+ #[serde(default)]
+ pub mismatch_only: bool,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct FormatPageRequest {
+ pub baseline_run_id: String,
+ #[serde(default)]
+ pub filters: FormatFilters,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub after_sha256: Option,
+ pub limit: u32,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct FormatObjectRow {
+ pub content_object_id: String,
+ pub sha256: String,
+ pub primary_filename: String,
+ pub expected_size_bytes: u64,
+ pub source_mime_type: String,
+ pub state: FormatState,
+ pub primary_identifier: Option,
+ pub primary_format_name: Option,
+ pub primary_format_version: Option,
+ pub primary_mime_type: Option,
+ pub match_count: u64,
+ pub extension_checked: bool,
+ pub extension_mismatch: bool,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct FormatPage {
+ pub items: Vec,
+ pub total_filtered: u64,
+ pub next_after_sha256: Option,
+ pub has_more: bool,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct FormatIdentificationResult {
+ pub format_run_id: String,
+ pub baseline_run_id: String,
+ pub configuration_fingerprint: String,
+ pub summary: FormatSummary,
+}
diff --git a/crates/profiler-core/src/lib.rs b/crates/profiler-core/src/lib.rs
index 4db26e1..70dc9a7 100644
--- a/crates/profiler-core/src/lib.rs
+++ b/crates/profiler-core/src/lib.rs
@@ -2,6 +2,7 @@ mod adapter;
mod error;
mod explorer;
mod file_stat;
+mod format;
mod inventory;
mod progress;
mod run;
@@ -16,6 +17,7 @@ pub use file_stat::{
FileStatResult, FileStatRunner, FileStatStore, FileStatSummary, FileStatWorkItem,
PhysicalObjectResolver, SizeState,
};
+pub use format::*;
pub use inventory::{
InventoryBatch, InventoryCheckpoint, InventoryOptions, InventoryRequest, InventoryResult,
InventorySink, InventorySource, InventorySummary, InventoryTable, SourceBlobRecord,
diff --git a/crates/profiler-core/src/workspace.rs b/crates/profiler-core/src/workspace.rs
index 1d2b6ff..ac6da9b 100644
--- a/crates/profiler-core/src/workspace.rs
+++ b/crates/profiler-core/src/workspace.rs
@@ -348,6 +348,33 @@ pub struct SanitizedFindingRow {
pub reviewed_at: Option,
}
+#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct SanitizedFormatSummary {
+ pub latest_format_run_id: Option,
+ pub latest_run_state: Option,
+ pub total_objects: u64,
+ pub eligible_objects: u64,
+ pub completed_objects: u64,
+ pub total_bytes: u64,
+ pub completed_bytes: u64,
+ pub identified: u64,
+ pub unknown: u64,
+ pub ambiguous: u64,
+ pub empty: u64,
+ pub skipped_unavailable: u64,
+ pub tool_errors: u64,
+ pub extension_mismatches: u64,
+ pub distinct_puids: u64,
+ pub tool_name: Option,
+ pub tool_version: Option,
+ pub executable_sha256: Option,
+ pub signature_version: Option,
+ pub signature_sha256: Option,
+ pub started_at: Option,
+ pub finished_at: Option,
+}
+
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct SanitizedRunSummary {
@@ -361,6 +388,7 @@ pub struct SanitizedRunSummary {
pub inventory: InventorySummary,
pub findings: FindingsSummary,
pub review: ReviewSummary,
+ pub exact_formats: SanitizedFormatSummary,
pub findings_by_code: BTreeMap,
pub review_by_status: BTreeMap,
}
diff --git a/crates/profiler-engine/Cargo.toml b/crates/profiler-engine/Cargo.toml
index bb0e8f2..5bddc23 100644
--- a/crates/profiler-engine/Cargo.toml
+++ b/crates/profiler-engine/Cargo.toml
@@ -13,6 +13,7 @@ time.workspace = true
hex.workspace = true
profiler-adapter-mailvault = { path = "../profiler-adapter-mailvault" }
profiler-core = { path = "../profiler-core" }
+profiler-format-siegfried = { path = "../profiler-format-siegfried" }
profiler-storage-sqlite = { path = "../profiler-storage-sqlite" }
rayon.workspace = true
serde.workspace = true
diff --git a/crates/profiler-engine/src/format.rs b/crates/profiler-engine/src/format.rs
new file mode 100644
index 0000000..dfba509
--- /dev/null
+++ b/crates/profiler-engine/src/format.rs
@@ -0,0 +1,604 @@
+use std::{
+ fs::{self, File, OpenOptions},
+ io::{Seek, SeekFrom, Write},
+ path::{Path, PathBuf},
+ time::{Duration, Instant},
+};
+
+use fs2::FileExt;
+
+use profiler_core::{
+ FormatIdentificationRequest, FormatIdentificationResult, FormatObservation, FormatPage,
+ FormatPageRequest, FormatRunRegistration, FormatRunStartRequest, FormatSummary,
+ FormatToolIdentity, FormatWorkItem, ProfilerError, ProfilerResult, ProgressEvent, ProgressSink,
+ ProgressUnit, RunStage, StageState,
+};
+use profiler_format_siegfried::{
+ BatchWorkspace, ResolvedFormatInput, SiegfriedOptions, SiegfriedRunner, skipped_observation,
+};
+use profiler_storage_sqlite::ProfilerStore;
+use sha2::{Digest, Sha256};
+
+#[derive(Debug, Default, Clone, Copy)]
+pub struct ExactFormatEngine;
+
+impl ExactFormatEngine {
+ pub fn probe_tool(
+ &self,
+ executable: Option,
+ signature: Option,
+ workers: u32,
+ ) -> ProfilerResult {
+ let options = SiegfriedOptions {
+ executable,
+ signature,
+ workers: resolve_worker_count(workers),
+ timeout: Duration::from_secs(30),
+ };
+ let runner = SiegfriedRunner::probe(&options)?;
+ Ok(runner.identity().clone())
+ }
+
+ pub fn identify(
+ &self,
+ request: &FormatIdentificationRequest,
+ progress: &dyn ProgressSink,
+ ) -> ProfilerResult {
+ validate_request(request)?;
+ let _format_lock = FormatRunLock::acquire(&request.workspace_root)?;
+ let mut prepared = prepare_format_run(request)?;
+ let started = Instant::now();
+
+ let execution = execute_format_run(
+ request,
+ progress,
+ &prepared.runner,
+ &mut prepared.store,
+ &prepared.registration,
+ prepared.workers,
+ started,
+ );
+ if let Err(error) = &execution {
+ let _ = prepared
+ .store
+ .fail_format_run(&prepared.registration.format_run_id, error);
+ }
+
+ let completed = execution?;
+ emit_completed_progress(
+ progress,
+ &prepared.registration.format_run_id,
+ completed.sequence,
+ &completed.summary,
+ started,
+ )?;
+
+ Ok(FormatIdentificationResult {
+ format_run_id: prepared.registration.format_run_id,
+ baseline_run_id: request.baseline_run_id.clone(),
+ configuration_fingerprint: prepared.configuration_fingerprint,
+ summary: completed.summary,
+ })
+ }
+
+ pub fn summary(
+ &self,
+ profiler_database: &Path,
+ baseline_run_id: &str,
+ ) -> ProfilerResult {
+ ProfilerStore::open_read_only(profiler_database)?.format_summary(baseline_run_id)
+ }
+
+ pub fn page(
+ &self,
+ profiler_database: &Path,
+ request: &FormatPageRequest,
+ ) -> ProfilerResult {
+ ProfilerStore::open_read_only(profiler_database)?.format_page(request)
+ }
+}
+
+struct PreparedFormatRun {
+ workers: u32,
+ runner: SiegfriedRunner,
+ store: ProfilerStore,
+ registration: FormatRunRegistration,
+ configuration_fingerprint: String,
+}
+
+struct CompletedFormatRun {
+ summary: FormatSummary,
+ sequence: u64,
+}
+
+struct FormatProgressState {
+ started: Instant,
+ last_progress_time: Instant,
+ last_completed: u64,
+ workers: u32,
+}
+
+struct ProcessedFormatBatch {
+ checkpoint: String,
+ summary: FormatSummary,
+}
+
+fn prepare_format_run(request: &FormatIdentificationRequest) -> ProfilerResult {
+ let workers = resolve_worker_count(request.options.workers);
+ let runner_options = SiegfriedOptions {
+ executable: request.siegfried_path.clone(),
+ signature: request.signature_path.clone(),
+ workers,
+ timeout: Duration::from_secs(request.options.timeout_seconds),
+ };
+ let runner = SiegfriedRunner::probe(&runner_options)?;
+ let configuration_fingerprint = configuration_fingerprint(request, runner.identity(), workers)?;
+ let mut store = ProfilerStore::open_existing(&request.profiler_database)?;
+ let registration = store.begin_format_run(&FormatRunStartRequest {
+ baseline_run_id: request.baseline_run_id.as_str(),
+ tool: runner.identity(),
+ configuration_fingerprint: configuration_fingerprint.as_str(),
+ batch_size: request.options.batch_size,
+ worker_count: workers,
+ timeout_seconds: request.options.timeout_seconds,
+ resume: request.options.resume,
+ })?;
+
+ Ok(PreparedFormatRun {
+ workers,
+ runner,
+ store,
+ registration,
+ configuration_fingerprint,
+ })
+}
+
+fn execute_format_run(
+ request: &FormatIdentificationRequest,
+ progress: &dyn ProgressSink,
+ runner: &SiegfriedRunner,
+ store: &mut ProfilerStore,
+ registration: &FormatRunRegistration,
+ workers: u32,
+ started: Instant,
+) -> ProfilerResult {
+ let mut after_sha256 = registration.resume_after_sha256.clone();
+ let mut sequence = registration.checkpoint_sequence;
+ let mut progress_state = FormatProgressState {
+ started,
+ last_progress_time: Instant::now(),
+ last_completed: registration.completed_objects,
+ workers,
+ };
+
+ loop {
+ let work = store.load_format_work_batch(
+ ®istration.collection_id,
+ after_sha256.as_deref(),
+ request.options.batch_size,
+ )?;
+ if work.is_empty() {
+ break;
+ }
+
+ sequence = sequence.saturating_add(1);
+ let processed =
+ process_format_batch(request, runner, store, registration, &work, sequence)?;
+ emit_progress(
+ progress,
+ ®istration.format_run_id,
+ sequence,
+ &processed.summary,
+ &mut progress_state,
+ )?;
+ after_sha256 = Some(processed.checkpoint);
+
+ if work.len() < request.options.batch_size as usize {
+ break;
+ }
+ }
+
+ let summary =
+ store.complete_format_run(®istration.format_run_id, &request.baseline_run_id)?;
+ Ok(CompletedFormatRun { summary, sequence })
+}
+
+fn process_format_batch(
+ request: &FormatIdentificationRequest,
+ runner: &SiegfriedRunner,
+ store: &mut ProfilerStore,
+ registration: &FormatRunRegistration,
+ work: &[FormatWorkItem],
+ sequence: u64,
+) -> ProfilerResult {
+ let checkpoint = work
+ .last()
+ .map(|item| item.sha256.clone())
+ .ok_or_else(|| ProfilerError::Internal("format batch was unexpectedly empty".into()))?;
+ let (mut observations, eligible) = resolve_batch_inputs(request, work);
+
+ if !eligible.is_empty() {
+ let staging = BatchWorkspace::new(
+ &request.workspace_root,
+ ®istration.format_run_id,
+ sequence,
+ )?;
+ observations.extend(runner.identify_batch(&eligible, &staging)?);
+ }
+
+ observations.sort_by(|left, right| left.sha256.cmp(&right.sha256));
+ let summary = store.commit_format_observations(
+ ®istration.format_run_id,
+ &request.baseline_run_id,
+ &observations,
+ &checkpoint,
+ sequence,
+ )?;
+
+ Ok(ProcessedFormatBatch {
+ checkpoint,
+ summary,
+ })
+}
+
+fn resolve_batch_inputs(
+ request: &FormatIdentificationRequest,
+ work: &[FormatWorkItem],
+) -> (Vec, Vec) {
+ let mut observations = Vec::with_capacity(work.len());
+ let mut eligible = Vec::new();
+
+ for item in work {
+ if item.expected_size_bytes == 0 || item.availability_state != "available" {
+ observations.push(skipped_observation(item));
+ continue;
+ }
+
+ match resolve_object_path(&request.archive_root, &item.canonical_path_display) {
+ Ok(source_path) => eligible.push(ResolvedFormatInput {
+ item: item.clone(),
+ source_path,
+ }),
+ Err(error) => observations.push(path_resolution_error(item, &error)),
+ }
+ }
+
+ (observations, eligible)
+}
+
+fn path_resolution_error(item: &FormatWorkItem, error: &ProfilerError) -> FormatObservation {
+ FormatObservation {
+ content_object_id: item.content_object_id.clone(),
+ sha256: item.sha256.clone(),
+ state: profiler_core::FormatState::ToolError,
+ source_mime_type: item.source_mime_type.clone(),
+ preferred_extension: item.preferred_extension.clone(),
+ staging_mode: "path_resolution_failed".into(),
+ primary_identifier: None,
+ primary_format_name: None,
+ primary_format_version: None,
+ primary_mime_type: None,
+ match_count: 0,
+ extension_checked: false,
+ extension_mismatch: false,
+ error_code: Some("SOURCE_PATH_UNAVAILABLE".into()),
+ error_message: Some(error.to_string()),
+ matches: Vec::new(),
+ observed_at: now_text(),
+ }
+}
+
+#[derive(Debug)]
+struct FormatRunLock {
+ file: File,
+}
+
+impl FormatRunLock {
+ fn acquire(workspace_root: &Path) -> ProfilerResult {
+ fs::create_dir_all(workspace_root).map_err(|source| ProfilerError::Io {
+ operation: "creating exact-format workspace root",
+ path: workspace_root.to_path_buf(),
+ source,
+ })?;
+ let path = workspace_root.join(".mailvault-profiler.format.lock");
+ let mut file = OpenOptions::new()
+ .read(true)
+ .write(true)
+ .create(true)
+ .truncate(false)
+ .open(&path)
+ .map_err(|source| ProfilerError::Io {
+ operation: "opening exact-format lock",
+ path: path.clone(),
+ source,
+ })?;
+ file.try_lock_exclusive().map_err(|source| {
+ if source.kind() == fs2::lock_contended_error().kind() {
+ ProfilerError::contract(
+ profiler_core::ErrorCode::FormatRunAlreadyActive,
+ "another exact-format identification process holds the workspace lock",
+ true,
+ )
+ } else {
+ ProfilerError::Io {
+ operation: "acquiring exact-format lock",
+ path: path.clone(),
+ source,
+ }
+ }
+ })?;
+ file.set_len(0).map_err(|source| ProfilerError::Io {
+ operation: "resetting exact-format lock metadata",
+ path: path.clone(),
+ source,
+ })?;
+ file.seek(SeekFrom::Start(0))
+ .map_err(|source| ProfilerError::Io {
+ operation: "seeking exact-format lock metadata",
+ path: path.clone(),
+ source,
+ })?;
+ let metadata = serde_json::to_vec(&serde_json::json!({
+ "pid": std::process::id(),
+ "startedAt": now_text(),
+ }))
+ .map_err(|error| {
+ ProfilerError::Internal(format!("serializing exact-format lock: {error}"))
+ })?;
+ file.write_all(&metadata)
+ .map_err(|source| ProfilerError::Io {
+ operation: "writing exact-format lock metadata",
+ path: path.clone(),
+ source,
+ })?;
+ file.sync_all().map_err(|source| ProfilerError::Io {
+ operation: "syncing exact-format lock metadata",
+ path,
+ source,
+ })?;
+ Ok(Self { file })
+ }
+}
+
+impl Drop for FormatRunLock {
+ fn drop(&mut self) {
+ let _ = FileExt::unlock(&self.file);
+ }
+}
+
+fn validate_request(request: &FormatIdentificationRequest) -> ProfilerResult<()> {
+ if request.baseline_run_id.trim().is_empty() {
+ return Err(ProfilerError::InvalidArgument(
+ "baseline run id cannot be empty".into(),
+ ));
+ }
+ if request.options.batch_size == 0 || request.options.batch_size > 10_000 {
+ return Err(ProfilerError::InvalidArgument(
+ "format batch size must be between 1 and 10000".into(),
+ ));
+ }
+ if request.options.timeout_seconds < 30 {
+ return Err(ProfilerError::InvalidArgument(
+ "format timeout must be at least 30 seconds".into(),
+ ));
+ }
+ if !request.profiler_database.is_file() {
+ return Err(ProfilerError::InvalidPath {
+ message: "profiler database does not exist".into(),
+ path: request.profiler_database.clone(),
+ });
+ }
+ if !request.archive_root.is_dir() {
+ return Err(ProfilerError::InvalidPath {
+ message: "MailVault archive root does not exist".into(),
+ path: request.archive_root.clone(),
+ });
+ }
+ Ok(())
+}
+
+fn resolve_worker_count(configured: u32) -> u32 {
+ if configured > 0 {
+ return configured.clamp(1, 16);
+ }
+ std::thread::available_parallelism()
+ .map_or(2, |value| u32::try_from(value.get()).unwrap_or(4))
+ .clamp(1, 4)
+}
+
+fn resolve_object_path(archive_root: &Path, locator: &str) -> ProfilerResult {
+ if locator.contains('\0') {
+ return Err(ProfilerError::InvalidPath {
+ message: "content locator contains NUL".into(),
+ path: PathBuf::from(locator),
+ });
+ }
+ let root = fs::canonicalize(archive_root).map_err(|source| ProfilerError::Io {
+ operation: "canonicalizing MailVault root for exact-format identification",
+ path: archive_root.to_path_buf(),
+ source,
+ })?;
+ let candidate = root.join(locator.replace('/', std::path::MAIN_SEPARATOR_STR));
+ let resolved = fs::canonicalize(&candidate).map_err(|source| ProfilerError::Io {
+ operation: "resolving MailVault content object for exact-format identification",
+ path: candidate,
+ source,
+ })?;
+ if !resolved.starts_with(&root) || !resolved.is_file() {
+ return Err(ProfilerError::InvalidPath {
+ message: "content object escapes archive root or is not a regular file".into(),
+ path: resolved,
+ });
+ }
+ Ok(resolved)
+}
+
+fn configuration_fingerprint(
+ request: &FormatIdentificationRequest,
+ tool: &FormatToolIdentity,
+ workers: u32,
+) -> ProfilerResult {
+ let payload = serde_json::to_vec(&serde_json::json!({
+ "contract": "mailvault-profiler-exact-format-v1",
+ "toolName": tool.tool_name,
+ "toolVersion": tool.tool_version,
+ "executableSha256": tool.executable_sha256,
+ "signatureVersion": tool.signature_version,
+ "signatureSha256": tool.signature_sha256,
+ "extensionEvidence": "symlink-alias-or-unchecked",
+ "batchSize": request.options.batch_size,
+ "workers": workers,
+ "timeoutSeconds": request.options.timeout_seconds,
+ "containerExpansion": false,
+ }))
+ .map_err(|error| {
+ ProfilerError::Internal(format!("serializing exact-format configuration: {error}"))
+ })?;
+ let mut digest = Sha256::new();
+ digest.update(b"mailvault-profiler-exact-format-v1\0");
+ digest.update(payload);
+ Ok(hex::encode(digest.finalize()))
+}
+
+fn emit_progress(
+ progress: &dyn ProgressSink,
+ format_run_id: &str,
+ sequence: u64,
+ summary: &FormatSummary,
+ state: &mut FormatProgressState,
+) -> ProfilerResult<()> {
+ let elapsed_since_last = state.last_progress_time.elapsed();
+ let completed_since_last = summary
+ .completed_objects
+ .saturating_sub(state.last_completed);
+ let instant = throughput_per_second(completed_since_last, elapsed_since_last);
+ let smoothed = throughput_per_second(summary.completed_objects, state.started.elapsed());
+ let eta_ms = estimate_eta_ms(
+ state.started.elapsed(),
+ summary.completed_objects,
+ summary.total_objects,
+ );
+
+ progress.send(ProgressEvent {
+ run_id: format_run_id.into(),
+ sequence,
+ stage: RunStage::FormatIdentification,
+ stage_state: StageState::Running,
+ unit: ProgressUnit::Objects,
+ completed_items: summary.completed_objects,
+ total_items: Some(summary.total_objects),
+ completed_bytes: summary.completed_bytes,
+ total_bytes: Some(summary.total_bytes),
+ elapsed_ms: elapsed_ms(state.started),
+ instant_throughput: instant,
+ smoothed_throughput: smoothed,
+ eta_ms,
+ active_workers: state.workers,
+ queue_depth: 0,
+ warnings: summary.unknown + summary.ambiguous + summary.extension_mismatches,
+ errors: summary.tool_errors,
+ current_object_display: None,
+ checkpoint_sequence: sequence,
+ })?;
+ state.last_progress_time = Instant::now();
+ state.last_completed = summary.completed_objects;
+ Ok(())
+}
+
+fn emit_completed_progress(
+ progress: &dyn ProgressSink,
+ format_run_id: &str,
+ sequence: u64,
+ summary: &FormatSummary,
+ started: Instant,
+) -> ProfilerResult<()> {
+ progress.send(ProgressEvent {
+ run_id: format_run_id.into(),
+ sequence: sequence.saturating_add(1),
+ stage: RunStage::FormatIdentification,
+ stage_state: StageState::Completed,
+ unit: ProgressUnit::Objects,
+ completed_items: summary.completed_objects,
+ total_items: Some(summary.total_objects),
+ completed_bytes: summary.completed_bytes,
+ total_bytes: Some(summary.total_bytes),
+ elapsed_ms: elapsed_ms(started),
+ instant_throughput: None,
+ smoothed_throughput: None,
+ eta_ms: Some(0),
+ active_workers: 0,
+ queue_depth: 0,
+ warnings: summary.unknown + summary.ambiguous + summary.extension_mismatches,
+ errors: summary.tool_errors,
+ current_object_display: None,
+ checkpoint_sequence: sequence.saturating_add(1),
+ })
+}
+
+fn throughput_per_second(completed: u64, elapsed: Duration) -> Option {
+ const SCALE: u128 = 1_000;
+
+ let elapsed_ms = elapsed.as_millis();
+ if completed == 0 || elapsed_ms == 0 {
+ return None;
+ }
+
+ let scaled_rate = u128::from(completed)
+ .saturating_mul(1_000)
+ .saturating_mul(SCALE)
+ / elapsed_ms;
+ let bounded_rate = u32::try_from(scaled_rate).unwrap_or(u32::MAX);
+ Some(f64::from(bounded_rate) / 1_000.0)
+}
+
+fn estimate_eta_ms(elapsed: Duration, completed: u64, total: u64) -> Option {
+ if completed < 3 {
+ return None;
+ }
+
+ let remaining = total.saturating_sub(completed);
+ if remaining == 0 {
+ return Some(0);
+ }
+
+ let elapsed_ms = elapsed.as_millis();
+ if elapsed_ms == 0 {
+ return None;
+ }
+
+ let completed_wide = u128::from(completed);
+ let numerator = elapsed_ms.saturating_mul(u128::from(remaining));
+ let rounded = numerator.saturating_add(completed_wide / 2) / completed_wide;
+ Some(u64::try_from(rounded).unwrap_or(u64::MAX))
+}
+
+fn elapsed_ms(started: Instant) -> u64 {
+ started.elapsed().as_millis().try_into().unwrap_or(u64::MAX)
+}
+
+fn now_text() -> String {
+ time::OffsetDateTime::now_utc()
+ .format(&time::format_description::well_known::Rfc3339)
+ .expect("RFC3339 formatting is infallible for OffsetDateTime")
+}
+
+#[cfg(test)]
+mod tests {
+ use super::{estimate_eta_ms, throughput_per_second};
+ use std::time::Duration;
+
+ #[test]
+ fn throughput_uses_integer_scaling_without_lossy_casts() {
+ assert_eq!(throughput_per_second(10, Duration::from_secs(2)), Some(5.0));
+ }
+
+ #[test]
+ fn eta_uses_saturating_integer_arithmetic() {
+ assert_eq!(estimate_eta_ms(Duration::from_secs(2), 4, 10), Some(3_000));
+ }
+
+ #[test]
+ fn eta_waits_for_a_minimum_sample() {
+ assert_eq!(estimate_eta_ms(Duration::from_secs(2), 2, 10), None);
+ }
+}
diff --git a/crates/profiler-engine/src/lib.rs b/crates/profiler-engine/src/lib.rs
index d4b4f17..ae543ef 100644
--- a/crates/profiler-engine/src/lib.rs
+++ b/crates/profiler-engine/src/lib.rs
@@ -1,4 +1,5 @@
mod file_stat;
+pub mod format;
pub mod workspace;
use std::{fs, path::PathBuf};
@@ -15,6 +16,8 @@ use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use uuid::Uuid;
+pub use format::ExactFormatEngine;
+
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ProfileOptions {
diff --git a/crates/profiler-format-siegfried/Cargo.toml b/crates/profiler-format-siegfried/Cargo.toml
new file mode 100644
index 0000000..55ae5a5
--- /dev/null
+++ b/crates/profiler-format-siegfried/Cargo.toml
@@ -0,0 +1,21 @@
+[package]
+name = "profiler-format-siegfried"
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+repository.workspace = true
+description = "Pinned Siegfried/PRONOM exact-format sidecar integration for MailVault Collection Profiler."
+
+[dependencies]
+hex.workspace = true
+profiler-core = { path = "../profiler-core" }
+serde.workspace = true
+serde_json.workspace = true
+sha2.workspace = true
+tempfile.workspace = true
+time.workspace = true
+tracing.workspace = true
+
+[lints]
+workspace = true
diff --git a/crates/profiler-format-siegfried/src/lib.rs b/crates/profiler-format-siegfried/src/lib.rs
new file mode 100644
index 0000000..445ab43
--- /dev/null
+++ b/crates/profiler-format-siegfried/src/lib.rs
@@ -0,0 +1,1055 @@
+use std::{
+ collections::{BTreeMap, HashMap},
+ fs::{self, File},
+ io::{BufReader, Read, Write},
+ path::{Path, PathBuf},
+ process::{Command, Stdio},
+ thread,
+ time::{Duration, Instant},
+};
+
+use profiler_core::{
+ ErrorCode, FormatIdentifierIdentity, FormatMatch, FormatObservation, FormatState,
+ FormatToolIdentity, FormatWorkItem, ProfilerError, ProfilerResult,
+};
+use serde::Deserialize;
+use sha2::{Digest, Sha256};
+use tempfile::TempDir;
+use time::OffsetDateTime;
+use tracing::{debug, warn};
+
+pub const RECOMMENDED_SIEGFRIED_VERSION: &str = "1.11.6";
+pub const RECOMMENDED_PRONOM_VERSION: &str = "v124";
+
+#[derive(Debug, Clone)]
+pub struct SiegfriedOptions {
+ pub executable: Option,
+ pub signature: Option,
+ pub workers: u32,
+ pub timeout: Duration,
+}
+
+#[derive(Debug, Clone)]
+pub struct SiegfriedRunner {
+ executable: PathBuf,
+ signature: Option,
+ workers: u32,
+ timeout: Duration,
+ identity: FormatToolIdentity,
+}
+
+#[derive(Debug, Clone)]
+pub struct ResolvedFormatInput {
+ pub item: FormatWorkItem,
+ pub source_path: PathBuf,
+}
+
+#[derive(Debug)]
+pub struct BatchWorkspace {
+ directory: TempDir,
+}
+
+impl BatchWorkspace {
+ pub fn new(parent: &Path, format_run_id: &str, sequence: u64) -> ProfilerResult {
+ let staging_root = parent
+ .join("format-staging")
+ .join(format_run_id)
+ .join(format!("batch-{sequence:08}"));
+ fs::create_dir_all(&staging_root).map_err(|source| ProfilerError::Io {
+ operation: "creating exact-format staging directory",
+ path: staging_root.clone(),
+ source,
+ })?;
+ let directory = tempfile::Builder::new()
+ .prefix("work-")
+ .tempdir_in(&staging_root)
+ .map_err(|source| ProfilerError::Io {
+ operation: "creating exact-format batch workspace",
+ path: staging_root,
+ source,
+ })?;
+ Ok(Self { directory })
+ }
+
+ pub fn path(&self) -> &Path {
+ self.directory.path()
+ }
+}
+
+impl SiegfriedRunner {
+ pub fn probe(options: &SiegfriedOptions) -> ProfilerResult {
+ let executable = discover_executable(options.executable.as_deref())?;
+ let signature = options
+ .signature
+ .clone()
+ .or_else(|| discover_signature(&executable))
+ .ok_or_else(|| {
+ ProfilerError::contract(
+ ErrorCode::FormatToolIncompatible,
+ "the pinned Siegfried default.sig signature file was not found",
+ false,
+ )
+ })?;
+ let probe_directory = tempfile::tempdir().map_err(|source| ProfilerError::Io {
+ operation: "creating Siegfried probe directory",
+ path: std::env::temp_dir(),
+ source,
+ })?;
+ let probe_file = probe_directory.path().join("probe.bin");
+ File::create(&probe_file).map_err(|source| ProfilerError::Io {
+ operation: "creating Siegfried probe file",
+ path: probe_file.clone(),
+ source,
+ })?;
+ let signature_path = fs::canonicalize(&signature).map_err(|source| ProfilerError::Io {
+ operation: "canonicalizing Siegfried signature file",
+ path: signature.clone(),
+ source,
+ })?;
+ let output = run_command(
+ &executable,
+ Some(signature_path.as_path()),
+ options.workers.max(1),
+ Duration::from_secs(30),
+ &[probe_file],
+ probe_directory.path(),
+ )?;
+ let parsed = parse_output(&output.stdout)?;
+ let executable_sha256 = sha256_file(&executable)?;
+ let signature_sha256 = Some(sha256_file(&signature_path)?);
+ let signature_version =
+ extract_pronom_version(&parsed.identifiers).unwrap_or_else(|| "unknown".into());
+ if parsed.siegfried.trim() != RECOMMENDED_SIEGFRIED_VERSION {
+ return Err(ProfilerError::contract_with_context(
+ ErrorCode::FormatToolIncompatible,
+ "Siegfried version does not match the profiler release contract",
+ false,
+ BTreeMap::from([
+ (
+ "requiredVersion".into(),
+ RECOMMENDED_SIEGFRIED_VERSION.into(),
+ ),
+ ("observedVersion".into(), parsed.siegfried.clone()),
+ ]),
+ ));
+ }
+ if signature_version != RECOMMENDED_PRONOM_VERSION {
+ return Err(ProfilerError::contract_with_context(
+ ErrorCode::FormatToolIncompatible,
+ "PRONOM signature version does not match the profiler release contract",
+ false,
+ BTreeMap::from([
+ (
+ "requiredSignature".into(),
+ RECOMMENDED_PRONOM_VERSION.into(),
+ ),
+ ("observedSignature".into(), signature_version.clone()),
+ ]),
+ ));
+ }
+ let identity = FormatToolIdentity {
+ tool_name: "siegfried".into(),
+ tool_version: parsed.siegfried,
+ executable_path: executable.to_string_lossy().into_owned(),
+ executable_sha256,
+ signature_path: signature_path.to_string_lossy().into_owned(),
+ signature_sha256,
+ signature_version,
+ signature_created: parsed.created,
+ identifiers: parsed
+ .identifiers
+ .into_iter()
+ .map(|identifier| FormatIdentifierIdentity {
+ name: identifier.name,
+ details: identifier.details,
+ })
+ .collect(),
+ probed_at: now_text(),
+ };
+ Ok(Self {
+ executable,
+ signature: Some(signature_path),
+ workers: options.workers.max(1),
+ timeout: options.timeout,
+ identity,
+ })
+ }
+
+ pub fn identity(&self) -> &FormatToolIdentity {
+ &self.identity
+ }
+
+ pub fn identify_batch(
+ &self,
+ inputs: &[ResolvedFormatInput],
+ staging: &BatchWorkspace,
+ ) -> ProfilerResult> {
+ if inputs.is_empty() {
+ return Ok(Vec::new());
+ }
+ self.identify_batch_adaptive(inputs, staging.path(), 0)
+ }
+
+ fn identify_batch_adaptive(
+ &self,
+ inputs: &[ResolvedFormatInput],
+ staging_root: &Path,
+ depth: u32,
+ ) -> ProfilerResult> {
+ match self.identify_batch_once(inputs, staging_root, depth) {
+ Ok(observations) => Ok(observations),
+ Err(error) if inputs.len() > 1 => {
+ warn!(count = inputs.len(), depth, error = %error, "splitting failed Siegfried batch");
+ let middle = inputs.len() / 2;
+ let mut left =
+ self.identify_batch_adaptive(&inputs[..middle], staging_root, depth + 1)?;
+ left.extend(self.identify_batch_adaptive(
+ &inputs[middle..],
+ staging_root,
+ depth + 1,
+ )?);
+ Ok(left)
+ }
+ Err(error) => Ok(vec![tool_error_observation(&inputs[0], &error)]),
+ }
+ }
+
+ fn identify_batch_once(
+ &self,
+ inputs: &[ResolvedFormatInput],
+ staging_root: &Path,
+ depth: u32,
+ ) -> ProfilerResult> {
+ let subdirectory = staging_root.join(format!("attempt-{depth}-{}", inputs.len()));
+ fs::create_dir_all(&subdirectory).map_err(|source| ProfilerError::Io {
+ operation: "creating Siegfried attempt directory",
+ path: subdirectory.clone(),
+ source,
+ })?;
+ let mut paths = Vec::with_capacity(inputs.len());
+ let mut by_key = HashMap::with_capacity(inputs.len());
+ let mut staging_modes = HashMap::with_capacity(inputs.len());
+ for input in inputs {
+ let alias = safe_alias_path(&subdirectory, &input.item);
+ let (scan_path, staging_mode) = match create_symlink_alias(&input.source_path, &alias) {
+ Ok(()) => (alias, "symlink_alias".to_owned()),
+ Err(error) => {
+ debug!(path = %input.source_path.display(), error = %error, "symbolic-link alias unavailable; using canonical object path without extension evidence");
+ (input.source_path.clone(), "canonical_path".to_owned())
+ }
+ };
+ let key = path_key(&scan_path);
+ by_key.insert(key.clone(), input);
+ staging_modes.insert(key, staging_mode);
+ paths.push(scan_path);
+ }
+ let output = run_command(
+ &self.executable,
+ self.signature.as_deref(),
+ self.workers,
+ self.timeout,
+ &paths,
+ &subdirectory,
+ )?;
+ let parsed = parse_output(&output.stdout)?;
+ let mut observations = Vec::with_capacity(inputs.len());
+ let mut seen = HashMap::::new();
+ for file in parsed.files {
+ let key = path_key(Path::new(&file.filename));
+ let Some(input) = by_key.get(&key) else {
+ continue;
+ };
+ *seen.entry(input.item.sha256.clone()).or_default() += 1;
+ observations.push(observation_from_file(
+ input,
+ &file,
+ staging_modes
+ .get(&key)
+ .map_or("canonical_path", String::as_str),
+ ));
+ }
+ for input in inputs {
+ if !seen.contains_key(&input.item.sha256) {
+ observations.push(FormatObservation {
+ content_object_id: input.item.content_object_id.clone(),
+ sha256: input.item.sha256.clone(),
+ state: FormatState::ToolError,
+ source_mime_type: input.item.source_mime_type.clone(),
+ preferred_extension: input.item.preferred_extension.clone(),
+ staging_mode: "unmatched_output".into(),
+ primary_identifier: None,
+ primary_format_name: None,
+ primary_format_version: None,
+ primary_mime_type: None,
+ match_count: 0,
+ extension_checked: false,
+ extension_mismatch: false,
+ error_code: Some("OUTPUT_RECORD_MISSING".into()),
+ error_message: Some("Siegfried returned no result for the input object".into()),
+ matches: Vec::new(),
+ observed_at: now_text(),
+ });
+ }
+ }
+ observations.sort_by(|left, right| left.sha256.cmp(&right.sha256));
+ Ok(observations)
+ }
+}
+
+#[derive(Debug)]
+struct ProcessOutput {
+ stdout: Vec,
+}
+
+fn append_signature_arguments(command: &mut Command, signature: &Path) -> ProfilerResult<()> {
+ let signature_home = signature
+ .parent()
+ .ok_or_else(|| ProfilerError::InvalidPath {
+ message: "Siegfried signature path has no parent directory".into(),
+ path: signature.to_path_buf(),
+ })?;
+ let signature_name = signature
+ .file_name()
+ .ok_or_else(|| ProfilerError::InvalidPath {
+ message: "Siegfried signature path has no filename".into(),
+ path: signature.to_path_buf(),
+ })?;
+
+ // Siegfried 1.11.6's JSON writer escapes scanned filenames but interpolates
+ // the header's signature value verbatim. An absolute Windows path therefore
+ // emits invalid JSON because backslashes are not escaped. Resolve the
+ // signature through -home and pass only the filename; this also keeps
+ // evidence output independent of private installation paths.
+ command
+ .arg("-home")
+ .arg(signature_home)
+ .arg("-sig")
+ .arg(signature_name);
+ Ok(())
+}
+
+fn run_command(
+ executable: &Path,
+ signature: Option<&Path>,
+ workers: u32,
+ timeout: Duration,
+ paths: &[PathBuf],
+ working_directory: &Path,
+) -> ProfilerResult {
+ let list_path = write_input_list(paths, working_directory)?;
+ let command = build_siegfried_command(
+ executable,
+ signature,
+ workers,
+ &list_path,
+ working_directory,
+ )?;
+ execute_siegfried(command, executable, timeout)
+}
+
+fn write_input_list(paths: &[PathBuf], working_directory: &Path) -> ProfilerResult {
+ let list_path = working_directory.join("inputs.txt");
+ let mut list_file = File::create(&list_path).map_err(|source| ProfilerError::Io {
+ operation: "creating Siegfried input list",
+ path: list_path.clone(),
+ source,
+ })?;
+ for path in paths {
+ let display = path.to_string_lossy();
+ if display.contains('\r') || display.contains('\n') {
+ return Err(ProfilerError::InvalidPath {
+ message: "Siegfried input path contains a line break".into(),
+ path: path.clone(),
+ });
+ }
+ writeln!(list_file, "{display}").map_err(|source| ProfilerError::Io {
+ operation: "writing Siegfried input list",
+ path: list_path.clone(),
+ source,
+ })?;
+ }
+ list_file.sync_all().map_err(|source| ProfilerError::Io {
+ operation: "syncing Siegfried input list",
+ path: list_path.clone(),
+ source,
+ })?;
+ Ok(list_path)
+}
+
+fn build_siegfried_command(
+ executable: &Path,
+ signature: Option<&Path>,
+ workers: u32,
+ list_path: &Path,
+ working_directory: &Path,
+) -> ProfilerResult {
+ let mut command = Command::new(executable);
+ command
+ .arg("-json")
+ .arg("-utc")
+ .arg("-coe")
+ .arg("-sym")
+ .arg("-multi")
+ .arg(workers.to_string());
+ if let Some(signature) = signature {
+ append_signature_arguments(&mut command, signature)?;
+ }
+ command
+ .arg("-f")
+ .arg(list_path)
+ .current_dir(working_directory)
+ .stdin(Stdio::null())
+ .stdout(Stdio::piped())
+ .stderr(Stdio::piped());
+ Ok(command)
+}
+
+fn execute_siegfried(
+ mut command: Command,
+ executable: &Path,
+ timeout: Duration,
+) -> ProfilerResult {
+ let mut child = command.spawn().map_err(|source| ProfilerError::Io {
+ operation: "starting Siegfried sidecar",
+ path: executable.to_path_buf(),
+ source,
+ })?;
+ let stdout = child
+ .stdout
+ .take()
+ .ok_or_else(|| ProfilerError::Internal("Siegfried stdout pipe was not available".into()))?;
+ let stderr = child
+ .stderr
+ .take()
+ .ok_or_else(|| ProfilerError::Internal("Siegfried stderr pipe was not available".into()))?;
+ let stdout_thread = spawn_limited_reader(stdout, 64 * 1024 * 1024);
+ let stderr_thread = spawn_limited_reader(stderr, 4 * 1024 * 1024);
+ let status = match wait_for_process(&mut child, executable, timeout) {
+ Ok(status) => status,
+ Err(error) => {
+ let _ = stdout_thread.join();
+ let _ = stderr_thread.join();
+ return Err(error);
+ }
+ };
+ let stdout_result = join_reader(stdout_thread, executable, "stdout");
+ let stderr_result = join_reader(stderr_thread, executable, "stderr");
+ let stdout = stdout_result?;
+ let stderr = stderr_result?;
+ if !status.success() {
+ return Err(ProfilerError::contract_with_context(
+ ErrorCode::FormatRunFailed,
+ "Siegfried sidecar returned a non-zero exit status",
+ true,
+ BTreeMap::from([
+ ("exitStatus".into(), status.to_string()),
+ (
+ "stderr".into(),
+ truncate_text(&String::from_utf8_lossy(&stderr), 4_096),
+ ),
+ ]),
+ ));
+ }
+ Ok(ProcessOutput { stdout })
+}
+
+fn spawn_limited_reader(reader: R, limit: usize) -> thread::JoinHandle>>
+where
+ R: Read + Send + 'static,
+{
+ thread::spawn(move || {
+ let mut bytes = Vec::new();
+ let mut reader = BufReader::new(reader);
+ read_to_end_limited(&mut reader, &mut bytes, limit).map(|()| bytes)
+ })
+}
+
+fn wait_for_process(
+ child: &mut std::process::Child,
+ executable: &Path,
+ timeout: Duration,
+) -> ProfilerResult {
+ let started = Instant::now();
+ loop {
+ if let Some(status) = child.try_wait().map_err(|source| ProfilerError::Io {
+ operation: "polling Siegfried sidecar",
+ path: executable.to_path_buf(),
+ source,
+ })? {
+ return Ok(status);
+ }
+ if started.elapsed() > timeout {
+ let _ = child.kill();
+ let _ = child.wait();
+ return Err(ProfilerError::contract_with_context(
+ ErrorCode::FormatRunFailed,
+ "Siegfried batch exceeded its configured timeout",
+ true,
+ BTreeMap::from([("timeoutSeconds".into(), timeout.as_secs().to_string())]),
+ ));
+ }
+ thread::sleep(Duration::from_millis(50));
+ }
+}
+
+fn join_reader(
+ handle: thread::JoinHandle>>,
+ executable: &Path,
+ stream: &'static str,
+) -> ProfilerResult> {
+ handle
+ .join()
+ .map_err(|_| ProfilerError::Internal(format!("Siegfried {stream} reader panicked")))?
+ .map_err(|source| ProfilerError::Io {
+ operation: if stream == "stdout" {
+ "reading Siegfried stdout"
+ } else {
+ "reading Siegfried stderr"
+ },
+ path: executable.to_path_buf(),
+ source,
+ })
+}
+
+#[derive(Debug, Deserialize)]
+struct RawOutput {
+ #[serde(default)]
+ siegfried: String,
+ #[serde(default)]
+ created: Option,
+ #[serde(default)]
+ identifiers: Vec,
+ #[serde(default)]
+ files: Vec,
+}
+
+#[derive(Debug, Deserialize)]
+struct RawIdentifier {
+ #[serde(default)]
+ name: String,
+ #[serde(default)]
+ details: String,
+}
+
+#[derive(Debug, Deserialize)]
+struct RawFile {
+ #[serde(default)]
+ filename: String,
+ #[serde(default)]
+ errors: serde_json::Value,
+ #[serde(default)]
+ matches: Vec,
+}
+
+#[derive(Debug, Deserialize)]
+struct RawMatch {
+ #[serde(default, rename = "ns")]
+ namespace: String,
+ #[serde(default, rename = "id")]
+ identifier: String,
+ #[serde(default, rename = "format")]
+ format_name: String,
+ #[serde(default, rename = "version")]
+ format_version: String,
+ #[serde(default, rename = "mime")]
+ mime_type: String,
+ #[serde(default, rename = "class")]
+ format_class: Option,
+ #[serde(default)]
+ basis: String,
+ #[serde(default)]
+ warning: String,
+}
+
+fn parse_output(bytes: &[u8]) -> ProfilerResult {
+ serde_json::from_slice(bytes).map_err(|error| {
+ ProfilerError::contract_with_context(
+ ErrorCode::FormatOutputInvalid,
+ "Siegfried returned invalid JSON output",
+ false,
+ BTreeMap::from([
+ ("parserError".into(), error.to_string()),
+ (
+ "outputPrefix".into(),
+ truncate_text(&String::from_utf8_lossy(bytes), 2_048),
+ ),
+ ]),
+ )
+ })
+}
+
+fn observation_from_file(
+ input: &ResolvedFormatInput,
+ file: &RawFile,
+ staging_mode: &str,
+) -> FormatObservation {
+ let error_message = normalize_errors(&file.errors);
+ let mut matches = file
+ .matches
+ .iter()
+ .map(|raw| FormatMatch {
+ namespace: raw.namespace.clone(),
+ identifier: raw.identifier.clone(),
+ format_name: raw.format_name.clone(),
+ format_version: raw.format_version.clone(),
+ mime_type: raw.mime_type.clone(),
+ format_class: raw.format_class.clone().filter(|value| !value.is_empty()),
+ basis: raw.basis.clone(),
+ warning: raw.warning.clone(),
+ is_primary: false,
+ })
+ .collect::>();
+ let primary_index = choose_primary_match(&matches);
+ if let Some(index) = primary_index
+ && let Some(primary) = matches.get_mut(index)
+ {
+ primary.is_primary = true;
+ }
+ let has_signature_evidence = matches
+ .iter()
+ .any(|format_match| is_viable_match(format_match) && !is_extension_only(format_match));
+ let viable_identifiers = matches
+ .iter()
+ .filter(|format_match| {
+ is_viable_match(format_match)
+ && (!has_signature_evidence || !is_extension_only(format_match))
+ })
+ .map(|format_match| (&format_match.namespace, &format_match.identifier))
+ .collect::>();
+ let state = if error_message.is_some() && matches.is_empty() {
+ FormatState::ToolError
+ } else if viable_identifiers.is_empty() {
+ FormatState::Unknown
+ } else if viable_identifiers.len() > 1 {
+ FormatState::Ambiguous
+ } else {
+ FormatState::Identified
+ };
+ let primary = primary_index.and_then(|index| matches.get(index));
+ let extension_checked =
+ input.item.preferred_extension.is_some() && staging_mode == "symlink_alias";
+ let extension_mismatch = extension_checked
+ && matches.iter().any(|format_match| {
+ let warning = format_match.warning.to_ascii_lowercase();
+ warning.contains("extension mismatch")
+ || warning.contains("filename mismatch")
+ || warning.contains("extension does not match")
+ });
+ FormatObservation {
+ content_object_id: input.item.content_object_id.clone(),
+ sha256: input.item.sha256.clone(),
+ state,
+ source_mime_type: input.item.source_mime_type.clone(),
+ preferred_extension: input.item.preferred_extension.clone(),
+ staging_mode: staging_mode.into(),
+ primary_identifier: primary.map(|value| value.identifier.clone()),
+ primary_format_name: primary.map(|value| value.format_name.clone()),
+ primary_format_version: primary.map(|value| value.format_version.clone()),
+ primary_mime_type: primary.map(|value| value.mime_type.clone()),
+ match_count: u64::try_from(matches.len()).unwrap_or(u64::MAX),
+ extension_checked,
+ extension_mismatch,
+ error_code: error_message
+ .as_ref()
+ .map(|_| "SIEGFRIED_FILE_ERROR".into()),
+ error_message,
+ matches,
+ observed_at: now_text(),
+ }
+}
+
+fn choose_primary_match(matches: &[FormatMatch]) -> Option {
+ matches
+ .iter()
+ .enumerate()
+ .filter(|(_, format_match)| is_viable_match(format_match))
+ .max_by_key(|(index, format_match)| (match_score(format_match), std::cmp::Reverse(*index)))
+ .map(|(index, _)| index)
+}
+
+fn is_viable_match(format_match: &FormatMatch) -> bool {
+ !format_match.identifier.trim().is_empty()
+ && !format_match.identifier.eq_ignore_ascii_case("UNKNOWN")
+}
+
+fn is_extension_only(format_match: &FormatMatch) -> bool {
+ let basis = format_match.basis.to_ascii_lowercase();
+ basis.contains("extension match")
+ && !basis.contains("byte match")
+ && !basis.contains("container match")
+ && !basis.contains("xml match")
+ && !basis.contains("text match")
+}
+
+fn match_score(format_match: &FormatMatch) -> u32 {
+ let basis = format_match.basis.to_ascii_lowercase();
+ let mut score = if basis.contains("container match") {
+ 500
+ } else if basis.contains("byte match") {
+ 400
+ } else if basis.contains("xml match") {
+ 350
+ } else if basis.contains("text match") {
+ 300
+ } else if basis.contains("extension match") {
+ 100
+ } else {
+ 200
+ };
+ if format_match.namespace.eq_ignore_ascii_case("pronom") {
+ score += 30;
+ }
+ if format_match.warning.trim().is_empty() {
+ score += 20;
+ }
+ score
+}
+
+fn tool_error_observation(input: &ResolvedFormatInput, error: &ProfilerError) -> FormatObservation {
+ FormatObservation {
+ content_object_id: input.item.content_object_id.clone(),
+ sha256: input.item.sha256.clone(),
+ state: FormatState::ToolError,
+ source_mime_type: input.item.source_mime_type.clone(),
+ preferred_extension: input.item.preferred_extension.clone(),
+ staging_mode: "batch_isolation".into(),
+ primary_identifier: None,
+ primary_format_name: None,
+ primary_format_version: None,
+ primary_mime_type: None,
+ match_count: 0,
+ extension_checked: false,
+ extension_mismatch: false,
+ error_code: Some(format!("{:?}", error.report().code).to_ascii_uppercase()),
+ error_message: Some(error.to_string()),
+ matches: Vec::new(),
+ observed_at: now_text(),
+ }
+}
+
+pub fn skipped_observation(item: &FormatWorkItem) -> FormatObservation {
+ let state = if item.expected_size_bytes == 0 {
+ FormatState::Empty
+ } else {
+ FormatState::SkippedUnavailable
+ };
+ FormatObservation {
+ content_object_id: item.content_object_id.clone(),
+ sha256: item.sha256.clone(),
+ state,
+ source_mime_type: item.source_mime_type.clone(),
+ preferred_extension: item.preferred_extension.clone(),
+ staging_mode: "not_invoked".into(),
+ primary_identifier: None,
+ primary_format_name: None,
+ primary_format_version: None,
+ primary_mime_type: None,
+ match_count: 0,
+ extension_checked: false,
+ extension_mismatch: false,
+ error_code: None,
+ error_message: None,
+ matches: Vec::new(),
+ observed_at: now_text(),
+ }
+}
+
+fn create_symlink_alias(source: &Path, alias: &Path) -> std::io::Result<()> {
+ #[cfg(windows)]
+ {
+ std::os::windows::fs::symlink_file(source, alias)
+ }
+ #[cfg(unix)]
+ {
+ std::os::unix::fs::symlink(source, alias)
+ }
+ #[cfg(not(any(windows, unix)))]
+ {
+ let _ = (source, alias);
+ Err(std::io::Error::new(
+ std::io::ErrorKind::Unsupported,
+ "symbolic file aliases are unsupported on this platform",
+ ))
+ }
+}
+
+fn read_to_end_limited(
+ reader: &mut R,
+ target: &mut Vec,
+ limit: usize,
+) -> std::io::Result<()> {
+ let mut buffer = vec![0_u8; 64 * 1024];
+ loop {
+ let read = reader.read(&mut buffer)?;
+ if read == 0 {
+ return Ok(());
+ }
+ if target.len().saturating_add(read) > limit {
+ return Err(std::io::Error::new(
+ std::io::ErrorKind::InvalidData,
+ "Siegfried output exceeded the configured safety limit",
+ ));
+ }
+ target.extend_from_slice(&buffer[..read]);
+ }
+}
+
+fn safe_alias_path(directory: &Path, item: &FormatWorkItem) -> PathBuf {
+ match item.preferred_extension.as_deref() {
+ Some(extension) => directory.join(format!("{}.{}", item.sha256, extension)),
+ None => directory.join(&item.sha256),
+ }
+}
+
+fn discover_executable(explicit: Option<&Path>) -> ProfilerResult {
+ if let Some(path) = explicit {
+ if path.is_file() {
+ return fs::canonicalize(path).map_err(|source| ProfilerError::Io {
+ operation: "canonicalizing Siegfried executable",
+ path: path.to_path_buf(),
+ source,
+ });
+ }
+ return Err(ProfilerError::contract_with_context(
+ ErrorCode::FormatToolNotFound,
+ "configured Siegfried executable does not exist",
+ false,
+ BTreeMap::from([("path".into(), path.to_string_lossy().into_owned())]),
+ ));
+ }
+ let binary = if cfg!(windows) { "sf.exe" } else { "sf" };
+ let mut candidates = Vec::new();
+ if let Ok(current) = std::env::current_exe()
+ && let Some(parent) = current.parent()
+ {
+ candidates.push(parent.join("tools").join("siegfried").join(binary));
+ candidates.push(
+ parent
+ .join("resources")
+ .join("tools")
+ .join("siegfried")
+ .join(binary),
+ );
+ candidates.push(parent.join(binary));
+ }
+ candidates.push(PathBuf::from("tools").join("siegfried").join(binary));
+ if let Some(path) = find_on_path(binary) {
+ candidates.push(path);
+ }
+ for candidate in candidates {
+ if candidate.is_file() {
+ return fs::canonicalize(&candidate).map_err(|source| ProfilerError::Io {
+ operation: "canonicalizing discovered Siegfried executable",
+ path: candidate,
+ source,
+ });
+ }
+ }
+ Err(ProfilerError::contract(
+ ErrorCode::FormatToolNotFound,
+ "Siegfried was not found; install the pinned sidecar or pass --siegfried",
+ false,
+ ))
+}
+
+fn discover_signature(executable: &Path) -> Option {
+ let parent = executable.parent()?;
+ [
+ parent.join("default.sig"),
+ parent.join("data").join("default.sig"),
+ parent.join("siegfried").join("default.sig"),
+ ]
+ .into_iter()
+ .find(|candidate| candidate.is_file())
+}
+
+fn find_on_path(binary: &str) -> Option {
+ let path = std::env::var_os("PATH")?;
+ std::env::split_paths(&path)
+ .map(|directory| directory.join(binary))
+ .find(|candidate| candidate.is_file())
+}
+
+fn path_key(path: &Path) -> String {
+ let normalized = path.to_string_lossy().replace('\\', "/");
+ if cfg!(windows) {
+ normalized.to_ascii_lowercase()
+ } else {
+ normalized
+ }
+}
+
+fn extract_pronom_version(identifiers: &[RawIdentifier]) -> Option {
+ identifiers.iter().find_map(|identifier| {
+ let marker = "DROID_SignatureFile_V";
+ let start = identifier.details.find(marker)? + marker.len();
+ let digits = identifier.details[start..]
+ .chars()
+ .take_while(char::is_ascii_digit)
+ .collect::();
+ (!digits.is_empty()).then(|| format!("v{digits}"))
+ })
+}
+
+fn normalize_errors(value: &serde_json::Value) -> Option {
+ match value {
+ serde_json::Value::Null => None,
+ serde_json::Value::String(text) => (!text.trim().is_empty()).then(|| text.clone()),
+ serde_json::Value::Array(values) => {
+ let joined = values
+ .iter()
+ .filter_map(serde_json::Value::as_str)
+ .filter(|value| !value.trim().is_empty())
+ .collect::>()
+ .join("; ");
+ (!joined.is_empty()).then_some(joined)
+ }
+ other => Some(other.to_string()),
+ }
+}
+
+fn sha256_file(path: &Path) -> ProfilerResult {
+ let file = File::open(path).map_err(|source| ProfilerError::Io {
+ operation: "opening format tool for SHA-256",
+ path: path.to_path_buf(),
+ source,
+ })?;
+ let mut reader = BufReader::with_capacity(1024 * 1024, file);
+ let mut digest = Sha256::new();
+ let mut buffer = vec![0_u8; 1024 * 1024];
+ loop {
+ let read = reader
+ .read(&mut buffer)
+ .map_err(|source| ProfilerError::Io {
+ operation: "hashing format tool",
+ path: path.to_path_buf(),
+ source,
+ })?;
+ if read == 0 {
+ break;
+ }
+ digest.update(&buffer[..read]);
+ }
+ Ok(hex::encode(digest.finalize()))
+}
+
+fn truncate_text(value: &str, max_chars: usize) -> String {
+ value.chars().take(max_chars).collect()
+}
+
+fn now_text() -> String {
+ OffsetDateTime::now_utc()
+ .format(&time::format_description::well_known::Rfc3339)
+ .expect("RFC3339 formatting is infallible for OffsetDateTime")
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn signature_argument_is_relative_to_siegfried_home() {
+ let mut command = Command::new("sf");
+ let signature = Path::new("/opt/mailvault/siegfried/default.sig");
+
+ append_signature_arguments(&mut command, signature).expect("signature arguments");
+ let arguments = command
+ .get_args()
+ .map(|value| value.to_string_lossy().into_owned())
+ .collect::>();
+
+ assert_eq!(
+ arguments,
+ vec!["-home", "/opt/mailvault/siegfried", "-sig", "default.sig"]
+ );
+ }
+
+ #[cfg(windows)]
+ #[test]
+ fn windows_signature_argument_never_contains_absolute_backslashes() {
+ let mut command = Command::new("sf.exe");
+ let signature =
+ Path::new(r"E:\github\mailvault-collection-profiler\tools\siegfried\default.sig");
+
+ append_signature_arguments(&mut command, signature).expect("signature arguments");
+ let arguments = command
+ .get_args()
+ .map(|value| value.to_string_lossy().into_owned())
+ .collect::>();
+
+ assert_eq!(arguments[0], "-home");
+ assert_eq!(
+ arguments[1],
+ r"E:\github\mailvault-collection-profiler\tools\siegfried"
+ );
+ assert_eq!(arguments[2], "-sig");
+ assert_eq!(arguments[3], "default.sig");
+ assert!(!arguments[3].contains('\\'));
+ }
+
+ #[test]
+ fn pronom_version_is_parsed_from_identifier_details() {
+ let identifiers = vec![RawIdentifier {
+ name: "pronom".into(),
+ details: "DROID_SignatureFile_V124.xml; container-signature-20260711.xml".into(),
+ }];
+ assert_eq!(
+ extract_pronom_version(&identifiers).as_deref(),
+ Some("v124")
+ );
+ }
+
+ #[test]
+ fn byte_or_container_evidence_beats_extension_only() {
+ let extension = FormatMatch {
+ namespace: "pronom".into(),
+ identifier: "fmt/1".into(),
+ format_name: "A".into(),
+ format_version: String::new(),
+ mime_type: String::new(),
+ format_class: None,
+ basis: "extension match doc".into(),
+ warning: String::new(),
+ is_primary: false,
+ };
+ let byte = FormatMatch {
+ basis: "byte match at 0".into(),
+ identifier: "fmt/2".into(),
+ ..extension.clone()
+ };
+ assert_eq!(choose_primary_match(&[extension, byte]), Some(1));
+ }
+
+ #[test]
+ fn extension_only_match_is_not_decisive_when_signature_evidence_exists() {
+ let extension = FormatMatch {
+ namespace: "pronom".into(),
+ identifier: "fmt/40".into(),
+ format_name: "Generic Word".into(),
+ format_version: String::new(),
+ mime_type: "application/msword".into(),
+ format_class: None,
+ basis: "extension match doc".into(),
+ warning: String::new(),
+ is_primary: false,
+ };
+ let signature = FormatMatch {
+ identifier: "fmt/609".into(),
+ format_name: "Microsoft Word 97-2003".into(),
+ basis: "container match with name WordDocument".into(),
+ ..extension.clone()
+ };
+ let matches = [extension, signature];
+ let has_signature_evidence = matches
+ .iter()
+ .any(|format_match| is_viable_match(format_match) && !is_extension_only(format_match));
+ let decisive = matches
+ .iter()
+ .filter(|format_match| {
+ is_viable_match(format_match)
+ && (!has_signature_evidence || !is_extension_only(format_match))
+ })
+ .count();
+ assert_eq!(decisive, 1);
+ }
+}
diff --git a/crates/profiler-storage-sqlite/migrations/0006_exact_format_identification.sql b/crates/profiler-storage-sqlite/migrations/0006_exact_format_identification.sql
new file mode 100644
index 0000000..e0a041a
--- /dev/null
+++ b/crates/profiler-storage-sqlite/migrations/0006_exact_format_identification.sql
@@ -0,0 +1,128 @@
+-- Exact format identification is a versioned downstream projection over a completed
+-- physical inventory. Existing format_assertions remain for backwards compatibility;
+-- all alpha.4 writes use the richer observation/match model below.
+
+ALTER TABLE content_objects ADD COLUMN format_state TEXT NOT NULL DEFAULT 'uninspected'
+ CHECK(format_state IN (
+ 'uninspected', 'identified', 'unknown', 'ambiguous', 'empty',
+ 'skipped_unavailable', 'tool_error'
+ ));
+ALTER TABLE content_objects ADD COLUMN primary_puid TEXT;
+ALTER TABLE content_objects ADD COLUMN primary_format_name TEXT;
+ALTER TABLE content_objects ADD COLUMN primary_format_version TEXT;
+ALTER TABLE content_objects ADD COLUMN primary_format_mime_type TEXT;
+ALTER TABLE content_objects ADD COLUMN format_match_count INTEGER NOT NULL DEFAULT 0
+ CHECK(format_match_count >= 0);
+ALTER TABLE content_objects ADD COLUMN extension_checked INTEGER NOT NULL DEFAULT 0
+ CHECK(extension_checked IN (0, 1));
+ALTER TABLE content_objects ADD COLUMN extension_mismatch INTEGER NOT NULL DEFAULT 0
+ CHECK(extension_mismatch IN (0, 1));
+ALTER TABLE content_objects ADD COLUMN last_format_run_id TEXT;
+ALTER TABLE content_objects ADD COLUMN last_format_at TEXT;
+
+CREATE TABLE format_tools (
+ id TEXT PRIMARY KEY,
+ tool_name TEXT NOT NULL,
+ tool_version TEXT NOT NULL,
+ executable_path TEXT NOT NULL,
+ executable_sha256 TEXT NOT NULL,
+ signature_path TEXT NOT NULL,
+ signature_sha256 TEXT NOT NULL DEFAULT '',
+ signature_version TEXT NOT NULL,
+ signature_created TEXT,
+ identifiers_json TEXT NOT NULL CHECK(json_valid(identifiers_json)),
+ probed_at TEXT NOT NULL,
+ UNIQUE(tool_name, executable_sha256, signature_version, signature_sha256)
+) STRICT;
+
+CREATE TABLE format_identification_runs (
+ id TEXT PRIMARY KEY,
+ baseline_run_id TEXT NOT NULL REFERENCES profiler_runs(id) ON DELETE RESTRICT,
+ collection_id TEXT NOT NULL REFERENCES collections(id) ON DELETE RESTRICT,
+ tool_id TEXT NOT NULL REFERENCES format_tools(id) ON DELETE RESTRICT,
+ state TEXT NOT NULL CHECK(state IN ('running', 'succeeded', 'failed', 'cancelled')),
+ configuration_fingerprint TEXT NOT NULL,
+ batch_size INTEGER NOT NULL CHECK(batch_size > 0),
+ worker_count INTEGER NOT NULL CHECK(worker_count > 0),
+ timeout_seconds INTEGER NOT NULL CHECK(timeout_seconds > 0),
+ total_objects INTEGER NOT NULL CHECK(total_objects >= 0),
+ eligible_objects INTEGER NOT NULL CHECK(eligible_objects >= 0),
+ completed_objects INTEGER NOT NULL DEFAULT 0 CHECK(completed_objects >= 0),
+ total_bytes INTEGER NOT NULL CHECK(total_bytes >= 0),
+ completed_bytes INTEGER NOT NULL DEFAULT 0 CHECK(completed_bytes >= 0),
+ identified INTEGER NOT NULL DEFAULT 0 CHECK(identified >= 0),
+ unknown INTEGER NOT NULL DEFAULT 0 CHECK(unknown >= 0),
+ ambiguous INTEGER NOT NULL DEFAULT 0 CHECK(ambiguous >= 0),
+ empty_objects INTEGER NOT NULL DEFAULT 0 CHECK(empty_objects >= 0),
+ skipped_unavailable INTEGER NOT NULL DEFAULT 0 CHECK(skipped_unavailable >= 0),
+ tool_errors INTEGER NOT NULL DEFAULT 0 CHECK(tool_errors >= 0),
+ extension_mismatches INTEGER NOT NULL DEFAULT 0 CHECK(extension_mismatches >= 0),
+ checkpoint_sha256 TEXT,
+ checkpoint_sequence INTEGER NOT NULL DEFAULT 0 CHECK(checkpoint_sequence >= 0),
+ started_at TEXT NOT NULL,
+ finished_at TEXT,
+ failure_code TEXT,
+ failure_message TEXT,
+ UNIQUE(baseline_run_id, configuration_fingerprint, started_at)
+) STRICT;
+
+CREATE TABLE format_observations (
+ id TEXT PRIMARY KEY,
+ format_run_id TEXT NOT NULL REFERENCES format_identification_runs(id) ON DELETE CASCADE,
+ baseline_run_id TEXT NOT NULL REFERENCES profiler_runs(id) ON DELETE RESTRICT,
+ content_object_id TEXT NOT NULL REFERENCES content_objects(id) ON DELETE CASCADE,
+ sha256 TEXT NOT NULL,
+ state TEXT NOT NULL CHECK(state IN (
+ 'identified', 'unknown', 'ambiguous', 'empty',
+ 'skipped_unavailable', 'tool_error'
+ )),
+ source_mime_type TEXT NOT NULL,
+ preferred_extension TEXT,
+ staging_mode TEXT NOT NULL,
+ primary_identifier TEXT,
+ primary_format_name TEXT,
+ primary_format_version TEXT,
+ primary_mime_type TEXT,
+ match_count INTEGER NOT NULL CHECK(match_count >= 0),
+ extension_checked INTEGER NOT NULL CHECK(extension_checked IN (0, 1)),
+ extension_mismatch INTEGER NOT NULL CHECK(extension_mismatch IN (0, 1)),
+ error_code TEXT,
+ error_message TEXT,
+ observed_at TEXT NOT NULL,
+ UNIQUE(format_run_id, content_object_id)
+) STRICT;
+
+CREATE TABLE format_matches (
+ id TEXT PRIMARY KEY,
+ observation_id TEXT NOT NULL REFERENCES format_observations(id) ON DELETE CASCADE,
+ ordinal INTEGER NOT NULL CHECK(ordinal >= 0),
+ namespace TEXT NOT NULL,
+ identifier TEXT NOT NULL,
+ format_name TEXT NOT NULL,
+ format_version TEXT NOT NULL,
+ mime_type TEXT NOT NULL,
+ format_class TEXT,
+ basis TEXT NOT NULL,
+ warning TEXT NOT NULL,
+ is_primary INTEGER NOT NULL CHECK(is_primary IN (0, 1)),
+ UNIQUE(observation_id, ordinal)
+) STRICT;
+
+CREATE INDEX idx_format_runs_baseline_started
+ ON format_identification_runs(baseline_run_id, started_at DESC);
+CREATE UNIQUE INDEX idx_format_runs_one_active
+ ON format_identification_runs(baseline_run_id)
+ WHERE state='running';
+CREATE INDEX idx_format_observations_run_state_sha
+ ON format_observations(format_run_id, state, sha256);
+CREATE INDEX idx_format_observations_content_run
+ ON format_observations(content_object_id, format_run_id);
+CREATE INDEX idx_format_matches_identifier
+ ON format_matches(identifier, observation_id);
+CREATE INDEX idx_content_objects_collection_format_state_sha
+ ON content_objects(collection_id, format_state, sha256);
+CREATE INDEX idx_content_objects_collection_puid_sha
+ ON content_objects(collection_id, primary_puid, sha256)
+ WHERE primary_puid IS NOT NULL;
+CREATE INDEX idx_content_objects_collection_format_mismatch_sha
+ ON content_objects(collection_id, extension_mismatch, sha256);
diff --git a/crates/profiler-storage-sqlite/src/format.rs b/crates/profiler-storage-sqlite/src/format.rs
new file mode 100644
index 0000000..3570dda
--- /dev/null
+++ b/crates/profiler-storage-sqlite/src/format.rs
@@ -0,0 +1,1136 @@
+use std::collections::BTreeMap;
+
+use profiler_core::{
+ ErrorCode, FormatMatch, FormatObjectRow, FormatObservation, FormatPage, FormatPageRequest,
+ FormatRunRegistration, FormatRunStartRequest, FormatState, FormatSummary, FormatToolIdentity,
+ FormatWorkItem, ProfilerError, ProfilerResult,
+};
+use rusqlite::{
+ OptionalExtension, Transaction, TransactionBehavior, named_params, params, types::Type,
+};
+use uuid::Uuid;
+
+use super::{ProfilerStore, now_text, sqlite_error, to_u64};
+
+impl ProfilerStore {
+ pub fn begin_format_run(
+ &mut self,
+ request: &FormatRunStartRequest<'_>,
+ ) -> ProfilerResult {
+ let transaction = self
+ .connection
+ .transaction_with_behavior(TransactionBehavior::Immediate)
+ .map_err(|source| sqlite_error("starting format run transaction", source))?;
+
+ let collection_id = load_format_baseline(&transaction, request.baseline_run_id)?;
+ let resumable = find_resumable_format_run(&transaction, request)?;
+ let tool_id = register_tool(&transaction, request.tool)?;
+ let workload = load_format_workload(&transaction, &collection_id)?;
+ let registration = match resumable {
+ Some(candidate) => {
+ resume_format_run(&transaction, &collection_id, &tool_id, workload, candidate)?
+ }
+ None => create_format_run(&transaction, request, &collection_id, &tool_id, workload)?,
+ };
+
+ transaction
+ .commit()
+ .map_err(|source| sqlite_error("committing exact-format run registration", source))?;
+ Ok(registration)
+ }
+
+ pub fn load_format_work_batch(
+ &self,
+ collection_id: &str,
+ after_sha256: Option<&str>,
+ limit: u32,
+ ) -> ProfilerResult> {
+ if limit == 0 {
+ return Err(ProfilerError::InvalidArgument(
+ "format batch limit must be greater than zero".into(),
+ ));
+ }
+ let mut statement = self
+ .connection
+ .prepare(
+ "SELECT content.id, content.sha256, content.expected_size_bytes,
+ content.source_detected_mime_type, content.canonical_path_display,
+ content.availability_state,
+ (SELECT variant.display_filename FROM filename_variants AS variant
+ WHERE variant.content_object_id=content.id
+ ORDER BY variant.occurrence_count DESC, variant.normalized_filename
+ LIMIT 1)
+ FROM content_objects AS content
+ WHERE content.collection_id=:collection_id
+ AND (:after_sha256 IS NULL OR content.sha256 > :after_sha256)
+ ORDER BY content.sha256
+ LIMIT :limit",
+ )
+ .map_err(|source| sqlite_error("preparing exact-format workload batch", source))?;
+ statement
+ .query_map(
+ named_params! {
+ ":collection_id": collection_id,
+ ":after_sha256": after_sha256,
+ ":limit": i64::from(limit),
+ },
+ |row| {
+ let filename: Option = row.get(6)?;
+ Ok(FormatWorkItem {
+ content_object_id: row.get(0)?,
+ sha256: row.get(1)?,
+ expected_size_bytes: row_u64(row, 2)?,
+ source_mime_type: row.get(3)?,
+ canonical_path_display: row.get(4)?,
+ availability_state: row.get(5)?,
+ preferred_extension: filename.as_deref().and_then(safe_extension),
+ })
+ },
+ )
+ .map_err(|source| sqlite_error("querying exact-format workload batch", source))?
+ .collect::, _>>()
+ .map_err(|source| sqlite_error("collecting exact-format workload batch", source))
+ }
+
+ pub fn commit_format_observations(
+ &mut self,
+ format_run_id: &str,
+ baseline_run_id: &str,
+ observations: &[FormatObservation],
+ checkpoint_sha256: &str,
+ checkpoint_sequence: u64,
+ ) -> ProfilerResult {
+ let transaction = self
+ .connection
+ .transaction_with_behavior(TransactionBehavior::Immediate)
+ .map_err(|source| sqlite_error("starting exact-format result transaction", source))?;
+
+ ensure_format_run_active(&transaction, format_run_id, baseline_run_id)?;
+ for observation in observations {
+ persist_format_observation(&transaction, format_run_id, baseline_run_id, observation)?;
+ }
+ refresh_format_run_projection(
+ &transaction,
+ format_run_id,
+ checkpoint_sha256,
+ checkpoint_sequence,
+ )?;
+ transaction
+ .commit()
+ .map_err(|source| sqlite_error("committing exact-format result batch", source))?;
+ self.format_summary(baseline_run_id)
+ }
+
+ pub fn complete_format_run(
+ &mut self,
+ format_run_id: &str,
+ baseline_run_id: &str,
+ ) -> ProfilerResult {
+ let transaction = self
+ .connection
+ .transaction_with_behavior(TransactionBehavior::Immediate)
+ .map_err(|source| {
+ sqlite_error("starting exact-format completion transaction", source)
+ })?;
+ let (completed, total): (i64, i64) = transaction
+ .query_row(
+ "SELECT completed_objects, total_objects FROM format_identification_runs
+ WHERE id=?1 AND baseline_run_id=?2 AND state='running'",
+ params![format_run_id, baseline_run_id],
+ |row| Ok((row.get(0)?, row.get(1)?)),
+ )
+ .optional()
+ .map_err(|source| sqlite_error("validating exact-format completion", source))?
+ .ok_or_else(|| {
+ ProfilerError::contract(
+ ErrorCode::FormatRunNotFound,
+ "active exact-format run was not found",
+ false,
+ )
+ })?;
+ if completed != total {
+ return Err(ProfilerError::contract_with_context(
+ ErrorCode::FormatRunFailed,
+ "exact-format run cannot be completed before every content object is accounted for",
+ true,
+ BTreeMap::from([
+ ("completedObjects".into(), completed.to_string()),
+ ("totalObjects".into(), total.to_string()),
+ ]),
+ ));
+ }
+ transaction
+ .execute(
+ "UPDATE format_identification_runs
+ SET state='succeeded', finished_at=?1
+ WHERE id=?2 AND baseline_run_id=?3 AND state='running'",
+ params![now_text(), format_run_id, baseline_run_id],
+ )
+ .map_err(|source| sqlite_error("completing exact-format run", source))?;
+ transaction
+ .commit()
+ .map_err(|source| sqlite_error("committing exact-format completion", source))?;
+ self.format_summary(baseline_run_id)
+ }
+
+ pub fn fail_format_run(
+ &mut self,
+ format_run_id: &str,
+ error: &ProfilerError,
+ ) -> ProfilerResult<()> {
+ let report = error.report();
+ self.connection
+ .execute(
+ "UPDATE format_identification_runs SET
+ state='failed', finished_at=?1, failure_code=?2, failure_message=?3
+ WHERE id=?4 AND state='running'",
+ params![
+ now_text(),
+ format!("{:?}", report.code).to_ascii_lowercase(),
+ report.message,
+ format_run_id,
+ ],
+ )
+ .map_err(|source| sqlite_error("failing exact-format run", source))?;
+ Ok(())
+ }
+
+ pub fn format_summary(&self, baseline_run_id: &str) -> ProfilerResult {
+ let collection_id = load_summary_collection_id(&self.connection, baseline_run_id)?;
+ let workload = load_format_workload(&self.connection, &collection_id)?;
+ let latest = load_latest_format_run(&self.connection, baseline_run_id)?;
+ let distinct_puids = count_distinct_puids(&self.connection, baseline_run_id)?;
+
+ let mut summary = FormatSummary {
+ baseline_run_id: baseline_run_id.into(),
+ total_objects: workload.total_objects()?,
+ eligible_objects: workload.eligible_objects()?,
+ total_bytes: workload.total_bytes()?,
+ distinct_puids,
+ ..FormatSummary::default()
+ };
+ if let Some(latest) = latest {
+ latest.apply_to(&mut summary)?;
+ }
+ Ok(summary)
+ }
+
+ pub fn format_page(&self, request: &FormatPageRequest) -> ProfilerResult {
+ validate_format_page_request(request)?;
+ let collection_id: String = self
+ .connection
+ .query_row(
+ "SELECT collection_id FROM profiler_runs WHERE id=?1",
+ [&request.baseline_run_id],
+ |row| row.get(0),
+ )
+ .optional()
+ .map_err(|source| sqlite_error("loading format page collection", source))?
+ .ok_or_else(|| {
+ ProfilerError::contract(ErrorCode::RunNotFound, "baseline run was not found", false)
+ })?;
+ let search_like = request
+ .filters
+ .search
+ .as_deref()
+ .map(str::trim)
+ .filter(|value| !value.is_empty())
+ .map(|value| format!("%{}%", value.to_ascii_lowercase()));
+ let state = request.filters.state.map(FormatState::as_str);
+ let puid = request
+ .filters
+ .puid
+ .as_deref()
+ .map(str::trim)
+ .filter(|value| !value.is_empty());
+ let mismatch = i64::from(request.filters.mismatch_only);
+ let total_i64: i64 = self
+ .connection
+ .query_row(
+ format_count_sql(),
+ named_params! {
+ ":baseline_run_id": request.baseline_run_id.as_str(),
+ ":collection_id": collection_id.as_str(),
+ ":search_like": search_like,
+ ":state": state,
+ ":puid": puid,
+ ":mismatch_only": mismatch,
+ },
+ |row| row.get(0),
+ )
+ .map_err(|source| sqlite_error("counting exact-format page", source))?;
+ let fetch_limit = i64::from(request.limit) + 1;
+ let mut statement = self
+ .connection
+ .prepare(format_page_sql())
+ .map_err(|source| sqlite_error("preparing exact-format page", source))?;
+ let mut items = statement
+ .query_map(
+ named_params! {
+ ":baseline_run_id": request.baseline_run_id.as_str(),
+ ":collection_id": collection_id.as_str(),
+ ":search_like": search_like,
+ ":state": state,
+ ":puid": puid,
+ ":mismatch_only": mismatch,
+ ":after_sha256": request.after_sha256.as_deref(),
+ ":fetch_limit": fetch_limit,
+ },
+ format_row,
+ )
+ .map_err(|source| sqlite_error("querying exact-format page", source))?
+ .collect::, _>>()
+ .map_err(|source| sqlite_error("collecting exact-format page", source))?;
+ let has_more = items.len() > request.limit as usize;
+ if has_more {
+ items.pop();
+ }
+ let next_after_sha256 = if has_more {
+ items.last().map(|item| item.sha256.clone())
+ } else {
+ None
+ };
+ Ok(FormatPage {
+ items,
+ total_filtered: to_u64(total_i64, "format filtered total")?,
+ next_after_sha256,
+ has_more,
+ })
+ }
+}
+
+#[derive(Debug, Clone, Copy)]
+struct FormatWorkload {
+ total_objects: i64,
+ eligible_objects: i64,
+ total_bytes: i64,
+}
+
+impl FormatWorkload {
+ fn total_objects(self) -> ProfilerResult {
+ to_u64(self.total_objects, "format total object count")
+ }
+
+ fn eligible_objects(self) -> ProfilerResult {
+ to_u64(self.eligible_objects, "format eligible object count")
+ }
+
+ fn total_bytes(self) -> ProfilerResult {
+ to_u64(self.total_bytes, "format total byte count")
+ }
+}
+
+#[derive(Debug)]
+struct ActiveFormatRun {
+ id: String,
+ configuration_fingerprint: String,
+ checkpoint_sha256: Option,
+ completed_objects: i64,
+ completed_bytes: i64,
+ checkpoint_sequence: i64,
+}
+
+#[derive(Debug)]
+struct ResumableFormatRun {
+ id: String,
+ checkpoint_sha256: Option,
+ completed_objects: i64,
+ completed_bytes: i64,
+ checkpoint_sequence: i64,
+}
+
+#[derive(Debug, Default)]
+struct FormatRunProgress {
+ checkpoint_sha256: Option,
+ completed_objects: i64,
+ completed_bytes: i64,
+ checkpoint_sequence: i64,
+}
+
+impl From for ResumableFormatRun {
+ fn from(active: ActiveFormatRun) -> Self {
+ Self {
+ id: active.id,
+ checkpoint_sha256: active.checkpoint_sha256,
+ completed_objects: active.completed_objects,
+ completed_bytes: active.completed_bytes,
+ checkpoint_sequence: active.checkpoint_sequence,
+ }
+ }
+}
+
+impl ResumableFormatRun {
+ fn into_parts(self) -> (String, FormatRunProgress) {
+ (
+ self.id,
+ FormatRunProgress {
+ checkpoint_sha256: self.checkpoint_sha256,
+ completed_objects: self.completed_objects,
+ completed_bytes: self.completed_bytes,
+ checkpoint_sequence: self.checkpoint_sequence,
+ },
+ )
+ }
+}
+
+#[derive(Debug)]
+struct PersistedFormatSummary {
+ format_run_id: String,
+ state: String,
+ completed_objects: i64,
+ completed_bytes: i64,
+ identified: i64,
+ unknown: i64,
+ ambiguous: i64,
+ empty: i64,
+ skipped_unavailable: i64,
+ tool_errors: i64,
+ extension_mismatches: i64,
+ started_at: String,
+ finished_at: Option,
+ tool_name: String,
+ tool_version: String,
+ executable_path: String,
+ executable_sha256: String,
+ signature_path: String,
+ signature_sha256: String,
+ signature_version: String,
+ signature_created: Option,
+ identifiers_json: String,
+ probed_at: String,
+}
+
+impl PersistedFormatSummary {
+ fn from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result {
+ Ok(Self {
+ format_run_id: row.get(0)?,
+ state: row.get(1)?,
+ completed_objects: row.get(2)?,
+ completed_bytes: row.get(3)?,
+ identified: row.get(4)?,
+ unknown: row.get(5)?,
+ ambiguous: row.get(6)?,
+ empty: row.get(7)?,
+ skipped_unavailable: row.get(8)?,
+ tool_errors: row.get(9)?,
+ extension_mismatches: row.get(10)?,
+ started_at: row.get(11)?,
+ finished_at: row.get(12)?,
+ tool_name: row.get(13)?,
+ tool_version: row.get(14)?,
+ executable_path: row.get(15)?,
+ executable_sha256: row.get(16)?,
+ signature_path: row.get(17)?,
+ signature_sha256: row.get(18)?,
+ signature_version: row.get(19)?,
+ signature_created: row.get(20)?,
+ identifiers_json: row.get(21)?,
+ probed_at: row.get(22)?,
+ })
+ }
+
+ fn apply_to(self, summary: &mut FormatSummary) -> ProfilerResult<()> {
+ summary.latest_format_run_id = Some(self.format_run_id);
+ summary.latest_run_state = Some(self.state);
+ summary.completed_objects = to_u64(self.completed_objects, "format completed objects")?;
+ summary.completed_bytes = to_u64(self.completed_bytes, "format completed bytes")?;
+ summary.identified = to_u64(self.identified, "identified formats")?;
+ summary.unknown = to_u64(self.unknown, "unknown formats")?;
+ summary.ambiguous = to_u64(self.ambiguous, "ambiguous formats")?;
+ summary.empty = to_u64(self.empty, "empty format objects")?;
+ summary.skipped_unavailable = to_u64(self.skipped_unavailable, "skipped format objects")?;
+ summary.tool_errors = to_u64(self.tool_errors, "format tool errors")?;
+ summary.extension_mismatches =
+ to_u64(self.extension_mismatches, "format extension mismatches")?;
+ summary.started_at = Some(self.started_at);
+ summary.finished_at = self.finished_at;
+ summary.tool = Some(FormatToolIdentity {
+ tool_name: self.tool_name,
+ tool_version: self.tool_version,
+ executable_path: self.executable_path,
+ executable_sha256: self.executable_sha256,
+ signature_path: self.signature_path,
+ signature_sha256: if self.signature_sha256.is_empty() {
+ None
+ } else {
+ Some(self.signature_sha256)
+ },
+ signature_version: self.signature_version,
+ signature_created: self.signature_created,
+ identifiers: serde_json::from_str(&self.identifiers_json).map_err(|error| {
+ ProfilerError::Internal(format!("parsing persisted format identifiers: {error}"))
+ })?,
+ probed_at: self.probed_at,
+ });
+ Ok(())
+ }
+}
+
+fn load_format_baseline(
+ transaction: &Transaction<'_>,
+ baseline_run_id: &str,
+) -> ProfilerResult {
+ let (collection_id, state): (String, String) = transaction
+ .query_row(
+ "SELECT collection_id, state FROM profiler_runs WHERE id=?1",
+ [baseline_run_id],
+ |row| Ok((row.get(0)?, row.get(1)?)),
+ )
+ .optional()
+ .map_err(|source| sqlite_error("loading baseline profiler run", source))?
+ .ok_or_else(|| {
+ ProfilerError::contract(ErrorCode::RunNotFound, "baseline run was not found", false)
+ })?;
+ if state != "succeeded" {
+ return Err(ProfilerError::contract(
+ ErrorCode::RunNotBrowsable,
+ "exact format identification requires a completed physical profile",
+ false,
+ ));
+ }
+ Ok(collection_id)
+}
+
+fn find_resumable_format_run(
+ transaction: &Transaction<'_>,
+ request: &FormatRunStartRequest<'_>,
+) -> ProfilerResult> {
+ if let Some(active) = load_active_format_run(transaction, request.baseline_run_id)? {
+ if request.resume && active.configuration_fingerprint == request.configuration_fingerprint {
+ return Ok(Some(active.into()));
+ }
+ return Err(ProfilerError::contract_with_context(
+ ErrorCode::FormatRunAlreadyActive,
+ "an exact-format run is already active for this physical profile",
+ true,
+ BTreeMap::from([("formatRunId".into(), active.id)]),
+ ));
+ }
+ if !request.resume {
+ return Ok(None);
+ }
+ load_failed_resumable_format_run(
+ transaction,
+ request.baseline_run_id,
+ request.configuration_fingerprint,
+ )
+}
+
+fn load_active_format_run(
+ transaction: &Transaction<'_>,
+ baseline_run_id: &str,
+) -> ProfilerResult> {
+ transaction
+ .query_row(
+ "SELECT id, configuration_fingerprint, checkpoint_sha256,
+ completed_objects, completed_bytes, checkpoint_sequence
+ FROM format_identification_runs
+ WHERE baseline_run_id=?1 AND state='running'
+ ORDER BY started_at DESC LIMIT 1",
+ [baseline_run_id],
+ |row| {
+ Ok(ActiveFormatRun {
+ id: row.get(0)?,
+ configuration_fingerprint: row.get(1)?,
+ checkpoint_sha256: row.get(2)?,
+ completed_objects: row.get(3)?,
+ completed_bytes: row.get(4)?,
+ checkpoint_sequence: row.get(5)?,
+ })
+ },
+ )
+ .optional()
+ .map_err(|source| sqlite_error("checking active format run", source))
+}
+
+fn load_failed_resumable_format_run(
+ transaction: &Transaction<'_>,
+ baseline_run_id: &str,
+ configuration_fingerprint: &str,
+) -> ProfilerResult> {
+ transaction
+ .query_row(
+ "SELECT id, checkpoint_sha256, completed_objects, completed_bytes,
+ checkpoint_sequence
+ FROM format_identification_runs
+ WHERE baseline_run_id=?1 AND configuration_fingerprint=?2
+ AND state IN ('failed', 'cancelled')
+ ORDER BY started_at DESC LIMIT 1",
+ params![baseline_run_id, configuration_fingerprint],
+ |row| {
+ Ok(ResumableFormatRun {
+ id: row.get(0)?,
+ checkpoint_sha256: row.get(1)?,
+ completed_objects: row.get(2)?,
+ completed_bytes: row.get(3)?,
+ checkpoint_sequence: row.get(4)?,
+ })
+ },
+ )
+ .optional()
+ .map_err(|source| sqlite_error("finding resumable format run", source))
+}
+
+fn load_format_workload(
+ connection: &rusqlite::Connection,
+ collection_id: &str,
+) -> ProfilerResult {
+ connection
+ .query_row(
+ "SELECT COUNT(*),
+ COALESCE(SUM(CASE WHEN availability_state='available'
+ AND expected_size_bytes>0 THEN 1 ELSE 0 END), 0),
+ COALESCE(SUM(CASE WHEN availability_state='available'
+ AND expected_size_bytes>0
+ THEN expected_size_bytes ELSE 0 END), 0)
+ FROM content_objects WHERE collection_id=?1",
+ [collection_id],
+ |row| {
+ Ok(FormatWorkload {
+ total_objects: row.get(0)?,
+ eligible_objects: row.get(1)?,
+ total_bytes: row.get(2)?,
+ })
+ },
+ )
+ .map_err(|source| sqlite_error("counting exact-format workload", source))
+}
+
+fn resume_format_run(
+ transaction: &Transaction<'_>,
+ collection_id: &str,
+ tool_id: &str,
+ workload: FormatWorkload,
+ candidate: ResumableFormatRun,
+) -> ProfilerResult {
+ transaction
+ .execute(
+ "UPDATE format_identification_runs SET
+ state='running', tool_id=?1, failure_code=NULL, failure_message=NULL,
+ finished_at=NULL
+ WHERE id=?2",
+ params![tool_id, candidate.id.as_str()],
+ )
+ .map_err(|source| sqlite_error("resuming exact-format run", source))?;
+ let (format_run_id, progress) = candidate.into_parts();
+ build_format_registration(format_run_id, collection_id, workload, progress)
+}
+
+fn create_format_run(
+ transaction: &Transaction<'_>,
+ request: &FormatRunStartRequest<'_>,
+ collection_id: &str,
+ tool_id: &str,
+ workload: FormatWorkload,
+) -> ProfilerResult {
+ let id = Uuid::now_v7().to_string();
+ transaction
+ .execute(
+ "INSERT INTO format_identification_runs(
+ id, baseline_run_id, collection_id, tool_id, state,
+ configuration_fingerprint, batch_size, worker_count, timeout_seconds,
+ total_objects, eligible_objects, total_bytes, started_at
+ ) VALUES(?1, ?2, ?3, ?4, 'running', ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)",
+ params![
+ id.as_str(),
+ request.baseline_run_id,
+ collection_id,
+ tool_id,
+ request.configuration_fingerprint,
+ i64::from(request.batch_size),
+ i64::from(request.worker_count),
+ i64::try_from(request.timeout_seconds).unwrap_or(i64::MAX),
+ workload.total_objects,
+ workload.eligible_objects,
+ workload.total_bytes,
+ now_text(),
+ ],
+ )
+ .map_err(|source| sqlite_error("creating exact-format run", source))?;
+ build_format_registration(id, collection_id, workload, FormatRunProgress::default())
+}
+
+fn build_format_registration(
+ format_run_id: String,
+ collection_id: &str,
+ workload: FormatWorkload,
+ progress: FormatRunProgress,
+) -> ProfilerResult {
+ Ok(FormatRunRegistration {
+ format_run_id,
+ collection_id: collection_id.into(),
+ total_objects: workload.total_objects()?,
+ eligible_objects: workload.eligible_objects()?,
+ total_bytes: workload.total_bytes()?,
+ resume_after_sha256: progress.checkpoint_sha256,
+ completed_objects: to_u64(progress.completed_objects, "resumed completed object count")?,
+ completed_bytes: to_u64(progress.completed_bytes, "resumed completed byte count")?,
+ checkpoint_sequence: to_u64(progress.checkpoint_sequence, "resumed checkpoint sequence")?,
+ })
+}
+
+fn persist_format_observation(
+ transaction: &Transaction<'_>,
+ format_run_id: &str,
+ baseline_run_id: &str,
+ observation: &FormatObservation,
+) -> ProfilerResult<()> {
+ let observation_id =
+ upsert_format_observation(transaction, format_run_id, baseline_run_id, observation)?;
+ replace_format_matches(transaction, &observation_id, &observation.matches)?;
+ update_content_format_projection(transaction, format_run_id, observation)
+}
+
+fn upsert_format_observation(
+ transaction: &Transaction<'_>,
+ format_run_id: &str,
+ baseline_run_id: &str,
+ observation: &FormatObservation,
+) -> ProfilerResult {
+ let observation_id = Uuid::now_v7().to_string();
+ transaction
+ .execute(
+ "INSERT INTO format_observations(
+ id, format_run_id, baseline_run_id, content_object_id, sha256, state,
+ source_mime_type, preferred_extension, staging_mode, primary_identifier,
+ primary_format_name, primary_format_version, primary_mime_type, match_count,
+ extension_checked, extension_mismatch, error_code, error_message, observed_at
+ ) VALUES(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19)
+ ON CONFLICT(format_run_id, content_object_id) DO UPDATE SET
+ sha256=excluded.sha256,
+ state=excluded.state,
+ source_mime_type=excluded.source_mime_type,
+ preferred_extension=excluded.preferred_extension,
+ staging_mode=excluded.staging_mode,
+ primary_identifier=excluded.primary_identifier,
+ primary_format_name=excluded.primary_format_name,
+ primary_format_version=excluded.primary_format_version,
+ primary_mime_type=excluded.primary_mime_type,
+ match_count=excluded.match_count,
+ extension_checked=excluded.extension_checked,
+ extension_mismatch=excluded.extension_mismatch,
+ error_code=excluded.error_code,
+ error_message=excluded.error_message,
+ observed_at=excluded.observed_at",
+ params![
+ observation_id,
+ format_run_id,
+ baseline_run_id,
+ observation.content_object_id,
+ observation.sha256,
+ observation.state.as_str(),
+ observation.source_mime_type,
+ observation.preferred_extension,
+ observation.staging_mode,
+ observation.primary_identifier,
+ observation.primary_format_name,
+ observation.primary_format_version,
+ observation.primary_mime_type,
+ i64::try_from(observation.match_count).unwrap_or(i64::MAX),
+ i64::from(observation.extension_checked),
+ i64::from(observation.extension_mismatch),
+ observation.error_code,
+ observation.error_message,
+ observation.observed_at,
+ ],
+ )
+ .map_err(|source| sqlite_error("upserting exact-format observation", source))?;
+ transaction
+ .query_row(
+ "SELECT id FROM format_observations
+ WHERE format_run_id=?1 AND content_object_id=?2",
+ params![format_run_id, observation.content_object_id],
+ |row| row.get(0),
+ )
+ .map_err(|source| sqlite_error("reading exact-format observation id", source))
+}
+
+fn replace_format_matches(
+ transaction: &Transaction<'_>,
+ observation_id: &str,
+ matches: &[FormatMatch],
+) -> ProfilerResult<()> {
+ transaction
+ .execute(
+ "DELETE FROM format_matches WHERE observation_id=?1",
+ [observation_id],
+ )
+ .map_err(|source| sqlite_error("replacing exact-format matches", source))?;
+ for (ordinal, format_match) in matches.iter().enumerate() {
+ insert_match(transaction, observation_id, ordinal, format_match)?;
+ }
+ Ok(())
+}
+
+fn update_content_format_projection(
+ transaction: &Transaction<'_>,
+ format_run_id: &str,
+ observation: &FormatObservation,
+) -> ProfilerResult<()> {
+ transaction
+ .execute(
+ "UPDATE content_objects SET
+ format_state=?1,
+ primary_puid=?2,
+ primary_format_name=?3,
+ primary_format_version=?4,
+ primary_format_mime_type=?5,
+ format_match_count=?6,
+ extension_checked=?7,
+ extension_mismatch=?8,
+ last_format_run_id=?9,
+ last_format_at=?10,
+ updated_at=?10
+ WHERE id=?11",
+ params![
+ observation.state.as_str(),
+ observation.primary_identifier,
+ observation.primary_format_name,
+ observation.primary_format_version,
+ observation.primary_mime_type,
+ i64::try_from(observation.match_count).unwrap_or(i64::MAX),
+ i64::from(observation.extension_checked),
+ i64::from(observation.extension_mismatch),
+ format_run_id,
+ observation.observed_at,
+ observation.content_object_id,
+ ],
+ )
+ .map_err(|source| sqlite_error("updating exact-format projection", source))?;
+ Ok(())
+}
+
+fn load_summary_collection_id(
+ connection: &rusqlite::Connection,
+ baseline_run_id: &str,
+) -> ProfilerResult {
+ connection
+ .query_row(
+ "SELECT collection_id FROM profiler_runs WHERE id=?1",
+ [baseline_run_id],
+ |row| row.get(0),
+ )
+ .optional()
+ .map_err(|source| sqlite_error("loading format summary baseline run", source))?
+ .ok_or_else(|| {
+ ProfilerError::contract(ErrorCode::RunNotFound, "baseline run was not found", false)
+ })
+}
+
+fn load_latest_format_run(
+ connection: &rusqlite::Connection,
+ baseline_run_id: &str,
+) -> ProfilerResult> {
+ connection
+ .query_row(
+ "SELECT run.id, run.state, run.completed_objects, run.completed_bytes,
+ run.identified, run.unknown, run.ambiguous, run.empty_objects,
+ run.skipped_unavailable, run.tool_errors, run.extension_mismatches,
+ run.started_at, run.finished_at,
+ tool.tool_name, tool.tool_version, tool.executable_path,
+ tool.executable_sha256, tool.signature_path, tool.signature_sha256,
+ tool.signature_version, tool.signature_created, tool.identifiers_json,
+ tool.probed_at
+ FROM format_identification_runs AS run
+ JOIN format_tools AS tool ON tool.id=run.tool_id
+ WHERE run.baseline_run_id=?1
+ ORDER BY run.started_at DESC LIMIT 1",
+ [baseline_run_id],
+ PersistedFormatSummary::from_row,
+ )
+ .optional()
+ .map_err(|source| sqlite_error("loading latest exact-format run", source))
+}
+
+fn count_distinct_puids(
+ connection: &rusqlite::Connection,
+ baseline_run_id: &str,
+) -> ProfilerResult {
+ let count = connection
+ .query_row(
+ "SELECT COUNT(DISTINCT primary_identifier)
+ FROM format_observations
+ WHERE format_run_id=(SELECT id FROM format_identification_runs
+ WHERE baseline_run_id=?1
+ ORDER BY started_at DESC LIMIT 1)
+ AND primary_identifier IS NOT NULL",
+ [baseline_run_id],
+ |row| row.get::<_, i64>(0),
+ )
+ .map_err(|source| sqlite_error("counting distinct format identifiers", source))?;
+ to_u64(count, "distinct format identifier count")
+}
+
+fn register_tool(
+ transaction: &rusqlite::Transaction<'_>,
+ tool: &FormatToolIdentity,
+) -> ProfilerResult {
+ let signature_sha256 = tool.signature_sha256.clone().unwrap_or_default();
+ let existing = transaction
+ .query_row(
+ "SELECT id FROM format_tools
+ WHERE tool_name=?1 AND executable_sha256=?2
+ AND signature_version=?3 AND signature_sha256=?4",
+ params![
+ tool.tool_name,
+ tool.executable_sha256,
+ tool.signature_version,
+ signature_sha256,
+ ],
+ |row| row.get::<_, String>(0),
+ )
+ .optional()
+ .map_err(|source| sqlite_error("finding persisted format tool", source))?;
+ if let Some(id) = existing {
+ return Ok(id);
+ }
+ let id = Uuid::now_v7().to_string();
+ let identifiers = serde_json::to_string(&tool.identifiers).map_err(|error| {
+ ProfilerError::Internal(format!("serializing format tool identifiers: {error}"))
+ })?;
+ transaction
+ .execute(
+ "INSERT INTO format_tools(
+ id, tool_name, tool_version, executable_path, executable_sha256,
+ signature_path, signature_sha256, signature_version, signature_created,
+ identifiers_json, probed_at
+ ) VALUES(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)",
+ params![
+ id,
+ tool.tool_name,
+ tool.tool_version,
+ tool.executable_path,
+ tool.executable_sha256,
+ tool.signature_path,
+ signature_sha256,
+ tool.signature_version,
+ tool.signature_created,
+ identifiers,
+ tool.probed_at,
+ ],
+ )
+ .map_err(|source| sqlite_error("persisting format tool identity", source))?;
+ Ok(id)
+}
+
+fn ensure_format_run_active(
+ transaction: &rusqlite::Transaction<'_>,
+ format_run_id: &str,
+ baseline_run_id: &str,
+) -> ProfilerResult<()> {
+ let active: bool = transaction
+ .query_row(
+ "SELECT EXISTS(SELECT 1 FROM format_identification_runs
+ WHERE id=?1 AND baseline_run_id=?2 AND state='running')",
+ params![format_run_id, baseline_run_id],
+ |row| row.get(0),
+ )
+ .map_err(|source| sqlite_error("validating active exact-format run", source))?;
+ if active {
+ Ok(())
+ } else {
+ Err(ProfilerError::contract(
+ ErrorCode::FormatRunNotFound,
+ "active exact-format run was not found",
+ false,
+ ))
+ }
+}
+
+fn insert_match(
+ transaction: &rusqlite::Transaction<'_>,
+ observation_id: &str,
+ ordinal: usize,
+ format_match: &FormatMatch,
+) -> ProfilerResult<()> {
+ transaction
+ .execute(
+ "INSERT INTO format_matches(
+ id, observation_id, ordinal, namespace, identifier, format_name,
+ format_version, mime_type, format_class, basis, warning, is_primary
+ ) VALUES(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)",
+ params![
+ Uuid::now_v7().to_string(),
+ observation_id,
+ i64::try_from(ordinal).unwrap_or(i64::MAX),
+ format_match.namespace,
+ format_match.identifier,
+ format_match.format_name,
+ format_match.format_version,
+ format_match.mime_type,
+ format_match.format_class,
+ format_match.basis,
+ format_match.warning,
+ i64::from(format_match.is_primary),
+ ],
+ )
+ .map_err(|source| sqlite_error("persisting exact-format match", source))?;
+ Ok(())
+}
+
+fn refresh_format_run_projection(
+ transaction: &rusqlite::Transaction<'_>,
+ format_run_id: &str,
+ checkpoint_sha256: &str,
+ checkpoint_sequence: u64,
+) -> ProfilerResult<()> {
+ transaction
+ .execute(
+ "UPDATE format_identification_runs SET
+ completed_objects=(SELECT COUNT(*) FROM format_observations WHERE format_run_id=?1),
+ completed_bytes=(SELECT COALESCE(SUM(CASE
+ WHEN content.availability_state='available'
+ AND content.expected_size_bytes>0
+ THEN content.expected_size_bytes ELSE 0 END), 0)
+ FROM format_observations AS observation
+ JOIN content_objects AS content ON content.id=observation.content_object_id
+ WHERE observation.format_run_id=?1),
+ identified=(SELECT COUNT(*) FROM format_observations WHERE format_run_id=?1 AND state='identified'),
+ unknown=(SELECT COUNT(*) FROM format_observations WHERE format_run_id=?1 AND state='unknown'),
+ ambiguous=(SELECT COUNT(*) FROM format_observations WHERE format_run_id=?1 AND state='ambiguous'),
+ empty_objects=(SELECT COUNT(*) FROM format_observations WHERE format_run_id=?1 AND state='empty'),
+ skipped_unavailable=(SELECT COUNT(*) FROM format_observations WHERE format_run_id=?1 AND state='skipped_unavailable'),
+ tool_errors=(SELECT COUNT(*) FROM format_observations WHERE format_run_id=?1 AND state='tool_error'),
+ extension_mismatches=(SELECT COUNT(*) FROM format_observations WHERE format_run_id=?1 AND extension_mismatch=1),
+ checkpoint_sha256=?2,
+ checkpoint_sequence=?3
+ WHERE id=?1",
+ params![
+ format_run_id,
+ checkpoint_sha256,
+ i64::try_from(checkpoint_sequence).unwrap_or(i64::MAX),
+ ],
+ )
+ .map_err(|source| sqlite_error("refreshing exact-format run projection", source))?;
+ Ok(())
+}
+
+fn safe_extension(filename: &str) -> Option {
+ let extension = std::path::Path::new(filename).extension()?.to_str()?;
+ let cleaned = extension
+ .chars()
+ .filter(char::is_ascii_alphanumeric)
+ .take(16)
+ .collect::()
+ .to_ascii_lowercase();
+ (!cleaned.is_empty()).then_some(cleaned)
+}
+
+fn validate_format_page_request(request: &FormatPageRequest) -> ProfilerResult<()> {
+ if request.baseline_run_id.trim().is_empty() {
+ return Err(ProfilerError::InvalidArgument(
+ "baseline run id cannot be empty".into(),
+ ));
+ }
+ if request.limit == 0 || request.limit > 500 {
+ return Err(ProfilerError::InvalidArgument(
+ "format page limit must be between 1 and 500".into(),
+ ));
+ }
+ Ok(())
+}
+
+fn parse_format_state(value: &str, column: usize) -> rusqlite::Result {
+ match value {
+ "uninspected" => Ok(FormatState::Uninspected),
+ "identified" => Ok(FormatState::Identified),
+ "unknown" => Ok(FormatState::Unknown),
+ "ambiguous" => Ok(FormatState::Ambiguous),
+ "empty" => Ok(FormatState::Empty),
+ "skipped_unavailable" => Ok(FormatState::SkippedUnavailable),
+ "tool_error" => Ok(FormatState::ToolError),
+ other => Err(rusqlite::Error::FromSqlConversionFailure(
+ column,
+ Type::Text,
+ Box::new(std::io::Error::new(
+ std::io::ErrorKind::InvalidData,
+ format!("unknown format state: {other}"),
+ )),
+ )),
+ }
+}
+
+fn row_u64(row: &rusqlite::Row<'_>, column: usize) -> rusqlite::Result {
+ let value = row.get::<_, i64>(column)?;
+ u64::try_from(value).map_err(|error| {
+ rusqlite::Error::FromSqlConversionFailure(column, Type::Integer, Box::new(error))
+ })
+}
+
+fn format_row(row: &rusqlite::Row<'_>) -> rusqlite::Result {
+ let state: String = row.get(5)?;
+ Ok(FormatObjectRow {
+ content_object_id: row.get(0)?,
+ sha256: row.get(1)?,
+ primary_filename: row.get(2)?,
+ expected_size_bytes: row_u64(row, 3)?,
+ source_mime_type: row.get(4)?,
+ state: parse_format_state(&state, 5)?,
+ primary_identifier: row.get(6)?,
+ primary_format_name: row.get(7)?,
+ primary_format_version: row.get(8)?,
+ primary_mime_type: row.get(9)?,
+ match_count: row_u64(row, 10)?,
+ extension_checked: row.get::<_, i64>(11)? != 0,
+ extension_mismatch: row.get::<_, i64>(12)? != 0,
+ })
+}
+
+fn format_count_sql() -> &'static str {
+ "WITH latest_run AS (
+ SELECT id FROM format_identification_runs
+ WHERE baseline_run_id=:baseline_run_id
+ ORDER BY started_at DESC LIMIT 1
+ )
+ SELECT COUNT(*) FROM content_objects AS content
+ LEFT JOIN format_observations AS observation
+ ON observation.content_object_id=content.id
+ AND observation.format_run_id=(SELECT id FROM latest_run)
+ WHERE content.collection_id=:collection_id
+ AND (:state IS NULL OR COALESCE(observation.state, 'uninspected')=:state)
+ AND (:puid IS NULL OR observation.primary_identifier=:puid)
+ AND (:mismatch_only=0 OR COALESCE(observation.extension_mismatch, 0)=1)
+ AND (:search_like IS NULL OR lower(content.sha256) LIKE :search_like
+ OR lower(content.source_detected_mime_type) LIKE :search_like
+ OR lower(COALESCE(observation.primary_identifier, '')) LIKE :search_like
+ OR lower(COALESCE(observation.primary_format_name, '')) LIKE :search_like
+ OR EXISTS(SELECT 1 FROM filename_variants AS variant
+ WHERE variant.content_object_id=content.id
+ AND lower(variant.display_filename) LIKE :search_like))"
+}
+
+fn format_page_sql() -> &'static str {
+ "WITH latest_run AS (
+ SELECT id FROM format_identification_runs
+ WHERE baseline_run_id=:baseline_run_id
+ ORDER BY started_at DESC LIMIT 1
+ )
+ SELECT content.id, content.sha256,
+ COALESCE((SELECT variant.display_filename FROM filename_variants AS variant
+ WHERE variant.content_object_id=content.id
+ ORDER BY variant.occurrence_count DESC, variant.normalized_filename LIMIT 1),
+ '[unnamed]'),
+ content.expected_size_bytes, content.source_detected_mime_type,
+ COALESCE(observation.state, 'uninspected'), observation.primary_identifier,
+ observation.primary_format_name, observation.primary_format_version,
+ observation.primary_mime_type, COALESCE(observation.match_count, 0),
+ COALESCE(observation.extension_checked, 0),
+ COALESCE(observation.extension_mismatch, 0)
+ FROM content_objects AS content
+ LEFT JOIN format_observations AS observation
+ ON observation.content_object_id=content.id
+ AND observation.format_run_id=(SELECT id FROM latest_run)
+ WHERE content.collection_id=:collection_id
+ AND (:after_sha256 IS NULL OR content.sha256>:after_sha256)
+ AND (:state IS NULL OR COALESCE(observation.state, 'uninspected')=:state)
+ AND (:puid IS NULL OR observation.primary_identifier=:puid)
+ AND (:mismatch_only=0 OR COALESCE(observation.extension_mismatch, 0)=1)
+ AND (:search_like IS NULL OR lower(content.sha256) LIKE :search_like
+ OR lower(content.source_detected_mime_type) LIKE :search_like
+ OR lower(COALESCE(observation.primary_identifier, '')) LIKE :search_like
+ OR lower(COALESCE(observation.primary_format_name, '')) LIKE :search_like
+ OR EXISTS(SELECT 1 FROM filename_variants AS variant
+ WHERE variant.content_object_id=content.id
+ AND lower(variant.display_filename) LIKE :search_like))
+ ORDER BY content.sha256 LIMIT :fetch_limit"
+}
diff --git a/crates/profiler-storage-sqlite/src/lib.rs b/crates/profiler-storage-sqlite/src/lib.rs
index a456958..4b2eb31 100644
--- a/crates/profiler-storage-sqlite/src/lib.rs
+++ b/crates/profiler-storage-sqlite/src/lib.rs
@@ -1,4 +1,5 @@
mod explorer;
+mod format;
mod review;
mod workspace;
@@ -32,7 +33,7 @@ use unicode_normalization::UnicodeNormalization;
use uuid::Uuid;
const APPLICATION_ID: i64 = 0x4D56_5046; // MVPF
-pub const CURRENT_USER_VERSION: i64 = 5;
+pub const CURRENT_USER_VERSION: i64 = 6;
#[derive(Debug, Clone, Copy)]
struct Migration {
@@ -67,6 +68,11 @@ const MIGRATIONS: &[Migration] = &[
user_version: 5,
sql: include_str!("../migrations/0005_workspace_reopen_reviews.sql"),
},
+ Migration {
+ id: "0006_exact_format_identification",
+ user_version: 6,
+ sql: include_str!("../migrations/0006_exact_format_identification.sql"),
+ },
];
#[derive(Debug)]
@@ -1687,18 +1693,19 @@ fn verify_required_workspace_tables(connection: &Connection) -> ProfilerResult<(
.query_row(
"SELECT COUNT(*) FROM sqlite_master
WHERE type='table' AND name IN (
- 'workspace_meta', 'finding_review_events', 'finding_review_state'
+ 'workspace_meta', 'finding_review_events', 'finding_review_state',
+ 'format_tools', 'format_identification_runs', 'format_observations', 'format_matches'
)",
[],
|row| row.get(0),
)
.map_err(|source| sqlite_error("checking workspace review tables", source))?;
- if count == 3 {
+ if count == 7 {
Ok(())
} else {
Err(ProfilerError::contract(
profiler_core::ErrorCode::WorkspaceInvalidLayout,
- "workspace review schema is incomplete",
+ "workspace review or format-identification schema is incomplete",
false,
))
}
@@ -2043,7 +2050,7 @@ mod tests {
row.get(0)
})
.unwrap();
- assert_eq!(count, 5);
+ assert_eq!(count, 6);
let user_version: i64 = second
.connection()
.pragma_query_value(None, "user_version", |row| row.get(0))
diff --git a/crates/profiler-storage-sqlite/src/review.rs b/crates/profiler-storage-sqlite/src/review.rs
index 129e4fc..6ee9fd9 100644
--- a/crates/profiler-storage-sqlite/src/review.rs
+++ b/crates/profiler-storage-sqlite/src/review.rs
@@ -3,8 +3,8 @@ use std::{collections::BTreeMap, str::FromStr};
use profiler_core::{
ErrorCode, FindingDetail, FindingReviewEvent, FindingReviewHistory, ProfilerError,
ProfilerResult, ReviewAction, ReviewActorKind, ReviewStatus, ReviewSummary,
- SanitizedFindingRow, SanitizedRunSummary, normalize_review_note, redaction_token,
- validate_status_note,
+ SanitizedFindingRow, SanitizedFormatSummary, SanitizedRunSummary, normalize_review_note,
+ redaction_token, validate_status_note,
};
use rusqlite::{OptionalExtension, Transaction, TransactionBehavior, params};
use uuid::Uuid;
@@ -343,6 +343,46 @@ impl ProfilerStore {
let inventory = self.run_inventory_summary(collection_id, snapshot_id)?;
let findings = self.findings_summary(run_id)?;
let review = self.review_summary(run_id)?;
+ let format_summary = self.format_summary(run_id)?;
+ let exact_formats = SanitizedFormatSummary {
+ latest_format_run_id: format_summary.latest_format_run_id,
+ latest_run_state: format_summary.latest_run_state,
+ total_objects: format_summary.total_objects,
+ eligible_objects: format_summary.eligible_objects,
+ completed_objects: format_summary.completed_objects,
+ total_bytes: format_summary.total_bytes,
+ completed_bytes: format_summary.completed_bytes,
+ identified: format_summary.identified,
+ unknown: format_summary.unknown,
+ ambiguous: format_summary.ambiguous,
+ empty: format_summary.empty,
+ skipped_unavailable: format_summary.skipped_unavailable,
+ tool_errors: format_summary.tool_errors,
+ extension_mismatches: format_summary.extension_mismatches,
+ distinct_puids: format_summary.distinct_puids,
+ tool_name: format_summary
+ .tool
+ .as_ref()
+ .map(|tool| tool.tool_name.clone()),
+ tool_version: format_summary
+ .tool
+ .as_ref()
+ .map(|tool| tool.tool_version.clone()),
+ executable_sha256: format_summary
+ .tool
+ .as_ref()
+ .map(|tool| tool.executable_sha256.clone()),
+ signature_version: format_summary
+ .tool
+ .as_ref()
+ .map(|tool| tool.signature_version.clone()),
+ signature_sha256: format_summary
+ .tool
+ .as_ref()
+ .and_then(|tool| tool.signature_sha256.clone()),
+ started_at: format_summary.started_at,
+ finished_at: format_summary.finished_at,
+ };
let findings_by_code = map_counts(
&self.connection,
"SELECT code, COUNT(*) FROM findings
@@ -361,7 +401,7 @@ impl ProfilerStore {
run_id,
)?;
Ok(SanitizedRunSummary {
- format_version: 1,
+ format_version: 2,
generated_at: now_text(),
run_id: run_id.to_owned(),
application_version: env!("CARGO_PKG_VERSION").into(),
@@ -371,6 +411,7 @@ impl ProfilerStore {
inventory,
findings,
review,
+ exact_formats,
findings_by_code,
review_by_status,
})
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index d04ceb9..0904071 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -1,87 +1,67 @@
# Architecture
+## Product boundary
+
```text
-MailVault archive (read-only)
- │
- â–¼
-Capability preflight
- │
- â–¼
-SQLite Online Backup snapshot
- │
- â–¼
-Streaming metadata inventory
- │
- â–¼
-SHA-256 content reconciliation
- │
- â–¼
-Bounded physical file-stat workers
- │
- â–¼
-Single-writer profiler SQLite
- │
- ├── read-only CLI/report queries
- └── state-bound Tauri desktop explorer
+MailVault canonical archive
+ ↓ read-only adapter and consistent snapshot
+Physical inventory
+ ↓ unique SHA-256 content objects and occurrences
+Exact format identification
+ ↓ versioned PUID/format assertions
+Future document corpus
+ ↓ safe extraction and selective OCR
+RMS intelligence
```
-## Crates
-
-- `profiler-core`: domain contracts, errors, run state, explorer types and progress events;
-- `profiler-storage-sqlite`: disposable profiler database, migrations, checkpoints and explorer queries;
-- `profiler-adapter-mailvault`: MailVault v2.0.6 schema/layout adapter and physical object resolver;
-- `profiler-engine`: ordered snapshot, inventory, reconciliation and file-stat pipeline;
-- `mailvault-profiler-cli`: headless preflight, snapshot and profile commands;
-- `mailvault-profiler-desktop`: Tauri command boundary and React application.
-
-## Source snapshot
-
-The source database is never copied with a raw filesystem copy. The adapter opens SQLite using
-read-only flags and uses the SQLite Online Backup API to create a consistent destination database.
-The destination is written only in the profiler workspace. Source metrics are compared before and
-after backup; a changing source invalidates the unpublished snapshot.
-
-## Physical inventory
-
-Messages, MIME parts, blob rows and relationships are streamed in stable source order. Attachment
-occurrences remain separate from SHA-256 content objects, so one binary may retain many filenames,
-messages and thread contexts without duplicate payload processing.
+MailVault remains the source of truth. The profiler database is derived and rebuildable, except for
+human review events that should be backed up.
-The file-stat stage works on unique content objects only. It uses bounded batches and a dedicated
-Rayon pool, validates the exact MailVault fan-out locator, rejects paths outside the archive root,
-and persists each result with the matching durable checkpoint transaction.
+## Workspace crates
-## Storage and explorer boundary
+- `profiler-core`: contracts, progress, errors, run and exact-format types;
+- `profiler-storage-sqlite`: migrations, inventory/review/format persistence and queries;
+- `profiler-adapter-mailvault`: schema-v3 read-only adapter and snapshot;
+- `profiler-engine`: physical-profile, workspace and exact-format orchestration;
+- `profiler-format-siegfried`: pinned sidecar probe, staging, bounded execution and JSON parsing;
+- `mailvault-profiler-cli`: headless operations and evidence streams;
+- `mailvault-profiler-desktop`: Tauri command boundary and React UI.
-The profiler database is disposable and rebuildable. One controlled writer is used during a run.
-The desktop explorer opens a separate SQLite connection in read-only/query-only mode and verifies
-the profiler application ID and exact schema version before executing queries.
+## Exact format stage
-Desktop explorer commands do not accept an arbitrary database path from the webview. A completed
-profile activates a server-side session containing the profiler database, collection and run IDs.
+The engine processes content objects in stable SHA-256 order. It skips unavailable and zero-byte
+objects, resolves each canonical locator beneath the archive root, invokes Siegfried in bounded
+batches and commits observations, all matches and a checkpoint in one transaction.
-## Compatibility
+The runner is deliberately outside `profiler-engine` so a future identifier can implement the same
+core contract without changing physical inventory or UI persistence.
-Compatibility is capability-based. The adapter checks required tables and columns, then records
-recommended-index warnings separately. A cosmetic package version string is not used as an archive
-contract.
+## Concurrency
-The current adapter accepts MailVault schema version 3 only. A newer source schema fails closed
-until the adapter is reviewed.
+- one exact-format writer per workspace, enforced by an OS file lock;
+- one sidecar process per batch;
+- Siegfried worker count is configured explicitly;
+- stdout/stderr readers are bounded and joined;
+- failed batches are split recursively to isolate per-object failures;
+- SQLite commits remain batch-scoped.
-## Progress
+## Evidence identity
-Progress events contain exact stage units:
+A completed format run stores:
-- snapshot: SQLite pages;
-- metadata inventory: source rows;
-- reconciliation and file stat: unique objects;
-- physical verification: expected bytes.
+- physical baseline run ID;
+- application contract version;
+- executable and signature hashes;
+- observed tool and signature versions;
+- configuration fingerprint;
+- counts and byte totals;
+- durable checkpoint;
+- object observations and all matches.
-The UI uses byte totals where available, does not fabricate an overall pipeline percentage and
-leaves ETA unavailable until the backend has enough measurable work. Warning, error, worker,
-queue and checkpoint counters come directly from the current stage event.
+A run cannot be marked complete unless committed object count equals the expected total.
-## Workspace reopen and review boundary
+## Security
-`profiler-engine::workspace` owns canonical path validation, operating-system locking, open modes and atomic export publication. `profiler-storage-sqlite` owns schema migration, backup, run catalog, review events/projection and integrity verification. Tauri and CLI call typed engine operations and do not execute SQL. Review state is derived metadata and never enters MailVault or original evidence files.
+Source paths are untrusted. Every file is canonicalized and required to remain beneath the MailVault
+root. No command shell is used. The sidecar receives argument arrays and a generated list file.
+Container expansion is disabled. See [Security model](SECURITY_MODEL.md).
diff --git a/docs/CLI_REFERENCE.md b/docs/CLI_REFERENCE.md
index c86a36d..2ccb841 100644
--- a/docs/CLI_REFERENCE.md
+++ b/docs/CLI_REFERENCE.md
@@ -1,216 +1,89 @@
# CLI reference
-Executable name:
+The CLI executable is `mailvault-profiler.exe`. Structured progress is written to `stderr`; final
+machine-readable results are written to `stdout`.
-```text
-mailvault-profiler.exe
-```
-
-Global behavior:
-
-- success exit code: `0`;
-- profiler error exit code: `2`;
-- structured errors are printed to `stderr`;
-- profile progress is printed as JSON Lines to `stderr`;
-- final command results are printed to `stdout`.
-
-
-
-## `preflight`
-
-```powershell
-mailvault-profiler.exe preflight --archive [--json]
-```
-
-Options:
-
-| Option | Required | Description |
-|---|---:|---|
-| `--archive ` | yes | MailVault archive root |
-| `--json` | no | Print the complete report as formatted JSON |
-
-Without `--json`, the command prints the archive, compatibility state, schema, source counts and
-each preflight check. An incompatible report returns exit code `2`.
-
-Example:
-
-```powershell
-.\target\release\mailvault-profiler.exe preflight `
- --archive "D:\MailVault-Demo" `
- --json > preflight.json
-```
-
-## `snapshot`
-
-```powershell
-mailvault-profiler.exe snapshot \
- --archive \
- --workspace \
- [--run-id ]
-```
-
-Options:
-
-| Option | Required | Description |
-|---|---:|---|
-| `--archive ` | yes | MailVault archive root |
-| `--workspace ` | yes | Separate profiler workspace |
-| `--run-id ` | no | Caller-supplied run identity; UUIDv7 generated when omitted |
-
-This command creates only the consistent source snapshot and prints the snapshot result as JSON.
-Progress events are emitted to `stderr`.
-
-## `profile`
-
-```powershell
-mailvault-profiler.exe profile \
- --archive \
- --workspace \
- [--batch-size ] \
- [--file-stat-workers ] \
- [--file-stat-batch-size ]
-```
-
-Options:
-
-| Option | Default | Description |
-|---|---:|---|
-| `--archive ` | required | MailVault archive root |
-| `--workspace ` | required | Separate profiler workspace |
-| `--batch-size ` | `1000` | Metadata inventory batch size |
-| `--file-stat-workers ` | `0` | `0` selects the conservative provisional automatic policy |
-| `--file-stat-batch-size ` | `512` | Durable file-stat batch/checkpoint size |
-
-Recommended capture:
-
-```powershell
-.\target\release\mailvault-profiler.exe profile `
- --archive "D:\MailVault-Demo" `
- --workspace "D:\MailVault-Profiler-Workspace" `
- --batch-size 1000 `
- --file-stat-workers 0 `
- --file-stat-batch-size 512 `
- 1> profile-result.json `
- 2> profile-progress.jsonl
-```
-
-## Progress JSON Lines
-
-Each line is a serialized `ProgressEvent` containing:
-
-```text
-runId
-sequence
-stage
-stageState
-unit
-completedItems / totalItems
-completedBytes / totalBytes
-elapsedMs
-instantThroughput / smoothedThroughput
-etaMs
-activeWorkers
-queueDepth
-warnings / errors
-currentObjectDisplay
-checkpointSequence
-```
-
-Stages currently used by the physical profile include:
+## Source and physical-profile commands
```text
-preflight
-source_snapshot
-metadata_inventory
-reconciliation
-file_stat
-aggregation
-publish
+preflight --archive [--json]
+snapshot --archive --workspace [--run-id ]
+profile --archive --workspace [options]
+workspace inspect --workspace [--json]
+runs list --workspace [--json]
+findings list|show|review|clear|note ...
+export sanitized-summary ...
```
-`fixity` and `format_identification` exist in the domain vocabulary but are not active capabilities
-in `0.1.0-alpha.3`.
-
-## Evidence wrapper
+## `formats probe`
-The repository includes a Windows PowerShell wrapper that builds the release CLI, runs JSON
-preflight and profile, records elapsed time, hashes output files and writes `run-manifest.json`.
+Validates the executable/signature pair and prints versioned tool identity.
```powershell
-powershell -NoProfile -ExecutionPolicy Bypass `
- -File .\scripts\run-real-archive-profile.ps1 `
- -ArchiveRoot "D:\MailVault-Demo" `
- -WorkspaceRoot "D:\MailVault-Profiler-Workspace" `
- -EvidenceRoot "D:\MailVault-Profiler-Evidence" `
- -InventoryBatchSize 1000 `
- -FileStatWorkers 0 `
- -FileStatBatchSize 512
+mailvault-profiler.exe formats probe `
+ --siegfried `
+ --signature `
+ [--workers 0] `
+ [--json]
```
-Use `-SkipBuild` only when the exact intended release CLI has already been built in the current
-working tree.
+The command fails when the pinned Siegfried or PRONOM version is not observed.
-## Workspace and review commands — alpha.3
-
-Inspect without modifying:
+## `formats identify`
```powershell
-.\target\release\mailvault-profiler.exe workspace inspect `
- --workspace "D:\MailVault-Profiler-Workspace" `
- --json
+mailvault-profiler.exe formats identify `
+ --workspace `
+ --run `
+ [--siegfried ] `
+ [--signature ] `
+ [--batch-size 2048] `
+ [--workers 0] `
+ [--timeout-seconds 900] `
+ [--resume true|false] `
+ [--allow-migration]
```
-List persisted runs:
+Notes:
-```powershell
-.\target\release\mailvault-profiler.exe runs list `
- --workspace "D:\MailVault-Profiler-Workspace" `
- --json
-```
+- `--allow-migration` is required to upgrade an older workspace schema;
+- `--workers 0` selects the conservative auto policy;
+- batch size must be 1–10,000;
+- timeout must be at least 30 seconds;
+- resume requires an identical configuration fingerprint;
+- `stdout` contains `FormatIdentificationResult` JSON;
+- `stderr` contains `ProgressEvent` JSONL.
-List findings with stable server-side filters:
+## `formats summary`
```powershell
-.\target\release\mailvault-profiler.exe findings list `
- --workspace "D:\MailVault-Profiler-Workspace" `
- --run "" `
- --category requires_attention `
- --review-status unreviewed `
- --limit 100 `
- --json
+mailvault-profiler.exe formats summary `
+ --workspace `
+ --run `
+ [--json]
```
-Show detail and review history:
+Returns totals for identified, unknown, ambiguous, empty, unavailable, tool errors, extension
+mismatches, PUIDs, objects and bytes.
-```powershell
-.\target\release\mailvault-profiler.exe findings show `
- --workspace "D:\MailVault-Profiler-Workspace" `
- --run "" `
- --finding "" `
- --json
-```
-
-Write review state. `--allow-migration` is required only when the selected workspace schema is older:
+## `formats list`
```powershell
-.\target\release\mailvault-profiler.exe findings review `
- --workspace "D:\MailVault-Profiler-Workspace" `
- --run "" `
- --finding "" `
- --status needs_investigation `
- --note "Verify against the retained backup." `
- --allow-migration
+mailvault-profiler.exe formats list `
+ --workspace `
+ --run `
+ [--state identified|unknown|ambiguous|empty|skipped_unavailable|tool_error] `
+ [--puid fmt/276] `
+ [--mismatch-only] `
+ [--search ] `
+ [--limit 100] `
+ [--json]
```
-Other write commands are `findings clear` and `findings note`. Only the process holding the workspace writer lock may use them.
+The desktop API additionally supports cursor continuation by SHA-256. CLI Alpha 4 returns the first
+requested page.
-Sanitized export:
-
-```powershell
-.\target\release\mailvault-profiler.exe export sanitized-summary `
- --workspace "D:\MailVault-Profiler-Workspace" `
- --run "" `
- --output ".\sanitized-summary.json"
-```
+## Exit behavior
-Use a `.csv` output path to export the sanitized finding rows instead of the aggregate JSON document.
+`0` means the command completed successfully. Non-zero exits serialize an `ErrorReport` to stderr.
+A partially committed format run is not reported as complete and may be resumed when eligible.
diff --git a/docs/FAQ.md b/docs/FAQ.md
index 15d5e0a..a12e292 100644
--- a/docs/FAQ.md
+++ b/docs/FAQ.md
@@ -17,7 +17,7 @@ physical availability and size. Full payload fixity modes are a later roadmap ca
## Can it reopen a previous workspace?
-Yes, through **Open existing workspace** in `0.1.0-alpha.3`.
+Yes, through **Open existing workspace** in `0.1.0-alpha.4`.
## Can I run it while MailVault is syncing?
@@ -47,4 +47,4 @@ are unsigned in the current alpha.
## Can I reopen a workspace after restarting the application?
-Yes, in `0.1.0-alpha.3`. Use **Open existing workspace**. The application inspects compatibility and lock state before loading the run catalog. Older alpha.2 workspaces require an explicit backed-up migration.
+Yes, in `0.1.0-alpha.4`. Use **Open existing workspace**. The application inspects compatibility and lock state before loading the run catalog. Older alpha.2 workspaces require an explicit backed-up migration.
diff --git a/docs/FINDINGS_REVIEW.md b/docs/FINDINGS_REVIEW.md
index 8c1a1e5..9040453 100644
--- a/docs/FINDINGS_REVIEW.md
+++ b/docs/FINDINGS_REVIEW.md
@@ -1,6 +1,6 @@
# Findings review
-Version `0.1.0-alpha.3` converts findings from a read-only list into a durable local review workflow while preserving MailVault and original run evidence.
+Version `0.1.0-alpha.4` converts findings from a read-only list into a durable local review workflow while preserving MailVault and original run evidence.
## Categories
diff --git a/docs/FORMAT_IDENTIFICATION.md b/docs/FORMAT_IDENTIFICATION.md
new file mode 100644
index 0000000..6be9453
--- /dev/null
+++ b/docs/FORMAT_IDENTIFICATION.md
@@ -0,0 +1,207 @@
+# Exact format identification
+
+## Purpose
+
+The physical inventory answers **which bytes exist and where they occurred**. Exact format
+identification answers **what technical format those bytes represent**. It remains part of the
+profiler because it operates on immutable content identity and technical evidence, not document
+meaning.
+
+Alpha 4 uses Siegfried with PRONOM signatures. A PUID is stored as an assertion produced by a named,
+hashed executable and signature database. It is not promoted to canonical MailVault metadata.
+
+## Product boundary
+
+Included:
+
+- exact format and version;
+- PUID and MIME assertion;
+- all candidate matches and deterministic primary match;
+- ambiguity, unknown and tool-error states;
+- extension/signature evidence;
+- tool/signature provenance;
+- progress, checkpoint, resume and filtering.
+
+Excluded:
+
+- archive extraction;
+- OCR or text extraction;
+- content preview;
+- malware execution or sandboxing;
+- semantic or procurement classification;
+- RMS writes.
+
+## Why Siegfried + PRONOM
+
+Siegfried supports machine-readable JSON, list-file input, multiple identifiers and configurable
+parallel scanning. PRONOM provides persistent format identifiers and signature data maintained by
+The National Archives. Alpha 4 pins:
+
+```text
+Siegfried: 1.11.6
+PRONOM signature: v124
+```
+
+The version pair is an evidence contract, not a floating dependency. Updating either requires a new
+configuration fingerprint and new assertions; historical assertions remain preserved.
+
+## Supply-chain contract
+
+`scripts/install-siegfried.ps1`:
+
+1. retrieves the named upstream GitHub release through the REST API;
+2. selects exactly one Windows x64 ZIP and rejects Win7 assets;
+3. requires a GitHub-provided `sha256:` asset digest;
+4. verifies asset size and SHA-256 before extraction;
+5. installs `sf.exe` and obtains `default.sig`;
+6. probes the runtime and requires Siegfried `1.11.6` and PRONOM `v124`;
+7. writes `tool-manifest.json` with provenance and local SHA-256 values.
+
+`scripts/verify-siegfried.ps1` repeats the local hash and runtime checks. Release installers bundle
+these verified files using Tauri resources. Generated binaries are ignored by Git.
+
+## Windows JSON compatibility
+
+Siegfried `1.11.6` escapes scanned filenames in JSON output, but its JSON header writes the
+configured signature value directly. Supplying an absolute Windows signature path therefore emits
+unescaped backslashes and invalid JSON. The profiler does not mutate or heuristically repair tool
+output. Instead, every probe and identification batch uses:
+
+```text
+-home
+-sig
+```
+
+The sidecar still loads the exact canonical, SHA-256-verified signature file, while the JSON header
+contains the portable value `default.sig`. Installer, verifier and Rust runner share this contract,
+and Windows CI executes a regression test for it.
+
+## Data model
+
+Migration `0006_exact_format_identification.sql` adds:
+
+- `format_runs`: stage-level identity, configuration and progress;
+- `format_observations`: one normalized current projection per content object/run;
+- `format_matches`: every returned match, including non-primary alternatives;
+- `format_checkpoints`: durable last committed SHA-256 and sequence;
+- format projection columns and indexes on `content_objects`.
+
+A format run is attached to a completed physical-profile baseline. The current projection can be
+rebuilt from versioned observations.
+
+## States
+
+| State | Meaning |
+|---|---|
+| `uninspected` | no completed observation exists |
+| `identified` | one decisive identifier remains |
+| `unknown` | no viable identifier was returned |
+| `ambiguous` | more than one decisive identifier remains |
+| `empty` | the content object is zero bytes; sidecar is not invoked |
+| `skipped_unavailable` | physical object is unavailable; sidecar is not invoked |
+| `tool_error` | path resolution, process, parse or per-file tool failure |
+
+Extension-only candidates do not create false ambiguity when stronger byte, container, XML or text
+signature evidence exists.
+
+## Primary assertion selection
+
+All matches are retained. The primary assertion is selected deterministically for display:
+
+1. viable identifiers only;
+2. container evidence;
+3. byte evidence;
+4. XML evidence;
+5. text evidence;
+6. other evidence;
+7. extension-only evidence;
+8. PRONOM and warning-free results receive deterministic tie-break preference.
+
+This is a presentation projection. It does not delete alternatives or claim certainty when the
+run remains ambiguous.
+
+## Extension evidence
+
+MailVault stores content-addressed objects without reliable filename extensions. The stage may
+create an ephemeral symbolic alias named with the preferred normalized extension and invokes
+Siegfried with symlink following enabled. The alias contains no file copy and is removed with the
+batch workspace.
+
+`extension_checked=true` is stored only when that alias was actually created. If the platform or
+permissions prevent alias creation, identification falls back to the canonical object path and the
+UI displays **Not checked**, not a false “No mismatch.”
+
+## Execution model
+
+```text
+read eligible content objects in SHA-256 order
+ → skip unavailable and zero-byte objects
+ → resolve canonical path beneath archive root
+ → create bounded batch workspace
+ → invoke one sidecar process for the batch
+ → stream and bound stdout/stderr
+ → parse JSON and reconcile every input path
+ → adaptively split a failed batch until the failing object is isolated
+ → commit observations + matches + checkpoint in one transaction
+ → emit structured progress
+```
+
+Defaults:
+
+```text
+batch size: 2048
+workers: auto, conservatively capped
+process timeout: 900 seconds
+resume: enabled
+container expansion: disabled
+```
+
+Batch size and worker defaults remain benchmark inputs; they are not performance claims for every
+storage device.
+
+## Safety controls
+
+- dedicated exclusive workspace lock;
+- archive-root containment check for every source path;
+- no shell command interpolation;
+- explicit executable/signature paths;
+- process timeout with kill/wait cleanup;
+- 64 MiB stdout and 4 MiB stderr bounds;
+- path lines containing CR/LF are rejected;
+- no `-z` archive expansion;
+- no attachment execution or renderer;
+- failed/incomplete run is not published as complete.
+
+## Resume and versioning
+
+A configuration fingerprint includes:
+
+- tool name/version and executable SHA-256;
+- signature version and SHA-256;
+- extension-evidence model;
+- batch size, worker count and timeout;
+- container-expansion setting.
+
+A crash-left `running` run may resume only when the fingerprint matches and the exclusive workspace
+lock can be acquired. A tool/signature/configuration change starts a separate run.
+
+## UI
+
+The **Exact formats** view provides:
+
+- tool probe and version display;
+- aggregate state counts and byte/object progress;
+- PUID count and extension mismatch count;
+- filters for state, PUID, text and mismatch;
+- cursor pagination;
+- object rows showing primary assertion and whether extension evidence was checked.
+
+Screenshots are sanitized release illustrations, not fabricated runtime claims.
+
+## References
+
+- Siegfried repository and release notes: https://github.com/richardlehane/siegfried
+- PRONOM registry: https://www.nationalarchives.gov.uk/PRONOM/
+- DROID and format identification guidance: https://www.nationalarchives.gov.uk/information-management/manage-information/preserving-digital-records/droid/
+- Tauri resource bundling: https://v2.tauri.app/develop/resources/
+- GitHub release-asset REST representation: https://docs.github.com/en/rest/releases/assets
diff --git a/docs/FORMAT_IDENTIFICATION_RUNBOOK.md b/docs/FORMAT_IDENTIFICATION_RUNBOOK.md
new file mode 100644
index 0000000..1061e0f
--- /dev/null
+++ b/docs/FORMAT_IDENTIFICATION_RUNBOOK.md
@@ -0,0 +1,167 @@
+# Exact format identification runbook
+
+## Preconditions
+
+- Alpha 3 or newer physical profile completed successfully.
+- Workspace is backed up before first schema migration.
+- MailVault archive is mounted at the same source root recorded by the workspace.
+- No second profiler writer is using the workspace.
+- Sufficient free space exists for profiler database growth and temporary batch metadata.
+- Pinned Siegfried resources are installed and verified.
+
+## 1. Update and build
+
+```powershell
+git checkout main
+git pull --ff-only
+npm ci
+.\scripts\install-siegfried.ps1
+.\scripts\verify-siegfried.ps1
+cargo build --release -p mailvault-profiler-cli --locked
+```
+
+The verifier intentionally passes `default.sig` relative to the configured Siegfried home. This is
+required on Windows because the upstream `1.11.6` JSON header does not escape backslashes in an
+absolute signature path. A successful probe must therefore produce parseable JSON before any
+workspace migration or real-archive run begins.
+
+## 2. Back up the workspace
+
+```powershell
+$Workspace = "E:\MailVault-Profiler-Alpha4"
+$Backup = "E:\MailVault-Profiler-Alpha4-PreMigration.zip"
+Compress-Archive -Path "$Workspace\*" -DestinationPath $Backup -Force
+Get-FileHash $Backup -Algorithm SHA256
+```
+
+## 3. Inspect and migrate explicitly
+
+```powershell
+.\target\release\mailvault-profiler.exe workspace inspect `
+ --workspace $Workspace `
+ --json
+```
+
+The `formats identify` command requires `--allow-migration` when an older workspace schema must be
+upgraded. Do not bypass the backup step.
+
+## 4. Find the physical baseline run
+
+```powershell
+.\target\release\mailvault-profiler.exe runs list `
+ --workspace $Workspace `
+ --json
+```
+
+Use a completed physical-profile run with the expected 17,296-message baseline.
+
+## 5. Probe the pinned toolchain
+
+```powershell
+$Tool = ".\tools\siegfried\windows-x86_64\sf.exe"
+$Signature = ".\tools\siegfried\windows-x86_64\default.sig"
+
+.\target\release\mailvault-profiler.exe formats probe `
+ --siegfried $Tool `
+ --signature $Signature `
+ --json |
+ Tee-Object -FilePath ".\format-tool-probe.json"
+```
+
+Require:
+
+```text
+toolVersion = 1.11.6
+signatureVersion = v124
+executableSha256 present
+signatureSha256 present
+```
+
+## 6. Run against the real archive
+
+```powershell
+$RunId = ""
+$Evidence = "E:\MailVault-Profiler-Evidence-Alpha4"
+New-Item -ItemType Directory -Path $Evidence -Force | Out-Null
+
+.\target\release\mailvault-profiler.exe formats identify `
+ --workspace $Workspace `
+ --run $RunId `
+ --siegfried $Tool `
+ --signature $Signature `
+ --batch-size 2048 `
+ --workers 0 `
+ --timeout-seconds 900 `
+ --resume true `
+ --allow-migration `
+ 1> "$Evidence\format-result.json" `
+ 2> "$Evidence\format-progress.jsonl"
+
+$LASTEXITCODE
+```
+
+Exit code must be `0` before treating the format run as complete.
+
+## 7. Inspect summary and high-risk queues
+
+```powershell
+.\target\release\mailvault-profiler.exe formats summary `
+ --workspace $Workspace `
+ --run $RunId `
+ --json |
+ Tee-Object -FilePath "$Evidence\format-summary.json"
+
+.\target\release\mailvault-profiler.exe formats list `
+ --workspace $Workspace `
+ --run $RunId `
+ --state tool_error `
+ --json |
+ Set-Content "$Evidence\format-tool-errors.json"
+
+.\target\release\mailvault-profiler.exe formats list `
+ --workspace $Workspace `
+ --run $RunId `
+ --state ambiguous `
+ --json |
+ Set-Content "$Evidence\format-ambiguous.json"
+
+.\target\release\mailvault-profiler.exe formats list `
+ --workspace $Workspace `
+ --run $RunId `
+ --mismatch-only `
+ --json |
+ Set-Content "$Evidence\format-extension-mismatches.json"
+```
+
+These files are private unless separately sanitized.
+
+## 8. Acceptance checks
+
+- run state is `succeeded`;
+- completed objects equals total content objects;
+- exactly one known missing object remains `skipped_unavailable`;
+- exactly one zero-byte object remains `empty`;
+- no path escapes or source mutation findings;
+- every identified/ambiguous observation has stored tool/signature identity;
+- no incomplete batch is counted as complete;
+- source MailVault database and blob tree remain unchanged;
+- unknown, ambiguous and tool-error counts are reviewed before parser planning.
+
+Do not prescribe expected identified/unknown/ambiguous counts before the real run.
+
+## 9. Resume after interruption
+
+Re-run the same command with identical tool, signature and options. The stage acquires the workspace
+lock, verifies the configuration fingerprint and continues after the last durable SHA-256
+checkpoint. A changed fingerprint starts a new run; it must not reuse prior completion claims.
+
+## 10. Public evidence
+
+Publish only aggregate sanitized outputs and toolchain provenance. Never publish:
+
+- profiler or snapshot SQLite databases;
+- real filenames, subjects, domains or local paths;
+- content hashes when they identify private documents;
+- raw progress logs;
+- review notes;
+- tool-error messages containing source paths.
diff --git a/docs/GETTING_STARTED.md b/docs/GETTING_STARTED.md
index 35771b5..094034e 100644
--- a/docs/GETTING_STARTED.md
+++ b/docs/GETTING_STARTED.md
@@ -1,153 +1,50 @@
# Getting started
-This guide performs one controlled profile of a MailVault archive on Windows without modifying the
-canonical source.
+## 1. Install
-## 1. Understand the three directories
+Download a Windows release, verify `SHA256SUMS.txt` and install the unsigned alpha package. Keep the
+archive and workspace separate.
-Use separate roots:
+## 2. Create the physical baseline
-```text
-D:\MailVault-Demo canonical source archive
-D:\MailVault-Profiler-Workspace disposable derived profiler data
-D:\MailVault-Profiler-Evidence run manifests and exported logs
-```
-
-The workspace and evidence roots must not be equal to, inside, or parents of the archive root.
-
-## 2. Prepare the source archive
-
-Before preflight:
-
-- stop MailVault synchronization, import, verification and maintenance tasks;
-- confirm the archive drive is stable and has no pending disconnect;
-- do not delete a lock file manually;
-- do not move database or object-store files;
-- ensure the profiler process can read the archive and write to the workspace.
-
-Expected MailVault layout:
-
-```text
-\database\mailvault.sqlite3
-\objects\raw\sha256\...
-\objects\blobs\sha256\...
-\state\...
-```
-
-## 3. Run read-only preflight
-
-Desktop:
+Run read-only preflight, create the source snapshot and complete the physical inventory. Reopen the
+workspace and confirm the expected baseline counts.
-1. Open **Collection setup**.
-2. Select the archive root.
-3. Choose **Run read-only preflight**.
+## 3. Install or verify exact-format resources
-CLI:
+Source builds:
```powershell
-.\target\release\mailvault-profiler.exe preflight `
- --archive "D:\MailVault-Demo"
+.\scripts\install-siegfried.ps1
+.\scripts\verify-siegfried.ps1
```
-Proceed only when:
-
-- `compatible` is true;
-- schema version is `3`;
-- writer lock is absent;
-- required path, schema and integrity checks pass.
-
-Warnings require review. Failed required checks block profiling.
-
-## 4. Select the profiler workspace
-
-Choose an empty or dedicated directory outside the source archive:
-
-```text
-D:\MailVault-Profiler-Workspace
-```
-
-The workspace stores:
-
-- the consistent source database snapshot;
-- profiler SQLite database and migrations;
-- run checkpoints;
-- derived inventory and findings.
-
-It does not need to be preserved as canonical evidence. It can contain sensitive derived metadata.
+Installed desktop builds resolve the bundled Tauri resources automatically.
-## 5. Create the physical inventory
+## 4. Migrate with a backup
-Desktop:
+Back up an Alpha 3 workspace before enabling migration. Migration 6 adds exact-format tables and
+indexes without changing MailVault.
-1. Select the workspace.
-2. Choose **Create physical inventory**.
-3. Leave the first-run defaults unchanged.
-4. Keep the application and archive drive available until the run reaches its terminal state.
+## 5. Run exact identification
-CLI defaults:
+Use the **Exact formats** view or follow the
+[exact format runbook](FORMAT_IDENTIFICATION_RUNBOOK.md).
-```text
-inventory batch size: 1000
-file-stat workers: 0 (conservative automatic policy)
-file-stat batch size: 512
-```
-
-Do not increase worker count before measuring the actual storage device. More concurrent file opens
-can reduce performance on HDDs, external drives and antivirus-scanned volumes.
-
-## 6. Review the result
-
-### Physical inventory
-
-Search by:
-
-- filename or filename variant;
-- full or partial SHA-256;
-- source-detected MIME type;
-- message subject;
-- sender domain.
-
-Filter by availability, size state or finding code. Pagination uses a stable SHA-256 cursor.
-
-### Findings
-
-Review errors first:
-
-- `MISSING_BLOB`;
-- `INVALID_BLOB_LOCATOR`;
-- unreadable or non-regular objects.
-
-Then review warnings:
-
-- `BLOB_SIZE_MISMATCH`;
-- `SAME_HASH_DIFFERENT_NAMES`;
-- `SAME_NAME_DIFFERENT_HASHES`;
-- zero-byte content evidence.
-
-Opening a finding with a content-object identity displays filename history, message occurrences and
-object-level technical evidence.
-
-## 7. Capture runtime evidence
-
-For benchmark or release evidence, use the wrapper:
-
-```powershell
-powershell -NoProfile -ExecutionPolicy Bypass `
- -File .\scripts\run-real-archive-profile.ps1 `
- -ArchiveRoot "D:\MailVault-Demo" `
- -WorkspaceRoot "D:\MailVault-Profiler-Workspace" `
- -EvidenceRoot "D:\MailVault-Profiler-Evidence"
-```
+## 6. Review output
-Read [Evidence outputs](EVIDENCE_OUTPUTS.md) before sharing any generated file.
+Start with:
-## 8. Finish safely
+1. tool errors;
+2. ambiguous results;
+3. unknown results;
+4. extension mismatches;
+5. generic formats such as OLE and octet-stream.
-- Record run state, elapsed time, warnings and errors.
-- Preserve evidence files only where access is controlled.
-- Do not publish filenames, domains, message subjects or local paths without sanitization.
-- Keep the canonical archive unchanged.
+Do not treat an extension match as content validation and do not infer procurement meaning from a
+PUID.
-## Reopen a completed workspace
+## 7. Keep evidence private
-After restart, choose **Open existing workspace**, select the same workspace directory, inspect compatibility, approve migration only after reviewing the retained-backup notice, select a run, and open it. Profiling does not rerun. Review writes are disabled automatically if another process owns the workspace lock or if history integrity validation fails.
+Profiler databases and raw logs contain sensitive metadata. Publish only sanitized aggregate
+exports and release screenshots built from synthetic data.
diff --git a/docs/GITHUB_PUBLISHING_GUIDE_FA.md b/docs/GITHUB_PUBLISHING_GUIDE_FA.md
index a30ec37..e8add49 100644
--- a/docs/GITHUB_PUBLISHING_GUIDE_FA.md
+++ b/docs/GITHUB_PUBLISHING_GUIDE_FA.md
@@ -90,7 +90,7 @@ git init
git branch -M main
git add .
git status --short
-git commit -m "Release MailVault Collection Profiler 0.1.0-alpha.3"
+git commit -m "Release MailVault Collection Profiler 0.1.0-alpha.4"
git remote add origin https://github.com/FireXCore/mailvault-collection-profiler.git
git push -u origin main
```
@@ -262,7 +262,7 @@ Release فقط با Tag نسخه اجرا میشود.
اگر CI قرمز است، Tag و Release نساز. انتهای Build موفق Frontend بهتنهایی به معنی Gate سبز نیست.
-## ۱۰. آمادهسازی Release `0.1.0-alpha.3`
+## ۱۰. آمادهسازی Release `0.1.0-alpha.4`
نسخه باید در این فایلها هماهنگ باشد:
@@ -278,13 +278,13 @@ apps/desktop/src-tauri/tauri.conf.json
```cmd
npm run check:release-config
-node scripts\check-tag-version.cjs v0.1.0-alpha.3
+node scripts\check-tag-version.cjs v0.1.0-alpha.4
```
Release Notes عمومی در این فایل آماده است:
```text
-docs/releases/v0.1.0-alpha.3.md
+docs/releases/v0.1.0-alpha.4.md
```
## ۱۱. ساخت Tag و اجرای Release Workflow
@@ -293,8 +293,8 @@ docs/releases/v0.1.0-alpha.3.md
git checkout main
git pull --ff-only
git status --short
-git tag -a v0.1.0-alpha.3 -m "MailVault Collection Profiler 0.1.0-alpha.3"
-git push origin v0.1.0-alpha.3
+git tag -a v0.1.0-alpha.4 -m "MailVault Collection Profiler 0.1.0-alpha.4"
+git push origin v0.1.0-alpha.4
```
Workflow زیر اجرا میشود:
@@ -325,7 +325,7 @@ Releases → Draft release
قبل از Publish کنترل کن:
-- Tag دقیقاً `v0.1.0-alpha.3` باشد
+- Tag دقیقاً `v0.1.0-alpha.4` باشد
- گزینه `This is a pre-release` روشن باشد
- Release Notes کامل باشد
- NSIS `-setup.exe` وجود داشته باشد
@@ -392,14 +392,14 @@ Publish release
اگر قبل از Publish مشکل وجود دارد، Draft را حذف کن، Tag را فقط در صورت منتشرنشدن حذف و اصلاح کن:
```cmd
-git push --delete origin v0.1.0-alpha.3
-git tag -d v0.1.0-alpha.3
+git push --delete origin v0.1.0-alpha.4
+git tag -d v0.1.0-alpha.4
```
اگر Release منتشر شده است، Tag را بازنویسی نکن. یک نسخه جدید مانند زیر بساز:
```text
-0.1.0-alpha.3
+0.1.0-alpha.4
```
Release منتشرشده باید Immutable تلقی شود.
diff --git a/docs/GUI_GUIDE.md b/docs/GUI_GUIDE.md
index fd56657..e52f8f1 100644
--- a/docs/GUI_GUIDE.md
+++ b/docs/GUI_GUIDE.md
@@ -1,101 +1,46 @@
-# Desktop GUI guide
+# GUI guide
-The desktop application has three primary views. Inventory and findings are enabled after a
-successful profile completes in the current application session.
+## Start and collection setup
-## 1. Collection setup
+Select the canonical MailVault root, run preflight and create or open a profiler workspace. The
+source is always read-only.
-
+## Runs
-### Archive root
+Choose a completed physical-profile run. Alpha 4 attaches exact format assertions to this baseline;
+it does not create a new archive.
-Select the MailVault archive root, not the database file itself. The application derives the
-canonical database, raw object store, blob object store and state directory from this root.
+## Physical inventory
-### Read-only preflight
+Search content objects and inspect filename/message occurrence history. SHA-256 is identity;
+filename is evidence.
-Preflight validates:
+## Findings
-- required paths and file types;
-- supported schema version;
-- required tables, columns and indexes;
-- source writer lock;
-- SQLite integrity and source metrics.
+Review physical findings using append-only statuses. Review state never modifies MailVault.
-The source contract panel lists every check and its required or advisory level.
+## Exact formats
-### Workspace
+The **Exact formats** view:
-The profiler workspace must be separate from the source archive. The action remains disabled until
-preflight is compatible and a workspace is selected.
+1. resolves the bundled or configured Siegfried sidecar;
+2. displays observed tool/signature identity;
+3. shows eligible object and byte totals;
+4. starts or resumes the versioned format run;
+5. streams exact object progress;
+6. exposes state, PUID, search and mismatch filters.
-## 2. Live profile progress
+Extension status meanings:
-
+- **Mismatch** — a safe extension alias was checked and the tool reported mismatch evidence;
+- **No mismatch** — an alias was checked and no mismatch warning was reported;
+- **Not checked** — no reliable extension alias was evaluated.
-Progress events expose:
+Unknown and ambiguous are evidence queues, not application failures. Tool errors require technical
+review.
-- current stage and stage state;
-- completed and total items;
-- completed and total bytes when available;
-- instantaneous and smoothed throughput;
-- elapsed time and ETA;
-- active workers and queue depth;
-- warnings, errors and checkpoint sequence;
-- current object display when appropriate.
+## Screenshots
-Progress is monotonic within a stage. A later event with lower item, byte or checkpoint counters is
-rejected by the core contract.
+
-## 3. Physical inventory
-
-
-
-The inventory represents exact binary content identities, not only attachment rows.
-
-### Search
-
-The search field matches:
-
-- primary and historical filenames;
-- SHA-256;
-- source-detected MIME type;
-- message subject;
-- sender domain.
-
-### Filters
-
-- Availability: available, missing, unreadable, invalid locator, non-regular.
-- Size state: match, mismatch, unavailable.
-- Finding code: zero byte, multiple filenames, missing blob, size mismatch.
-
-### Pagination
-
-The explorer uses stable SHA-256 keyset pagination. It does not use unstable offset pagination for
-large inventory browsing.
-
-## 4. Findings
-
-
-
-Findings can be filtered by severity and structured code. An object-level finding opens its content
-object. Collection-level findings remain reviewable without an object identity.
-
-## 5. Content-object detail
-
-
-
-The detail drawer includes:
-
-- complete SHA-256 identity;
-- availability and size state;
-- expected size, occurrence count, message count and thread count;
-- technical findings;
-- filename variant history;
-- message occurrence history with source part path.
-
-The drawer intentionally shows metadata only. It does not render or execute attachment payloads.
-
-## Existing workspace and review journey
-
-The start page provides **Profile new archive** and **Open existing workspace**. Inspection shows schema, compatibility, lock mode and run count. Older compatible workspaces require explicit migration confirmation. The run catalog opens previous inventory and findings. The finding drawer exposes status, required-note validation and append-only history. Informational filename/content relationships are separated from attention-required findings.
+
diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md
index 34d8cef..4635657 100644
--- a/docs/IMPLEMENTATION_STATUS.md
+++ b/docs/IMPLEMENTATION_STATUS.md
@@ -1,99 +1,39 @@
-# Implementation status — 0.1.0-alpha.3
-
-**Validation date:** 2026-07-20
-**Release state:** implementation complete for the alpha.3 scope; Windows quality gate and controlled real-workspace acceptance passed.
-
-## Runtime status
-
-```text
-read-only preflight and physical inventory
-→ durable profiler workspace
-→ workspace inspection and explicit migration
-→ run catalog after process restart
-→ reopened inventory/findings/object detail
-→ append-only review events and projection
-→ review hash-chain verification
-→ sanitized aggregate export
-→ Windows and real-archive validation
-```
-
-Recorded alpha.3 status:
-
-- Rust format, Clippy and tests: green;
-- TypeScript and Vite production build: green;
-- native Tauri desktop Clippy/compile gate: green;
-- real MailVault schema-v3 profile: completed;
-- source/snapshot aggregate comparison: matched;
-- workspace reopen after full restart: passed;
-- Windows single-writer/read-only fallback: passed;
-- review persistence and append-only integrity: passed;
-- sanitized JSON/CSV acceptance: passed;
-- source mutation: none.
-
-See [Validation evidence](VALIDATION_0.1.0-alpha.3.md).
-
-## Completed implementation
-
-### Workspace reopening
-
-- Existing profiler databases open with explicit read-only/read-write flags.
-- Missing databases are not created.
-- Schema 4 migrates to schema 5 only after confirmation and a retained SQLite backup.
-- Newer, corrupted, overlapping or partially migrated workspaces fail closed.
-- One operating-system lock owns review writes; concurrent instances fall back to read-only.
-- Completed, failed and interrupted run records appear in the run catalog after process restart.
-- Completed runs reopen without profiling the MailVault source again.
-
-### Findings review
-
-- statuses: acknowledged, expected, needs investigation and resolved externally;
-- status clearing and notes append new events;
-- historical update/delete is blocked by SQLite triggers;
-- event sequence, prior status, prior hash, event hash and projection are validated on open;
-- notes are normalized, bounded and excluded from logs and sanitized exports;
-- informational evidence is separated from warnings and errors;
-- integrity failure disables writes while retaining read-only browsing.
-
-### Inventory and evidence
-
-- MailVault schema-v3 read-only preflight;
-- consistent SQLite Online Backup snapshot;
-- streaming metadata and physical object inventory;
-- SHA-256 content identity and occurrence reconciliation;
-- filename history and duplicate-relationship evidence;
-- bounded file-stat inspection;
-- missing, unreadable, invalid-locator, non-regular and size-mismatch findings;
-- cursor-paginated inventory and finding search;
-- content-object detail and related message evidence;
-- sanitized JSON summary and CSV findings export.
-
-### User surfaces
-
-- desktop start screen for a new profile or existing workspace;
-- compatibility, migration and lock-state result;
-- run catalog;
-- reopened inventory and findings;
-- content-object and finding detail drawers;
-- review controls and append-only history;
-- sanitized export;
-- equivalent CLI inspection, listing, review and export commands.
-
-## Explicitly incomplete
-
-- resuming interrupted profiling;
-- user-facing pause/cancel controls;
-- archive repair, deletion, renaming or deduplication;
-- full payload fixity pass;
-- exact format identification, JHOVE and container expansion;
-- OCR or semantic classification;
-- cloud synchronization, authentication and multi-user review;
-- automatic updates;
-- public Windows code signing.
-
-## Release classification
-
-`0.1.0-alpha.3` is a development pre-release. Runtime Green means the implemented alpha scope passed
-its declared gates; it does not claim the deferred capabilities listed above.
-
-See [Workspace format](WORKSPACE_FORMAT.md), [Findings review](FINDINGS_REVIEW.md),
-[Validation evidence](VALIDATION_0.1.0-alpha.3.md) and [Roadmap](ROADMAP.md).
+# Implementation status — 0.1.0-alpha.4
+
+## Implemented
+
+- read-only MailVault v3 preflight and SQLite snapshot;
+- physical inventory, content/occurrence separation and file-stat checks;
+- findings, append-only review, sanitized export and explorer UI;
+- exact format core/storage/engine/sidecar/CLI/desktop modules;
+- pinned Siegfried/PRONOM acquisition, verification and Tauri resource bundling;
+- schema migration 6 and format filtering/progress/resume contract;
+- publication-ready English/Persian documentation and sanitized screenshots.
+
+## Validated
+
+- frontend type/build;
+- Rust syntax parse;
+- real Alpha 3 profiler DB migration and count preservation;
+- SQLite integrity after migration;
+- synthetic format projection;
+- Windows PowerShell 5.1 sidecar installation and independent verification;
+- pinned Siegfried `1.11.6` and PRONOM `v124` runtime identity;
+- Windows-safe home-relative signature argument contract;
+- five `profiler-format-siegfried` unit tests;
+- workspace Rust formatting check.
+
+## Required before runtime-green release
+
+- rerun strict workspace Clippy after the structural refactor;
+- run the complete Rust workspace test suite on CI;
+- Windows Tauri build and installers;
+- verified sidecar install in CI;
+- private real-archive exact-format run;
+- benchmark, resume-interruption and source-nonmutation evidence;
+- sanitized aggregate format report.
+
+## Deferred
+
+Physical-profile pause/resume, full fixity, container expansion, JHOVE, extraction, OCR, semantic
+search, procurement classification and RMS integration.
diff --git a/docs/INDEX.md b/docs/INDEX.md
index 0324d50..66c100e 100644
--- a/docs/INDEX.md
+++ b/docs/INDEX.md
@@ -4,36 +4,33 @@
- [Getting started](GETTING_STARTED.md)
- [Windows installation](INSTALLATION_WINDOWS.md)
-- [Desktop GUI guide](GUI_GUIDE.md)
+- [GUI guide](GUI_GUIDE.md)
- [CLI reference](CLI_REFERENCE.md)
-- [Workspace format and reopening](WORKSPACE_FORMAT.md)
+- [Exact format identification](FORMAT_IDENTIFICATION.md)
+- [Exact format runbook](FORMAT_IDENTIFICATION_RUNBOOK.md)
- [Findings review](FINDINGS_REVIEW.md)
-- [Evidence outputs](EVIDENCE_OUTPUTS.md)
- [Troubleshooting](TROUBLESHOOTING.md)
- [FAQ](FAQ.md)
-## Safety and trust
-
-- [Security model](SECURITY_MODEL.md)
-- [Privacy and data handling](PRIVACY.md)
-- [Adapter contract](ADAPTER_CONTRACT.md)
-- [Screenshot and public-media policy](SCREENSHOTS.md)
-
-## Technical design
+## Architecture and evidence
- [Architecture](ARCHITECTURE.md)
+- [Adapter contract](ADAPTER_CONTRACT.md)
+- [Workspace format](WORKSPACE_FORMAT.md)
- [Inventory explorer](INVENTORY_EXPLORER.md)
-- [Implementation status](IMPLEMENTATION_STATUS.md)
+- [Evidence outputs](EVIDENCE_OUTPUTS.md)
- [Real archive baseline](REAL_ARCHIVE_BASELINE.md)
-- [Technical specification 0.1.1](TECHNICAL_SPEC_0.1.1.md)
+- [Security model](SECURITY_MODEL.md)
+- [Privacy](PRIVACY.md)
-## Maintainers
+## Development and release
- [Development](DEVELOPMENT.md)
+- [Implementation status](IMPLEMENTATION_STATUS.md)
+- [Roadmap](ROADMAP.md)
- [Release process](RELEASE_PROCESS.md)
-- [Validation evidence — alpha.3](VALIDATION_0.1.0-alpha.3.md)
-- [Validation evidence — alpha.2](VALIDATION_0.1.0-alpha.2.md)
+- [Screenshot policy](SCREENSHOTS.md)
+- [Alpha 4 validation](VALIDATION_0.1.0-alpha.4.md)
+- [Alpha 4 release notes](releases/v0.1.0-alpha.4.md)
- [GitHub publishing guide — Persian](GITHUB_PUBLISHING_GUIDE_FA.md)
- [Repository release handoff — Persian](REPOSITORY_RELEASE_HANDOFF_FA.md)
-- [Release notes — alpha.3](releases/v0.1.0-alpha.3.md)
-- [Release notes — alpha.2](releases/v0.1.0-alpha.2.md)
diff --git a/docs/INSTALLATION_WINDOWS.md b/docs/INSTALLATION_WINDOWS.md
index f6b42e9..6c8ae50 100644
--- a/docs/INSTALLATION_WINDOWS.md
+++ b/docs/INSTALLATION_WINDOWS.md
@@ -2,7 +2,7 @@
## Supported target
-`0.1.0-alpha.3` is validated for Windows x64 using the MSVC Rust target. The project may compile on
+`0.1.0-alpha.4` is validated for Windows x64 using the MSVC Rust target. The project may compile on
other desktop platforms after installing normal Tauri system libraries, but the canonical native
release gate is Windows.
@@ -108,7 +108,7 @@ target\release\bundle\nsis
target\release\bundle\msi
```
-The public SemVer remains `0.1.0-alpha.3`. WiX/MSI uses the numeric package version `0.1.0.3`
+The public SemVer remains `0.1.0-alpha.4`. WiX/MSI uses the numeric package version `0.1.0.4`
because Windows Installer does not accept an arbitrary prerelease label in ProductVersion.
## Run in development
@@ -126,3 +126,16 @@ npm run tauri -- dev
- MSI prerelease version error: run `npm run check:release-config` and keep the numeric WiX mapping.
See [Troubleshooting](TROUBLESHOOTING.md).
+
+
+## Exact-format sidecar for source builds
+
+Before compiling a release bundle, run:
+
+```powershell
+.\scripts\install-siegfried.ps1
+.\scripts\verify-siegfried.ps1
+```
+
+The installer build embeds the verified `sf.exe`, `default.sig` and `tool-manifest.json` as Tauri
+resources. Do not commit generated sidecar binaries.
diff --git a/docs/PRIVACY.md b/docs/PRIVACY.md
index d12993a..561fffe 100644
--- a/docs/PRIVACY.md
+++ b/docs/PRIVACY.md
@@ -1,47 +1,22 @@
-# Privacy and data handling
+# Privacy
-MailVault Collection Profiler is designed for local processing.
+Processing is local. The application does not require cloud upload or telemetry.
-## Network behavior
+Sensitive derived data can include:
-The profiling engine and desktop runtime do not require a cloud API, account or telemetry service.
-Network access may still occur during installation or development when downloading npm crates,
-Rust crates, Tauri tooling, WebView2, WiX or release artifacts.
-
-## Data that remains local
-
-- canonical MailVault archive;
-- consistent source database snapshot;
-- profiler database and checkpoints;
+- archive and workspace paths;
- filenames and filename variants;
-- message subject and sender-domain metadata;
-- SHA-256 identities;
-- technical findings;
-- progress and runtime evidence.
-
-## Derived data is still sensitive
-
-A profiler database contains no need for attachment payload execution, but it can disclose business
-relationships and collection structure. Protect it with the same access discipline used for the
-source archive.
-
-## Public issue policy
-
-Do not attach:
-
-- archive databases;
-- profiler databases;
-- EML files;
-- attachments;
-- private paths;
-- message subjects;
-- sender or recipient addresses;
-- credentials;
-- unredacted logs.
+- sender domains and message subjects;
+- SHA-256 content identities;
+- PUID/format assertions tied to private objects;
+- tool errors and review notes.
-Use synthetic fixtures and aggregate counts. Documentation screenshots in this repository use demo
-paths and synthetic metadata.
+Exact-format identification invokes a local bundled sidecar only. The sidecar receives local file
+paths and writes machine-readable output to the profiler process. No attachment bytes are sent to
+GitHub, PRONOM or another service during runtime identification.
-## Review data and sanitized exports
+The build-time sidecar installation script accesses upstream release services to obtain the pinned
+tool/signature resources. This happens during development/CI, not while profiling an archive.
-Finding decisions and notes are local profiler metadata. They are not uploaded and are not stored in MailVault. Sanitized export omits absolute paths, filenames, addresses, subjects, full locators and raw notes; short SHA-256-derived tokens provide correlation without exposing source identifiers.
+Publish only sanitized aggregate exports. Do not publish profiler databases, source snapshots, raw
+progress logs or content-level format lists from a private collection.
diff --git a/docs/REAL_ARCHIVE_BASELINE.md b/docs/REAL_ARCHIVE_BASELINE.md
index 7314981..2d709d3 100644
--- a/docs/REAL_ARCHIVE_BASELINE.md
+++ b/docs/REAL_ARCHIVE_BASELINE.md
@@ -1,71 +1,34 @@
# Real archive baseline
-The initial architecture and acceptance gates are grounded in the supplied private MailVault
-collection. These values are not sample data and are not hard-coded as general compatibility
-requirements.
-
-## Historical supplied baseline
-
-| Metric | Observed |
-|---|---:|
-| Archive scale | approximately 20–30 GB |
-| Messages | 17,296 |
-| Message occurrences | 17,307 |
-| MIME parts | 54,450 |
-| Attachment occurrences in the broader supplied metadata inventory | 21,946 |
-| Messages with attachment metadata | 10,020 |
-| Unique attachment SHA-256 values | 13,592 |
-| Blob rows | 13,684 |
-| Message relationships | 12,115 |
-| Participant rows | 51,101 |
-| Known security-excluded message | 1 |
-
-Derived from the supplied attachment inventory:
-
-| Metric | Observed |
-|---|---:|
-| Attachment occurrence bytes | 9,954,841,724 bytes |
-| Unique attachment payload bytes | 6,442,427,318 bytes |
-| Repeated attachment occurrences | 8,354 |
-| Exact duplicate occurrence ratio | 38.07% |
-| Unique binaries with multiple normalized filenames | 367 |
-| Normalized filenames referring to multiple hashes | 1,123 |
-| Unique zero-byte binary | 1 |
-| Zero-byte occurrences | 16 |
-
-## Validated alpha.3 adapter run
-
-The controlled `0.1.0-alpha.3` run recorded the following source-contract and inventory aggregates:
+The project is designed and validated against a private MailVault collection with approximately
+20–30 GB of evidence.
| Metric | Recorded |
|---|---:|
-| Accounts | 1 |
| Messages | 17,296 |
| Message occurrences | 17,307 |
| MIME parts | 54,450 |
-| Attachment occurrences under the current adapter contract | 18,552 |
-| Blob rows / content objects | 13,684 |
-| Blob bytes | 6,467,253,277 |
+| Explicit attachment-role occurrences | 18,552 |
+| Historical broader attachment metadata baseline | 21,946 |
+| Content objects / blob rows | 13,684 |
| Content occurrences | 22,068 |
| Message relationships | 12,115 |
-| Participant rows | 51,101 |
-| Same hash with different names | 375 |
-| Same normalized name with different hashes | 1,107 |
-| Zero-byte content objects | 1 |
-| Missing physical objects | 1 |
-
-## Reconciliation boundary
+| Participants | 51,101 |
+| Blob bytes | 6,467,253,277 |
+| Available objects | 13,683 |
+| Known missing security-excluded object | 1 |
+| Zero-byte content object | 1 |
+| Physical findings | 1,484 |
-The historical value `21,946` and the alpha.3 adapter value `18,552` are retained as separate
-metrics. The project does not silently replace one with the other. The difference must remain
-traceable to the role, filtering and occurrence boundaries used by the source inventory and the
-current physical-attachment contract.
+The three attachment-related counts represent different scopes and must not be collapsed:
-A future compatibility or benchmark report must:
+- 18,552: MIME parts with explicit `role=attachment` in the current adapter contract;
+- 21,946: historical attachment metadata inventory scope;
+- 22,068: all content-bearing occurrences linked to content objects.
-1. name the exact occurrence definition used;
-2. report excluded roles or records explicitly;
-3. preserve source and snapshot aggregate comparisons;
-4. avoid presenting either count as a universal compatibility constant.
+Alpha 3 completed the physical profile in roughly 400 seconds on the user's Windows system. Alpha 4
+must record its own exact-format duration and memory evidence before format performance is claimed.
-See [Validation evidence — 0.1.0-alpha.3](VALIDATION_0.1.0-alpha.3.md).
+Current source MIME distribution includes approximately 2,890 generic OLE objects and 137
+`application/octet-stream` objects, which is the principal reason exact PRONOM identification is
+required.
diff --git a/docs/RELEASE_PROCESS.md b/docs/RELEASE_PROCESS.md
index 0b11b53..a67b625 100644
--- a/docs/RELEASE_PROCESS.md
+++ b/docs/RELEASE_PROCESS.md
@@ -36,7 +36,7 @@ apps/desktop/src-tauri/tauri.conf.json bundle.windows.wix.version
```
The public version may contain a prerelease suffix. MSI uses a separate numeric four-part version.
-For `0.1.0-alpha.3`, the MSI version is `0.1.0.3`.
+For `0.1.0-alpha.4`, the MSI version is `0.1.0.4`.
## 2. Write release notes and changelog
@@ -67,7 +67,7 @@ Open an x64 Visual Studio Developer Command Prompt:
call "C:\Program Files\Microsoft Visual Studio\18\Community\Common7\Tools\VsDevCmd.bat" -arch=amd64 -host_arch=amd64
cd /d D:\mailvault-collection-profiler
npm ci
-node scripts\check-tag-version.cjs v0.1.0-alpha.3
+node scripts\check-tag-version.cjs v0.1.0-alpha.4
powershell -NoProfile -ExecutionPolicy Bypass -File .\scripts\quality.ps1
npm run tauri:desktop:bundle
powershell -NoProfile -ExecutionPolicy Bypass -File .\scripts\generate-release-checksums.ps1
@@ -101,10 +101,10 @@ Repeat the install/uninstall check with the MSI package.
```cmd
git add .
-git commit -m "Release MailVault Collection Profiler 0.1.0-alpha.3"
+git commit -m "Release MailVault Collection Profiler 0.1.0-alpha.4"
git push origin main
-git tag -a v0.1.0-alpha.3 -m "MailVault Collection Profiler 0.1.0-alpha.3"
-git push origin v0.1.0-alpha.3
+git tag -a v0.1.0-alpha.4 -m "MailVault Collection Profiler 0.1.0-alpha.4"
+git push origin v0.1.0-alpha.4
```
The tag starts `.github/workflows/release.yml`.
@@ -156,3 +156,10 @@ After publication, fix defects in a new version and retain the original release
- Confirm GitHub's source archives contain no generated directories or private paths.
- Open the next milestone and move deferred work to the roadmap.
- Record the tag, commit SHA, installer SHA-256 and publication time in the release record.
+
+
+## Sidecar release gate
+
+The Windows release workflow must run `install-siegfried.ps1` and `verify-siegfried.ps1` before
+quality checks and Tauri bundling. Release artifacts are invalid if the required tool/signature
+identity is not reproduced.
diff --git a/docs/REPOSITORY_RELEASE_HANDOFF_FA.md b/docs/REPOSITORY_RELEASE_HANDOFF_FA.md
index 3afdb1c..6bb078f 100644
--- a/docs/REPOSITORY_RELEASE_HANDOFF_FA.md
+++ b/docs/REPOSITORY_RELEASE_HANDOFF_FA.md
@@ -1,147 +1,167 @@
-# تحویل نهایی آمادهسازی مخزن GitHub
+# تحویل انتشار GitHub — `0.1.0-alpha.4`
-این سند مشخص میکند چه چیزی داخل بسته آماده شده و ترتیب اجرای مالک مخزن چیست.
+این سند ترتیب امن انتقال Candidate آلفا ۴ به مخزن عمومی را مشخص میکند. آلفا ۴ قابلیت
+**Exact Format Identification** را به Profiler اضافه میکند، اما تا قبل از سبز شدن Windows CI و
+اجرای خصوصی روی آرشیو واقعی، `runtime green` محسوب نمیشود.
## خروجی آمادهشده
-مخزن اکنون شامل این لایهها است:
+مخزن اکنون شامل این موارد است:
-1. README عمومی با معرفی، Safety Contract، نصب، GUI، CLI، محدودیتها و تصاویر Sanitized؛
-2. مستند نصب Windows و Build از Source؛
-3. راهنمای کامل GUI و CLI؛
-4. معماری، Security Model، Privacy و Evidence Outputs؛
-5. Troubleshooting و FAQ؛
-6. راهنمای توسعه و Release Process؛
-7. راهنمای فارسی تنظیم GitHub، Ruleset، Actions، Security و انتشار؛
-8. Issue Formهای Bug، Feature و Compatibility؛
-9. Pull Request Template حرفهای؛
-10. CI چهارمرحلهای برای Rust، Frontend، Windows Desktop و Privacy Scrub؛
-11. CodeQL، Dependency Review و Dependabot؛
-12. Release Workflow برای Build خودکار MSI/NSIS، Draft Release، SHA-256 و Attestation؛
-13. Social Preview و شش تصویر عمومی بدون مسیر یا اطلاعات خصوصی؛
-14. Gate خودکار برای Broken Link، فایلهای ضروری، Registry خصوصی و اطلاعات حساس؛
-15. Release Notes نسخه `0.1.0-alpha.3`.
+1. موتور Read-only فهرستبرداری فیزیکی Alpha 3؛
+2. ماژول مستقل `profiler-format-siegfried`؛
+3. قرارداد Pin شده Siegfried `1.11.6` و PRONOM `v124`؛
+4. ثبت PUID، نام و نسخه فرمت، MIME، Basis، Warning و همه Matchها؛
+5. انتخاب Primary Assertion بهصورت Deterministic بدون حذف Ambiguity؛
+6. Batch محدود، Timeout، سقف خروجی، Adaptive Failure Isolation و Resume؛
+7. Workspace Lock مستقل برای جلوگیری از اجرای همزمان Format Stage؛
+8. Migration شماره `0006` و Queryهای Baseline-scoped؛
+9. CLIهای `formats probe/identify/summary/list`؛
+10. صفحه **Exact formats** در Desktop UI؛
+11. Sanitized Export Schema 2؛
+12. مستندات انگلیسی و فارسی، Runbook، Security، Privacy و Release Notes؛
+13. نه تصویر Sanitized با وضوح بالا و Social Preview؛
+14. CI برای Rust، Frontend، Windows Desktop و Documentation/Privacy؛
+15. Release Workflow برای MSI/NSIS، SHA-256 و Attestation.
-## مرحله ۱: جایگزینی مخزن محلی
+## وضعیت Validation همین بسته
-پوشه نهایی را در مسیر عمومی و بدون نام خصوصی استخراج کن:
+در محیط آمادهسازی فعلی این Gateها اجرا و سبز شدهاند:
```text
-D:\mailvault-collection-profiler
+Release configuration check
+Documentation / privacy / local-link check
+Rust Tree-sitter syntax parse: 34 files
+TypeScript strict type-check
+Vite production build
+npm production audit: 0 vulnerabilities
+npm complete audit: 0 vulnerabilities
+Real Alpha 3 database migration: schema 5 → 6
+SQLite quick_check / foreign_key_check
+Baseline-scoped format projection isolation
```
-پوشه قبلی را مبنا قرار نده. فایلهای `target`، `node_modules` و `dist` داخل بسته Release Repository
-نیستند و باید محلی ساخته شوند.
+در این محیط Rust toolchain موجود نبود. بنابراین موارد زیر هنوز ادعا نمیشوند:
-## مرحله ۲: Gate نهایی Windows
-
-داخل **x64 Native Tools Command Prompt for Visual Studio**:
-
-```cmd
-call "C:\Program Files\Microsoft Visual Studio\18\Community\Common7\Tools\VsDevCmd.bat" -arch=amd64 -host_arch=amd64
-cd /d D:\mailvault-collection-profiler
-npm ci
-npm run check:docs
-npm run check:release-config
-node scripts\check-tag-version.cjs v0.1.0-alpha.3
-powershell -NoProfile -ExecutionPolicy Bypass -File .\scripts\quality.ps1
-npm run tauri:desktop:bundle
+```text
+cargo check
+rustfmt semantic gate
+Clippy
+Rust unit/integration tests
+Native Tauri compile
+MSI/NSIS build
+Live Siegfried installation/probe
+Real 13,684-object format run
```
-هیچ Tag یا Release تا قبل از سبز شدن کامل این Gate ساخته نشود.
+مدرک کامل: `docs/VALIDATION_0.1.0-alpha.4.md`.
-## مرحله ۳: ساخت مخزن
+## مرحله ۱: ساخت Branch
-Repository:
+بسته Source را در یک مسیر عمومی و بدون اطلاعات خصوصی استخراج کن و داخل Repository فعلی Branch
+جدید بساز:
-```text
-FireXCore/mailvault-collection-profiler
+```powershell
+git checkout main
+git pull --ff-only
+git checkout -b feat/alpha4-exact-format-identification
```
-Description:
+فایلهای بسته را روی Repository جایگزین کن. پوشههای `node_modules`، `target`، `dist`، دیتابیسهای
+Profiler و Runtime Evidence نباید Commit شوند.
-```text
-Local-first, read-only physical inventory and technical evidence explorer for MailVault archives.
-```
+## مرحله ۲: Gate کامل Windows
-Social Preview:
+داخل **x64 Native Tools Command Prompt for Visual Studio** یا PowerShellی که Toolchainهای لازم را
+میبیند:
-```text
-docs/assets/social-preview.png
+```powershell
+cd E:\github\mailvault-collection-profiler
+npm ci
+powershell -NoProfile -ExecutionPolicy Bypass -File .\scripts\install-siegfried.ps1
+powershell -NoProfile -ExecutionPolicy Bypass -File .\scripts\quality.ps1
+npm run tauri:desktop:bundle
```
-## مرحله ۴: Push اولیه
+همچنین Tag Mapping را قبل از Release کنترل کن:
-```cmd
-git init
-git branch -M main
-git add .
-git status --short
-git commit -m "Release MailVault Collection Profiler 0.1.0-alpha.3"
-git remote add origin https://github.com/FireXCore/mailvault-collection-profiler.git
-git push -u origin main
+```powershell
+npm run check:release-config
+node .\scripts\check-tag-version.cjs v0.1.0-alpha.4
```
-## مرحله ۵: ساخت Labelها
+هیچ Tag یا Release تا قبل از سبز شدن این Gate ساخته نشود.
-بعد از `gh auth login`:
+## مرحله ۳: Commit و Pull Request
```powershell
-powershell -NoProfile -ExecutionPolicy Bypass `
- -File .\scripts\configure-github-labels.ps1 `
- -Repository "FireXCore/mailvault-collection-profiler"
+git add .
+git status --short
+git commit -m "feat: add exact PRONOM format identification"
+git push -u origin feat/alpha4-exact-format-identification
```
-## مرحله ۶: تنظیمات GitHub
+Pull Request باید تمام Required Checkهای Branch Protection را پاس کند. Diff اصلی شامل ۱۸ فایل
+جدید و ۵۱ فایل اصلاحشده است.
+
+## مرحله ۴: Smoke Test خصوصی
-راهنمای خطبهخط:
+پس از ساخت Release CLI، Runbook زیر را روی یک Copy/Backup از Workspace آلفا ۳ اجرا کن:
```text
-docs/GITHUB_PUBLISHING_GUIDE_FA.md
+docs/FORMAT_IDENTIFICATION_RUNBOOK.md
```
-این بخشها باید تنظیم شوند:
+حداقل شواهد لازم:
-- About، Topics و Social Preview؛
-- Ruleset شاخه `main`؛
-- Required Status Checks؛
-- Workflow Permission حداقلی؛
-- Dependabot، Secret Scanning و Push Protection؛
-- Private Vulnerability Reporting؛
-- Community Standards.
+- Probe دقیق Siegfried `1.11.6` و PRONOM `v124`؛
+- Hash ابزار و Signature؛
+- Migration موفق Workspace؛
+- اجرای کامل یا Resume موفق Format Stage؛
+- تعداد Identified، Unknown، Ambiguous، Mismatch و Tool Error؛
+- `quick_check` و `foreign_key_check`؛
+- Source non-mutation evidence؛
+- Sanitized aggregate export.
-## مرحله ۷: Release
+Raw Profiler DB، Snapshot، Path، Filename، Subject، Domain و Progress JSONL عمومی نشوند.
-```cmd
-git tag -a v0.1.0-alpha.3 -m "MailVault Collection Profiler 0.1.0-alpha.3"
-git push origin v0.1.0-alpha.3
-```
+## مرحله ۵: Merge و Tag
-Workflow فایل زیر Draft Release میسازد:
+فقط پس از سبز شدن CI و Smoke Test:
-```text
-.github/workflows/release.yml
+```powershell
+git checkout main
+git pull --ff-only
+git tag -a v0.1.0-alpha.4 -m "MailVault Collection Profiler 0.1.0-alpha.4"
+git push origin v0.1.0-alpha.4
```
-Draft باید شامل NSIS، MSI و `SHA256SUMS.txt` باشد. تا قبل از Smoke Test روی یک Windows تمیز Publish
-نشود.
+Workflow `release.yml` یک Draft Release میسازد. Draft باید شامل NSIS، MSI و
+`SHA256SUMS.txt` باشد. قبل از Publish، Installerها را روی Windows تمیز نصب و باز کن و Bundled
+Siegfried Resources را Probe کن.
-## مرز ادعا
+## فایلهای GitHub
-موارد تأییدشده:
+Repository Description:
-- Quality Gate و تستهای Rust سبز؛
-- Windows Snapshot Regression سبز؛
-- Tauri Desktop Compile سبز؛
-- MSI و NSIS ساخته شدهاند؛
-- برنامه نصب و اجرا شده است؛
-- Preflight آرشیو واقعی Schema v3 سازگار است؛
-- اجرای واقعی وارد Metadata Inventory شده است.
+```text
+Local-first, read-only physical inventory and exact file-format evidence for MailVault archives.
+```
-موردی که هنوز نباید در README یا Release بهعنوان قطعی ادعا شود:
+Social Preview:
```text
-Full 20–30 GB benchmark completed
+docs/assets/social-preview.png
```
-این ادعا فقط پس از پایان Run و ثبت Evidence Manifest، زمان، حافظه، شمارندهها و Hash قبل/بعد مجاز است.
+Release Notes:
+
+```text
+docs/releases/v0.1.0-alpha.4.md
+```
+
+## مرز ادعا
+
+بعد از CI میتوان ادعا کرد که Source و Installer Build Gate سبز است. فقط بعد از اجرای واقعی
+میتوان آمار Format Distribution، Throughput، ETA، Peak Memory و Runtime Green بودن آلفا ۴ را
+منتشر کرد.
diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md
index d8b9af7..9adacf7 100644
--- a/docs/ROADMAP.md
+++ b/docs/ROADMAP.md
@@ -1,59 +1,33 @@
# Roadmap
-The roadmap is capability-gated. A listed future capability is not implemented merely because its
-domain enum or schema placeholder exists.
-
-## Current: `0.1-C` physical inventory and explorer
-
-Delivered:
-
-- read-only source contract and preflight;
-- consistent SQLite snapshot;
-- metadata inventory and attachment reconciliation;
-- bounded file-stat inspection;
-- durable findings and checkpoints;
-- inventory, findings and content-object desktop exploration;
-- CLI and runtime evidence wrapper.
-
-## Required closure before the next public capability
-
-- complete the canonical private 20–30 GB benchmark;
-- record elapsed time, throughput, warning and error counts;
-- reconcile observed counts against the documented baseline;
-- tune worker and batch defaults from measured storage behavior;
-- publish only sanitized aggregate evidence.
-
-## Planned next slice: exact format identification
-
-Candidate scope:
-
-- frozen Siegfried binary provenance;
-- frozen PRONOM signature provenance;
-- resumable format-identification batches;
-- no duplicate payload rereads where identity permits reuse;
-- structured assertions and format findings.
-
-## Deferred
-
-- reopening existing workspaces;
-- interactive pause/resume/cancel;
-- full fixity modes;
-- container expansion;
-- JHOVE validation;
-- OCR;
-- semantic extraction and classification;
-- graph analysis;
-- procurement-specific classification;
-- automatic updates;
-- multi-platform signed distribution.
-## 0.1.0-alpha.3 — completed source scope
-
-- Workspace reopen and compatibility inspection.
-- Backed-up schema migration.
-- Run catalog after restart.
-- Append-only finding review and history integrity.
-- Sanitized review exports.
-
-## Next candidate
-
-`0.1.0-alpha.4` should focus on resumable execution controls and durable run recovery, without expanding into archive repair or semantic processing.
+## Completed baseline
+
+### Alpha 3
+
+- physical inventory and explorer;
+- findings review and workspace reopen;
+- real 17,296-message runtime evidence.
+
+### Alpha 4 implementation candidate
+
+- exact format identification with pinned Siegfried/PRONOM;
+- PUID and all-match evidence;
+- bounded/resumable batch execution;
+- exact-format UI and CLI;
+- schema 6 and complete documentation.
+
+Alpha 4 becomes runtime green only after Windows CI and the private real-archive format run pass.
+
+## Next profiler slices
+
+- benchmark-informed format worker/batch defaults;
+- richer technical characterization derived from exact formats;
+- optional JHOVE structural validation for supported formats;
+- privacy-safe aggregate format exports;
+- improved run control and interruption testing.
+
+## Separate future layer
+
+Safe text extraction, selective OCR and procurement classification should consume profiler
+manifests as a separate document-corpus/intelligence boundary. They should not mutate MailVault or
+weaken the profiler's technical evidence contract.
diff --git a/docs/SCREENSHOTS.md b/docs/SCREENSHOTS.md
index b8adf27..3b14d4a 100644
--- a/docs/SCREENSHOTS.md
+++ b/docs/SCREENSHOTS.md
@@ -1,37 +1,21 @@
-# Screenshot and documentation media policy
-
-Repository screenshots must not reveal private archive or workstation information.
-
-## Required sanitization
-
-Remove or replace:
-
-- drive labels and private volume names;
-- user profile paths;
-- company, client or supplier names;
-- email addresses and domains;
-- message subjects;
-- real filenames;
-- full content hashes from private material;
-- taskbar, notifications and unrelated applications;
-- timestamps that identify a private incident when unnecessary.
-
-## Repository images
-
-- `01-collection-setup-preflight.png` and `02-profile-running.png` are sanitized application
- captures with demo paths.
-- Explorer, findings, content-detail and CLI images use synthetic documentation data modeled on the
- implemented interface and command contracts.
-- `alpha3/01-start.png`, `alpha3/03-runs.png` and `alpha3/06-findings.png` are public release-tour
- images with sanitized or synthetic paths and metadata.
-- Baseline counts may reflect documented aggregate evidence; they are not compatibility constants.
-
-## Path convention
-
-Documentation uses:
-
-```text
-D:\MailVault-Demo
-D:\MailVault-Profiler-Workspace
-D:\MailVault-Profiler-Evidence
-```
+# Screenshot policy
+
+All repository screenshots are sanitized or synthetic. They must not contain real filenames,
+subjects, email addresses, domains, local paths, content hashes or review notes.
+
+Deterministic publication assets:
+
+- `01-collection-setup-preflight.png`
+- `02-profile-running.png`
+- `03-inventory-explorer.png`
+- `04-findings-explorer.png`
+- `05-cli-workflow.png`
+- `06-content-object-detail.png`
+- `07-exact-format-identification-ready.png`
+- `08-format-assertion-detail.png`
+- `09-alpha4-architecture.png`
+- `../social-preview.png`
+
+Alpha 4 screenshots are 2560×1440 PNG files. The social preview is 1280×640. Screenshots showing
+format results are explicitly labeled sanitized examples and must not be interpreted as real run
+metrics.
diff --git a/docs/SECURITY_MODEL.md b/docs/SECURITY_MODEL.md
index 4a5ac60..ca92a4a 100644
--- a/docs/SECURITY_MODEL.md
+++ b/docs/SECURITY_MODEL.md
@@ -2,74 +2,42 @@
## Trust boundaries
-### Canonical MailVault archive
+Untrusted inputs include MailVault database metadata, canonical locators, filenames, sidecar output
+and local configuration. The canonical archive is read-only; the profiler workspace is writable.
-High-value, read-only source. The profiler must never use it as a workspace, write temporary files
-inside it, modify its database or repair its object stores.
+## Invariants
-### Profiler workspace
+- source SQLite is opened read-only and snapshotted consistently;
+- workspace must be outside the source archive;
+- every object path is canonicalized and constrained beneath the source root;
+- no attachment is executed or rendered;
+- no command shell is built from archive data;
+- exact-format sidecar and signature are versioned and SHA-256 recorded;
+- process output and runtime are bounded;
+- archive/container expansion is disabled;
+- one exact-format writer may hold a workspace lock;
+- incomplete runs cannot be marked complete;
+- telemetry is disabled by default.
-Writable, disposable derived data. It contains a source database snapshot, profiler database,
-checkpoints and derived metadata. It is outside the source archive by invariant.
+## Sidecar supply chain
-### Runtime evidence directory
+The Windows acquisition script requires a GitHub release-asset SHA-256 digest, verifies size/hash,
+probes the observed tool/signature versions and generates a local manifest. The release workflow
+bundles the verified resources. Runtime probing fails closed when the expected version contract is
+not met.
-Writable output for logs, manifests and JSON results. It must also remain outside the source
-archive.
+## Extension aliases
-### Archive metadata
+Ephemeral symbolic aliases may expose a preferred extension to the identifier without copying the
+content object. Alias failure falls back to canonical path identification and records extension as
+not checked. Alias paths are generated from SHA-256 and a normalized extension.
-All paths, filenames, MIME labels, subjects, domains and database values are untrusted input. They
-may be malformed, hostile, unexpectedly large or privacy-sensitive.
+## Data exposure
-## Enforced invariants
+Profiler SQLite, raw logs and tool errors may contain paths, filenames, domains and subjects. They
+must not be attached to public issues or GitHub releases.
-- SQLite source connections use read-only behavior.
-- Workspace and evidence roots are checked for path overlap with the archive.
-- The source database snapshot uses SQLite's backup API instead of copying an active database file.
-- Snapshot publication uses a durable temporary file and atomic rename sequence.
-- On Windows, durable file sync uses a write-capable handle required by `FlushFileBuffers`.
-- Blob locators are parsed and validated against the expected content-addressed layout.
-- Resolved file paths must remain contained by the canonical object-store root.
-- Locator mismatches are not opened.
-- Physical files are inspected as files; payloads are not executed or rendered.
-- Explorer commands use the active profiler run and do not accept arbitrary source paths.
-- Read-side profiler connections are opened read-only/query-only.
-- Progress counters must be monotonic within a stage.
+## Out of scope
-## Failure policy
-
-The profiler fails closed for:
-
-- unsupported or newer source schema;
-- required path or structure failures;
-- active writer lock where consistency cannot be guaranteed;
-- source integrity failures;
-- workspace/source overlap;
-- invalid persisted run state;
-- non-monotonic progress events;
-- snapshot or publication I/O failure.
-
-Physical content findings such as missing blobs and size mismatches are generally non-fatal because
-they are evidence the profiler is designed to record.
-
-## Explicit non-capabilities
-
-`0.1.0-alpha.3` does not:
-
-- execute attachments;
-- render HTML email or embedded active content;
-- extract archives or containers;
-- run OCR, LLMs or semantic classifiers;
-- send telemetry;
-- upload data;
-- repair the MailVault archive.
-
-## Vulnerability reporting
-
-Use GitHub private vulnerability reporting. Do not publish exploit details or real archive samples.
-See the root [security policy](../SECURITY.md).
-
-## Review-store security
-
-Only one process may write review state. Concurrent sessions open read-only. Review events are append-only, hash chained and validated on open. Corruption disables review writes. Notes are excluded from routine logs and sanitized exports. Existing workspace open uses SQLite read-only/read-write flags without create.
+Alpha 4 is not an antivirus sandbox, document renderer, archive extractor or malware detonation
+environment. A successful format identification does not mean content is safe to open.
diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md
index 9aff888..0b2b054 100644
--- a/docs/TROUBLESHOOTING.md
+++ b/docs/TROUBLESHOOTING.md
@@ -69,13 +69,13 @@ Use the corrected desktop bridge without `#[derive(Debug)]` on `ChannelProgressS
Tauri app SemVer can remain:
```text
-0.1.0-alpha.3
+0.1.0-alpha.4
```
WiX/MSI version must remain numeric:
```text
-0.1.0.3
+0.1.0.4
```
Run:
diff --git a/docs/VALIDATION_0.1.0-alpha.4.md b/docs/VALIDATION_0.1.0-alpha.4.md
new file mode 100644
index 0000000..c891126
--- /dev/null
+++ b/docs/VALIDATION_0.1.0-alpha.4.md
@@ -0,0 +1,143 @@
+# Validation evidence — 0.1.0-alpha.4
+
+## Scope
+
+This document separates completed evidence from work that still requires Windows CI or a private
+real-archive execution. It must not be read as a fabricated claim that exact format identification
+has already been run across the production collection.
+
+## Source examined
+
+- user-supplied `mailvault-collection-profiler` Alpha 3 source;
+- private Alpha 3 real-profile workspace;
+- Alpha 4 implementation candidate in this package;
+- target application version `0.1.0-alpha.4`;
+- profiler schema migration `0006_exact_format_identification.sql`.
+
+## Research decisions
+
+Alpha 4 uses a pinned Siegfried/PRONOM sidecar rather than an in-process magic-byte table or DROID
+runtime because it provides machine-readable batch identification and persistent PRONOM IDs while
+remaining practical to bundle in a Windows desktop application. The design records tool/signature
+identity and retains all matches.
+
+Pinned contract:
+
+```text
+Siegfried 1.11.6
+PRONOM v124
+container expansion disabled
+```
+
+## Completed static and frontend gates
+
+| Gate | Result |
+|---|---|
+| TypeScript strict type-check | passed |
+| Vite production build | passed |
+| Rust source Tree-sitter syntax parse | passed, 34 Rust files |
+| Release configuration parser | passed after MSI mapping correction |
+| Documentation/privacy/link gate | passed in packaged candidate |
+| Screenshot dimensions and deterministic names | passed |
+| npm production audit | passed, 0 vulnerabilities |
+| npm complete audit | passed, 0 vulnerabilities |
+
+Tree-sitter proves syntactic parsability only; it is not a substitute for Rust type checking.
+
+## Real Alpha 3 database migration test
+
+A private copy of the real Alpha 3 profiler database was migrated from schema 5 to schema 6.
+
+| Check | Result |
+|---|---|
+| source database SHA-256 before/after test | unchanged |
+| migrated database `PRAGMA quick_check(1)` | `ok` |
+| `PRAGMA foreign_key_check` | no violations |
+| content objects preserved | 13,684 |
+| content occurrences preserved | 22,068 |
+| findings preserved | 1,484 |
+| synthetic format projection | passed |
+| baseline-scoped format projection isolation | passed |
+
+Source database SHA-256 used in the private migration test:
+
+```text
+d73b040cc7a136a80d0fcab6d8194fa66c2496958bfe4e6d69f06b849d1766b8
+```
+
+This hash is evidence for the provided private profiler database, not a public MailVault content
+hash.
+
+## Implemented safety tests in code
+
+The source includes unit/integration coverage for:
+
+- PRONOM version parsing;
+- stronger signature evidence outranking extension-only evidence;
+- exclusion of extension-only alternatives from false ambiguity;
+- migration idempotency and expected migration count;
+- format projection, baseline isolation and cursor query behavior;
+- checkpoint/run completion constraints;
+- physical-profile source immutability inherited from Alpha 3.
+
+These Rust tests require semantic compilation in CI before release.
+
+## Windows runtime evidence supplied by the maintainer
+
+The following gates were executed on Windows PowerShell 5.1 with Rust 1.97.1 tooling:
+
+| Gate | Result |
+|---|---|
+| pinned Siegfried installation | passed |
+| independent Siegfried verification | passed |
+| Siegfried version | `1.11.6` |
+| PRONOM signature | `v124` |
+| Windows-safe relative signature JSON contract | passed |
+| `cargo test -p profiler-format-siegfried --locked` | passed, 5 tests |
+| `cargo fmt --all -- --check` | passed |
+
+The first strict workspace Clippy run reached semantic compilation and reported structural/style
+findings in the new exact-format modules. The implementation was then refactored without lint
+suppression: options are borrowed, run-start parameters are grouped, large functions are decomposed,
+large read buffers are heap-backed, and SQLite row/projection work is split into focused helpers.
+This refactor still requires a fresh Windows `cargo clippy` execution before it can be recorded as
+passed.
+
+## Not yet completed
+
+These results are deliberately **not claimed** until the updated source is rerun:
+
+- strict workspace Clippy after the structural refactor;
+- complete Rust workspace test suite;
+- native Tauri compile or installer build;
+- real 13,684-object Siegfried identification run;
+- real throughput, ETA accuracy, format distribution or peak memory.
+
+## Release CI requirements
+
+A publishable tag must pass the configured Windows workflow:
+
+1. install Rust `1.97.1` and Node `24`;
+2. install and verify pinned Siegfried resources;
+3. run formatting, Clippy and all Rust tests with `--locked`;
+4. run TypeScript checks and production build;
+5. run documentation/privacy gate;
+6. compile Tauri desktop targets;
+7. build NSIS and MSI installers;
+8. generate `SHA256SUMS.txt` and artifact attestations where supported.
+
+## Runtime gate after CI
+
+Before calling Alpha 4 **runtime green**, execute the private runbook against the real workspace and
+record:
+
+- exact tool and signature hashes;
+- elapsed time and peak memory;
+- identified, unknown, ambiguous, mismatch and tool-error counts;
+- PUID/format distribution;
+- checkpoint/resume evidence;
+- source non-mutation evidence;
+- sanitized aggregate export.
+
+Until that evidence exists, this package is an implementation/release candidate, not a completed
+real-archive Alpha 4 benchmark.
diff --git a/docs/WORKSPACE_FORMAT.md b/docs/WORKSPACE_FORMAT.md
index c757cab..6ea4f23 100644
--- a/docs/WORKSPACE_FORMAT.md
+++ b/docs/WORKSPACE_FORMAT.md
@@ -1,68 +1,41 @@
-# Workspace format — schema 5
-
-MailVault Collection Profiler stores all derived state outside the canonical MailVault archive. A workspace is a local directory owned by the profiler, not by MailVault.
+# Workspace format
## Layout
```text
-D:\MailVault-Profiler-Workspace\
-├── .mailvault-profiler.workspace.lock
-├── profiler\
-│ └── profiler.sqlite3
-├── snapshots\
-│ └── \
-│ ├── mailvault.sqlite3
-│ └── snapshot-manifest.json
-└── backups\
- └── profiler-before-workspace-schema-.sqlite3
+/
+ profiler/profiler.sqlite3
+ snapshots//mailvault.sqlite3
+ .mailvault-profiler.lock
+ .mailvault-profiler.format.lock
+ format-staging//... # temporary, removable
```
-The lock file contains temporary local process metadata. It is not evidence and must not be published. The profiler database contains run metadata, derived inventory, findings, checkpoints and finding-review state.
-
-## Compatibility inspection
-
-Workspace inspection opens `profiler/profiler.sqlite3` read-only and checks:
-
-- SQLite application ID;
-- `PRAGMA user_version`;
-- database integrity;
-- required metadata and review tables;
-- migration state;
-- registered source archive roots;
-- source/workspace path overlap;
-- active operating-system file lock;
-- review history integrity after open.
-
-A missing profiler database is reported as `WORKSPACE_DATABASE_MISSING`; it is never created by an open-existing operation. A workspace newer than the application fails closed with `WORKSPACE_SCHEMA_NEWER_THAN_APPLICATION`.
-
-## Schema migration
-
-Schema 5 adds workspace metadata and append-only finding review storage. Migration is explicit at the application boundary. Before any schema change, the profiler uses the SQLite Backup API to create:
-
-```text
-backups/profiler-before-workspace-schema-.sqlite3
-```
+Actual temporary directory names are implementation details and must not be used as API contracts.
-The migration runs transactionally. A failure retains the backup and writes a local migration-failure marker. A workspace containing that marker opens neither read-write nor silently downgraded.
+## Schema versions
-## Locking and access modes
+- 1–4: initial physical inventory and explorer;
+- 5: workspace reopen and append-only finding review;
+- 6: exact format runs, observations, all matches, checkpoints and indexes.
-The operating-system lock, not the mere presence of a lock file, determines ownership.
+Migrations are append-only. Back up the workspace before explicit migration. The MailVault source is
+not migrated.
-| Access mode | Browse | Review writes | Migration |
-|---|---:|---:|---:|
-| `read_write` | yes | yes | yes |
-| `read_only_locked` | yes | no | no |
-| `read_only_compatibility` | yes | no | no |
+## Exact-format persistence
-Only one process may hold the review-writer lock. A second process may browse the workspace but cannot change review state.
+A physical run can have multiple format runs with different fingerprints. Current object projection
+is indexed for UI queries; historical run metadata and matches remain versioned.
-## Review storage
+A format checkpoint is committed in the same transaction as its observation batch. Resume begins
+after the last durable SHA-256.
-`finding_review_events` is append-only. Update and delete triggers reject historical mutation. `finding_review_state` is a transactionally maintained projection of the latest event for each finding.
+## Locks
-Each event stores a SHA-256 hash over stable canonical fields and links to the previous event hash. Opening a workspace validates sequence continuity, identity, action semantics, prior status, prior hash, recomputed hash and projection consistency. Integrity failure disables review writes while preserving read-only browsing.
+The normal workspace lock protects review/migration writers. The dedicated format lock prevents
+concurrent exact-format jobs. Another session may open read-only when appropriate.
-## Evidence boundary
+## Backup
-Original run evidence and source snapshots are not rewritten by findings review. Review decisions live only in the profiler database. Sanitized export creates a new file outside the source archive and does not include full paths, filenames, email addresses or review notes.
+Before migration or review-heavy use, back up `profiler.sqlite3`. Snapshots and inventory metadata
+are rebuildable; human review history is not automatically recoverable from MailVault.
diff --git a/docs/assets/screenshots/07-exact-format-identification-ready.png b/docs/assets/screenshots/07-exact-format-identification-ready.png
new file mode 100644
index 0000000..43b0946
Binary files /dev/null and b/docs/assets/screenshots/07-exact-format-identification-ready.png differ
diff --git a/docs/assets/screenshots/08-format-assertion-detail.png b/docs/assets/screenshots/08-format-assertion-detail.png
new file mode 100644
index 0000000..0fc746f
Binary files /dev/null and b/docs/assets/screenshots/08-format-assertion-detail.png differ
diff --git a/docs/assets/screenshots/09-alpha4-architecture.png b/docs/assets/screenshots/09-alpha4-architecture.png
new file mode 100644
index 0000000..5f64d6c
Binary files /dev/null and b/docs/assets/screenshots/09-alpha4-architecture.png differ
diff --git a/docs/assets/social-preview.png b/docs/assets/social-preview.png
index e9ed834..bf9f6f7 100644
Binary files a/docs/assets/social-preview.png and b/docs/assets/social-preview.png differ
diff --git a/docs/releases/v0.1.0-alpha.4.md b/docs/releases/v0.1.0-alpha.4.md
new file mode 100644
index 0000000..0445bba
--- /dev/null
+++ b/docs/releases/v0.1.0-alpha.4.md
@@ -0,0 +1,61 @@
+# MailVault Collection Profiler `v0.1.0-alpha.4`
+
+Alpha 4 adds exact technical file-format identification to the existing read-only physical profiler.
+It does not create a second archive and does not add OCR or procurement classification.
+
+## Highlights
+
+- pinned Siegfried `1.11.6` + PRONOM `v124` Windows sidecar;
+- verified release-asset acquisition and local tool/signature manifest;
+- PUID, format/version, MIME, evidence basis and all alternative matches;
+- deterministic primary assertion with explicit ambiguity preservation;
+- safe extension-evidence aliases and honest **Not checked** state;
+- bounded batch runner with timeout, output limits and adaptive failure isolation;
+- durable format checkpoints and same-fingerprint resume;
+- exclusive format-stage workspace lock;
+- desktop **Exact formats** dashboard and CLI commands;
+- profiler schema migration 6;
+- sanitized aggregate format summary schema 2;
+- updated English/Persian docs and publication-ready screenshots.
+
+## CLI
+
+```text
+formats probe
+formats identify
+formats summary
+formats list
+```
+
+See [CLI reference](../CLI_REFERENCE.md) and
+[format runbook](../FORMAT_IDENTIFICATION_RUNBOOK.md).
+
+## Security and privacy
+
+- MailVault remains read-only.
+- Sidecar resources are versioned and hashed.
+- Containers are not expanded.
+- No attachment is executed or rendered.
+- Raw profiler databases, paths, filenames and progress evidence remain private.
+
+## Upgrade
+
+Back up an Alpha 3 workspace before using `--allow-migration`. Migration `0006` preserves existing
+objects, occurrences, findings and review history while adding exact-format tables and indexes.
+
+## Validation status
+
+Static/frontend gates and migration of a private real Alpha 3 workspace passed. Full Rust semantic
+compilation, Windows installer creation and the first private real-archive Siegfried run are required
+before publishing runtime-green performance claims. See
+[Alpha 4 validation evidence](../VALIDATION_0.1.0-alpha.4.md).
+
+## Known limitations
+
+- no physical-profile pause/cancel/resume;
+- no container expansion;
+- no JHOVE validation;
+- no extraction, OCR or preview;
+- no document classification or RMS integration;
+- unsigned Windows installers;
+- no public real-archive format distribution is bundled in this source candidate.
diff --git a/package-lock.json b/package-lock.json
index 83c6e13..1d07296 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "mailvault-profiler-workspace",
- "version": "0.1.0-alpha.3",
+ "version": "0.1.0-alpha.4",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "mailvault-profiler-workspace",
- "version": "0.1.0-alpha.3",
+ "version": "0.1.0-alpha.4",
"license": "Apache-2.0",
"workspaces": [
"apps/desktop"
@@ -22,7 +22,7 @@
},
"apps/desktop": {
"name": "@firexcore/mailvault-profiler-desktop",
- "version": "0.1.0-alpha.3",
+ "version": "0.1.0-alpha.4",
"license": "Apache-2.0",
"dependencies": {
"@tauri-apps/api": "^2.11.1",
diff --git a/package.json b/package.json
index 9d5a07c..a85ad1c 100644
--- a/package.json
+++ b/package.json
@@ -1,7 +1,7 @@
{
"name": "mailvault-profiler-workspace",
"private": true,
- "version": "0.1.0-alpha.3",
+ "version": "0.1.0-alpha.4",
"workspaces": [
"apps/desktop"
],
diff --git a/scripts/check-docs.cjs b/scripts/check-docs.cjs
index 12ecf8a..e442490 100644
--- a/scripts/check-docs.cjs
+++ b/scripts/check-docs.cjs
@@ -25,7 +25,12 @@ const requiredFiles = [
"docs/WORKSPACE_FORMAT.md",
"docs/FINDINGS_REVIEW.md",
"docs/VALIDATION_0.1.0-alpha.3.md",
+ "docs/VALIDATION_0.1.0-alpha.4.md",
"docs/releases/v0.1.0-alpha.3.md",
+ "docs/releases/v0.1.0-alpha.4.md",
+ "docs/FORMAT_IDENTIFICATION.md",
+ "docs/FORMAT_IDENTIFICATION_RUNBOOK.md",
+ "THIRD_PARTY_NOTICES.md",
"docs/SECURITY_MODEL.md",
"docs/PRIVACY.md",
"docs/TROUBLESHOOTING.md",
@@ -40,6 +45,9 @@ const requiredFiles = [
"docs/assets/screenshots/04-findings-explorer.png",
"docs/assets/screenshots/05-cli-workflow.png",
"docs/assets/screenshots/06-content-object-detail.png",
+ "docs/assets/screenshots/07-exact-format-identification-ready.png",
+ "docs/assets/screenshots/08-format-assertion-detail.png",
+ "docs/assets/screenshots/09-alpha4-architecture.png",
"docs/assets/screenshots/alpha3/01-start.png",
"docs/assets/screenshots/alpha3/03-runs.png",
"docs/assets/screenshots/alpha3/06-findings.png",
diff --git a/scripts/install-siegfried.ps1 b/scripts/install-siegfried.ps1
new file mode 100644
index 0000000..420a433
--- /dev/null
+++ b/scripts/install-siegfried.ps1
@@ -0,0 +1,815 @@
+#Requires -Version 5.1
+
+
+[CmdletBinding()]
+param(
+ [Parameter()]
+ [ValidateNotNullOrEmpty()]
+ [string] $Tag = "v1.11.6",
+
+ [Parameter()]
+ [ValidateNotNullOrEmpty()]
+ [string] $RequiredVersion = "1.11.6",
+
+ [Parameter()]
+ [ValidateNotNullOrEmpty()]
+ [string] $RequiredPronomVersion = "v124",
+
+ # Deliberately has no default expression that references $PSScriptRoot.
+
+ # Windows PowerShell can evaluate parameter defaults before $PSScriptRoot
+
+ # is reliably available.
+
+ [Parameter()]
+ [string] $Destination,
+
+ [Parameter()]
+ [switch] $Force,
+
+ [Parameter()]
+ [switch] $ValidateEnvironmentOnly
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = "Stop"
+$ProgressPreference = "SilentlyContinue"
+
+Write-Host "Starting pinned Siegfried installer..."
+
+# Resolve the running script path after the param block. $PSCommandPath describes
+
+# the current script; $MyInvocation.PSScriptRoot describes the caller and must
+
+# not be used as a replacement.
+
+$scriptPath = [string]$PSCommandPath
+if ([string]::IsNullOrWhiteSpace($scriptPath)) {
+ $scriptPath = [string]$MyInvocation.MyCommand.Path
+}
+if ([string]::IsNullOrWhiteSpace($scriptPath)) {
+ throw "Unable to resolve the current script path. Run this file with PowerShell -File or invoke it as a .ps1 script."
+}
+
+$scriptRoot = [System.IO.Path]::GetDirectoryName([System.IO.Path]::GetFullPath($scriptPath))
+if ([string]::IsNullOrWhiteSpace($scriptRoot)) {
+ throw "Unable to resolve the directory containing install-siegfried.ps1."
+}
+
+$repositoryRoot = [System.IO.Path]::GetFullPath((Join-Path $scriptRoot ".."))
+
+if ([string]::IsNullOrWhiteSpace($Destination)) {
+ $Destination = Join-Path $repositoryRoot "tools\siegfried\windows-x86_64"
+}
+
+# Resolve relative PowerShell paths without requiring the destination to exist.
+
+try {
+ $destinationPath = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($Destination)
+}
+catch {
+ throw "Unable to resolve destination path '$Destination': $($_.Exception.Message)"
+}
+$destinationPath = [System.IO.Path]::GetFullPath($destinationPath)
+
+$destinationInfo = [System.IO.DirectoryInfo]::new($destinationPath)
+$destinationParent = if ($null -ne $destinationInfo.Parent) { $destinationInfo.Parent.FullName } else { $null }
+$destinationLeaf = $destinationInfo.Name
+if ([string]::IsNullOrWhiteSpace($destinationParent) -or
+ [string]::IsNullOrWhiteSpace($destinationLeaf)) {
+ throw "Unsafe destination path '$destinationPath'. A non-root directory is required."
+}
+
+$verifyScriptPath = Join-Path $scriptRoot "verify-siegfried.ps1"
+if (-not (Test-Path -LiteralPath $verifyScriptPath -PathType Leaf)) {
+ throw "Required verification script was not found: $verifyScriptPath"
+}
+
+if ($ValidateEnvironmentOnly) {
+ Write-Host "Siegfried installer environment validation passed:"
+ Write-Host " PowerShell: $($PSVersionTable.PSVersion)"
+ Write-Host " Script root: $scriptRoot"
+ Write-Host " Repository: $repositoryRoot"
+ Write-Host " Destination: $destinationPath"
+ Write-Host " Verifier: $verifyScriptPath"
+ return
+}
+
+# GitHub requires modern TLS. This is especially important for Windows
+
+# PowerShell 5.1 on older .NET Framework configurations.
+
+try {
+ [Net.ServicePointManager]::SecurityProtocol =
+ [Net.ServicePointManager]::SecurityProtocol -bor
+ [Net.SecurityProtocolType]::Tls12
+}
+catch {
+ throw "Unable to enable TLS 1.2 for GitHub downloads: $($_.Exception.Message)"
+}
+
+function Get-Sha256 {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory = $true)]
+ [ValidateNotNullOrEmpty()]
+ [string] $Path
+ )
+
+ if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) {
+ throw "Cannot hash missing file: $Path"
+ }
+
+ return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
+}
+
+function Write-Utf8NoBom {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory = $true)]
+ [ValidateNotNullOrEmpty()]
+ [string] $Path,
+
+ [Parameter(Mandatory = $true)]
+ [AllowEmptyString()]
+ [string] $Content
+ )
+
+ # Set-Content -Encoding utf8NoBOM is not available in Windows PowerShell
+
+ # 5.1, so use the .NET encoder explicitly.
+
+ $encoding = [System.Text.UTF8Encoding]::new($false)
+ [System.IO.File]::WriteAllText($Path, $Content, $encoding)
+}
+
+function Invoke-GitHubDownload {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory = $true)]
+ [ValidateNotNullOrEmpty()]
+ [string] $Uri,
+
+ [Parameter(Mandatory = $true)]
+ [ValidateNotNullOrEmpty()]
+ [string] $OutFile,
+
+ [Parameter(Mandatory = $true)]
+ [hashtable] $Headers
+ )
+
+ $parameters = @{
+ Uri = $Uri
+ Headers = $Headers
+ OutFile = $OutFile
+ ErrorAction = "Stop"
+ }
+
+ # Windows PowerShell 5.1 may otherwise depend on the legacy Internet
+
+ # Explorer parsing engine.
+
+ if ($PSVersionTable.PSVersion.Major -le 5) {
+ $parameters.UseBasicParsing = $true
+ }
+
+ Invoke-WebRequest @parameters
+}
+
+function ConvertTo-NativeCommandLineArgument {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory = $true)]
+ [AllowEmptyString()]
+ [string] $Value
+ )
+
+ # ProcessStartInfo.ArgumentList is unavailable on the .NET Framework used
+ # by Windows PowerShell 5.1. Quote arguments according to the Windows
+ # CommandLineToArgvW rules so paths containing spaces, quotes, or trailing
+ # backslashes reach the native executable unchanged.
+ if ($Value.Length -gt 0 -and $Value -notmatch '[\s"]') {
+ return $Value
+ }
+
+ $builder = [System.Text.StringBuilder]::new()
+ [void] $builder.Append([char]34)
+ $backslashCount = 0
+
+ foreach ($character in $Value.ToCharArray()) {
+ $codePoint = [int] $character
+
+ if ($codePoint -eq 92) {
+ $backslashCount++
+ continue
+ }
+
+ if ($codePoint -eq 34) {
+ if ($backslashCount -gt 0) {
+ [void] $builder.Append([char]92, (($backslashCount * 2) + 1))
+ }
+ else {
+ [void] $builder.Append([char]92)
+ }
+
+ [void] $builder.Append([char]34)
+ $backslashCount = 0
+ continue
+ }
+
+ if ($backslashCount -gt 0) {
+ [void] $builder.Append([char]92, $backslashCount)
+ $backslashCount = 0
+ }
+
+ [void] $builder.Append($character)
+ }
+
+ # Backslashes immediately before the closing quote must be doubled.
+ if ($backslashCount -gt 0) {
+ [void] $builder.Append([char]92, ($backslashCount * 2))
+ }
+
+ [void] $builder.Append([char]34)
+ return $builder.ToString()
+}
+
+function Invoke-NativeProcess {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory = $true)]
+ [ValidateNotNullOrEmpty()]
+ [string] $Executable,
+
+ [Parameter(Mandatory = $true)]
+ [AllowEmptyCollection()]
+ [string[]] $Arguments,
+
+ [Parameter()]
+ [ValidateRange(1, 3600)]
+ [int] $TimeoutSeconds = 300
+ )
+
+ if (-not (Test-Path -LiteralPath $Executable -PathType Leaf)) {
+ throw "Native executable was not found: $Executable"
+ }
+
+ $quotedArguments = @(
+ foreach ($argument in $Arguments) {
+ ConvertTo-NativeCommandLineArgument -Value ([string] $argument)
+ }
+ )
+
+ $startInfo = [System.Diagnostics.ProcessStartInfo]::new()
+ $startInfo.FileName = $Executable
+ $startInfo.Arguments = $quotedArguments -join " "
+ $startInfo.UseShellExecute = $false
+ $startInfo.CreateNoWindow = $true
+ $startInfo.RedirectStandardOutput = $true
+ $startInfo.RedirectStandardError = $true
+
+ $process = [System.Diagnostics.Process]::new()
+ $process.StartInfo = $startInfo
+
+ try {
+ if (-not $process.Start()) {
+ throw "Failed to start native executable: $Executable"
+ }
+
+ # Start both asynchronous reads before waiting. Reading either stream
+ # synchronously first can deadlock when the child fills the other pipe.
+ $stdoutTask = $process.StandardOutput.ReadToEndAsync()
+ $stderrTask = $process.StandardError.ReadToEndAsync()
+
+ if (-not $process.WaitForExit($TimeoutSeconds * 1000)) {
+ try {
+ $process.Kill()
+ }
+ catch {
+ # Preserve the timeout as the primary failure.
+ }
+
+ $process.WaitForExit()
+ throw "Native command timed out after $TimeoutSeconds seconds: $Executable $($startInfo.Arguments)"
+ }
+
+ # WaitForExit() without a timeout ensures async stream handlers have
+ # finished after the process handle is signaled.
+ $process.WaitForExit()
+
+ $stdout = $stdoutTask.GetAwaiter().GetResult()
+ $stderr = $stderrTask.GetAwaiter().GetResult()
+
+ return [pscustomobject]@{
+ ExitCode = [int] $process.ExitCode
+ StdOut = [string] $stdout
+ StdErr = [string] $stderr
+ CommandLine = "$Executable $($startInfo.Arguments)"
+ }
+ }
+ finally {
+ $process.Dispose()
+ }
+}
+
+function Invoke-JsonProbe {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory = $true)]
+ [ValidateNotNullOrEmpty()]
+ [string] $Executable,
+
+ [Parameter(Mandatory = $true)]
+ [ValidateNotNullOrEmpty()]
+ [string] $Signature,
+
+ [Parameter(Mandatory = $true)]
+ [ValidateNotNullOrEmpty()]
+ # Do not name this parameter `Home`: PowerShell variable names are
+ # case-insensitive, and `$HOME` is a read-only automatic variable.
+ [string] $ToolHome
+ )
+
+ $probePath = Join-Path $ToolHome (
+ "probe-" + [guid]::NewGuid().ToString("N") + ".txt"
+ )
+
+ # A zero-byte file causes Siegfried to emit a normal file diagnostic on
+ # stderr. Windows PowerShell 5.1 promotes native stderr to its Error stream,
+ # which previously aborted the installer under ErrorActionPreference=Stop.
+ # Use a deterministic non-empty text object and capture native streams via
+ # System.Diagnostics.Process instead of PowerShell redirection.
+ $probeEncoding = [System.Text.UTF8Encoding]::new($false)
+ [System.IO.File]::WriteAllText(
+ $probePath,
+ "MailVault Collection Profiler Siegfried verification probe.",
+ $probeEncoding
+ )
+
+ $signatureFullPath = [System.IO.Path]::GetFullPath($Signature)
+ $toolHomeFullPath = [System.IO.Path]::GetFullPath($ToolHome)
+ $signatureDirectory = [System.IO.Path]::GetDirectoryName($signatureFullPath)
+ $signatureArgument = [System.IO.Path]::GetFileName($signatureFullPath)
+
+ if ([string]::IsNullOrWhiteSpace($signatureDirectory) -or
+ [string]::IsNullOrWhiteSpace($signatureArgument) -or
+ -not [string]::Equals(
+ $signatureDirectory.TrimEnd([char[]]"\/"),
+ $toolHomeFullPath.TrimEnd([char[]]"\/"),
+ [System.StringComparison]::OrdinalIgnoreCase
+ )) {
+ throw "Siegfried probe signature must be a direct child of the configured tool home. Signature: '$signatureFullPath'; home: '$toolHomeFullPath'."
+ }
+
+ try {
+ # Siegfried 1.11.6's JSON writer escapes scanned filenames but writes
+ # the JSON header's signature value verbatim. Passing an absolute
+ # Windows path therefore produces invalid JSON (e.g. "E:\tools\...").
+ # Resolve the file through -home and pass only its basename.
+ $result = Invoke-NativeProcess `
+ -Executable $Executable `
+ -Arguments @(
+ "-home", $toolHomeFullPath,
+ "-sig", $signatureArgument,
+ "-json",
+ "-utc",
+ $probePath
+ ) `
+ -TimeoutSeconds 120
+
+ if ($result.ExitCode -ne 0) {
+ $diagnostic = (
+ $result.StdOut +
+ [Environment]::NewLine +
+ $result.StdErr
+ ).Trim()
+
+ throw "Siegfried probe failed with exit code $($result.ExitCode): $diagnostic"
+ }
+
+ $jsonText = $result.StdOut.Trim()
+ if ([string]::IsNullOrWhiteSpace($jsonText)) {
+ throw "Siegfried probe returned an empty JSON response. stderr: $($result.StdErr.Trim())"
+ }
+
+ try {
+ # ConvertFrom-Json -Depth is unavailable in Windows PowerShell 5.1.
+ $probe = $jsonText | ConvertFrom-Json
+ }
+ catch {
+ throw "Siegfried probe returned invalid JSON: $($_.Exception.Message). Raw output: $jsonText"
+ }
+
+ if ($null -eq $probe -or [string]::IsNullOrWhiteSpace([string] $probe.siegfried)) {
+ throw "Siegfried probe JSON did not contain the expected header fields."
+ }
+
+ if (-not [string]::IsNullOrWhiteSpace($result.StdErr)) {
+ Write-Verbose "Siegfried probe diagnostics: $($result.StdErr.Trim())"
+ }
+
+ return $probe
+ }
+ finally {
+ Remove-Item -LiteralPath $probePath -Force -ErrorAction SilentlyContinue
+ }
+}
+
+function Invoke-SignatureUpdate {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory = $true)]
+ [ValidateNotNullOrEmpty()]
+ [string] $Executable,
+
+ [Parameter(Mandatory = $true)]
+ [ValidateNotNullOrEmpty()]
+ # Do not name this parameter `Home`: PowerShell variable names are
+ # case-insensitive, and `$HOME` is a read-only automatic variable.
+ [string] $ToolHome
+ )
+
+ $result = Invoke-NativeProcess `
+ -Executable $Executable `
+ -Arguments @("-home", $ToolHome, "-update") `
+ -TimeoutSeconds 300
+
+ if ($result.ExitCode -ne 0) {
+ $diagnostic = (
+ $result.StdOut +
+ [Environment]::NewLine +
+ $result.StdErr
+ ).Trim()
+
+ throw "Siegfried signature update failed with exit code $($result.ExitCode): $diagnostic"
+ }
+
+ if (-not [string]::IsNullOrWhiteSpace($result.StdErr)) {
+ Write-Verbose "Siegfried signature update diagnostics: $($result.StdErr.Trim())"
+ }
+
+ return $result.StdOut.Trim()
+}
+
+function Invoke-InstalledToolVerification {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory = $true)]
+ [ValidateNotNullOrEmpty()]
+ [string] $Directory
+ )
+
+ # This is a PowerShell script, not a native executable. $LASTEXITCODE can
+
+ # contain a stale value from an earlier native process, so verification
+
+ # must rely on normal PowerShell error propagation instead.
+
+ & $verifyScriptPath -Directory $Directory
+}
+
+$manifestPath = Join-Path $destinationPath "tool-manifest.json"
+$sfPath = Join-Path $destinationPath "sf.exe"
+$signaturePath = Join-Path $destinationPath "default.sig"
+
+if (-not $Force -and
+ (Test-Path -LiteralPath $manifestPath -PathType Leaf) -and
+ (Test-Path -LiteralPath $sfPath -PathType Leaf) -and
+ (Test-Path -LiteralPath $signaturePath -PathType Leaf)) {
+
+ try {
+ Invoke-InstalledToolVerification -Directory $destinationPath
+ }
+ catch {
+ throw "An existing Siegfried installation was found but failed verification. Re-run with -Force to replace it. Verification error: $($_.Exception.Message)"
+ }
+
+ Write-Host "Pinned Siegfried toolchain is already installed and verified:"
+ Write-Host " $destinationPath"
+ return
+}
+
+New-Item -ItemType Directory -Path $destinationParent -Force | Out-Null
+
+$tempRoot = Join-Path (
+ [System.IO.Path]::GetTempPath()
+) ("mailvault-profiler-siegfried-download-" + [guid]::NewGuid().ToString("N"))
+
+$stagingPath = Join-Path $destinationParent (
+ ".$destinationLeaf.install-" + [guid]::NewGuid().ToString("N")
+)
+
+$backupPath = Join-Path $destinationParent (
+ ".$destinationLeaf.backup-" + [guid]::NewGuid().ToString("N")
+)
+
+New-Item -ItemType Directory -Path $tempRoot -Force | Out-Null
+New-Item -ItemType Directory -Path $stagingPath -Force | Out-Null
+
+$installed = $false
+$destinationBackedUp = $false
+
+try {
+ $headers = @{
+ Accept = "application/vnd.github+json"
+ "X-GitHub-Api-Version" = "2022-11-28"
+ "User-Agent" = "FireXCore-MailVault-Collection-Profiler"
+ }
+
+ if (-not [string]::IsNullOrWhiteSpace([string]$env:GITHUB_TOKEN)) {
+ $headers.Authorization = "Bearer $($env:GITHUB_TOKEN)"
+ }
+
+ $escapedTag = [Uri]::EscapeDataString($Tag)
+ $releaseUri = "https://api.github.com/repos/richardlehane/siegfried/releases/tags/$escapedTag"
+
+ Write-Host "Resolving Siegfried release $Tag from GitHub..."
+ $release = Invoke-RestMethod `
+ -Uri $releaseUri `
+ -Headers $headers `
+ -Method Get `
+ -ErrorAction Stop
+
+ if ([string]$release.tag_name -ne $Tag) {
+ throw "GitHub release tag mismatch: requested '$Tag', received '$($release.tag_name)'."
+ }
+ if ([bool]$release.draft) {
+ throw "Refusing to install from draft GitHub release '$Tag'."
+ }
+
+ # Pin the non-static Windows x64 archive by its exact upstream name.
+ # Siegfried also publishes a *_win64_static.zip build whose signatures are
+ # embedded in the executable. The profiler deliberately uses the regular
+ # build because it treats sf.exe and default.sig as separately hashed,
+ # versioned supply-chain artifacts.
+ $releaseVersionSlug = $RequiredVersion -replace '\.', '-'
+ $expectedAssetName = "siegfried_${releaseVersionSlug}_win64.zip"
+
+ $assets = @(
+ $release.assets | Where-Object {
+ [string]$_.name -ieq $expectedAssetName
+ }
+ )
+
+ if ($assets.Count -ne 1) {
+ $available = @(
+ $release.assets |
+ ForEach-Object { [string]$_.name } |
+ Sort-Object
+ ) -join ", "
+
+ throw "Expected exactly one pinned Siegfried release asset '$expectedAssetName' for '$Tag'; found $($assets.Count). Available assets: $available"
+ }
+
+ $asset = $assets[0]
+ Write-Host "Selected pinned release asset:"
+ Write-Host " $expectedAssetName"
+ $assetName = [string]$asset.name
+ $assetDigest = [string]$asset.digest
+ $assetDownloadUrl = [string]$asset.browser_download_url
+ $assetSize = [int64]$asset.size
+
+ if ([string]::IsNullOrWhiteSpace($assetName) -or
+ [string]::IsNullOrWhiteSpace($assetDownloadUrl) -or
+ $assetSize -le 0) {
+ throw "GitHub returned incomplete metadata for the selected release asset."
+ }
+
+ if ([string]::IsNullOrWhiteSpace($assetDigest) -or
+ $assetDigest -notmatch '^sha256:[0-9a-fA-F]{64}$') {
+ throw "GitHub did not provide a valid SHA-256 digest for release asset '$assetName'; refusing an unverifiable download."
+ }
+
+ $archivePath = Join-Path $tempRoot $assetName
+
+ Write-Host "Downloading verified release asset:"
+ Write-Host " $assetName"
+ Invoke-GitHubDownload `
+ -Uri $assetDownloadUrl `
+ -OutFile $archivePath `
+ -Headers $headers
+
+ $downloadedSize = (Get-Item -LiteralPath $archivePath).Length
+ if ($downloadedSize -ne $assetSize) {
+ throw "Downloaded asset size mismatch: expected $assetSize bytes, got $downloadedSize bytes."
+ }
+
+ $expectedAssetSha = $assetDigest.Substring(7).ToLowerInvariant()
+ $actualAssetSha = Get-Sha256 -Path $archivePath
+ if ($actualAssetSha -ne $expectedAssetSha) {
+ throw "Downloaded asset SHA-256 mismatch: expected $expectedAssetSha, got $actualAssetSha."
+ }
+
+ Write-Host "Release asset SHA-256 verified."
+
+ $expandedPath = Join-Path $tempRoot "expanded"
+ New-Item -ItemType Directory -Path $expandedPath -Force | Out-Null
+ Expand-Archive -LiteralPath $archivePath -DestinationPath $expandedPath -Force
+
+ $sourceExecutables = @(
+ Get-ChildItem -LiteralPath $expandedPath -Recurse -File -Filter "sf.exe"
+ )
+ if ($sourceExecutables.Count -ne 1) {
+ throw "The verified release asset must contain exactly one sf.exe; found $($sourceExecutables.Count)."
+ }
+
+ $stagedSfPath = Join-Path $stagingPath "sf.exe"
+ $stagedSignaturePath = Join-Path $stagingPath "default.sig"
+ $stagedManifestPath = Join-Path $stagingPath "tool-manifest.json"
+
+ Copy-Item `
+ -LiteralPath $sourceExecutables[0].FullName `
+ -Destination $stagedSfPath `
+ -Force
+
+ $sourceSignatures = @(
+ Get-ChildItem -LiteralPath $expandedPath -Recurse -File -Filter "default.sig"
+ )
+
+ if ($sourceSignatures.Count -gt 1) {
+ throw "The verified release asset contained multiple default.sig files; refusing an ambiguous installation."
+ }
+
+ if ($sourceSignatures.Count -eq 1) {
+ Copy-Item `
+ -LiteralPath $sourceSignatures[0].FullName `
+ -Destination $stagedSignaturePath `
+ -Force
+ $signatureSource = "verified-release-asset"
+ }
+ else {
+ Write-Host "The release archive did not contain default.sig; invoking Siegfried's verified signature updater..."
+ $null = Invoke-SignatureUpdate `
+ -Executable $stagedSfPath `
+ -ToolHome $stagingPath
+
+ if (-not (Test-Path -LiteralPath $stagedSignaturePath -PathType Leaf)) {
+ throw "Siegfried update completed without creating default.sig in '$stagingPath'."
+ }
+
+ $signatureSource = "siegfried-verified-update"
+ }
+
+ Write-Host "Probing installed tool and PRONOM signature..."
+ $probe = Invoke-JsonProbe `
+ -Executable $stagedSfPath `
+ -Signature $stagedSignaturePath `
+ -ToolHome $stagingPath
+
+ $observedVersion = [string]$probe.siegfried
+ if ($observedVersion -ne $RequiredVersion) {
+ throw "Siegfried version mismatch: required '$RequiredVersion', observed '$observedVersion'."
+ }
+
+ $requiredPronomNumber = $RequiredPronomVersion.TrimStart([char[]]"vV")
+ if ([string]::IsNullOrWhiteSpace($requiredPronomNumber)) {
+ throw "RequiredPronomVersion '$RequiredPronomVersion' is invalid."
+ }
+
+ $identifierDetails = @(
+ $probe.identifiers | ForEach-Object { [string]$_.details }
+ )
+
+ $expectedPronomFilePattern = "DROID_SignatureFile_V$([regex]::Escape($requiredPronomNumber))\.xml"
+ $pronomMatched = @(
+ $identifierDetails | Where-Object {
+ $_ -match $expectedPronomFilePattern
+ }
+ ).Count -gt 0
+
+ if (-not $pronomMatched) {
+ throw "PRONOM signature mismatch: required '$RequiredPronomVersion'; identifier details were: $($identifierDetails -join '; ')"
+ }
+
+ $manifest = [ordered]@{
+ schemaVersion = 1
+ sourceRepository = "https://github.com/richardlehane/siegfried"
+ releaseTag = $Tag
+ releaseAsset = [ordered]@{
+ name = $assetName
+ browserDownloadUrl = $assetDownloadUrl
+ sizeBytes = $assetSize
+ githubDigest = $assetDigest
+ downloadedSha256 = $actualAssetSha
+ }
+ tool = [ordered]@{
+ name = "siegfried"
+ version = $observedVersion
+ filename = "sf.exe"
+ sha256 = Get-Sha256 -Path $stagedSfPath
+ }
+ signature = [ordered]@{
+ filename = "default.sig"
+ version = $RequiredPronomVersion
+ sha256 = Get-Sha256 -Path $stagedSignaturePath
+ created = $probe.created
+ source = $signatureSource
+ identifiers = @($probe.identifiers)
+ }
+ containerExpansion = $false
+ acquiredAtUtc = [DateTimeOffset]::UtcNow.ToString("O")
+ installer = [ordered]@{
+ script = "scripts/install-siegfried.ps1"
+ powershellVersion = $PSVersionTable.PSVersion.ToString()
+ destination = $destinationPath
+ }
+ }
+
+ $manifestJson = $manifest | ConvertTo-Json -Depth 100
+ Write-Utf8NoBom `
+ -Path $stagedManifestPath `
+ -Content ($manifestJson + [Environment]::NewLine)
+
+ # Verify the complete staged installation before replacing any working copy.
+
+ Invoke-InstalledToolVerification -Directory $stagingPath
+
+ if (Test-Path -LiteralPath $destinationPath) {
+ Move-Item `
+ -LiteralPath $destinationPath `
+ -Destination $backupPath
+ $destinationBackedUp = $true
+ }
+
+ try {
+ Move-Item `
+ -LiteralPath $stagingPath `
+ -Destination $destinationPath
+ $installed = $true
+ }
+ catch {
+ if ($destinationBackedUp -and
+ (Test-Path -LiteralPath $backupPath) -and
+ -not (Test-Path -LiteralPath $destinationPath)) {
+ Move-Item `
+ -LiteralPath $backupPath `
+ -Destination $destinationPath
+ $destinationBackedUp = $false
+ }
+
+ throw
+ }
+
+ # Verify once more at the final path so path-sensitive mistakes cannot pass
+
+ # staging validation.
+
+ Invoke-InstalledToolVerification -Directory $destinationPath
+
+ if ($destinationBackedUp -and (Test-Path -LiteralPath $backupPath)) {
+ Remove-Item -LiteralPath $backupPath -Recurse -Force
+ $destinationBackedUp = $false
+ }
+
+ Write-Host ""
+ Write-Host "Installed and verified pinned format-identification toolchain:"
+ Write-Host " Siegfried: $RequiredVersion"
+ Write-Host " PRONOM: $RequiredPronomVersion"
+ Write-Host " Directory: $destinationPath"
+}
+catch {
+ # If the final-path verification failed, remove the newly installed copy.
+
+ if ($installed -and (Test-Path -LiteralPath $destinationPath)) {
+ Remove-Item `
+ -LiteralPath $destinationPath `
+ -Recurse `
+ -Force `
+ -ErrorAction SilentlyContinue
+ }
+
+ # Restore the previously verified/installed directory when one existed.
+
+ if ($destinationBackedUp -and (Test-Path -LiteralPath $backupPath)) {
+ Move-Item `
+ -LiteralPath $backupPath `
+ -Destination $destinationPath
+
+ $destinationBackedUp = $false
+ }
+
+ throw
+}
+finally {
+ Remove-Item `
+ -LiteralPath $tempRoot `
+ -Recurse `
+ -Force `
+ -ErrorAction SilentlyContinue
+
+ Remove-Item `
+ -LiteralPath $stagingPath `
+ -Recurse `
+ -Force `
+ -ErrorAction SilentlyContinue
+
+ # A backup is deliberately retained if restoration could not be completed.
+
+ if (-not $destinationBackedUp) {
+ Remove-Item `
+ -LiteralPath $backupPath `
+ -Recurse `
+ -Force `
+ -ErrorAction SilentlyContinue
+ }
+}
diff --git a/scripts/quality.ps1 b/scripts/quality.ps1
index b67af0a..e346fef 100644
--- a/scripts/quality.ps1
+++ b/scripts/quality.ps1
@@ -61,6 +61,15 @@ Invoke-CheckedNative -FilePath $npmExecutable -ArgumentList @("run", "check:rust
Invoke-CheckedNative -FilePath $npmExecutable -ArgumentList @("run", "type-check")
Invoke-CheckedNative -FilePath $npmExecutable -ArgumentList @("run", "build")
+# The native desktop bundle includes a pinned Siegfried/PRONOM runtime. Acquire and
+# verify it before compiling the Tauri context, because bundle resources are part of
+# the release contract rather than an optional developer dependency.
+if ($env:OS -eq "Windows_NT") {
+ # PowerShell scripts report failure through terminating errors. Do not read
+ # $LASTEXITCODE here: it is owned by native executables and may be stale.
+ & (Join-Path $PSScriptRoot "install-siegfried.ps1")
+}
+
# The core Rust gate intentionally excludes the native Tauri crate so the same
# workspace can be validated on hosts without desktop system libraries. This
# Windows gate runs after the frontend build and must compile every desktop
diff --git a/scripts/run-real-archive-profile.ps1 b/scripts/run-real-archive-profile.ps1
index 4b90ea3..9402f4d 100644
--- a/scripts/run-real-archive-profile.ps1
+++ b/scripts/run-real-archive-profile.ps1
@@ -420,7 +420,7 @@ finally {
$manifest = [ordered]@{
schemaVersion = 1
- profilerVersion = "0.1.0-alpha.3"
+ profilerVersion = "0.1.0-alpha.4"
startedAt = $startedAt.ToString("o")
finishedAt = $finishedAt.ToString("o")
elapsedMilliseconds = [long] $elapsedMilliseconds
diff --git a/scripts/verify-siegfried.ps1 b/scripts/verify-siegfried.ps1
new file mode 100644
index 0000000..f90ed23
--- /dev/null
+++ b/scripts/verify-siegfried.ps1
@@ -0,0 +1,347 @@
+#Requires -Version 5.1
+
+
+[CmdletBinding()]
+param(
+ # Deliberately resolved after the param block so Windows PowerShell 5.1
+
+ # never has to evaluate a default expression that depends on $PSScriptRoot.
+
+ [Parameter()]
+ [string] $Directory
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = "Stop"
+$ProgressPreference = "SilentlyContinue"
+
+$scriptPath = [string]$PSCommandPath
+if ([string]::IsNullOrWhiteSpace($scriptPath)) {
+ $scriptPath = [string]$MyInvocation.MyCommand.Path
+}
+if ([string]::IsNullOrWhiteSpace($scriptPath)) {
+ throw "Unable to resolve the current verification script path."
+}
+
+$scriptRoot = [System.IO.Path]::GetDirectoryName(
+ [System.IO.Path]::GetFullPath($scriptPath)
+)
+if ([string]::IsNullOrWhiteSpace($scriptRoot)) {
+ throw "Unable to resolve the directory containing verify-siegfried.ps1."
+}
+
+$repositoryRoot = [System.IO.Path]::GetFullPath(
+ (Join-Path $scriptRoot "..")
+)
+
+if ([string]::IsNullOrWhiteSpace($Directory)) {
+ $Directory = Join-Path $repositoryRoot "tools\siegfried\windows-x86_64"
+}
+
+try {
+ $root = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($Directory)
+}
+catch {
+ throw "Unable to resolve Siegfried directory '$Directory': $($_.Exception.Message)"
+}
+$root = [System.IO.Path]::GetFullPath($root)
+
+function Get-Sha256 {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory = $true)]
+ [ValidateNotNullOrEmpty()]
+ [string] $Path
+ )
+
+ if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) {
+ throw "Cannot hash missing file: $Path"
+ }
+
+ return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
+}
+
+function Read-JsonFile {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory = $true)]
+ [ValidateNotNullOrEmpty()]
+ [string] $Path
+ )
+
+ try {
+ $jsonText = [System.IO.File]::ReadAllText($Path)
+ if ([string]::IsNullOrWhiteSpace($jsonText)) {
+ throw "JSON file is empty."
+ }
+
+ # ConvertFrom-Json -Depth is not available in Windows PowerShell 5.1.
+
+ return ($jsonText | ConvertFrom-Json)
+ }
+ catch {
+ throw "Unable to parse JSON file '$Path': $($_.Exception.Message)"
+ }
+}
+
+function ConvertTo-NativeCommandLineArgument {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory = $true)]
+ [AllowEmptyString()]
+ [string] $Value
+ )
+
+ # ProcessStartInfo.ArgumentList is unavailable on the .NET Framework used
+ # by Windows PowerShell 5.1. Quote arguments according to the Windows
+ # CommandLineToArgvW rules so paths containing spaces, quotes, or trailing
+ # backslashes reach the native executable unchanged.
+ if ($Value.Length -gt 0 -and $Value -notmatch '[\s"]') {
+ return $Value
+ }
+
+ $builder = [System.Text.StringBuilder]::new()
+ [void] $builder.Append([char]34)
+ $backslashCount = 0
+
+ foreach ($character in $Value.ToCharArray()) {
+ $codePoint = [int] $character
+
+ if ($codePoint -eq 92) {
+ $backslashCount++
+ continue
+ }
+
+ if ($codePoint -eq 34) {
+ if ($backslashCount -gt 0) {
+ [void] $builder.Append([char]92, (($backslashCount * 2) + 1))
+ }
+ else {
+ [void] $builder.Append([char]92)
+ }
+
+ [void] $builder.Append([char]34)
+ $backslashCount = 0
+ continue
+ }
+
+ if ($backslashCount -gt 0) {
+ [void] $builder.Append([char]92, $backslashCount)
+ $backslashCount = 0
+ }
+
+ [void] $builder.Append($character)
+ }
+
+ # Backslashes immediately before the closing quote must be doubled.
+ if ($backslashCount -gt 0) {
+ [void] $builder.Append([char]92, ($backslashCount * 2))
+ }
+
+ [void] $builder.Append([char]34)
+ return $builder.ToString()
+}
+
+function Invoke-NativeProcess {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory = $true)]
+ [ValidateNotNullOrEmpty()]
+ [string] $Executable,
+
+ [Parameter(Mandatory = $true)]
+ [AllowEmptyCollection()]
+ [string[]] $Arguments,
+
+ [Parameter()]
+ [ValidateRange(1, 3600)]
+ [int] $TimeoutSeconds = 300
+ )
+
+ if (-not (Test-Path -LiteralPath $Executable -PathType Leaf)) {
+ throw "Native executable was not found: $Executable"
+ }
+
+ $quotedArguments = @(
+ foreach ($argument in $Arguments) {
+ ConvertTo-NativeCommandLineArgument -Value ([string] $argument)
+ }
+ )
+
+ $startInfo = [System.Diagnostics.ProcessStartInfo]::new()
+ $startInfo.FileName = $Executable
+ $startInfo.Arguments = $quotedArguments -join " "
+ $startInfo.UseShellExecute = $false
+ $startInfo.CreateNoWindow = $true
+ $startInfo.RedirectStandardOutput = $true
+ $startInfo.RedirectStandardError = $true
+
+ $process = [System.Diagnostics.Process]::new()
+ $process.StartInfo = $startInfo
+
+ try {
+ if (-not $process.Start()) {
+ throw "Failed to start native executable: $Executable"
+ }
+
+ # Start both asynchronous reads before waiting. Reading either stream
+ # synchronously first can deadlock when the child fills the other pipe.
+ $stdoutTask = $process.StandardOutput.ReadToEndAsync()
+ $stderrTask = $process.StandardError.ReadToEndAsync()
+
+ if (-not $process.WaitForExit($TimeoutSeconds * 1000)) {
+ try {
+ $process.Kill()
+ }
+ catch {
+ # Preserve the timeout as the primary failure.
+ }
+
+ $process.WaitForExit()
+ throw "Native command timed out after $TimeoutSeconds seconds: $Executable $($startInfo.Arguments)"
+ }
+
+ # WaitForExit() without a timeout ensures async stream handlers have
+ # finished after the process handle is signaled.
+ $process.WaitForExit()
+
+ $stdout = $stdoutTask.GetAwaiter().GetResult()
+ $stderr = $stderrTask.GetAwaiter().GetResult()
+
+ return [pscustomobject]@{
+ ExitCode = [int] $process.ExitCode
+ StdOut = [string] $stdout
+ StdErr = [string] $stderr
+ CommandLine = "$Executable $($startInfo.Arguments)"
+ }
+ }
+ finally {
+ $process.Dispose()
+ }
+}
+
+$manifestPath = Join-Path $root "tool-manifest.json"
+$sfPath = Join-Path $root "sf.exe"
+$signaturePath = Join-Path $root "default.sig"
+
+foreach ($requiredPath in @($manifestPath, $sfPath, $signaturePath)) {
+ if (-not (Test-Path -LiteralPath $requiredPath -PathType Leaf)) {
+ throw "Pinned Siegfried resource is missing: $requiredPath"
+ }
+}
+
+$manifest = Read-JsonFile -Path $manifestPath
+
+if ([int]$manifest.schemaVersion -ne 1) {
+ throw "Unsupported tool-manifest.json schema version '$($manifest.schemaVersion)'."
+}
+
+$manifestToolVersion = [string]$manifest.tool.version
+$manifestToolHash = ([string]$manifest.tool.sha256).ToLowerInvariant()
+$manifestSignatureVersion = [string]$manifest.signature.version
+$manifestSignatureHash = ([string]$manifest.signature.sha256).ToLowerInvariant()
+
+if ([string]::IsNullOrWhiteSpace($manifestToolVersion) -or
+ $manifestToolHash -notmatch '^[0-9a-f]{64}$' -or
+ [string]::IsNullOrWhiteSpace($manifestSignatureVersion) -or
+ $manifestSignatureHash -notmatch '^[0-9a-f]{64}$') {
+ throw "tool-manifest.json is missing required version or SHA-256 fields."
+}
+
+$sfHash = Get-Sha256 -Path $sfPath
+$sigHash = Get-Sha256 -Path $signaturePath
+
+if ($sfHash -ne $manifestToolHash) {
+ throw "sf.exe SHA-256 does not match tool-manifest.json. Expected $manifestToolHash, observed $sfHash."
+}
+if ($sigHash -ne $manifestSignatureHash) {
+ throw "default.sig SHA-256 does not match tool-manifest.json. Expected $manifestSignatureHash, observed $sigHash."
+}
+
+$probePath = Join-Path $root (
+ ".verify-probe-" + [guid]::NewGuid().ToString("N") + ".txt"
+)
+
+$probeEncoding = [System.Text.UTF8Encoding]::new($false)
+[System.IO.File]::WriteAllText(
+ $probePath,
+ "MailVault Collection Profiler Siegfried verification probe.",
+ $probeEncoding
+)
+
+try {
+ $signatureArgument = [System.IO.Path]::GetFileName($signaturePath)
+ if ([string]::IsNullOrWhiteSpace($signatureArgument)) {
+ throw "Unable to derive a home-relative Siegfried signature filename from '$signaturePath'."
+ }
+
+ # Keep the JSON header portable and valid on Windows. Siegfried 1.11.6
+ # writes the header's signature value without escaping backslashes.
+ $result = Invoke-NativeProcess `
+ -Executable $sfPath `
+ -Arguments @(
+ "-home", $root,
+ "-sig", $signatureArgument,
+ "-json",
+ "-utc",
+ $probePath
+ ) `
+ -TimeoutSeconds 120
+
+ if ($result.ExitCode -ne 0) {
+ $diagnostic = (
+ $result.StdOut +
+ [Environment]::NewLine +
+ $result.StdErr
+ ).Trim()
+
+ throw "Siegfried verification probe failed with exit code $($result.ExitCode): $diagnostic"
+ }
+
+ $jsonText = $result.StdOut.Trim()
+ if ([string]::IsNullOrWhiteSpace($jsonText)) {
+ throw "Siegfried verification probe returned empty JSON. stderr: $($result.StdErr.Trim())"
+ }
+
+ try {
+ $probe = $jsonText | ConvertFrom-Json
+ }
+ catch {
+ throw "Siegfried verification probe returned invalid JSON: $($_.Exception.Message). Raw output: $jsonText"
+ }
+
+ $runtimeToolVersion = [string] $probe.siegfried
+ if ($runtimeToolVersion -ne $manifestToolVersion) {
+ throw "Runtime Siegfried version '$runtimeToolVersion' differs from manifest version '$manifestToolVersion'."
+ }
+
+ $requiredPronomNumber = $manifestSignatureVersion.TrimStart([char[]] "vV")
+ if ([string]::IsNullOrWhiteSpace($requiredPronomNumber)) {
+ throw "Manifest PRONOM version '$manifestSignatureVersion' is invalid."
+ }
+
+ $identifierDetails = @(
+ $probe.identifiers | ForEach-Object { [string] $_.details }
+ )
+
+ $expectedPattern = "DROID_SignatureFile_V$([regex]::Escape($requiredPronomNumber))\.xml"
+ $matched = @(
+ $identifierDetails | Where-Object { $_ -match $expectedPattern }
+ ).Count -gt 0
+
+ if (-not $matched) {
+ throw "Runtime PRONOM signature differs from tool-manifest.json. Required '$manifestSignatureVersion'; identifier details were: $($identifierDetails -join '; ')"
+ }
+
+ if (-not [string]::IsNullOrWhiteSpace($result.StdErr)) {
+ Write-Verbose "Siegfried verification diagnostics: $($result.StdErr.Trim())"
+ }
+}
+finally {
+ Remove-Item -LiteralPath $probePath -Force -ErrorAction SilentlyContinue
+}
+
+Write-Host "Siegfried resources verified:"
+Write-Host " Siegfried: $manifestToolVersion"
+Write-Host " PRONOM: $manifestSignatureVersion"
+Write-Host " Directory: $root"
diff --git a/tools/siegfried/windows-x86_64/README.md b/tools/siegfried/windows-x86_64/README.md
new file mode 100644
index 0000000..7ac3a8c
--- /dev/null
+++ b/tools/siegfried/windows-x86_64/README.md
@@ -0,0 +1,12 @@
+# Generated Siegfried release resources
+
+This directory is populated by `scripts/install-siegfried.ps1` during Windows CI and release builds.
+The generated files are intentionally not committed:
+
+- `sf.exe`
+- `default.sig`
+- `tool-manifest.json`
+
+The installer script downloads the pinned v1.11.6 Windows x64 release asset through the GitHub REST
+API, verifies the asset digest and size, acquires the PRONOM v124 signature, probes the resulting
+runtime and records the executable/signature SHA-256 values in `tool-manifest.json`.
-" `
- --finding "" `
- --status needs_investigation `
- --note "Verify the physical object against the retained backup."
+.\target\release\mailvault-profiler.exe formats identify `
+ --workspace "E:\MailVault-Profiler-Alpha4" `
+ --run "" `
+ --siegfried ".\tools\siegfried\windows-x86_64\sf.exe" `
+ --signature ".\tools\siegfried\windows-x86_64\default.sig" `
+ --batch-size 2048 `
+ --workers 0 `
+ --timeout-seconds 900 `
+ --resume true `
+ --allow-migration `
+ 1> format-result.json `
+ 2> format-progress.jsonl
```
-Export خلاصه Sanitized:
+راهنمای کامل: [تشخیص دقیق فرمت](docs/FORMAT_IDENTIFICATION.md) و
+[Runbook اجرایی](docs/FORMAT_IDENTIFICATION_RUNBOOK.md).
+
+## Build از Source
```powershell
-.\target\release\mailvault-profiler.exe export sanitized-summary `
- --workspace "D:\MailVault-Profiler-Workspace" `
- --run "" `
- --output ".\mailvault-profile-sanitized-summary.json"
+npm ci
+.\scripts\install-siegfried.ps1
+.\scripts\quality.ps1
+npm run tauri:desktop:bundle
```
-وضعیت Review داخل Profiler Database ذخیره میشود و MailVault Source یا Evidence اصلی را تغییر
-نمیدهد. جزئیات در [ساختار Workspace](docs/WORKSPACE_FORMAT.md) و
-[Review یافتهها](docs/FINDINGS_REVIEW.md) آمده است.
+فایلهای تولیدشده `sf.exe`، `default.sig` و `tool-manifest.json` داخل Git Commit نمیشوند؛ Release
+Workflow آنها را از Release رسمی دریافت، Hash را بررسی و بهعنوان Resource داخل Installer قرار میدهد.
-## نتیجه Validation واقعی `0.1.0-alpha.3`
+## وضعیت Validation
-نسخه Alpha.3 روی یک آرشیو خصوصی واقعی در Windows x64 اجرا و از ابتدا تا Reopen، Review و Export
-اعتبارسنجی شده است. فقط Aggregateها در Repository منتشر میشوند و Evidence خام خصوصی باقی میماند.
+در محیط فعلی این موارد Pass شدهاند:
-| معیار | مقدار ثبتشده در اجرای Alpha.3 |
-|---|---:|
-| Account | 1 |
-| پیام | 17,296 |
-| Message occurrence | 17,307 |
-| MIME part | 54,450 |
-| Attachment occurrence در قرارداد فعلی Adapter | 18,552 |
-| Blob row / Content object | 13,684 |
-| Content occurrence | 22,068 |
-| Message relationship | 12,115 |
-| Participant row | 51,101 |
-| Blob bytes ثبتشده | 6,467,253,277 |
-| Findings | 1,484 |
-| Errors | 0 |
-| Warnings | 2 |
-| Missing object | 1 |
-| Unreadable object | 0 |
-| Size mismatch | 0 |
-
-نتایج مهم:
-
-- Source metrics و Snapshot metrics یکسان بودند؛
-- 13,683 Object از 13,684 Object فیزیکی در دسترس بود؛
-- یک Blob مفقود و یک Zero-byte content object شناسایی شد؛
-- 375 رابطه Same-hash/Different-name ثبت شد؛
-- 1,107 رابطه Same-name/Different-hash ثبت شد؛
-- Workspace پس از Restart دوباره باز شد؛
-- Reviewها و Hash chain پس از Restart باقی ماندند؛
-- Session دوم در زمان Lock فعال به حالت Read-only منتقل شد؛
-- Export Sanitized فاقد Path، Filename، Email و Review note خصوصی بود؛
-- Source MailVault تغییر نکرد.
-
-Baseline تاریخی خصوصی، 21,946 Attachment occurrence را در دامنه گستردهتر Metadata ثبت کرده است؛
-اجرای Alpha.3 مقدار 18,552 را طبق قرارداد فعلی Adapter گزارش میکند. این دو عدد نباید بهعنوان یک
-Metric واحد جایگزین یکدیگر شوند و Reconciliation آنها در
-[Baseline آرشیو واقعی](docs/REAL_ARCHIVE_BASELINE.md) نگهداری میشود.
-
-گزارش کامل: [Validation نسخه Alpha.3](docs/VALIDATION_0.1.0-alpha.3.md).
+- TypeScript type-check و Vite production build؛
+- Parse نحوی تمام فایلهای Rust با Tree-sitter؛
+- Migration دیتابیس واقعی Alpha 3 از Schema 5 به 6؛
+- `quick_check` و `foreign_key_check`؛
+- حفظ 13,684 Object، 22,068 Occurrence و 1,484 Finding؛
+- ثابتماندن SHA-256 دیتابیس Source؛
+- Projection آزمایشی Format Assertion روی Schema مهاجرتدادهشده.
-## Build از Source
+در Container فعلی Rust toolchain وجود نداشت؛ بنابراین ادعای دروغ درباره `cargo check`، Clippy، تستهای
+Rust، Build Native Tauri یا اجرای واقعی Siegfried نشده است. CI ویندوز برای اجرای همین Gateهای کامل
+تنظیم شده است.
-پیشنیازها:
+گزارش: [Validation Alpha 4](docs/VALIDATION_0.1.0-alpha.4.md).
-- Rust `1.97.1` مطابق `rust-toolchain.toml`؛
-- Node.js نسخه `24.x` الزامی است؛
-- npm نسخه `11+`؛
-- Visual Studio 2026 یا Build Tools با workload **Desktop development with C++**؛
-- Windows 10/11 SDK و WebView2 Runtime.
+## مستندات اصلی
-Build را در x64 Visual Studio Developer Command Prompt اجرا کن:
+- [فهرست کامل](docs/INDEX.md)
+- [شروع سریع](docs/GETTING_STARTED.md)
+- [نصب Windows](docs/INSTALLATION_WINDOWS.md)
+- [راهنمای GUI](docs/GUI_GUIDE.md)
+- [مرجع CLI](docs/CLI_REFERENCE.md)
+- [تشخیص دقیق فرمت](docs/FORMAT_IDENTIFICATION.md)
+- [Runbook تشخیص فرمت](docs/FORMAT_IDENTIFICATION_RUNBOOK.md)
+- [معماری](docs/ARCHITECTURE.md)
+- [مدل امنیتی](docs/SECURITY_MODEL.md)
+- [حریم خصوصی](docs/PRIVACY.md)
+- [Baseline واقعی](docs/REAL_ARCHIVE_BASELINE.md)
+- [Release Notes Alpha 4](docs/releases/v0.1.0-alpha.4.md)
-```cmd
-call "C:\Program Files\Microsoft Visual Studio\18\Community\Common7\Tools\VsDevCmd.bat" -arch=amd64 -host_arch=amd64
-cd /d D:\mailvault-collection-profiler
-npm ci
-powershell -NoProfile -ExecutionPolicy Bypass -File .\scripts\quality.ps1
-npm run tauri:desktop:bundle
-```
+## خارج از Scope فعلی
-جزئیات: [Development](docs/DEVELOPMENT.md) و [نصب Windows](docs/INSTALLATION_WINDOWS.md).
-
-## مستندات
-
-| مستند | کاربرد |
-|---|---|
-| [فهرست مستندات](docs/INDEX.md) | نقشه کامل مستندات کاربر، امنیت، فنی و Maintainer |
-| [شروع سریع](docs/GETTING_STARTED.md) | اجرای امن از نصب تا Review یافتهها |
-| [نصب Windows](docs/INSTALLATION_WINDOWS.md) | پیشنیاز Installer و Build از Source |
-| [راهنمای GUI](docs/GUI_GUIDE.md) | راهاندازی Collection، Inventory، Findings و Object detail |
-| [مرجع CLI](docs/CLI_REFERENCE.md) | Commandها، Optionها، Streamهای خروجی و Exit behavior |
-| [خروجیهای Evidence](docs/EVIDENCE_OUTPUTS.md) | فایلهای Evidence wrapper و قواعد نگهداری |
-| [ساختار Workspace](docs/WORKSPACE_FORMAT.md) | Layout، Schema، Migration، Lock و Reopen |
-| [Review یافتهها](docs/FINDINGS_REVIEW.md) | وضعیتها، Note policy و Append-only history |
-| [معماری](docs/ARCHITECTURE.md) | مرز Crateها و جریان داده |
-| [مدل امنیتی](docs/SECURITY_MODEL.md) | Trust boundaryها، Invariantها و Threat handling |
-| [حریم خصوصی](docs/PRIVACY.md) | پردازش محلی و Metadata مشتقشده حساس |
-| [رفع اشکال](docs/TROUBLESHOOTING.md) | خطاهای شناختهشده Windows، Rust، npm، Tauri و Archive |
-| [وضعیت پیادهسازی](docs/IMPLEMENTATION_STATUS.md) | قابلیتهای تکمیلشده و موارد Deferred |
-| [نقشه راه](docs/ROADMAP.md) | قابلیتهای برنامهریزیشده و خارج از Scope فعلی |
-| [فرایند Release](docs/RELEASE_PROCESS.md) | Release و Verification برای Maintainer |
-| [راهنمای انتشار در GitHub](docs/GITHUB_PUBLISHING_GUIDE_FA.md) | تنظیم Repository، Ruleset، CI و Release |
-| [تحویل آمادهسازی مخزن](docs/REPOSITORY_RELEASE_HANDOFF_FA.md) | ترتیب Push، Labelها، Gateها و Release |
-
-## وضعیت پروژه و محدودیتها
-
-در این Release پیادهسازی نشده است:
-
-- Resume اجرای Profiling قطعشده و کنترلهای Pause/Cancel در UI؛
-- Full payload SHA-256 fixity pass؛
-- شناسایی دقیق Format با Siegfried/PRONOM؛
-- Container expansion و اعتبارسنجی JHOVE؛
-- OCR، Semantic extraction، Embedding، LLM یا Classification تجاری؛
-- Auto-update برنامه؛
-- امضای دیجیتال عمومی Installerهای Windows.
-
-برای وضعیت دقیق به [وضعیت پیادهسازی](docs/IMPLEMENTATION_STATUS.md) و
-[نقشه راه](docs/ROADMAP.md) مراجعه کن.
-
-## مشارکت، پشتیبانی و گزارش امنیتی
-
-پیش از تغییر قرارداد Evidence، Snapshot behavior، Canonical locator یا Migrationهای Profiler،
-[CONTRIBUTING.md](CONTRIBUTING.md) را مطالعه کن. در Issueها هیچ Archive واقعی، فایل EML، Attachment،
-Profiler database یا Log حساس را پیوست نکن.
-
-- پشتیبانی: [SUPPORT.md](SUPPORT.md)
-- گزارش محرمانه آسیبپذیری: [SECURITY.md](SECURITY.md)
-- آییننامه رفتاری: [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md)
+Resume اجرای Physical Profile قطعشده، Full Fixity Hash، Container expansion، JHOVE، استخراج متن،
+OCR، Embedding، LLM، تشخیص Invoice/Quotation، اتصال خودکار به RFQ و هر نوع نوشتن در RMS.
## مجوز
-این پروژه تحت Apache License 2.0 منتشر میشود. فایلهای [LICENSE](LICENSE) و [NOTICE](NOTICE)
-مرجع حقوقی هستند.
+Apache License 2.0. فایلهای [LICENSE](LICENSE)، [NOTICE](NOTICE) و
+[THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md) را ببین.
diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md
new file mode 100644
index 0000000..0aa24ce
--- /dev/null
+++ b/THIRD_PARTY_NOTICES.md
@@ -0,0 +1,19 @@
+# Third-party notices
+
+MailVault Collection Profiler can bundle the following independently maintained component in its
+Windows installers.
+
+## Siegfried
+
+- Project: `richardlehane/siegfried`
+- Pinned release: `v1.11.6`
+- License: Apache License 2.0
+- Copyright: Richard Lehane, Ross Spencer and contributors
+- Purpose: signature-based exact file-format identification
+
+The generated Windows release also includes a pinned `default.sig` signature database built from
+The National Archives' PRONOM registry, version `v124`. The build records SHA-256 digests for both
+the executable and signature file in `tool-manifest.json`.
+
+Siegfried and PRONOM are not affiliated with FireXCore. Their names identify the upstream tools and
+registries used to produce technical format assertions.
diff --git a/apps/cli/src/main.rs b/apps/cli/src/main.rs
index 3d4fd4c..c457d75 100644
--- a/apps/cli/src/main.rs
+++ b/apps/cli/src/main.rs
@@ -7,12 +7,13 @@ use std::{
use clap::{Parser, Subcommand};
use profiler_adapter_mailvault::MailVaultAdapter;
use profiler_core::{
- CollectionAdapter, FindingCategory, FindingsPageRequest, ProfilerResult, ProgressEvent,
- ProgressSink, ReviewActorKind, ReviewStatus, SnapshotOptions, SnapshotRequest,
+ CollectionAdapter, FindingCategory, FindingsPageRequest, FormatFilters,
+ FormatIdentificationRequest, FormatOptions, FormatPageRequest, FormatState, ProfilerResult,
+ ProgressEvent, ProgressSink, ReviewActorKind, ReviewStatus, SnapshotOptions, SnapshotRequest,
WorkspaceOpenMode,
};
use profiler_engine::{
- ProfileEngine, ProfileOptions, ProfileRequest,
+ ExactFormatEngine, ProfileEngine, ProfileOptions, ProfileRequest,
workspace::{
WorkspaceSession, add_review_note, clear_review_status, export_sanitized_run,
finding_detail, findings_page, list_runs, set_review_status,
@@ -68,6 +69,10 @@ enum Command {
#[command(subcommand)]
command: FindingsCommand,
},
+ Formats {
+ #[command(subcommand)]
+ command: FormatsCommand,
+ },
Export {
#[command(subcommand)]
command: ExportCommand,
@@ -166,6 +171,91 @@ enum FindingsCommand {
},
}
+#[derive(Debug, Subcommand)]
+enum FormatsCommand {
+ Probe {
+ #[arg(long)]
+ siegfried: Option
+
-
-> **نسخه آزمایشی توسعهای:** نسخه `0.1.0-alpha.3` برای ارزیابی فنی کنترلشده آماده است.
-> Installerهای Windows در این نسخه هنوز امضای دیجیتال عمومی ندارند. Workspaceهای قبلی اکنون
-> قابل Inspect، Migration و Reopen هستند؛ تصمیمهای Review بهصورت Append-only ذخیره میشوند و
-> خروجی Sanitized در قالب JSON و CSV در دسترس است. Resume اجرای قطعشده و تعمیر Archive عمداً
-> در این نسخه فعال نیست.
-
-
-
-## چرا این پروژه ساخته شده است؟
-
-آرشیو MailVault فقط پوشهای از Attachmentها نیست. یک آرشیو واقعی شامل دیتابیس Canonical از نوع
-SQLite، پیامها، MIME Partها، Participantها، روابط بین پیامها، Object Storeهای Content-addressed،
-نامهای تاریخی فایلها و فایلهای فیزیکی است. هرکدام از این لایهها ممکن است با Metadata ثبتشده
-ناسازگار، مفقود، غیرقابلخواندن یا ناقص باشند.
+## این برنامه دقیقاً چه کاری میکند؟
-MailVault Collection Profiler بدون تغییر آرشیو Canonical، یک موجودی فنی تکرارپذیر میسازد:
+MailVault مدرک اصلی ایمیل را نگه میدارد. Profiler یک Index فنی و قابل بازسازی روی همان مدرک میسازد:
```text
-Read-only preflight
-→ ساخت Snapshot سازگار SQLite در Workspace جدا
-→ Inventory جریانی Metadata
-→ تطبیق Attachment occurrence و هویت SHA-256
-→ بررسی محدود و کنترلشده فایلهای فیزیکی
-→ Findings و Checkpointهای ماندگار
-→ مرور Inventory، Findings و شواهد در GUI و CLI
+MailVault فقطخواندنی
+ ← Snapshot سازگار SQLite
+ ← Physical Inventory
+ ← هویت دقیق SHA-256 و تاریخچه occurrence
+ ← کنترل فیزیکی فایلها
+ ← تشخیص دقیق فرمت و PUID
+ ← Explorer دسکتاپ و CLI
```
-## قرارداد اصلی ایمنی
-
-- دیتابیس Canonical MailVault فقط در حالت Read-only باز میشود؛
-- Workspace باید خارج از Archive Root قرار داشته باشد؛
-- پیش از پردازش Inventory، یک Snapshot سازگار SQLite داخل Workspace ساخته میشود؛
-- Locator مربوط به Blob پیش از بازشدن فایل Canonical و از نظر Path containment بررسی میشود؛
-- شکست containment به Finding تبدیل میشود و Profiler مسیر نامعتبر را دنبال نمیکند؛
-- هیچ Attachmentی اجرا، Render، Extract یا Upload نمیشود؛
-- هیچ سرویس Cloud، حساب کاربری یا Telemetry اجباری وجود ندارد؛
-- خروجیهای Profiler مشتقشدهاند و ممکن است شامل Filename، Domain یا Path حساس باشند؛
-- فایلهای خام Evidence و دیتابیسهای SQLite نباید بدون Sanitization منتشر شوند.
-
-جزئیات در [مدل امنیتی](docs/SECURITY_MODEL.md)، [حریم خصوصی](docs/PRIVACY.md) و
-[خروجیهای Evidence](docs/EVIDENCE_OUTPUTS.md) مستند شده است.
-
-## قابلیتهای پیادهسازیشده در `0.1.0-alpha.3`
-
-- قرارداد سازگاری MailVault Schema v3 و Preflight فقطخواندنی؛
-- بررسی مسیرها، Tableها، Columnها، Indexها، Writer Lock و سلامت ساختاری SQLite؛
-- ساخت Snapshot با SQLite Online Backup API همراه با Progress و تشخیص تغییر Source؛
-- Inventory جریانی پیامها، MIME Partها، Participantها، Relationها و Blobها؛
-- حفظ Attachment occurrenceها و هویت دقیق Content بر اساس SHA-256؛
-- Normalization نام فایل و نگهداری Filename History؛
-- تشخیص Same-hash/Different-name و Same-name/Different-hash؛
-- بررسی محدود File-stat با انتخاب محافظهکارانه تعداد Workerها؛
-- تشخیص Missing، Unreadable، Invalid locator، Non-regular object و Size mismatch؛
-- Search و Cursor pagination بر اساس Filename، SHA-256، MIME، Subject و Sender domain؛
-- Content Object Detail شامل Filename variantها، Message occurrenceها و Findingهای فنی؛
-- Progress eventهای ساختیافته برای Row، Object، Page و Byte؛
-- برنامه Desktop ویندوز با Tauri و CLI مستقل؛
-- Inspect سازگاری Workspace و Migration صریح همراه با Backup نگهداریشده؛
-- Reopen کردن Runهای Completed، Failed یا Interrupted پس از Restart کامل برنامه؛
-- Single-writer lock برای Workspace با Read-only fallback در Session همزمان؛
-- Review eventهای Append-only با Hash chain مبتنی بر SHA-256؛
-- وضعیتهای Review شامل `acknowledged`، `expected`، `needs_investigation` و `resolved_externally`؛
-- الزام Note برای وضعیتهای Investigation و Resolved externally؛
-- جداسازی Findings به Requires attention، Informational evidence، Reviewed و All findings؛
-- Export Sanitized در قالب JSON و CSV بدون Path، Filename، Email address و Review note؛
-- تستهای Integration برای Restart persistence، Lock fallback، Export Sanitized و عدم تغییر Source.
-
-## نمای محصول
-
-تمام تصاویر مستندات با دادههای Sanitized یا Synthetic ساخته شدهاند. هیچ آرشیو خصوصی، پیام،
-Attachment یا دیتابیس واقعی در Repository نگهداری نمیشود. سیاست کامل تصاویر در
-[سیاست Screenshotها](docs/SCREENSHOTS.md) قرار دارد.
-
-### شروع، Preflight و انتخاب Workspace
-
-
-
-### اجرای Profile و Progress ساختیافته
-
-
-
-### بازکردن Runهای قبلی پس از Restart
-
-
+Profiler ایمیل دانلود نمیکند، MailVault را تغییر نمیدهد، فایل را اجرا نمیکند، OCR انجام نمیدهد،
+ZIP/RAR را باز نمیکند، سند را Invoice یا Quotation تشخیص نمیدهد و چیزی در RMS نمینویسد.
-### موجودی دقیق Binaryها
+## مبنای واقعی طراحی
-
-
-### بررسی Findings نیازمند توجه
-
-
-
-### Findings Explorer با Filterهای فنی
-
-
-
-### جزئیات Content Object و تاریخچه occurrence
-
-
-
-### Workflow امن CLI
-
-
-
-## دریافت و نصب در Windows
-
-1. صفحه [GitHub Releases](https://github.com/FireXCore/mailvault-collection-profiler/releases) را باز کن؛
-2. Installer ویندوز x64 با پسوند `-setup.exe` یا بسته MSI را دریافت کن؛
-3. Hash فایل را با `SHA256SUMS.txt` همراه Release مقایسه کن؛
-4. برنامه را نصب کن؛ نسخه Alpha بدون امضای عمومی ممکن است پیام SmartScreen نشان دهد؛
-5. Archive، Workspace و Runtime Evidence را در سه مسیر جدا نگه دار.
-
-ساختار پیشنهادی:
-
-```text
-D:\MailVault-Demo
-D:\MailVault-Profiler-Workspace
-D:\MailVault-Profiler-Evidence
-```
+| معیار | مقدار ثبتشده |
+|---|---:|
+| حجم آرشیو | حدود ۲۰ تا ۳۰ گیگابایت |
+| پیام | 17,296 |
+| MIME part | 54,450 |
+| Content object | 13,684 |
+| Content occurrence | 22,068 |
+| Message relationship | 12,115 |
+| حجم Blobهای منحصربهفرد | 6,467,253,277 بایت |
+| Findings مرحله فیزیکی | 1,484 |
+| Warning / Error | 2 / 0 |
-راهنمای کامل: [نصب Windows](docs/INSTALLATION_WINDOWS.md).
+Alpha 3 روی آرشیو واقعی Windows اعتبارسنجی شد. Alpha 4 همان Baseline را حفظ میکند، Migration شماره
+`0006` و قرارداد اجرای تشخیص دقیق فرمت را اضافه میکند. قبل از انتشار آمار سرعت و توزیع فرمتها،
+یک اجرای خصوصی واقعی Alpha 4 هنوز لازم است.
-## اولین Profile با برنامه Desktop
+## قابلیتهای Alpha 4
-1. عملیات Sync، Import یا Maintenance مربوط به MailVault را متوقف کن؛
-2. Archive Root را انتخاب کن؛
-3. گزینه **Run read-only preflight** را اجرا کن؛
-4. سازگاری Source، Schema `v3` و نبود Writer lock فعال را بررسی کن؛
-5. یک Workspace خالی و خارج از Archive Root انتخاب کن؛
-6. Physical Inventory را ایجاد کن؛
-7. بخشهای **Physical inventory**، **Findings review** و Content Object Detail را بررسی کن.
+- اجرای Identification فقط روی فایلهای منحصربهفرد، نه هر occurrence تکراری؛
+- Siegfried `1.11.6` و PRONOM `v124` Pinشده؛
+- ثبت SHA-256 ابزار و Signature، نسخهها، زمان ساخت و Identifierها؛
+- نگهداری تمام Matchها و انتخاب Primary Assertion بدون حذف Ambiguity؛
+- ذخیره PUID، نام و نسخه فرمت، MIME، Evidence basis، Warning و Format class؛
+- وضعیتهای `identified`، `unknown`، `ambiguous`، `empty`، `skipped_unavailable` و `tool_error`؛
+- ثبت Extension mismatch فقط وقتی Alias امن با Extension واقعاً بررسی شده باشد؛
+- Batch محدود، Timeout، محدودیت خروجی Process و جداسازی خودکار Batch خراب؛
+- Checkpoint/Resume براساس Configuration fingerprint؛
+- Lock اختصاصی Workspace برای جلوگیری از دو Writer همزمان؛
+- عدم Container expansion، OCR، Preview و Classification تجاری.
-پس از پایان Profile میتوان Workspace را از صفحه Start دوباره باز کرد. پیش از بازشدن Run، برنامه
-Schema، نیاز به Migration، وضعیت Lock و سلامت Review history را بررسی میکند. Migration از Schema
-قدیمی فقط با تأیید صریح انجام میشود و پیش از آن Backup دیتابیس Workspace نگهداری خواهد شد.
+
+
+ );
+}
+
+export default function FormatsView({
+ summary,
+ page,
+ tool,
+ progress,
+ loading,
+ running,
+ writable,
+ search,
+ state,
+ puid,
+ mismatchOnly,
+ onSearch,
+ onState,
+ onPuid,
+ onMismatchOnly,
+ onReload,
+ onProbe,
+ onRun,
+ onNext,
+}: {
+ summary: FormatSummary | null;
+ page: FormatPage | null;
+ tool: FormatToolIdentity | null;
+ progress: ProgressEvent | null;
+ loading: boolean;
+ running: boolean;
+ writable: boolean;
+ search: string;
+ state: FormatState | '';
+ puid: string;
+ mismatchOnly: boolean;
+ onSearch: (value: string) => void;
+ onState: (value: FormatState | '') => void;
+ onPuid: (value: string) => void;
+ onMismatchOnly: (value: boolean) => void;
+ onReload: () => void;
+ onProbe: () => void;
+ onRun: () => void;
+ onNext: () => void;
+}) {
+ const percent = progress?.totalItems
+ ? Math.min(100, (progress.completedItems / progress.totalItems) * 100)
+ : summary?.totalObjects
+ ? Math.min(100, (summary.completedObjects / summary.totalObjects) * 100)
+ : 0;
+ const effectiveTool = tool ?? summary?.tool ?? null;
+ return (
+ <>
+
+ {item.primaryFilename}
+ {shortHash(item.sha256)} · {formatBytes(item.expectedSizeBytes)}
+
+ {titleCase(item.state)}
+
+ {item.primaryFormatName ?? 'Not identified'}
+ {item.primaryFormatVersion || item.primaryMimeType || item.sourceMimeType}
+
+ {item.primaryIdentifier ?? '—'}
+ {extensionLabel}
+
+ EXACT FORMAT IDENTIFICATION
+
+ Identify real file formats with reproducible PRONOM evidence.
+No OCR · no container expansion
+
+ PINNED SIDECAR
+
+ Siegfried + PRONOM
++ One versioned assertion per unique SHA-256 object. Results preserve every match, + the primary decision, basis, warning, executable digest, and signature release. +
+
+
+ Tool{effectiveTool ? `${effectiveTool.toolName} ${effectiveTool.toolVersion}` : 'Not probed'}
+ Signature{effectiveTool?.signatureVersion ?? '—'}
+ Executable digest
+ {effectiveTool ? shortHash(effectiveTool.executableSha256) : '—'}
+
+
+
+ {!writable ? Open the workspace read-write to run this stage. : null}
+
+
+ DURABLE PROGRESS
+ {decimal.format(percent)}%
+ {summary?.latestRunState ? titleCase(summary.latestRunState) : 'Ready'}
+
-
+
- Objects
- {number.format(progress?.completedItems ?? summary?.completedObjects ?? 0)} / {number.format(progress?.totalItems ?? summary?.totalObjects ?? 0)}
- Bytes considered
- {formatBytes(progress?.completedBytes ?? summary?.completedBytes ?? 0)} / {formatBytes(progress?.totalBytes ?? summary?.totalBytes ?? 0)}
- Workers
- {progress?.activeWorkers ?? '—'}
- Checkpoint
- {progress?.checkpointSequence ?? '—'}
+ onSearch(event.target.value)} placeholder="Search filename, SHA-256, PUID or format" />
+
+ onPuid(event.target.value)} placeholder="PUID, e.g. fmt/61" />
+
+
+
+ Content objectStateExact formatPUIDExtension
+
+ {page?.items.map((item) =>
+
+ No objects match the current format filters.
: null}
+