From 65c4ecbf17123715fe0e5163a6c73dd621d1bd24 Mon Sep 17 00:00:00 2001 From: Rafael Araujo Lehmkuhl Date: Thu, 12 Mar 2026 17:04:34 -0300 Subject: [PATCH] wip --- src/components/MainMenu.vue | 7 + src/components/VideoLibraryModal.vue | 26 ++- src/composables/videoChunkManager.ts | 22 +- src/libs/live-video-processor.ts | 20 +- src/libs/sensors-logging.ts | 245 +++++++++++++++++++++- src/stores/appInterface.ts | 1 + src/stores/video.ts | 27 ++- src/types/video.ts | 8 + src/utils/video.ts | 20 ++ src/views/ToolsLogsView.vue | 290 +++++++++++++++++++++++++++ 10 files changed, 643 insertions(+), 23 deletions(-) create mode 100644 src/views/ToolsLogsView.vue diff --git a/src/components/MainMenu.vue b/src/components/MainMenu.vue index ce8bb76ab7..4b65c91da9 100644 --- a/src/components/MainMenu.vue +++ b/src/components/MainMenu.vue @@ -236,6 +236,7 @@ import ConfigurationMissionView from '@/views/ConfigurationMissionView.vue' import ConfigurationUIView from '@/views/ConfigurationUIView.vue' import ConfigurationVideoView from '@/views/ConfigurationVideoView.vue' import ToolsDataLakeView from '@/views/ToolsDataLakeView.vue' +import ToolsLogsView from '@/views/ToolsLogsView.vue' import ToolsMAVLinkView from '@/views/ToolsMAVLinkView.vue' const route = useRoute() @@ -435,6 +436,12 @@ const toolsMenu = computed(() => { componentName: SubMenuComponentName.ToolsDataLake, component: markRaw(ToolsDataLakeView) as SubMenuComponent, }, + { + icon: 'mdi-file-chart-outline', + title: 'Data Logs', + componentName: SubMenuComponentName.ToolsLogs, + component: markRaw(ToolsLogsView) as SubMenuComponent, + }, ] if (interfaceStore.pirateMode) { diff --git a/src/components/VideoLibraryModal.vue b/src/components/VideoLibraryModal.vue index 168c6f03f1..5756a0adf3 100644 --- a/src/components/VideoLibraryModal.vue +++ b/src/components/VideoLibraryModal.vue @@ -834,7 +834,12 @@ import { useSnapshotStore } from '@/stores/snapshot' import { useVideoStore } from '@/stores/video' import { SnapshotLibraryFile } from '@/types/snapshot' import { VideoLibraryFile, VideoLibraryLogFile } from '@/types/video' -import { videoSubtitlesFilename, videoThumbnailFilename } from '@/utils/video' +import { + videoSubtitlesFilename, + videoTelemetryCsvFilename, + videoTelemetryJsonFilename, + videoThumbnailFilename, +} from '@/utils/video' const videoStore = useVideoStore() const interfaceStore = useAppInterfaceStore() @@ -1146,9 +1151,24 @@ const deselectAllVideos = (): void => { // Add the log files to the list of files to be downloaded/discarded const addLogDataToFileList = (fileNames: string[]): string[] => { const filesWithLogData = fileNames.flatMap((fileName) => { + const result = [fileName] + const subtitleFileName = videoSubtitlesFilename(fileName) - const subtitleExists = availableLogFiles.value.some((video) => video.fileName === subtitleFileName) - return subtitleExists ? [fileName, subtitleFileName] : [fileName] + if (availableLogFiles.value.some((video) => video.fileName === subtitleFileName)) { + result.push(subtitleFileName) + } + + const jsonFileName = videoTelemetryJsonFilename(fileName) + if (availableLogFiles.value.some((video) => video.fileName === jsonFileName)) { + result.push(jsonFileName) + } + + const csvFileName = videoTelemetryCsvFilename(fileName) + if (availableLogFiles.value.some((video) => video.fileName === csvFileName)) { + result.push(csvFileName) + } + + return result }) return filesWithLogData } diff --git a/src/composables/videoChunkManager.ts b/src/composables/videoChunkManager.ts index aeb1f7f8c1..5274474ceb 100644 --- a/src/composables/videoChunkManager.ts +++ b/src/composables/videoChunkManager.ts @@ -4,7 +4,12 @@ import { LiveVideoProcessor } from '@/libs/live-video-processor' import { formatBytes, isElectron } from '@/libs/utils' import { useVideoStore } from '@/stores/video' import { type FileDescriptor } from '@/types/video' -import { videoFilename, videoSubtitlesFilename } from '@/utils/video' +import { + videoFilename, + videoSubtitlesFilename, + videoTelemetryCsvFilename, + videoTelemetryJsonFilename, +} from '@/utils/video' import { useInteractionDialog } from './interactionDialog' import { useSnackbar } from './snackbar' @@ -501,15 +506,26 @@ export const useVideoChunkManager = (): { // Finalize the streaming process await window.electronAPI.finalizeVideoRecording(processId) - // Find and copy telemetry file if it exists + // Find and copy telemetry files if they exist (.ass, .json, .csv) try { const videoKeys = await videoStore.videoStorage.keys() + const assFile = videoKeys.find((key) => key.includes(group.hash) && key.endsWith('.ass')) if (assFile) { await window.electronAPI.copyTelemetryFile(assFile, videoSubtitlesFilename(outputPath)) } + + const jsonFile = videoKeys.find((key) => key.includes(group.hash) && key.endsWith('.json')) + if (jsonFile) { + await window.electronAPI.copyTelemetryFile(jsonFile, videoTelemetryJsonFilename(outputPath)) + } + + const csvFile = videoKeys.find((key) => key.includes(group.hash) && key.endsWith('.csv')) + if (csvFile) { + await window.electronAPI.copyTelemetryFile(csvFile, videoTelemetryCsvFilename(outputPath)) + } } catch (error) { - console.warn('Failed to copy telemetry file:', error) + console.warn('Failed to copy telemetry files:', error) } openSnackbar({ diff --git a/src/libs/live-video-processor.ts b/src/libs/live-video-processor.ts index e69c1321b2..e371d4416f 100644 --- a/src/libs/live-video-processor.ts +++ b/src/libs/live-video-processor.ts @@ -1,6 +1,6 @@ import { isElectron } from '@/libs/utils' import type { VideoChunkQueueItem, ZipExtractionResult } from '@/types/video' -import { videoSubtitlesFilename } from '@/utils/video' +import { videoSubtitlesFilename, videoTelemetryCsvFilename, videoTelemetryJsonFilename } from '@/utils/video' /** * Error class for LiveVideoProcessor initialization errors @@ -237,7 +237,7 @@ export class LiveVideoProcessor { // Extract ZIP and get chunk information const extractionResult: ZipExtractionResult = await window.electronAPI.extractVideoChunksZip(zipFilePath) - const { chunkPaths, assFilePath, hash, fileName, tempDir } = extractionResult + const { chunkPaths, assFilePath, jsonFilePath, csvFilePath, hash, fileName, tempDir } = extractionResult console.log(`Extracted ${chunkPaths.length} chunks from ZIP file`) onProgress?.(30, 'Starting video processing...') @@ -276,10 +276,18 @@ export class LiveVideoProcessor { // Finalize the streaming process await window.electronAPI.finalizeVideoRecording(processId) - // Copy telemetry file if it exists (using the full output path) - if (assFilePath) { - onProgress?.(95, 'Copying telemetry file...') - await window.electronAPI.copyTelemetryFile(assFilePath, videoSubtitlesFilename(outputPath)) + // Copy telemetry files if they exist (using the full output path) + if (assFilePath || jsonFilePath || csvFilePath) { + onProgress?.(95, 'Copying telemetry files...') + if (assFilePath) { + await window.electronAPI.copyTelemetryFile(assFilePath, videoSubtitlesFilename(outputPath)) + } + if (jsonFilePath) { + await window.electronAPI.copyTelemetryFile(jsonFilePath, videoTelemetryJsonFilename(outputPath)) + } + if (csvFilePath) { + await window.electronAPI.copyTelemetryFile(csvFilePath, videoTelemetryCsvFilename(outputPath)) + } } // Clean up temporary extraction directory diff --git a/src/libs/sensors-logging.ts b/src/libs/sensors-logging.ts index 9d0b6eb377..38ad556892 100644 --- a/src/libs/sensors-logging.ts +++ b/src/libs/sensors-logging.ts @@ -128,6 +128,40 @@ export interface CockpitStandardLogPoint { */ export type CockpitStandardLog = CockpitStandardLogPoint[] +/** + * Information about a data session (detected from raw log entries) + */ +export interface DataSessionInfo { + /** + * Unique identifier for the session + */ + id: string + /** + * Start time of the session (epoch ms) + */ + startTime: number + /** + * End time of the session (epoch ms) + */ + endTime: number + /** + * Formatted date/time string + */ + dateTimeFormatted: string + /** + * Number of data points in the session + */ + dataPointCount: number + /** + * Duration of the session in seconds + */ + durationSeconds: number + /** + * Whether this is the current active session + */ + isCurrentSession: boolean +} + /** * Position for telemetry data on the video overlay */ @@ -213,7 +247,7 @@ export class CurrentlyLoggedVariables { /** * Manager logging vehicle data and others */ -class DataLogger { +export class DataLogger { logRequesters: string[] = [] datetimeLastLogPoint: Date | null = null variablesBeingUsed: DatalogVariable[] = [] @@ -287,7 +321,7 @@ class DataLogger { description: 'Local backups of Cockpit sensor logs, to be retrieved in case of failure.', }) - cockpitTemporaryLogsDB = localforage.createInstance({ + static cockpitTemporaryLogsDB = localforage.createInstance({ driver: localforage.INDEXEDDB, name: 'Cockpit - Temporary Sensor Log points', storeName: 'cockpit-temporary-sensor-logs-db', @@ -351,7 +385,7 @@ class DataLogger { data: structuredClone(variablesData), } - await this.cockpitTemporaryLogsDB.setItem(`epoch=${logPoint.epoch}`, logPoint) + await DataLogger.cockpitTemporaryLogsDB.setItem(`epoch=${logPoint.epoch}`, logPoint) this.datetimeLastLogPoint = new Date() if (this.shouldBeLogging()) { @@ -501,7 +535,7 @@ class DataLogger { const logDateTimeFmt = `${logDateFormat} / HH꞉mm꞉ss O` const fileName = `Cockpit (${format(initialTime, logDateTimeFmt)} - ${format(finalTime, logDateTimeFmt)}).clog` - const availableLogsKeys = await this.cockpitTemporaryLogsDB.keys() + const availableLogsKeys = await DataLogger.cockpitTemporaryLogsDB.keys() // The key is in the format epoch=. We extract the epoch and compare it to the initial and final times // to see if the log point is in the range of the desired log. @@ -536,7 +570,7 @@ class DataLogger { const logPointsInRange: CockpitStandardLogPoint[] = [] for (const info of infoLogPointsInRange) { - const log = (await this.cockpitTemporaryLogsDB.getItem(info.key)) as CockpitStandardLogPoint + const log = (await DataLogger.cockpitTemporaryLogsDB.getItem(info.key)) as CockpitStandardLogPoint // Only consider real log points(objects with an epoch and data property, and non-empty data) if (log.epoch === undefined || log.data === undefined || Object.keys(log.data).length === 0) continue @@ -561,6 +595,62 @@ class DataLogger { return finalLog } + /** + * Convert Cockpit standard log to JSON format + * @param {CockpitStandardLog} log - The telemetry log data + * @returns {string} JSON string representation of the telemetry data + */ + toJson(log: CockpitStandardLog): string { + return JSON.stringify(log, null, 2) + } + + /** + * Convert Cockpit standard log to CSV format + * @param {CockpitStandardLog} log - The telemetry log data + * @returns {string} CSV string representation of the telemetry data + */ + toCsv(log: CockpitStandardLog): string { + if (log.length === 0) return '' + + // Helper to escape CSV values (handle quotes and special characters) + const escapeCSV = (value: string | undefined | null): string => { + if (value === undefined || value === null) return '' + const str = String(value).trim() + // Escape quotes by doubling them and wrap in quotes if contains special chars + if (str.includes('"') || str.includes(',') || str.includes('\n') || str.includes('\r')) { + return `"${str.replace(/"/g, '""')}"` + } + return str + } + + // Get all unique variable names from all log points + const allVariables = new Set() + log.forEach((logPoint) => { + if (logPoint.data) { + Object.keys(logPoint.data).forEach((key) => allVariables.add(key)) + } + }) + + // Create header row with epoch and all variable names + const sortedVariables = Array.from(allVariables).sort() + const headers = ['epoch', 'timestamp', ...sortedVariables] + + // Create CSV rows + const rows = log.map((logPoint) => { + const timestamp = new Date(logPoint.epoch).toISOString() + const values = sortedVariables.map((variable) => { + const data = logPoint.data?.[variable] + if (data && data.value !== undefined) { + return escapeCSV(data.value) + } + return '' + }) + return [logPoint.epoch, timestamp, ...values].join(',') + }) + + return [headers.join(','), ...rows].join('\n') + } + /** * Convert Cockpit standard log files to Advanced SubStation Alpha subtitle overlays * @param {CockpitStandardLog} log @@ -673,6 +763,151 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text` return assFile } + + // Gap threshold to consider a new session (5 minutes of no data) + static SESSION_GAP_THRESHOLD_MS = 5 * 60 * 1000 + + /** + * Get all data sessions detected from raw log entries + * Sessions are determined by gaps in the data (>5 minutes between points = new session) + * @returns {Promise} Array of session information + */ + static async getDataSessions(): Promise { + const allKeys = await DataLogger.cockpitTemporaryLogsDB.keys() + + if (allKeys.length === 0) return [] + + // Extract epochs from keys and sort them + const epochs = allKeys + .map((key) => { + const match = key.match(/^epoch=(\d+)$/) + return match ? parseInt(match[1], 10) : null + }) + .filter((epoch): epoch is number => epoch !== null) + .sort((a, b) => a - b) + + if (epochs.length === 0) return [] + + // Group epochs into sessions based on gaps + const sessions: DataSessionInfo[] = [] + let sessionStart = epochs[0] + let sessionEnd = epochs[0] + let pointCount = 1 + + for (let i = 1; i < epochs.length; i++) { + const gap = epochs[i] - epochs[i - 1] + + if (gap > DataLogger.SESSION_GAP_THRESHOLD_MS) { + // End current session and start a new one + sessions.push({ + id: `session-${sessionStart}`, + startTime: sessionStart, + endTime: sessionEnd, + dateTimeFormatted: format(new Date(sessionStart), 'LLL dd, yyyy - HH:mm:ss'), + dataPointCount: pointCount, + durationSeconds: Math.round((sessionEnd - sessionStart) / 1000), + isCurrentSession: false, + }) + sessionStart = epochs[i] + sessionEnd = epochs[i] + pointCount = 1 + } else { + sessionEnd = epochs[i] + pointCount++ + } + } + + // Add the last session + const lastSession: DataSessionInfo = { + id: `session-${sessionStart}`, + startTime: sessionStart, + endTime: sessionEnd, + dateTimeFormatted: format(new Date(sessionStart), 'LLL dd, yyyy - HH:mm:ss'), + dataPointCount: pointCount, + durationSeconds: Math.round((sessionEnd - sessionStart) / 1000), + isCurrentSession: false, + } + + // Check if this is the current session (last log point within the last minute) + const now = Date.now() + if (now - sessionEnd < 60000) { + lastSession.isCurrentSession = true + } + + sessions.push(lastSession) + + // Sort by date, newest first + return sessions.sort((a, b) => b.startTime - a.startTime) + } + + /** + * Generate a data log from a session's raw data + * @param {DataSessionInfo} session - The session to generate log from + * @returns {Promise} The generated log + */ + static async generateLogFromSession(session: DataSessionInfo): Promise { + const log: CockpitStandardLog = [] + + // Get all keys and filter by session time range + const allKeys = await DataLogger.cockpitTemporaryLogsDB.keys() + + for (const key of allKeys) { + const match = key.match(/^epoch=(\d+)$/) + if (!match) continue + + const epoch = parseInt(match[1], 10) + if (epoch >= session.startTime && epoch <= session.endTime) { + const logPoint = (await DataLogger.cockpitTemporaryLogsDB.getItem(key)) as CockpitStandardLogPoint | null + if (logPoint && logPoint.epoch !== undefined && logPoint.data !== undefined) { + log.push(logPoint) + } + } + } + + // Sort by epoch + return log.sort((a, b) => a.epoch - b.epoch) + } + + /** + * Delete a data session's raw data + * @param {DataSessionInfo} session - The session to delete + */ + static async deleteDataSession(session: DataSessionInfo): Promise { + const allKeys = await DataLogger.cockpitTemporaryLogsDB.keys() + + for (const key of allKeys) { + const match = key.match(/^epoch=(\d+)$/) + if (!match) continue + + const epoch = parseInt(match[1], 10) + if (epoch >= session.startTime && epoch <= session.endTime) { + await DataLogger.cockpitTemporaryLogsDB.removeItem(key) + } + } + } + + /** + * Delete all data sessions older than a specified number of days + * @param {number} daysOld - Number of days (sessions older than this will be deleted) + * @returns {Promise} Number of deleted sessions + */ + static async deleteOldDataSessions(daysOld = 1): Promise { + const cutoffDate = new Date() + cutoffDate.setDate(cutoffDate.getDate() - daysOld) + const cutoffMs = cutoffDate.getTime() + + const sessions = await DataLogger.getDataSessions() + let deletedCount = 0 + + for (const session of sessions) { + if (session.endTime < cutoffMs && !session.isCurrentSession) { + await DataLogger.deleteDataSession(session) + deletedCount++ + } + } + + return deletedCount + } } export const datalogger = new DataLogger() diff --git a/src/stores/appInterface.ts b/src/stores/appInterface.ts index efa8a64fdc..06b46c9783 100644 --- a/src/stores/appInterface.ts +++ b/src/stores/appInterface.ts @@ -33,6 +33,7 @@ export enum SubMenuComponentName { SettingsMAVLink = 'settings-mavlink', ToolsMAVLink = 'tools-mavlink', ToolsDataLake = 'tools-datalake', + ToolsLogs = 'tools-logs', } export const useAppInterfaceStore = defineStore('responsive', { diff --git a/src/stores/video.ts b/src/stores/video.ts index 7d9c8c2143..d99b20b3ed 100644 --- a/src/stores/video.ts +++ b/src/stores/video.ts @@ -37,7 +37,13 @@ import { VideoExtensionContainer, VideoStreamCorrespondency, } from '@/types/video' -import { videoFilename, videoSubtitlesFilename, videoThumbnailFilename } from '@/utils/video' +import { + videoFilename, + videoSubtitlesFilename, + videoTelemetryCsvFilename, + videoTelemetryJsonFilename, + videoThumbnailFilename, +} from '@/utils/video' import { useAlertStore } from './alert' const { openSnackbar } = useSnackbar() @@ -346,7 +352,7 @@ export const useVideoStore = defineStore('video', () => { } /** - * Generate .ass telemetry overlay file for a video recording + * Generate telemetry files (.ass, .json, .csv) for a video recording * @param {string} recordingHash - The hash of the recording */ const generateTelemetryOverlay = async (recordingHash: string): Promise => { @@ -360,16 +366,25 @@ export const useVideoStore = defineStore('video', () => { const telemetryLog = await datalogger.generateLog(recordingData.dateStart!, recordingData.dateFinish!) if (telemetryLog !== undefined) { + // Generate and save .ass overlay file const assLog = datalogger.toAssOverlay( telemetryLog, recordingData.vWidth!, recordingData.vHeight!, recordingData.dateStart!.getTime() ) - const logBlob = new Blob([assLog], { type: 'text/plain' }) - - // Save the .ass file - await videoStorage.setItem(videoSubtitlesFilename(recordingData.fileName), logBlob) + const assBlob = new Blob([assLog], { type: 'text/plain' }) + await videoStorage.setItem(videoSubtitlesFilename(recordingData.fileName), assBlob) + + // Generate and save .json data file + const jsonLog = datalogger.toJson(telemetryLog) + const jsonBlob = new Blob([jsonLog], { type: 'application/json' }) + await videoStorage.setItem(videoTelemetryJsonFilename(recordingData.fileName), jsonBlob) + + // Generate and save .csv data file + const csvLog = datalogger.toCsv(telemetryLog) + const csvBlob = new Blob([csvLog], { type: 'text/csv' }) + await videoStorage.setItem(videoTelemetryCsvFilename(recordingData.fileName), csvBlob) } } catch (error) { throw new Error(`Failed to generate telemetry for recording '${recordingHash}': ${error}`) diff --git a/src/types/video.ts b/src/types/video.ts index a470bebc4c..e49167a96e 100644 --- a/src/types/video.ts +++ b/src/types/video.ts @@ -362,6 +362,14 @@ export interface ZipExtractionResult { * Path to extracted telemetry .ass file (if exists) */ assFilePath?: string + /** + * Path to extracted telemetry .json file (if exists) + */ + jsonFilePath?: string + /** + * Path to extracted telemetry .csv file (if exists) + */ + csvFilePath?: string /** * Recording hash extracted from chunk filenames */ diff --git a/src/utils/video.ts b/src/utils/video.ts index dbb0852d29..8b6aea8cea 100644 --- a/src/utils/video.ts +++ b/src/utils/video.ts @@ -41,3 +41,23 @@ export const videoThumbnailFilename = (videoFileName: string): string => { export const videoSubtitlesFilename = (videoFileName: string): string => { return `${videoFilenameWithoutExtension(videoFileName)}.ass` } + +/** + * Returns the filename for the JSON telemetry data of a video. + * Can be used with complete paths or just the filename. It will just replace the extension with .json. + * @param {string} videoFileName - The filename of the video, with or without the extension. + * @returns {string} The filename for the JSON telemetry data of the video. + */ +export const videoTelemetryJsonFilename = (videoFileName: string): string => { + return `${videoFilenameWithoutExtension(videoFileName)}.json` +} + +/** + * Returns the filename for the CSV telemetry data of a video. + * Can be used with complete paths or just the filename. It will just replace the extension with .csv. + * @param {string} videoFileName - The filename of the video, with or without the extension. + * @returns {string} The filename for the CSV telemetry data of the video. + */ +export const videoTelemetryCsvFilename = (videoFileName: string): string => { + return `${videoFilenameWithoutExtension(videoFileName)}.csv` +} diff --git a/src/views/ToolsLogsView.vue b/src/views/ToolsLogsView.vue new file mode 100644 index 0000000000..30380388b9 --- /dev/null +++ b/src/views/ToolsLogsView.vue @@ -0,0 +1,290 @@ + + + + +