From b1fc02338051f01c85695b02816f2e710e41b249 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 01:26:42 +0000 Subject: [PATCH 1/8] Initial plan From ad57d59f923b7d468356735cf44863b018a55739 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 01:29:21 +0000 Subject: [PATCH 2/8] Initial analysis and plan for data mirroring feature Co-authored-by: busse <18599+busse@users.noreply.github.com> --- package-lock.json | 16 +++++++++++++--- playwright-report/index.html | 2 +- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index e6301fe..519c8c2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -523,6 +523,7 @@ "dev": true, "license": "BSD-2-Clause", "optional": true, + "peer": true, "dependencies": { "cross-dirname": "^0.1.0", "debug": "^4.3.4", @@ -544,6 +545,7 @@ "dev": true, "license": "MIT", "optional": true, + "peer": true, "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", @@ -560,6 +562,7 @@ "dev": true, "license": "MIT", "optional": true, + "peer": true, "dependencies": { "universalify": "^2.0.0" }, @@ -574,6 +577,7 @@ "dev": true, "license": "MIT", "optional": true, + "peer": true, "engines": { "node": ">= 10.0.0" } @@ -1069,7 +1073,6 @@ "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -1887,7 +1890,8 @@ "integrity": "sha512-+R08/oI0nl3vfPcqftZRpytksBXDzOUveBq/NBVx0sUp1axwzPQrKinNx5yd5sxPu8j1wIy8AfnVQ+5eFdha6Q==", "dev": true, "license": "MIT", - "optional": true + "optional": true, + "peer": true }, "node_modules/cross-spawn": { "version": "7.0.6", @@ -2069,7 +2073,6 @@ "integrity": "sha512-59CAAjAhTaIMCN8y9kD573vDkxbs1uhDcrFLHSgutYdPcGOU35Rf95725snvzEOy4BFB7+eLJ8djCNPmGwG67w==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "app-builder-lib": "26.0.12", "builder-util": "26.0.11", @@ -2435,6 +2438,7 @@ "dev": true, "hasInstallScript": true, "license": "MIT", + "peer": true, "dependencies": { "@electron/asar": "^3.2.1", "debug": "^4.1.1", @@ -2455,6 +2459,7 @@ "integrity": "sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "graceful-fs": "^4.1.2", "jsonfile": "^4.0.0", @@ -4139,6 +4144,7 @@ "dev": true, "license": "MIT", "optional": true, + "peer": true, "dependencies": { "commander": "^9.4.0" }, @@ -4156,6 +4162,7 @@ "dev": true, "license": "MIT", "optional": true, + "peer": true, "engines": { "node": "^12.20.0 || >=14" } @@ -4766,6 +4773,7 @@ "integrity": "sha512-yYrrsWnrXMcdsnu/7YMYAofM1ktpL5By7vZhf15CrXijWWrEYZks5AXBudalfSWJLlnen/QUJUB5aoB0kqZUGA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "mkdirp": "^0.5.1", "rimraf": "~2.6.2" @@ -4829,6 +4837,7 @@ "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "minimist": "^1.2.6" }, @@ -4843,6 +4852,7 @@ "deprecated": "Rimraf versions prior to v4 are no longer supported", "dev": true, "license": "ISC", + "peer": true, "dependencies": { "glob": "^7.1.3" }, diff --git a/playwright-report/index.html b/playwright-report/index.html index aca9b6b..b5c5e68 100644 --- a/playwright-report/index.html +++ b/playwright-report/index.html @@ -82,4 +82,4 @@
- \ No newline at end of file + \ No newline at end of file From 6479f8fe7edf8effd1702ce53bf2f0851254094f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 01:41:39 +0000 Subject: [PATCH 3/8] feat: Add data mirror feature to sync data to a second directory - Add feature flag 'dataMirror' to enable/disable the feature - Add IPC handlers for mirror directory selection and sync - Add preload.js API methods for mirror feature - Implement automatic sync on data writes - Add UI in Settings modal for configuring mirror directory - Add tests for the data mirror feature - Update README with documentation Co-authored-by: busse <18599+busse@users.noreply.github.com> --- README.md | 20 +++ index.html | 92 +++++++++++ main.js | 156 +++++++++++++++++- playwright-report/index.html | 2 +- preload.js | 6 + tests/data-mirror.test.js | 250 +++++++++++++++++++++++++++++ tests/helpers/mock-electron-api.js | 9 +- tests/keyboard-shortcuts.test.js | 8 +- 8 files changed, 539 insertions(+), 4 deletions(-) create mode 100644 tests/data-mirror.test.js diff --git a/README.md b/README.md index 1281e78..4055be1 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,7 @@ A simple desktop RSS reader application for Mac built with Electron. The applica - **Automatic updates** - Background update checking and installation via electron-updater - **Data migration** - Automatic data structure migrations when updating to new versions - **Data persistence** - All feeds, articles, settings, and read states are stored locally +- **Data Mirror** - Mirror all data to a second directory for integration with personal knowledge management tools like Obsidian (requires feature flag) ### Coming Soon - OPML import/export support @@ -174,9 +175,28 @@ To configure your API key: The Settings area includes a feature flag system that allows you to enable or disable specific features: - **AI Article Summary**: Enable AI-powered article summaries +- **Data Mirror**: Enable mirroring of all data to a second directory Feature flags can be toggled on or off using the toggle switches in the Settings modal. Changes are saved immediately when you click "Save". +#### Data Mirror + +When the Data Mirror feature flag is enabled, a new section appears in Settings that allows you to: + +1. **Select a Mirror Directory**: Click "Browse..." to choose a directory where all RSS data will be mirrored +2. **Clear Directory**: Click the "✕" button to clear the selected directory +3. **Sync Now**: Manually trigger a sync of all data to the mirror directory + +The data is automatically synced to the mirror directory whenever any data changes (feeds, articles, read states, settings, etc.). This is useful for integration with personal knowledge management tools like Obsidian, Notion, or any other tool that can read JSON files. + +**Mirrored Files:** +- `feeds.json` - Feed subscriptions +- `categories.json` - Category/folder organization +- `read-states.json` - Article read status +- `settings.json` - Application settings +- `ai-summaries.json` - AI-generated summaries (if enabled) +- `articles-{feedId}.json` - Articles for each feed + ### Feed Management The Feed Management page allows you to: diff --git a/index.html b/index.html index 25a6ff6..f9689f0 100644 --- a/index.html +++ b/index.html @@ -1041,6 +1041,21 @@

Settings

+
@@ -1965,6 +1980,17 @@

${escapeHtml(article.title || 'Untitled')}

${escapeHtml(article.title || 'Untitled')}${escapeHtml(article.title || 'Untitled')} `; }).join(''); + + // Show/hide data mirror section based on feature flag + const dataMirrorSection = document.getElementById('dataMirrorSection'); + if (dataMirrorSection) { + dataMirrorSection.style.display = featureFlags.dataMirror ? 'block' : 'none'; + } } async function saveSettings() { @@ -2022,6 +2059,16 @@

${escapeHtml(article.title || 'Untitled')}

${escapeHtml(article.title || 'Untitled')} { + if (window.electronAPI && window.electronAPI.selectMirrorDirectory) { + try { + const result = await window.electronAPI.selectMirrorDirectory(); + if (result.success && result.data) { + document.getElementById('mirrorDirectoryInput').value = result.data; + } + } catch (error) { + console.error('Error selecting mirror directory:', error); + showToast('Failed to select directory'); + } + } + }); + + document.getElementById('clearMirrorDirectoryBtn').addEventListener('click', () => { + document.getElementById('mirrorDirectoryInput').value = ''; + }); + + document.getElementById('syncNowBtn').addEventListener('click', async () => { + if (window.electronAPI && window.electronAPI.syncToMirror) { + try { + const result = await window.electronAPI.syncToMirror(); + if (result.success) { + showToast('Data synced to mirror directory'); + } else { + showToast('Sync failed: ' + (result.error || 'Unknown error')); + } + } catch (error) { + console.error('Error syncing to mirror:', error); + showToast('Sync failed'); + } + } + }); + + // Listen for feature flag changes to show/hide data mirror section + featureFlagList.addEventListener('change', (e) => { + if (e.target.dataset.flagName === 'dataMirror') { + const dataMirrorSection = document.getElementById('dataMirrorSection'); + if (dataMirrorSection) { + dataMirrorSection.style.display = e.target.checked ? 'block' : 'none'; + } + } + }); // Update Notification Management const updateNotification = document.getElementById('updateNotification'); diff --git a/main.js b/main.js index a61ded4..f63baca 100644 --- a/main.js +++ b/main.js @@ -1,4 +1,4 @@ -const { app, BrowserWindow, ipcMain } = require('electron'); +const { app, BrowserWindow, ipcMain, dialog } = require('electron'); const path = require('path'); const fs = require('fs').promises; const crypto = require('crypto'); @@ -144,6 +144,8 @@ ipcMain.handle('write-json', async (event, filename, data) => { const filePath = path.join(dataDir, filename); try { await fs.writeFile(filePath, JSON.stringify(data, null, 2), 'utf-8'); + // Trigger mirror sync in background (don't await to avoid blocking) + syncToMirror().catch(err => console.error('Mirror sync error:', err)); return { success: true }; } catch (error) { console.error('Error writing JSON file:', error); @@ -170,6 +172,8 @@ ipcMain.handle('write-feeds', async (event, feeds) => { const filePath = path.join(dataDir, 'feeds.json'); try { await fs.writeFile(filePath, JSON.stringify(feeds, null, 2), 'utf-8'); + // Trigger mirror sync in background (don't await to avoid blocking) + syncToMirror().catch(err => console.error('Mirror sync error:', err)); return { success: true }; } catch (error) { console.error('Error writing feeds:', error); @@ -196,6 +200,8 @@ ipcMain.handle('write-categories', async (event, categories) => { const filePath = path.join(dataDir, 'categories.json'); try { await fs.writeFile(filePath, JSON.stringify(categories, null, 2), 'utf-8'); + // Trigger mirror sync in background (don't await to avoid blocking) + syncToMirror().catch(err => console.error('Mirror sync error:', err)); return { success: true }; } catch (error) { console.error('Error writing categories:', error); @@ -222,6 +228,8 @@ ipcMain.handle('write-read-states', async (event, readStates) => { const filePath = path.join(dataDir, 'read-states.json'); try { await fs.writeFile(filePath, JSON.stringify(readStates, null, 2), 'utf-8'); + // Trigger mirror sync in background (don't await to avoid blocking) + syncToMirror().catch(err => console.error('Mirror sync error:', err)); return { success: true }; } catch (error) { console.error('Error writing read states:', error); @@ -248,6 +256,8 @@ ipcMain.handle('write-settings', async (event, settings) => { const filePath = path.join(dataDir, 'settings.json'); try { await fs.writeFile(filePath, JSON.stringify(settings, null, 2), 'utf-8'); + // Trigger mirror sync in background (don't await to avoid blocking) + syncToMirror().catch(err => console.error('Mirror sync error:', err)); return { success: true }; } catch (error) { console.error('Error writing settings:', error); @@ -299,6 +309,8 @@ ipcMain.handle('set-encrypted-api-key', async (event, apiKey) => { } await fs.writeFile(filePath, JSON.stringify(settings, null, 2), 'utf-8'); + // Trigger mirror sync in background (don't await to avoid blocking) + syncToMirror().catch(err => console.error('Mirror sync error:', err)); return { success: true }; } catch (error) { console.error('Error setting encrypted API key:', error); @@ -327,6 +339,9 @@ ipcMain.handle('get-feature-flags', async (event) => { if (featureFlags.aiArticleSummary === undefined) { featureFlags.aiArticleSummary = false; } + if (featureFlags.dataMirror === undefined) { + featureFlags.dataMirror = false; + } return { success: true, data: featureFlags }; } catch (error) { @@ -356,6 +371,8 @@ ipcMain.handle('set-feature-flag', async (event, flagName, enabled) => { settings.featureFlags[flagName] = enabled; await fs.writeFile(filePath, JSON.stringify(settings, null, 2), 'utf-8'); + // Trigger mirror sync in background (don't await to avoid blocking) + syncToMirror().catch(err => console.error('Mirror sync error:', err)); return { success: true }; } catch (error) { console.error('Error setting feature flag:', error); @@ -363,6 +380,139 @@ ipcMain.handle('set-feature-flag', async (event, flagName, enabled) => { } }); +// Data Mirror operations +ipcMain.handle('select-mirror-directory', async (event) => { + try { + const result = await dialog.showOpenDialog(mainWindow, { + title: 'Select Mirror Directory', + properties: ['openDirectory', 'createDirectory'] + }); + + if (result.canceled || !result.filePaths || result.filePaths.length === 0) { + return { success: true, data: null }; + } + + return { success: true, data: result.filePaths[0] }; + } catch (error) { + console.error('Error selecting mirror directory:', error); + return { success: false, error: error.message }; + } +}); + +ipcMain.handle('get-mirror-directory', async (event) => { + try { + const filePath = path.join(dataDir, 'settings.json'); + let settings = {}; + + try { + const data = await fs.readFile(filePath, 'utf-8'); + settings = JSON.parse(data); + } catch (error) { + if (error.code !== 'ENOENT') { + throw error; + } + } + + return { success: true, data: settings.mirrorDirectory || null }; + } catch (error) { + console.error('Error getting mirror directory:', error); + return { success: false, error: error.message }; + } +}); + +ipcMain.handle('set-mirror-directory', async (event, mirrorDirectory) => { + try { + const filePath = path.join(dataDir, 'settings.json'); + let settings = {}; + + try { + const data = await fs.readFile(filePath, 'utf-8'); + settings = JSON.parse(data); + } catch (error) { + if (error.code !== 'ENOENT') { + throw error; + } + } + + if (mirrorDirectory) { + settings.mirrorDirectory = mirrorDirectory; + } else { + delete settings.mirrorDirectory; + } + + await fs.writeFile(filePath, JSON.stringify(settings, null, 2), 'utf-8'); + // Trigger immediate sync if directory is set and feature is enabled + if (mirrorDirectory && settings.featureFlags?.dataMirror) { + syncToMirror().catch(err => console.error('Mirror sync error:', err)); + } + return { success: true }; + } catch (error) { + console.error('Error setting mirror directory:', error); + return { success: false, error: error.message }; + } +}); + +// Helper function to sync data to mirror directory +async function syncToMirror() { + try { + // Check if feature is enabled + const settingsPath = path.join(dataDir, 'settings.json'); + let settings = {}; + + try { + const data = await fs.readFile(settingsPath, 'utf-8'); + settings = JSON.parse(data); + } catch (error) { + if (error.code !== 'ENOENT') { + throw error; + } + return; // No settings file, nothing to sync + } + + // Check if data mirror feature is enabled and directory is set + if (!settings.featureFlags?.dataMirror || !settings.mirrorDirectory) { + return; + } + + const mirrorDir = settings.mirrorDirectory; + + // Ensure mirror directory exists + await fs.mkdir(mirrorDir, { recursive: true }); + + // Get all files in the data directory + const files = await fs.readdir(dataDir); + + // Copy each JSON file to mirror directory + for (const file of files) { + if (file.endsWith('.json')) { + const sourcePath = path.join(dataDir, file); + const destPath = path.join(mirrorDir, file); + + try { + const content = await fs.readFile(sourcePath, 'utf-8'); + await fs.writeFile(destPath, content, 'utf-8'); + } catch (error) { + console.error(`Error copying ${file} to mirror:`, error); + } + } + } + + console.log('Data synced to mirror directory:', mirrorDir); + } catch (error) { + console.error('Error syncing to mirror:', error); + } +} + +ipcMain.handle('sync-to-mirror', async (event) => { + try { + await syncToMirror(); + return { success: true }; + } catch (error) { + console.error('Error syncing to mirror:', error); + return { success: false, error: error.message }; + } +}); + // AI Summary operations - store summaries by article ID ipcMain.handle('read-ai-summary', async (event, articleId) => { const filePath = path.join(dataDir, 'ai-summaries.json'); @@ -392,6 +542,8 @@ ipcMain.handle('write-ai-summary', async (event, articleId, summaryData) => { summaries[articleId] = summaryData; await fs.writeFile(filePath, JSON.stringify(summaries, null, 2), 'utf-8'); + // Trigger mirror sync in background (don't await to avoid blocking) + syncToMirror().catch(err => console.error('Mirror sync error:', err)); return { success: true }; } catch (error) { console.error('Error writing AI summary:', error); @@ -488,6 +640,8 @@ ipcMain.handle('write-articles', async (event, feedId, articlesData) => { feedId: feedId }; await fs.writeFile(filePath, JSON.stringify(dataToWrite, null, 2), 'utf-8'); + // Trigger mirror sync in background (don't await to avoid blocking) + syncToMirror().catch(err => console.error('Mirror sync error:', err)); return { success: true }; } catch (error) { console.error('Error writing articles:', error); diff --git a/playwright-report/index.html b/playwright-report/index.html index b5c5e68..54dadf5 100644 --- a/playwright-report/index.html +++ b/playwright-report/index.html @@ -82,4 +82,4 @@
- \ No newline at end of file + \ No newline at end of file diff --git a/preload.js b/preload.js index fa2ff31..24083e6 100644 --- a/preload.js +++ b/preload.js @@ -39,6 +39,12 @@ contextBridge.exposeInMainWorld('electronAPI', { getFeatureFlags: () => ipcRenderer.invoke('get-feature-flags'), setFeatureFlag: (flagName, enabled) => ipcRenderer.invoke('set-feature-flag', flagName, enabled), + // Data Mirror operations + selectMirrorDirectory: () => ipcRenderer.invoke('select-mirror-directory'), + getMirrorDirectory: () => ipcRenderer.invoke('get-mirror-directory'), + setMirrorDirectory: (mirrorDirectory) => ipcRenderer.invoke('set-mirror-directory', mirrorDirectory), + syncToMirror: () => ipcRenderer.invoke('sync-to-mirror'), + // AI Summary operations readAISummary: (articleId) => ipcRenderer.invoke('read-ai-summary', articleId), writeAISummary: (articleId, summaryData) => ipcRenderer.invoke('write-ai-summary', articleId, summaryData), diff --git a/tests/data-mirror.test.js b/tests/data-mirror.test.js new file mode 100644 index 0000000..68c9864 --- /dev/null +++ b/tests/data-mirror.test.js @@ -0,0 +1,250 @@ +// Data Mirror Feature Test Suite +// Tests the data mirror feature that enables mirroring data to a second directory + +const { test, expect } = require('@playwright/test'); +const path = require('path'); + +test.describe('Data Mirror Feature', () => { + let page; + + test.beforeEach(async ({ browser }) => { + // Create a new page for each test + page = await browser.newPage(); + + // Inject mock Electron API before page loads + await page.addInitScript(() => { + const testFeeds = [ + { + id: 'feed-1', + title: 'Test Feed 1', + url: 'https://example.com/feed1.xml', + category: null, + status: 'healthy', + lastUpdated: new Date().toISOString() + } + ]; + + const testCategories = []; + const testArticles = { + 'feed-1': [ + { + id: 'article-1', + title: 'Article 1 from Feed 1', + link: 'https://example.com/article1', + description: 'Description of article 1', + content: '

Content of article 1

', + pubDate: new Date(Date.now() - 86400000).toISOString(), + author: 'Author 1', + categories: [] + } + ] + }; + const testReadStates = {}; + + // Create mock API with functions that work in browser context + window.electronAPI = { + // Navigation + navigateTo: async (file) => ({ success: true }), + + // Feeds operations + readFeeds: async () => ({ success: true, data: testFeeds }), + writeFeeds: async (feeds) => ({ success: true }), + + // Categories operations + readCategories: async () => ({ success: true, data: testCategories }), + writeCategories: async (categories) => ({ success: true }), + + // Read states operations + readReadStates: async () => ({ success: true, data: testReadStates }), + writeReadStates: async (readStates) => ({ success: true }), + + // Settings operations + readSettings: async () => ({ success: true, data: { featureFlags: { aiArticleSummary: false, dataMirror: true } } }), + writeSettings: async (settings) => ({ success: true }), + + // Encrypted API key operations + getEncryptedApiKey: async () => ({ success: true, data: null }), + setEncryptedApiKey: async (apiKey) => ({ success: true }), + + // Feature flags operations + getFeatureFlags: async () => ({ success: true, data: { aiArticleSummary: false, dataMirror: true } }), + setFeatureFlag: async (flagName, enabled) => ({ success: true }), + + // Data Mirror operations + selectMirrorDirectory: async () => ({ success: true, data: '/test/mirror/directory' }), + getMirrorDirectory: async () => ({ success: true, data: '/existing/mirror/path' }), + setMirrorDirectory: async (mirrorDirectory) => ({ success: true }), + syncToMirror: async () => ({ success: true }), + + // AI Summary operations + readAISummary: async (articleId) => ({ success: true, data: null }), + writeAISummary: async (articleId, summaryData) => ({ success: true }), + + // RSS Feed operations + fetchFeed: async (url) => ({ success: true, data: { feedTitle: 'Test Feed', articles: [] } }), + + // Articles operations + readArticles: async (feedId) => { + const articles = testArticles[feedId] || []; + return { + success: true, + data: { + feedId: feedId, + lastFetched: new Date().toISOString(), + articles: articles + } + }; + }, + writeArticles: async (feedId, articlesData) => ({ success: true }), + + // Update operations + checkForUpdates: async () => ({ success: true }), + getUpdateStatus: async () => ({ success: true, status: 'idle' }), + installUpdate: async () => ({ success: true }), + + // Update event listeners (no-ops for testing) + onUpdateStatus: () => {}, + onUpdateAvailable: () => {}, + onUpdateDownloaded: () => {}, + onUpdateProgress: () => {}, + onUpdateError: () => {}, + removeUpdateListeners: () => {}, + + // Get app version + getAppVersion: async () => ({ success: true, version: '1.0.2' }) + }; + }); + + // Load the index.html file + const htmlPath = path.join(__dirname, '..', 'index.html'); + await page.goto(`file://${htmlPath}`); + + // Wait for the app to initialize + await page.waitForSelector('.sidebar', { timeout: 5000 }); + + // Wait a bit more for all async operations to complete + await page.waitForTimeout(500); + }); + + test.afterEach(async () => { + await page.close(); + }); + + test('Data Mirror feature flag appears in settings', async () => { + // Open settings modal + await page.click('#settingsBtn'); + await page.waitForTimeout(200); + + // Check that the settings modal is visible + const settingsModal = await page.$('#settingsModal'); + const isHidden = await settingsModal.evaluate(el => el.classList.contains('hidden')); + expect(isHidden).toBe(false); + + // Check that the Data Mirror feature flag exists + const dataMirrorToggle = await page.$('input[data-flag-name="dataMirror"]'); + expect(dataMirrorToggle).toBeTruthy(); + }); + + test('Data Mirror section is visible when feature flag is enabled', async () => { + // Open settings modal + await page.click('#settingsBtn'); + await page.waitForTimeout(200); + + // The data mirror section should be visible since feature flag is enabled + const dataMirrorSection = await page.$('#dataMirrorSection'); + expect(dataMirrorSection).toBeTruthy(); + + const isVisible = await dataMirrorSection.evaluate(el => el.style.display !== 'none'); + expect(isVisible).toBe(true); + }); + + test('Mirror directory input displays current path', async () => { + // Open settings modal + await page.click('#settingsBtn'); + await page.waitForTimeout(200); + + // Check that the mirror directory input has the expected value + const mirrorInput = await page.$('#mirrorDirectoryInput'); + expect(mirrorInput).toBeTruthy(); + + const value = await mirrorInput.inputValue(); + expect(value).toBe('/existing/mirror/path'); + }); + + test('Browse button triggers directory selection', async () => { + // Open settings modal + await page.click('#settingsBtn'); + await page.waitForTimeout(200); + + // Click the browse button + await page.click('#selectMirrorDirectoryBtn'); + await page.waitForTimeout(100); + + // Check that the input was updated with the selected directory + const mirrorInput = await page.$('#mirrorDirectoryInput'); + const value = await mirrorInput.inputValue(); + expect(value).toBe('/test/mirror/directory'); + }); + + test('Clear button clears the directory path', async () => { + // Open settings modal + await page.click('#settingsBtn'); + await page.waitForTimeout(200); + + // Verify input has a value + const mirrorInput = await page.$('#mirrorDirectoryInput'); + let value = await mirrorInput.inputValue(); + expect(value).toBe('/existing/mirror/path'); + + // Click the clear button + await page.click('#clearMirrorDirectoryBtn'); + await page.waitForTimeout(100); + + // Check that the input was cleared + value = await mirrorInput.inputValue(); + expect(value).toBe(''); + }); + + test('Sync Now button triggers sync', async () => { + // Open settings modal + await page.click('#settingsBtn'); + await page.waitForTimeout(200); + + // Click the sync now button + await page.click('#syncNowBtn'); + await page.waitForTimeout(500); + + // Check that a toast notification appeared + const toast = await page.$('.toast'); + expect(toast).toBeTruthy(); + }); + + test('Toggling Data Mirror feature flag shows/hides section', async () => { + // Open settings modal + await page.click('#settingsBtn'); + await page.waitForTimeout(200); + + // The data mirror section should be visible (flag is enabled in mock) + let dataMirrorSection = await page.$('#dataMirrorSection'); + let isVisible = await dataMirrorSection.evaluate(el => el.style.display !== 'none'); + expect(isVisible).toBe(true); + + // Toggle the feature flag off - click on the toggle-slider (parent label contains the checkbox) + const dataMirrorToggle = await page.$('input[data-flag-name="dataMirror"]'); + const toggleSwitch = await dataMirrorToggle.evaluateHandle(el => el.parentElement); + await toggleSwitch.click(); + await page.waitForTimeout(100); + + // The section should now be hidden + isVisible = await dataMirrorSection.evaluate(el => el.style.display !== 'none'); + expect(isVisible).toBe(false); + + // Toggle it back on + await toggleSwitch.click(); + await page.waitForTimeout(100); + + // The section should be visible again + isVisible = await dataMirrorSection.evaluate(el => el.style.display !== 'none'); + expect(isVisible).toBe(true); + }); +}); diff --git a/tests/helpers/mock-electron-api.js b/tests/helpers/mock-electron-api.js index a706166..91a3c0b 100644 --- a/tests/helpers/mock-electron-api.js +++ b/tests/helpers/mock-electron-api.js @@ -84,7 +84,8 @@ function createMockElectronAPI() { const testReadStates = {}; const testSettings = { featureFlags: { - aiArticleSummary: false + aiArticleSummary: false, + dataMirror: false } }; @@ -140,6 +141,12 @@ function createMockElectronAPI() { return { success: true, data: testSettings.featureFlags }; }, setFeatureFlag: async (flagName, enabled) => ({ success: true }), + + // Data Mirror operations + selectMirrorDirectory: async () => ({ success: true, data: null }), + getMirrorDirectory: async () => ({ success: true, data: null }), + setMirrorDirectory: async (mirrorDirectory) => ({ success: true }), + syncToMirror: async () => ({ success: true }), // AI Summary operations readAISummary: async (articleId) => ({ success: true, data: null }), diff --git a/tests/keyboard-shortcuts.test.js b/tests/keyboard-shortcuts.test.js index 6889628..7ba0fd1 100644 --- a/tests/keyboard-shortcuts.test.js +++ b/tests/keyboard-shortcuts.test.js @@ -61,8 +61,14 @@ test.describe('Keyboard Shortcuts', () => { setEncryptedApiKey: async (apiKey) => ({ success: true }), // Feature flags operations - getFeatureFlags: async () => ({ success: true, data: { aiArticleSummary: false } }), + getFeatureFlags: async () => ({ success: true, data: { aiArticleSummary: false, dataMirror: false } }), setFeatureFlag: async (flagName, enabled) => ({ success: true }), + + // Data Mirror operations + selectMirrorDirectory: async () => ({ success: true, data: null }), + getMirrorDirectory: async () => ({ success: true, data: null }), + setMirrorDirectory: async (mirrorDirectory) => ({ success: true }), + syncToMirror: async () => ({ success: true }), // AI Summary operations readAISummary: async (articleId) => ({ success: true, data: null }), From fc958fab16d1fd21019b8bde45fdb3a6e698390d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 01:43:58 +0000 Subject: [PATCH 4/8] refactor: Extract mirror sync trigger into helper function and use parallel file copy Co-authored-by: busse <18599+busse@users.noreply.github.com> --- main.js | 37 +++++++++++++++++++++--------------- playwright-report/index.html | 2 +- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/main.js b/main.js index f63baca..4f4431d 100644 --- a/main.js +++ b/main.js @@ -145,7 +145,7 @@ ipcMain.handle('write-json', async (event, filename, data) => { try { await fs.writeFile(filePath, JSON.stringify(data, null, 2), 'utf-8'); // Trigger mirror sync in background (don't await to avoid blocking) - syncToMirror().catch(err => console.error('Mirror sync error:', err)); + triggerMirrorSync(); return { success: true }; } catch (error) { console.error('Error writing JSON file:', error); @@ -173,7 +173,7 @@ ipcMain.handle('write-feeds', async (event, feeds) => { try { await fs.writeFile(filePath, JSON.stringify(feeds, null, 2), 'utf-8'); // Trigger mirror sync in background (don't await to avoid blocking) - syncToMirror().catch(err => console.error('Mirror sync error:', err)); + triggerMirrorSync(); return { success: true }; } catch (error) { console.error('Error writing feeds:', error); @@ -201,7 +201,7 @@ ipcMain.handle('write-categories', async (event, categories) => { try { await fs.writeFile(filePath, JSON.stringify(categories, null, 2), 'utf-8'); // Trigger mirror sync in background (don't await to avoid blocking) - syncToMirror().catch(err => console.error('Mirror sync error:', err)); + triggerMirrorSync(); return { success: true }; } catch (error) { console.error('Error writing categories:', error); @@ -229,7 +229,7 @@ ipcMain.handle('write-read-states', async (event, readStates) => { try { await fs.writeFile(filePath, JSON.stringify(readStates, null, 2), 'utf-8'); // Trigger mirror sync in background (don't await to avoid blocking) - syncToMirror().catch(err => console.error('Mirror sync error:', err)); + triggerMirrorSync(); return { success: true }; } catch (error) { console.error('Error writing read states:', error); @@ -257,7 +257,7 @@ ipcMain.handle('write-settings', async (event, settings) => { try { await fs.writeFile(filePath, JSON.stringify(settings, null, 2), 'utf-8'); // Trigger mirror sync in background (don't await to avoid blocking) - syncToMirror().catch(err => console.error('Mirror sync error:', err)); + triggerMirrorSync(); return { success: true }; } catch (error) { console.error('Error writing settings:', error); @@ -310,7 +310,7 @@ ipcMain.handle('set-encrypted-api-key', async (event, apiKey) => { await fs.writeFile(filePath, JSON.stringify(settings, null, 2), 'utf-8'); // Trigger mirror sync in background (don't await to avoid blocking) - syncToMirror().catch(err => console.error('Mirror sync error:', err)); + triggerMirrorSync(); return { success: true }; } catch (error) { console.error('Error setting encrypted API key:', error); @@ -372,7 +372,7 @@ ipcMain.handle('set-feature-flag', async (event, flagName, enabled) => { await fs.writeFile(filePath, JSON.stringify(settings, null, 2), 'utf-8'); // Trigger mirror sync in background (don't await to avoid blocking) - syncToMirror().catch(err => console.error('Mirror sync error:', err)); + triggerMirrorSync(); return { success: true }; } catch (error) { console.error('Error setting feature flag:', error); @@ -443,7 +443,7 @@ ipcMain.handle('set-mirror-directory', async (event, mirrorDirectory) => { await fs.writeFile(filePath, JSON.stringify(settings, null, 2), 'utf-8'); // Trigger immediate sync if directory is set and feature is enabled if (mirrorDirectory && settings.featureFlags?.dataMirror) { - syncToMirror().catch(err => console.error('Mirror sync error:', err)); + triggerMirrorSync(); } return { success: true }; } catch (error) { @@ -452,6 +452,11 @@ ipcMain.handle('set-mirror-directory', async (event, mirrorDirectory) => { } }); +// Helper function to trigger mirror sync with error handling +function triggerMirrorSync() { + syncToMirror().catch(err => console.error('Mirror sync error:', err)); +} + // Helper function to sync data to mirror directory async function syncToMirror() { try { @@ -482,9 +487,10 @@ async function syncToMirror() { // Get all files in the data directory const files = await fs.readdir(dataDir); - // Copy each JSON file to mirror directory - for (const file of files) { - if (file.endsWith('.json')) { + // Copy JSON files to mirror directory in parallel + const copyPromises = files + .filter(file => file.endsWith('.json')) + .map(async (file) => { const sourcePath = path.join(dataDir, file); const destPath = path.join(mirrorDir, file); @@ -494,8 +500,9 @@ async function syncToMirror() { } catch (error) { console.error(`Error copying ${file} to mirror:`, error); } - } - } + }); + + await Promise.all(copyPromises); console.log('Data synced to mirror directory:', mirrorDir); } catch (error) { @@ -543,7 +550,7 @@ ipcMain.handle('write-ai-summary', async (event, articleId, summaryData) => { summaries[articleId] = summaryData; await fs.writeFile(filePath, JSON.stringify(summaries, null, 2), 'utf-8'); // Trigger mirror sync in background (don't await to avoid blocking) - syncToMirror().catch(err => console.error('Mirror sync error:', err)); + triggerMirrorSync(); return { success: true }; } catch (error) { console.error('Error writing AI summary:', error); @@ -641,7 +648,7 @@ ipcMain.handle('write-articles', async (event, feedId, articlesData) => { }; await fs.writeFile(filePath, JSON.stringify(dataToWrite, null, 2), 'utf-8'); // Trigger mirror sync in background (don't await to avoid blocking) - syncToMirror().catch(err => console.error('Mirror sync error:', err)); + triggerMirrorSync(); return { success: true }; } catch (error) { console.error('Error writing articles:', error); diff --git a/playwright-report/index.html b/playwright-report/index.html index 54dadf5..e42a3e2 100644 --- a/playwright-report/index.html +++ b/playwright-report/index.html @@ -82,4 +82,4 @@
- \ No newline at end of file + \ No newline at end of file From edc56b4b84f605f88fa77f761bdde7f7e23024fe Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 01:45:39 +0000 Subject: [PATCH 5/8] Final cleanup and documentation Co-authored-by: busse <18599+busse@users.noreply.github.com> --- playwright-report/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playwright-report/index.html b/playwright-report/index.html index e42a3e2..4f3ae2d 100644 --- a/playwright-report/index.html +++ b/playwright-report/index.html @@ -82,4 +82,4 @@
- \ No newline at end of file + \ No newline at end of file From 42b980005662ab3ed5fe922a2cec77a4b3c581a7 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 03:34:40 +0000 Subject: [PATCH 6/8] feat: Add Sync as Markdown option for Obsidian integration - Add syncAsMarkdown setting (enabled by default when dataMirror is enabled) - Convert articles, feeds, and categories to Markdown with Obsidian frontmatter - Create organized folder structure: articles/{feed}/, feeds/, categories/ - Add YAML frontmatter with title, link, author, pubDate, read status, tags - Add toggle in Settings UI to switch between Markdown and JSON modes - Add proper YAML string escaping to prevent injection issues - Fix HTML stripping to handle nested/malformed tags safely - Add 2 new tests for the Sync as Markdown toggle - Update README with detailed Markdown sync documentation Co-authored-by: busse <18599+busse@users.noreply.github.com> --- README.md | 35 +++- index.html | 30 +++ main.js | 307 +++++++++++++++++++++++++++-- playwright-report/index.html | 2 +- preload.js | 2 + tests/data-mirror.test.js | 43 ++++ tests/helpers/mock-electron-api.js | 2 + tests/keyboard-shortcuts.test.js | 2 + 8 files changed, 401 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 4055be1..0b8966c 100644 --- a/README.md +++ b/README.md @@ -184,12 +184,39 @@ Feature flags can be toggled on or off using the toggle switches in the Settings When the Data Mirror feature flag is enabled, a new section appears in Settings that allows you to: 1. **Select a Mirror Directory**: Click "Browse..." to choose a directory where all RSS data will be mirrored -2. **Clear Directory**: Click the "✕" button to clear the selected directory -3. **Sync Now**: Manually trigger a sync of all data to the mirror directory +2. **Sync as Markdown**: Toggle to convert data to Markdown files with Obsidian-compatible frontmatter (enabled by default) +3. **Clear Directory**: Click the "✕" button to clear the selected directory +4. **Sync Now**: Manually trigger a sync of all data to the mirror directory + +The data is automatically synced to the mirror directory whenever any data changes (feeds, articles, read states, settings, etc.). This is useful for integration with personal knowledge management tools like Obsidian, Notion, or any other tool that can read Markdown or JSON files. + +**Sync as Markdown Mode (Default):** + +When "Sync as Markdown" is enabled, the mirrored data is organized into folders and converted to Markdown files with [Obsidian-compatible frontmatter properties](https://help.obsidian.md/properties): + +- `articles/{feed-name}/` - Individual article files with title, link, author, pubDate, read status, and tags in frontmatter +- `feeds/` - Feed subscription files with URL, status, and category info +- `categories/` - Category/folder files + +Each Markdown file includes YAML frontmatter with properties like: +```yaml +--- +title: "Article Title" +link: "https://example.com/article" +author: "Author Name" +feed: "Feed Name" +pubDate: "2024-01-15T10:30:00Z" +read: true +tags: + - "Technology" + - "News" +type: article +--- +``` -The data is automatically synced to the mirror directory whenever any data changes (feeds, articles, read states, settings, etc.). This is useful for integration with personal knowledge management tools like Obsidian, Notion, or any other tool that can read JSON files. +**JSON Mode:** -**Mirrored Files:** +When "Sync as Markdown" is disabled, the following JSON files are mirrored directly: - `feeds.json` - Feed subscriptions - `categories.json` - Category/folder organization - `read-states.json` - Article read status diff --git a/index.html b/index.html index f9689f0..d4bc57f 100644 --- a/index.html +++ b/index.html @@ -1052,6 +1052,18 @@

Settings

Select a directory to mirror all RSS data. Useful for integration with personal knowledge management tools like Obsidian.

+
+
+
+
Sync as Markdown (.md)
+
Convert data to Markdown files with Obsidian-compatible frontmatter properties instead of JSON
+
+ +
+
@@ -1991,6 +2003,15 @@

${escapeHtml(article.title || 'Untitled')}

${escapeHtml(article.title || 'Untitled')} { } }); +ipcMain.handle('get-sync-as-markdown', async (event) => { + try { + const filePath = path.join(dataDir, 'settings.json'); + let settings = {}; + + try { + const data = await fs.readFile(filePath, 'utf-8'); + settings = JSON.parse(data); + } catch (error) { + if (error.code !== 'ENOENT') { + throw error; + } + } + + // Default to true (enabled) when dataMirror is enabled + const syncAsMarkdown = settings.syncAsMarkdown !== undefined ? settings.syncAsMarkdown : true; + return { success: true, data: syncAsMarkdown }; + } catch (error) { + console.error('Error getting syncAsMarkdown setting:', error); + return { success: false, error: error.message }; + } +}); + +ipcMain.handle('set-sync-as-markdown', async (event, enabled) => { + try { + const filePath = path.join(dataDir, 'settings.json'); + let settings = {}; + + try { + const data = await fs.readFile(filePath, 'utf-8'); + settings = JSON.parse(data); + } catch (error) { + if (error.code !== 'ENOENT') { + throw error; + } + } + + settings.syncAsMarkdown = enabled; + + await fs.writeFile(filePath, JSON.stringify(settings, null, 2), 'utf-8'); + // Trigger sync after changing the setting + if (settings.featureFlags?.dataMirror && settings.mirrorDirectory) { + triggerMirrorSync(); + } + return { success: true }; + } catch (error) { + console.error('Error setting syncAsMarkdown:', error); + return { success: false, error: error.message }; + } +}); + ipcMain.handle('set-mirror-directory', async (event, mirrorDirectory) => { try { const filePath = path.join(dataDir, 'settings.json'); @@ -457,6 +508,143 @@ function triggerMirrorSync() { syncToMirror().catch(err => console.error('Mirror sync error:', err)); } +// Helper function to strip HTML tags and decode entities safely +function stripHtml(html) { + if (!html) return ''; + // Repeatedly remove HTML tags until none remain (handles nested/malformed tags) + let text = html; + let previous = ''; + while (previous !== text) { + previous = text; + text = text.replace(/<[^>]*>/g, ''); + } + // Decode common HTML entities (in correct order - & last to avoid double-decoding) + text = text.replace(/</g, '<'); + text = text.replace(/>/g, '>'); + text = text.replace(/"/g, '"'); + text = text.replace(/'/g, "'"); + text = text.replace(/ /g, ' '); + text = text.replace(/&/g, '&'); + // Normalize whitespace + text = text.replace(/\s+/g, ' ').trim(); + return text; +} + +// Helper function to escape a string for YAML double-quoted strings +function escapeYamlString(str) { + if (!str) return ''; + // Escape backslashes first, then double quotes, then newlines + return str + .replace(/\\/g, '\\\\') + .replace(/"/g, '\\"') + .replace(/\n/g, '\\n') + .replace(/\r/g, '\\r') + .replace(/\t/g, '\\t'); +} + +// Helper function to convert article to Markdown with Obsidian frontmatter +function articleToMarkdown(article, feedTitle, readState) { + const frontmatter = { + title: article.title || 'Untitled', + link: article.link || '', + author: article.author || '', + feed: feedTitle || '', + pubDate: article.pubDate || '', + read: readState?.read || false, + readAt: readState?.readAt || '', + tags: article.categories || [], + type: 'article' + }; + + // Build YAML frontmatter with proper escaping + let yaml = '---\n'; + yaml += `title: "${escapeYamlString(frontmatter.title)}"\n`; + yaml += `link: "${escapeYamlString(frontmatter.link)}"\n`; + yaml += `author: "${escapeYamlString(frontmatter.author)}"\n`; + yaml += `feed: "${escapeYamlString(frontmatter.feed)}"\n`; + yaml += `pubDate: "${escapeYamlString(frontmatter.pubDate)}"\n`; + yaml += `read: ${frontmatter.read}\n`; + if (frontmatter.readAt) { + yaml += `readAt: "${escapeYamlString(frontmatter.readAt)}"\n`; + } + if (frontmatter.tags && frontmatter.tags.length > 0) { + yaml += `tags:\n`; + frontmatter.tags.forEach(tag => { + yaml += ` - "${escapeYamlString(tag || '')}"\n`; + }); + } + yaml += `type: article\n`; + yaml += '---\n\n'; + + // Add article title as heading + let content = `# ${article.title || 'Untitled'}\n\n`; + + // Add link to original + if (article.link) { + content += `[View Original Article](${article.link})\n\n`; + } + + // Add description/summary + if (article.description) { + content += `## Summary\n\n${stripHtml(article.description)}\n\n`; + } + + // Add full content (converted from HTML to plain text) + if (article.content) { + content += `## Content\n\n${stripHtml(article.content)}\n`; + } + + return yaml + content; +} + +// Helper function to convert feed to Markdown with Obsidian frontmatter +function feedToMarkdown(feed) { + let yaml = '---\n'; + yaml += `title: "${escapeYamlString(feed.title || 'Untitled Feed')}"\n`; + yaml += `url: "${escapeYamlString(feed.url || '')}"\n`; + yaml += `category: "${escapeYamlString(feed.category || '')}"\n`; + yaml += `icon: "${escapeYamlString(feed.icon || '')}"\n`; + yaml += `lastUpdated: "${escapeYamlString(feed.lastUpdated || '')}"\n`; + yaml += `status: "${escapeYamlString(feed.status || '')}"\n`; + yaml += `type: feed\n`; + yaml += '---\n\n'; + + let content = `# ${feed.title || 'Untitled Feed'}\n\n`; + content += `**URL:** ${feed.url || 'N/A'}\n\n`; + content += `**Status:** ${feed.status || 'unknown'}\n\n`; + if (feed.lastUpdated) { + content += `**Last Updated:** ${feed.lastUpdated}\n\n`; + } + + return yaml + content; +} + +// Helper function to convert category to Markdown with Obsidian frontmatter +function categoryToMarkdown(category) { + let yaml = '---\n'; + yaml += `name: "${escapeYamlString(category.name || 'Untitled Category')}"\n`; + yaml += `icon: "${escapeYamlString(category.icon || '')}"\n`; + yaml += `type: category\n`; + yaml += '---\n\n'; + + let content = `# ${category.icon || '📁'} ${category.name || 'Untitled Category'}\n\n`; + content += `This is a category/folder for organizing RSS feeds.\n`; + + return yaml + content; +} + +// Helper function to sanitize filename for file system +function sanitizeFilename(name) { + if (!name) return 'untitled'; + // Remove or replace invalid characters + return name + .replace(/[<>:"/\\|?*]/g, '-') + .replace(/\s+/g, '-') + .replace(/-+/g, '-') + .replace(/^-|-$/g, '') + .substring(0, 100) || 'untitled'; +} + // Helper function to sync data to mirror directory async function syncToMirror() { try { @@ -480,6 +668,7 @@ async function syncToMirror() { } const mirrorDir = settings.mirrorDirectory; + const syncAsMarkdown = settings.syncAsMarkdown !== false; // Default to true // Ensure mirror directory exists await fs.mkdir(mirrorDir, { recursive: true }); @@ -487,24 +676,108 @@ async function syncToMirror() { // Get all files in the data directory const files = await fs.readdir(dataDir); - // Copy JSON files to mirror directory in parallel - const copyPromises = files - .filter(file => file.endsWith('.json')) - .map(async (file) => { - const sourcePath = path.join(dataDir, file); - const destPath = path.join(mirrorDir, file); - - try { - const content = await fs.readFile(sourcePath, 'utf-8'); - await fs.writeFile(destPath, content, 'utf-8'); - } catch (error) { - console.error(`Error copying ${file} to mirror:`, error); + if (syncAsMarkdown) { + // Create subdirectories for organized Markdown files + const articlesDir = path.join(mirrorDir, 'articles'); + const feedsDir = path.join(mirrorDir, 'feeds'); + const categoriesDir = path.join(mirrorDir, 'categories'); + + await Promise.all([ + fs.mkdir(articlesDir, { recursive: true }), + fs.mkdir(feedsDir, { recursive: true }), + fs.mkdir(categoriesDir, { recursive: true }) + ]); + + // Load read states for article metadata + let readStates = {}; + try { + const readStatesData = await fs.readFile(path.join(dataDir, 'read-states.json'), 'utf-8'); + readStates = JSON.parse(readStatesData); + } catch (error) { + if (error.code !== 'ENOENT') { + console.error('Error reading read states:', error); } - }); - - await Promise.all(copyPromises); - - console.log('Data synced to mirror directory:', mirrorDir); + } + + // Load feeds for reference + let feeds = []; + try { + const feedsData = await fs.readFile(path.join(dataDir, 'feeds.json'), 'utf-8'); + feeds = JSON.parse(feedsData); + } catch (error) { + if (error.code !== 'ENOENT') { + console.error('Error reading feeds:', error); + } + } + + // Process files and convert to Markdown + const syncPromises = files + .filter(file => file.endsWith('.json')) + .map(async (file) => { + const sourcePath = path.join(dataDir, file); + + try { + const content = await fs.readFile(sourcePath, 'utf-8'); + const data = JSON.parse(content); + + if (file === 'feeds.json' && Array.isArray(data)) { + // Convert each feed to a Markdown file + const feedPromises = data.map(async (feed) => { + const mdContent = feedToMarkdown(feed); + const filename = sanitizeFilename(feed.title || feed.id) + '.md'; + await fs.writeFile(path.join(feedsDir, filename), mdContent, 'utf-8'); + }); + await Promise.all(feedPromises); + } else if (file === 'categories.json' && Array.isArray(data)) { + // Convert each category to a Markdown file + const catPromises = data.map(async (category) => { + const mdContent = categoryToMarkdown(category); + const filename = sanitizeFilename(category.name || category.id) + '.md'; + await fs.writeFile(path.join(categoriesDir, filename), mdContent, 'utf-8'); + }); + await Promise.all(catPromises); + } else if (file.startsWith('articles-') && data.articles) { + // Convert each article to a Markdown file + const feed = feeds.find(f => f.id === data.feedId); + const feedTitle = feed?.title || 'Unknown Feed'; + const feedFolder = path.join(articlesDir, sanitizeFilename(feedTitle)); + await fs.mkdir(feedFolder, { recursive: true }); + + const articlePromises = data.articles.map(async (article) => { + const readState = readStates[article.id]; + const mdContent = articleToMarkdown(article, feedTitle, readState); + const filename = sanitizeFilename(article.title || article.id) + '.md'; + await fs.writeFile(path.join(feedFolder, filename), mdContent, 'utf-8'); + }); + await Promise.all(articlePromises); + } + // Skip other files like settings.json, read-states.json, ai-summaries.json in Markdown mode + } catch (error) { + console.error(`Error converting ${file} to Markdown:`, error); + } + }); + + await Promise.all(syncPromises); + console.log('Data synced to mirror directory as Markdown:', mirrorDir); + } else { + // Copy JSON files to mirror directory in parallel (original behavior) + const copyPromises = files + .filter(file => file.endsWith('.json')) + .map(async (file) => { + const sourcePath = path.join(dataDir, file); + const destPath = path.join(mirrorDir, file); + + try { + const content = await fs.readFile(sourcePath, 'utf-8'); + await fs.writeFile(destPath, content, 'utf-8'); + } catch (error) { + console.error(`Error copying ${file} to mirror:`, error); + } + }); + + await Promise.all(copyPromises); + console.log('Data synced to mirror directory as JSON:', mirrorDir); + } } catch (error) { console.error('Error syncing to mirror:', error); } diff --git a/playwright-report/index.html b/playwright-report/index.html index 4f3ae2d..d552533 100644 --- a/playwright-report/index.html +++ b/playwright-report/index.html @@ -82,4 +82,4 @@
- \ No newline at end of file + \ No newline at end of file diff --git a/preload.js b/preload.js index 24083e6..bd3f4b2 100644 --- a/preload.js +++ b/preload.js @@ -43,6 +43,8 @@ contextBridge.exposeInMainWorld('electronAPI', { selectMirrorDirectory: () => ipcRenderer.invoke('select-mirror-directory'), getMirrorDirectory: () => ipcRenderer.invoke('get-mirror-directory'), setMirrorDirectory: (mirrorDirectory) => ipcRenderer.invoke('set-mirror-directory', mirrorDirectory), + getSyncAsMarkdown: () => ipcRenderer.invoke('get-sync-as-markdown'), + setSyncAsMarkdown: (enabled) => ipcRenderer.invoke('set-sync-as-markdown', enabled), syncToMirror: () => ipcRenderer.invoke('sync-to-mirror'), // AI Summary operations diff --git a/tests/data-mirror.test.js b/tests/data-mirror.test.js index 68c9864..3da9346 100644 --- a/tests/data-mirror.test.js +++ b/tests/data-mirror.test.js @@ -74,6 +74,8 @@ test.describe('Data Mirror Feature', () => { selectMirrorDirectory: async () => ({ success: true, data: '/test/mirror/directory' }), getMirrorDirectory: async () => ({ success: true, data: '/existing/mirror/path' }), setMirrorDirectory: async (mirrorDirectory) => ({ success: true }), + getSyncAsMarkdown: async () => ({ success: true, data: true }), + setSyncAsMarkdown: async (enabled) => ({ success: true }), syncToMirror: async () => ({ success: true }), // AI Summary operations @@ -219,6 +221,47 @@ test.describe('Data Mirror Feature', () => { expect(toast).toBeTruthy(); }); + test('Sync as Markdown toggle is visible and checked by default', async () => { + // Open settings modal + await page.click('#settingsBtn'); + await page.waitForTimeout(200); + + // Check that the Sync as Markdown toggle exists and is checked by default + const syncAsMarkdownToggle = await page.$('#syncAsMarkdownToggle'); + expect(syncAsMarkdownToggle).toBeTruthy(); + + const isChecked = await syncAsMarkdownToggle.isChecked(); + expect(isChecked).toBe(true); + }); + + test('Sync as Markdown toggle can be toggled', async () => { + // Open settings modal + await page.click('#settingsBtn'); + await page.waitForTimeout(200); + + // Get the toggle and verify it's checked + const syncAsMarkdownToggle = await page.$('#syncAsMarkdownToggle'); + let isChecked = await syncAsMarkdownToggle.isChecked(); + expect(isChecked).toBe(true); + + // Toggle it off by clicking the parent label + const toggleSwitch = await syncAsMarkdownToggle.evaluateHandle(el => el.parentElement); + await toggleSwitch.click(); + await page.waitForTimeout(100); + + // Verify it's now unchecked + isChecked = await syncAsMarkdownToggle.isChecked(); + expect(isChecked).toBe(false); + + // Toggle it back on + await toggleSwitch.click(); + await page.waitForTimeout(100); + + // Verify it's checked again + isChecked = await syncAsMarkdownToggle.isChecked(); + expect(isChecked).toBe(true); + }); + test('Toggling Data Mirror feature flag shows/hides section', async () => { // Open settings modal await page.click('#settingsBtn'); diff --git a/tests/helpers/mock-electron-api.js b/tests/helpers/mock-electron-api.js index 91a3c0b..4a0015f 100644 --- a/tests/helpers/mock-electron-api.js +++ b/tests/helpers/mock-electron-api.js @@ -146,6 +146,8 @@ function createMockElectronAPI() { selectMirrorDirectory: async () => ({ success: true, data: null }), getMirrorDirectory: async () => ({ success: true, data: null }), setMirrorDirectory: async (mirrorDirectory) => ({ success: true }), + getSyncAsMarkdown: async () => ({ success: true, data: true }), + setSyncAsMarkdown: async (enabled) => ({ success: true }), syncToMirror: async () => ({ success: true }), // AI Summary operations diff --git a/tests/keyboard-shortcuts.test.js b/tests/keyboard-shortcuts.test.js index 7ba0fd1..4307335 100644 --- a/tests/keyboard-shortcuts.test.js +++ b/tests/keyboard-shortcuts.test.js @@ -68,6 +68,8 @@ test.describe('Keyboard Shortcuts', () => { selectMirrorDirectory: async () => ({ success: true, data: null }), getMirrorDirectory: async () => ({ success: true, data: null }), setMirrorDirectory: async (mirrorDirectory) => ({ success: true }), + getSyncAsMarkdown: async () => ({ success: true, data: true }), + setSyncAsMarkdown: async (enabled) => ({ success: true }), syncToMirror: async () => ({ success: true }), // AI Summary operations From 0131e8b8284db3911b6ef059e046be18eca21901 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 03:56:48 +0000 Subject: [PATCH 7/8] fix: Preserve line breaks in Markdown content when stripping HTML - Convert block-level elements (p, div, h1-h6, li, tr, blockquote) to line breaks - Convert br tags to single newlines - Convert hr tags to markdown horizontal rules - Normalize multiple consecutive newlines to paragraph breaks - Trim each line while preserving paragraph structure Co-authored-by: busse <18599+busse@users.noreply.github.com> --- main.js | 34 ++++++++++++++++++++++++++++++---- playwright-report/index.html | 2 +- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/main.js b/main.js index 3230cb0..cf5449c 100644 --- a/main.js +++ b/main.js @@ -508,16 +508,33 @@ function triggerMirrorSync() { syncToMirror().catch(err => console.error('Mirror sync error:', err)); } -// Helper function to strip HTML tags and decode entities safely +// Helper function to strip HTML tags and decode entities safely while preserving line breaks function stripHtml(html) { if (!html) return ''; - // Repeatedly remove HTML tags until none remain (handles nested/malformed tags) let text = html; + + // Convert block-level elements to line breaks before removing tags + // Replace closing block tags with double newlines for paragraph breaks + text = text.replace(/<\/p>/gi, '\n\n'); + text = text.replace(/<\/div>/gi, '\n\n'); + text = text.replace(/<\/h[1-6]>/gi, '\n\n'); + text = text.replace(/<\/li>/gi, '\n'); + text = text.replace(/<\/tr>/gi, '\n'); + text = text.replace(/<\/blockquote>/gi, '\n\n'); + + // Replace
tags with single newlines + text = text.replace(//gi, '\n'); + + // Replace
with a line + text = text.replace(//gi, '\n---\n'); + + // Repeatedly remove remaining HTML tags until none remain (handles nested/malformed tags) let previous = ''; while (previous !== text) { previous = text; text = text.replace(/<[^>]*>/g, ''); } + // Decode common HTML entities (in correct order - & last to avoid double-decoding) text = text.replace(/</g, '<'); text = text.replace(/>/g, '>'); @@ -525,8 +542,17 @@ function stripHtml(html) { text = text.replace(/'/g, "'"); text = text.replace(/ /g, ' '); text = text.replace(/&/g, '&'); - // Normalize whitespace - text = text.replace(/\s+/g, ' ').trim(); + + // Normalize multiple consecutive newlines to max 2 (paragraph break) + text = text.replace(/\n{3,}/g, '\n\n'); + + // Normalize spaces within lines (but preserve newlines) + text = text.replace(/[^\S\n]+/g, ' '); + + // Trim each line and remove empty lines at start/end + text = text.split('\n').map(line => line.trim()).join('\n'); + text = text.replace(/^\n+|\n+$/g, ''); + return text; } diff --git a/playwright-report/index.html b/playwright-report/index.html index d552533..539ed33 100644 --- a/playwright-report/index.html +++ b/playwright-report/index.html @@ -82,4 +82,4 @@
- \ No newline at end of file + \ No newline at end of file From 830ce00750ffc8393e4a22f04e4caa5e4f2ee690 Mon Sep 17 00:00:00 2001 From: busse Date: Wed, 3 Dec 2025 23:12:03 -0500 Subject: [PATCH 8/8] updated README for mirror data to second directory --- README.md | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 0b8966c..82c314c 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ A simple desktop RSS reader application for Mac built with Electron. The applica - **OpenAI API key management** - Secure, encrypted storage of API keys using AES-256-GCM - **Feature flags** - Toggle individual features on/off to customize your experience - **Settings modal** - Centralized configuration interface accessible from the sidebar +- **Data Mirror with Markdown sync** - Mirror all RSS data to an external directory, with option to convert to Markdown files with Obsidian-compatible frontmatter ![Settings](screenshots/settings.png) @@ -184,19 +185,27 @@ Feature flags can be toggled on or off using the toggle switches in the Settings When the Data Mirror feature flag is enabled, a new section appears in Settings that allows you to: 1. **Select a Mirror Directory**: Click "Browse..." to choose a directory where all RSS data will be mirrored -2. **Sync as Markdown**: Toggle to convert data to Markdown files with Obsidian-compatible frontmatter (enabled by default) +2. **Sync as Markdown** (enabled by default): Toggle to convert data to Markdown files with Obsidian-compatible frontmatter instead of raw JSON files 3. **Clear Directory**: Click the "✕" button to clear the selected directory 4. **Sync Now**: Manually trigger a sync of all data to the mirror directory The data is automatically synced to the mirror directory whenever any data changes (feeds, articles, read states, settings, etc.). This is useful for integration with personal knowledge management tools like Obsidian, Notion, or any other tool that can read Markdown or JSON files. +**Note:** The "Sync as Markdown" option is enabled by default, providing a more readable and organized format for your mirrored data. You can disable it to mirror raw JSON files instead. + **Sync as Markdown Mode (Default):** -When "Sync as Markdown" is enabled, the mirrored data is organized into folders and converted to Markdown files with [Obsidian-compatible frontmatter properties](https://help.obsidian.md/properties): +When "Sync as Markdown" is enabled (the default), the mirrored data is organized into a clean folder structure and converted to Markdown files with [Obsidian-compatible frontmatter properties](https://help.obsidian.md/properties): + +- `articles/{feed-name}/` - Individual article files, one per article, with full content and metadata +- `feeds/` - Feed subscription files with URL, status, category, and update information +- `categories/` - Category/folder definition files -- `articles/{feed-name}/` - Individual article files with title, link, author, pubDate, read status, and tags in frontmatter -- `feeds/` - Feed subscription files with URL, status, and category info -- `categories/` - Category/folder files +This format makes it easy to: +- Browse and search articles in your knowledge management tool +- Link between articles, feeds, and categories +- Use Obsidian's graph view to visualize your reading patterns +- Export or backup your RSS data in a human-readable format Each Markdown file includes YAML frontmatter with properties like: ```yaml