From 7e04e1abf43b19f8197f4060e761950d4e67fbc5 Mon Sep 17 00:00:00 2001
From: chenkel-data <chenkel.data@gmail.com>
Date: Fri, 27 Mar 2026 21:16:11 +0100
Subject: [PATCH 1/3] feat: add caching-logic, extend data-model for shared
 listings

---
 client/src/App.jsx                    |  78 +++-----
 client/src/components/FilterBar.jsx   |  12 +-
 client/src/components/ListingCard.jsx |  20 ++
 client/src/hooks/useListings.js       |  17 +-
 client/src/index.css                  |   1 +
 client/src/utils/formatting.js        |  11 +-
 src/db/database.js                    | 267 +++++++++++++++++---------
 src/providers/immoscout24/index.js    |  36 +++-
 src/scrapers/engine.js                |  22 +++
 src/services/scraperService.js        |  27 ++-
 10 files changed, 313 insertions(+), 178 deletions(-)

diff --git a/client/src/App.jsx b/client/src/App.jsx
index c1661cd..4a92697 100644
--- a/client/src/App.jsx
+++ b/client/src/App.jsx
@@ -18,21 +18,6 @@ import { api } from './api.js';
 import { TABS, ITEMS_PER_PAGE, LISTING_TYPE_LABELS, LISTING_TYPE_COLORS, PROVIDER_COLORS, PROVIDER_LABELS } from './constants.js';
 
 const FILTERS_STORAGE_KEY = 'immo.filters.v1';
-const EMPTY_STATS = { total: 0, unseen: 0, favorites: 0, blacklisted: 0 };
-
-function buildStatsFromListings(list) {
-  const stats = { ...EMPTY_STATS };
-  for (const listing of list) {
-    if (listing.is_blacklisted) {
-      stats.blacklisted += 1;
-      continue;
-    }
-    stats.total += 1;
-    if (!listing.is_seen) stats.unseen += 1;
-    if (listing.is_favorite) stats.favorites += 1;
-  }
-  return stats;
-}
 
 function readPersistedFilters() {
   const defaults = {
@@ -69,7 +54,7 @@ export default function App() {
   }, []);
 
   const {
-    listings, loading, orphanStats,
+    listings, loading, stats, orphanStats, configStats,
     loadListings, loadStats, loadConfigStats,
     handleSeen, handleFavorite, handleBlacklist, handleUnblacklist,
     handleMarkAllSeen, handleReset, handleResetConfig,
@@ -142,19 +127,24 @@ export default function App() {
   /* config selection handler – always resets tab to ALL when switching */
   const handleSelectConfig = useCallback((configId) => {
     setActiveConfigId(configId ?? null);
-    currentListingParamsRef.current = { include_blacklisted: true };
+    const params = { include_blacklisted: true };
+    if (configId) params.search_config_id = configId;
+    currentListingParamsRef.current = params;
+    loadListings(params);
     setActiveTab(TABS.ALL);
     setPage(1);
-  }, []);
+  }, [loadListings]);
 
   /* navigate home: deselect agent, reset to unseen tab */
   const handleNavigateHome = useCallback(() => {
     setActiveConfigId(null);
-    currentListingParamsRef.current = { include_blacklisted: true };
+    const params = { include_blacklisted: true };
+    currentListingParamsRef.current = params;
+    loadListings(params);
     setActiveTab(TABS.UNSEEN);
     setPage(1);
     setSearchQuery(''); setMinPrice(''); setMaxPrice(''); setMinSize(''); setMinRooms(''); setListingTypeFilter(''); setProviderFilter(''); setPublisherFilter(''); setMaxAvailableFrom('');
-  }, []);
+  }, [loadListings]);
 
   const isProviderFilterActive = !activeConfigId && activeTab === TABS.ALL;
 
@@ -183,10 +173,9 @@ export default function App() {
   }, [listings, listingTypeFilter, publisherFilter, searchQuery, minPrice, maxPrice, minSize, minRooms, maxAvailableFrom, scrapeConfig]);
 
   const filteredBase = useMemo(() => {
-    if (activeConfigId) return uiFilteredListings.filter(l => l.search_config_id === activeConfigId);
     if (isProviderFilterActive && providerFilter) return uiFilteredListings.filter(l => l.provider === providerFilter);
     return uiFilteredListings;
-  }, [uiFilteredListings, activeConfigId, isProviderFilterActive, providerFilter]);
+  }, [uiFilteredListings, isProviderFilterActive, providerFilter]);
 
   const filtered = useMemo(() => {
     let list = [...filteredBase];
@@ -212,13 +201,17 @@ export default function App() {
   }, [filteredBase, activeTab]);
 
   const tabCounts = useMemo(() => {
+    let base = listings;
+    if (listingTypeFilter) base = base.filter(l => l.listing_type === listingTypeFilter);
+    if (isProviderFilterActive && providerFilter) base = base.filter(l => l.provider === providerFilter);
+    const nonBlacklisted = base.filter(l => !l.is_blacklisted);
     return {
-      [TABS.ALL]: filteredBase.filter(l => !l.is_blacklisted).length,
-      [TABS.UNSEEN]: filteredBase.filter(l => !l.is_blacklisted && !l.is_seen).length,
-      [TABS.FAVORITES]: filteredBase.filter(l => !l.is_blacklisted && l.is_favorite).length,
-      [TABS.BLACKLISTED]: filteredBase.filter(l => l.is_blacklisted).length,
+      [TABS.ALL]: nonBlacklisted.length,
+      [TABS.UNSEEN]: nonBlacklisted.filter(l => !l.is_seen).length,
+      [TABS.FAVORITES]: nonBlacklisted.filter(l => l.is_favorite).length,
+      [TABS.BLACKLISTED]: base.filter(l => l.is_blacklisted).length,
     };
-  }, [filteredBase]);
+  }, [listings, listingTypeFilter, isProviderFilterActive, providerFilter]);
 
   const activeConfigStats = useMemo(() => ({
     total: tabCounts[TABS.ALL],
@@ -227,30 +220,6 @@ export default function App() {
     blacklisted: tabCounts[TABS.BLACKLISTED],
   }), [tabCounts]);
 
-  const sidebarConfigStats = useMemo(() => {
-    const map = {};
-    for (const cfg of configs) map[cfg.id] = { ...EMPTY_STATS };
-    for (const listing of uiFilteredListings) {
-      const configId = listing.search_config_id;
-      if (configId == null) continue;
-      if (!map[configId]) map[configId] = { ...EMPTY_STATS };
-      const bucket = map[configId];
-      if (listing.is_blacklisted) {
-        bucket.blacklisted += 1;
-        continue;
-      }
-      bucket.total += 1;
-      if (!listing.is_seen) bucket.unseen += 1;
-      if (listing.is_favorite) bucket.favorites += 1;
-    }
-    return map;
-  }, [configs, uiFilteredListings]);
-
-  const sidebarGlobalStats = useMemo(() => (
-    buildStatsFromListings(uiFilteredListings)
-  ), [uiFilteredListings]);
-
-
   const pages = Math.max(1, Math.ceil(filtered.length / ITEMS_PER_PAGE));
   const safePage = Math.min(page, pages);
   const paginated = filtered.slice((safePage - 1) * ITEMS_PER_PAGE, safePage * ITEMS_PER_PAGE);
@@ -301,9 +270,9 @@ export default function App() {
           <AgentSidebar
             configs={configs}
             providers={providers}
-            configStats={sidebarConfigStats}
+            configStats={configStats}
             orphanStats={orphanStats}
-            globalStats={sidebarGlobalStats}
+            globalStats={stats}
             activeConfigId={activeConfigId}
             onSelectConfig={handleSelectConfig}
             onAdd={addConfig}
@@ -430,6 +399,7 @@ export default function App() {
               providerFilter={providerFilter}
               providers={providers}
               showProviderFilter={isProviderFilterActive}
+              showListingTypeFilter={!activeConfigId}
               onTabChange={setActiveTab} onListingTypeChange={setListingTypeFilter}
               onSearch={setSearchQuery} onMinPrice={setMinPrice} onMaxPrice={setMaxPrice} onMinSize={setMinSize} onMinRooms={setMinRooms}
               onPublisherFilter={setPublisherFilter}
@@ -457,7 +427,7 @@ export default function App() {
           onClose={() => setSidebarOpen(false)}
           onReset={() => handleReset(currentListingParamsRef.current)}
           showToast={showToast}
-          onSaved={loadScrapeConfig}
+          onSaved={() => { loadScrapeConfig(); reloadAll(); }}
           onClearFavorites={() =>
             askConfirm({
               title: 'Alle Favoriten löschen?',
diff --git a/client/src/components/FilterBar.jsx b/client/src/components/FilterBar.jsx
index fff61e2..8108f86 100644
--- a/client/src/components/FilterBar.jsx
+++ b/client/src/components/FilterBar.jsx
@@ -3,12 +3,12 @@ import { TABS, LISTING_TYPE_LABELS, PROVIDER_LABELS } from '../constants.js';
 export default function FilterBar({
   activeTab, stats, listingTypeFilter,
   searchQuery, minPrice, maxPrice, minSize, minRooms,
-  publisherFilter, providerFilter, providers, showProviderFilter,
+  publisherFilter, providerFilter, providers, showProviderFilter, showListingTypeFilter = true,
   tabCounts,
   maxAvailableFrom,
   onTabChange, onListingTypeChange, onSearch, onMinPrice, onMaxPrice, onMinSize, onMinRooms, onPublisherFilter, onProviderFilter, onMaxAvailableFrom, onReset,
 }) {
-  const hasFilters = searchQuery || minPrice || maxPrice || minSize || minRooms || publisherFilter || maxAvailableFrom || (showProviderFilter && providerFilter);
+  const hasFilters = searchQuery || minPrice || maxPrice || minSize || minRooms || publisherFilter || maxAvailableFrom || (showProviderFilter && providerFilter) || (showListingTypeFilter && listingTypeFilter);
 
   const tabs = [
     { id: TABS.ALL, label: 'Alle', count: stats.total, icon: (
@@ -48,9 +48,11 @@ export default function FilterBar({
       </nav>
 
       <div className="filter-controls">
-        <select className="filter-select" value={listingTypeFilter} onChange={(e) => onListingTypeChange(e.target.value)}>
-          {typeOptions.map((t) => <option key={t.id} value={t.id}>{t.label}</option>)}
-        </select>
+        {showListingTypeFilter && (
+          <select className="filter-select" value={listingTypeFilter} onChange={(e) => onListingTypeChange(e.target.value)}>
+            {typeOptions.map((t) => <option key={t.id} value={t.id}>{t.label}</option>)}
+          </select>
+        )}
 
         {showProviderFilter && (
           <select className="filter-select" value={providerFilter} onChange={(e) => onProviderFilter(e.target.value)}>
diff --git a/client/src/components/ListingCard.jsx b/client/src/components/ListingCard.jsx
index 412a09c..afcbe99 100644
--- a/client/src/components/ListingCard.jsx
+++ b/client/src/components/ListingCard.jsx
@@ -55,6 +55,9 @@ const ListingCard = memo(function ListingCard({ listing: l, onSeen, onFavorite,
   const providerLabel = PROVIDER_LABELS[l.provider] || l.provider;
   const publishedLabel = formatListingDate(l.listed_at);
   const availableFromLabel = formatAvailableFrom(l.available_from);
+  const firstSeenLabel = formatListingDate(l.first_seen);
+  const lastSeenLabel = formatListingDate(l.last_seen);
+  const seenMultipleTimes = l.first_seen && l.last_seen && l.first_seen !== l.last_seen;
 
   return (
     <article className={`card ${isNew ? 'card--new' : ''} ${l.is_seen ? 'card--seen' : ''} ${l.is_blacklisted ? 'card--blacklisted' : ''}`}>
@@ -114,6 +117,9 @@ const ListingCard = memo(function ListingCard({ listing: l, onSeen, onFavorite,
           </div>
         </div>
         <h2 className="card-title">{l.title}</h2>
+        {l.link?.includes('/expose/') && (
+          <p className="card-expose-id">ID: {l.link.split('/expose/')[1]}</p>
+        )}
         {l.address && (
           <p className="card-address">
             <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" className="addr-icon"><path d="M21 10c0 7-9 13-9 13S3 17 3 10a9 9 0 1 1 18 0z"/><circle cx="12" cy="10" r="3"/></svg>
@@ -144,6 +150,20 @@ const ListingCard = memo(function ListingCard({ listing: l, onSeen, onFavorite,
               <span>{l.publisher}</span>
             </span>
           )}
+          {l.first_seen && (
+            <span className="card-date card-date--scraped" title={new Date(l.first_seen).toLocaleString('de-DE')}>
+              <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" className="date-icon"><circle cx="11" cy="11" r="8"/><line x1="21" y1="21" x2="16.65" y2="16.65"/></svg>
+              <span className="card-date-label">Entdeckt:</span>
+              <span>{firstSeenLabel}</span>
+            </span>
+          )}
+          {seenMultipleTimes && (
+            <span className="card-date card-date--scraped" title={new Date(l.last_seen).toLocaleString('de-DE')}>
+              <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" className="date-icon"><polyline points="23 4 23 10 17 10"/><polyline points="1 20 1 14 7 14"/><path d="M3.51 9a9 9 0 0 1 14.85-3.36L23 10M1 14l4.64 4.36A9 9 0 0 0 20.49 15"/></svg>
+              <span className="card-date-label">Zuletzt gesehen:</span>
+              <span>{lastSeenLabel}</span>
+            </span>
+          )}
         </div>
         <a href={l.link} target="_blank" rel="noopener noreferrer" className="card-open-btn" onClick={handleOpen}>
           Öffnen
diff --git a/client/src/hooks/useListings.js b/client/src/hooks/useListings.js
index 2b3dacb..70b8502 100644
--- a/client/src/hooks/useListings.js
+++ b/client/src/hooks/useListings.js
@@ -82,11 +82,16 @@ export function useListings(showToast) {
     setListings((prev) => prev.map((l) => (l.id === id ? { ...l, is_seen: willBeSeen ? 1 : 0 } : l)));
     const delta = willBeSeen ? -1 : 1;
     setStats((prev) => ({ ...prev, unseen: Math.max(0, (prev.unseen ?? 0) + delta) }));
-    if (current.search_config_id != null) {
+    const agentIds = current.agent_ids || [];
+    if (agentIds.length > 0) {
       setConfigStats((prev) => {
-        const existing = prev[current.search_config_id];
-        if (!existing) return prev;
-        return { ...prev, [current.search_config_id]: { ...existing, unseen: Math.max(0, (existing.unseen ?? 0) + delta) } };
+        const next = { ...prev };
+        for (const configId of agentIds) {
+          const existing = next[configId];
+          if (!existing) continue;
+          next[configId] = { ...existing, unseen: Math.max(0, (existing.unseen ?? 0) + delta) };
+        }
+        return next;
       });
     }
   }, [listings]);
@@ -175,7 +180,7 @@ export function useListings(showToast) {
   const handleClearFavoritesByConfig = useCallback(async (configId, loadParams = {}) => {
     await api.listings.clearFavoritesByConfig(configId);
     setListings((prev) => prev.map((l) =>
-      l.search_config_id === configId ? { ...l, is_favorite: 0 } : l
+      (l.agent_ids || []).includes(configId) ? { ...l, is_favorite: 0 } : l
     ));
     await refreshStats();
     showToast?.('Favoriten dieses Agenten entfernt.', 'info');
@@ -191,7 +196,7 @@ export function useListings(showToast) {
   const handleClearBlacklistByConfig = useCallback(async (configId) => {
     await api.listings.clearBlacklistByConfig(configId);
     setListings((prev) => prev.map((l) => (
-      l.search_config_id === configId && l.is_blacklisted
+      (l.agent_ids || []).includes(configId) && l.is_blacklisted
         ? { ...l, is_blacklisted: 0 }
         : l
     )));
diff --git a/client/src/index.css b/client/src/index.css
index 5b03dcd..0371106 100644
--- a/client/src/index.css
+++ b/client/src/index.css
@@ -563,6 +563,7 @@ a { color: inherit; text-decoration: none; }
 .pill { font-size: .65rem; padding: 2px 7px; border-radius: var(--radius-full); background: var(--gray-100); color: var(--gray-600); font-weight: 600; }
 .card-title { font-size: .875rem; font-weight: 600; line-height: 1.4; display: -webkit-box; line-clamp: 2; -webkit-line-clamp: 2; -webkit-box-orient: vertical; overflow: hidden; margin-bottom: 4px; color: var(--gray-800); }
 .card-address { display: flex; align-items: flex-start; gap: 4px; font-size: .78rem; color: var(--text-muted); margin-bottom: 6px; }
+.card-expose-id { font-size: .72rem; color: var(--text-muted); opacity: .65; margin: -2px 0 6px; font-family: monospace; }
 .addr-icon { width: 13px; height: 13px; flex-shrink: 0; margin-top: 1px; color: var(--gray-400); }
 .card-desc { font-size: .78rem; color: var(--gray-400); line-height: 1.45; display: -webkit-box; line-clamp: 2; -webkit-line-clamp: 2; -webkit-box-orient: vertical; overflow: hidden; flex: 1; }
 .card-footer { display: flex; align-items: center; justify-content: space-between; padding: 10px 16px; border-top: 1px solid var(--gray-100); }
diff --git a/client/src/utils/formatting.js b/client/src/utils/formatting.js
index 042f72c..76d8c54 100644
--- a/client/src/utils/formatting.js
+++ b/client/src/utils/formatting.js
@@ -14,8 +14,7 @@ export function parseNum(str) {
 /**
  * Formats an ISO date as a human-readable relative date (German locale).
  * - < 1h:    "Vor 23 Minuten"
- * - < 12h:   "Vor 3 Stunden"
- * - < 24h:   "Heute, 14:33"
+ * - < 24h:   "Vor 16 Stunden"
  * - Yesterday: "Gestern, 09:12"
  * - Older:   "7. März 2026" (date only, time irrelevant after > 1 day)
  */
@@ -35,17 +34,13 @@ export function formatListingDate(iso) {
     return 'Gerade eben';
   } else if (diffMin < 60) {
     return `Vor ${diffMin} Minute${diffMin === 1 ? '' : 'n'}`;
-  } else if (diffH < 12) {
+  } else if (diffH < 24) {
     return `Vor ${diffH} Stunde${diffH === 1 ? '' : 'n'}`;
   } else {
     const hh = String(date.getHours()).padStart(2, '0');
     const mm = String(date.getMinutes()).padStart(2, '0');
-    const todayStart = new Date(now.getFullYear(), now.getMonth(), now.getDate());
-    const yesterdayStart = new Date(todayStart.getTime() - 86400000);
+    const yesterdayStart = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 1);
 
-    if (date >= todayStart) {
-      return `Heute, ${hh}:${mm}`;
-    }
     if (date >= yesterdayStart) {
       return `Gestern, ${hh}:${mm}`;
     }
diff --git a/src/db/database.js b/src/db/database.js
index 1a80a58..dea86dc 100644
--- a/src/db/database.js
+++ b/src/db/database.js
@@ -68,6 +68,14 @@ db.exec(`
     created_at TEXT NOT NULL DEFAULT (datetime('now')),
     UNIQUE(listing_id)
   );
+
+  CREATE TABLE IF NOT EXISTS listing_agents (
+    listing_id       TEXT NOT NULL,
+    search_config_id INTEGER NOT NULL,
+    PRIMARY KEY (listing_id, search_config_id),
+    FOREIGN KEY (listing_id) REFERENCES listings(id) ON DELETE CASCADE,
+    FOREIGN KEY (search_config_id) REFERENCES search_configs(id) ON DELETE CASCADE
+  );
 `);
 
 // ── Migrations for existing DBs ───────────────────────────────────────────
@@ -102,10 +110,38 @@ safeAlter('ALTER TABLE search_configs DROP COLUMN radius');
 safeAlter('ALTER TABLE listings ADD COLUMN blacklisted_at TEXT');
 safeAlter('ALTER TABLE listings ADD COLUMN available_from TEXT');
 safeAlter('ALTER TABLE listings ADD COLUMN favorited_at TEXT');
+safeAlter('ALTER TABLE listings ADD COLUMN scrape_rank INTEGER');
+
+// listing_agents per-agent rank (sort order of provider)
+safeAlter('ALTER TABLE listing_agents ADD COLUMN scrape_rank INTEGER');
+
+// per-agent: tracks which scrape run last actively scraped this listing (for partition sort)
+safeAlter('ALTER TABLE listing_agents ADD COLUMN last_scraped_run_id INTEGER');
 
 // Index for fast link-based deduplication
 db.exec('CREATE INDEX IF NOT EXISTS idx_listings_link ON listings(link)');
 
+// Index for fast agent lookups on the junction table
+db.exec(
+  'CREATE INDEX IF NOT EXISTS idx_listing_agents_config ON listing_agents(search_config_id)',
+);
+
+// Migration: populate listing_agents from existing search_config_id values
+{
+  const result = db
+    .prepare(
+      `INSERT OR IGNORE INTO listing_agents (listing_id, search_config_id)
+       SELECT id, search_config_id FROM listings
+       WHERE search_config_id IS NOT NULL
+         AND search_config_id IN (SELECT id FROM search_configs)`,
+    )
+    .run();
+  if (result.changes > 0)
+    console.log(
+      `[db] ${result.changes} listing-agent link(s) migrated to listing_agents table.`,
+    );
+}
+
 // Migration: set search_config_id = NULL for orphaned listings (deleted agents)
 {
   const result = db
@@ -210,12 +246,14 @@ export function updateSearchConfig(id, data) {
 }
 
 export function deleteSearchConfig(id) {
-  // Detach favorited & blacklisted listings – they survive agent deletion
+  // Remove junction entries for this agent
+  db.prepare('DELETE FROM listing_agents WHERE search_config_id = ?').run(id);
+  // Delete listings with no remaining agent associations and not pinned
   db.prepare(
-    'UPDATE listings SET search_config_id = NULL WHERE search_config_id = ? AND (is_favorite = 1 OR is_blacklisted = 1)',
-  ).run(id);
-  // Delete remaining listings for this agent
-  db.prepare('DELETE FROM listings WHERE search_config_id = ?').run(id);
+    `DELETE FROM listings
+     WHERE is_favorite = 0 AND is_blacklisted = 0
+       AND id NOT IN (SELECT listing_id FROM listing_agents)`,
+  ).run();
   // Delete scrape run history for this agent
   db.prepare('DELETE FROM scrape_runs WHERE search_config_id = ?').run(id);
   // Delete the config itself
@@ -261,7 +299,9 @@ export function clearAllFavorites() {
 
 export function clearFavoritesByConfig(searchConfigId) {
   db.prepare(
-    'UPDATE listings SET is_favorite = 0, favorited_at = NULL WHERE search_config_id = ? AND is_favorite = 1',
+    `UPDATE listings SET is_favorite = 0, favorited_at = NULL
+     WHERE is_favorite = 1
+       AND id IN (SELECT listing_id FROM listing_agents WHERE search_config_id = ?)`,
   ).run(searchConfigId);
 }
 
@@ -272,14 +312,14 @@ export function clearAllBlacklist() {
 
 export function clearBlacklistByConfig(searchConfigId) {
   db.prepare(
-    `
-    DELETE FROM blacklist WHERE listing_id IN (
-      SELECT id FROM listings WHERE search_config_id = ?
-    )
-  `,
+    `DELETE FROM blacklist WHERE listing_id IN (
+       SELECT listing_id FROM listing_agents WHERE search_config_id = ?
+     )`,
   ).run(searchConfigId);
   db.prepare(
-    'UPDATE listings SET is_blacklisted = 0 WHERE search_config_id = ? AND is_blacklisted = 1',
+    `UPDATE listings SET is_blacklisted = 0
+     WHERE is_blacklisted = 1
+       AND id IN (SELECT listing_id FROM listing_agents WHERE search_config_id = ?)`,
   ).run(searchConfigId);
 }
 
@@ -310,47 +350,10 @@ export function upsertListing(l) {
   if (l.link && isUrlBlacklisted(l.link)) {
     l.is_blacklisted = 1;
   }
-  // Auto-migration: if the ID schema has changed, identify the existing entry by URL.
-  // Only migrate if the existing entry belongs to the same agent
-  // or is an unpinned orphan (no favorite, no blacklist entry).
-  // Listings from other agents (favorited/blacklisted by another agent) remain
-  // untouched – no cross-agent carry-over.
-  let carrySeen = 0,
-    carryFav = 0,
-    carryFirstSeen = null;
-  if (l.link) {
-    const existingByLink = db.prepare('SELECT * FROM listings WHERE link = ? LIMIT 1').get(l.link);
-    if (existingByLink && existingByLink.id !== l.id) {
-      const isSameAgent = existingByLink.search_config_id === l.search_config_id;
-      const isSafeOrphan =
-        existingByLink.search_config_id == null &&
-        !existingByLink.is_favorite &&
-        !existingByLink.is_blacklisted;
-      if (isSameAgent || isSafeOrphan) {
-        // Same agent or unpinned orphan → carry over flags, replace old entry
-        carrySeen = existingByLink.is_seen ?? 0;
-        carryFav = existingByLink.is_favorite ?? 0;
-        carryFirstSeen = existingByLink.first_seen ?? null;
-        try {
-          db.prepare('UPDATE blacklist SET listing_id = ? WHERE listing_id = ?').run(
-            l.id,
-            existingByLink.id,
-          );
-        } catch {}
-        try {
-          db.prepare('UPDATE blacklist SET listing_id = ? WHERE url = ?').run(l.id, l.link);
-        } catch {}
-        try {
-          db.prepare('DELETE FROM listings WHERE id = ?').run(existingByLink.id);
-        } catch {}
-      }
-      // Otherwise (different agent with a pinned entry): leave the old entry untouched,
-      // the new listing will be inserted normally for the current agent.
-    }
-  }
+
   db.prepare(
     `
-    INSERT INTO listings (id, source, provider, listing_type, search_config_id, title, price, size, rooms, address, description, publisher, link, image, is_blacklisted, listed_at, available_from, first_seen, last_seen)
+    INSERT INTO listings (id, source, provider, listing_type, title, price, size, rooms, address, description, publisher, link, image, is_blacklisted, listed_at, available_from, first_seen, last_seen, scrape_rank)
     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
     ON CONFLICT(id) DO UPDATE SET
       price           = excluded.price,
@@ -363,14 +366,13 @@ export function upsertListing(l) {
       listed_at       = COALESCE(excluded.listed_at, listed_at),
       available_from  = COALESCE(excluded.available_from, available_from),
       last_seen       = excluded.last_seen,
-      search_config_id = COALESCE(search_config_id, excluded.search_config_id)
+      scrape_rank     = excluded.scrape_rank
   `,
   ).run(
     l.id,
     l.source ?? l.provider ?? 'kleinanzeigen',
     l.provider ?? 'kleinanzeigen',
     l.listing_type ?? 'miete',
-    l.search_config_id ?? null,
     l.title,
     l.price ?? null,
     l.size ?? null,
@@ -383,16 +385,21 @@ export function upsertListing(l) {
     l.is_blacklisted ?? 0,
     l.listed_at ?? null,
     l.available_from ?? null,
-    carryFirstSeen && (!l.first_seen || carryFirstSeen < l.first_seen)
-      ? carryFirstSeen
-      : l.first_seen,
+    l.first_seen,
     l.last_seen,
+    l.scrape_rank ?? null,
   );
 
-  if (carrySeen || carryFav) {
+  // Create/update junction entry for the agent that scraped this listing,
+  // storing the per-agent scrape_rank so each agent has its own sort order (and does not overwrite others for shared listings).
+  if (l.search_config_id) {
     db.prepare(
-      'UPDATE listings SET is_seen = COALESCE(?, is_seen), is_favorite = COALESCE(?, is_favorite) WHERE id = ?',
-    ).run(carrySeen ? 1 : null, carryFav ? 1 : null, l.id);
+      `INSERT INTO listing_agents (listing_id, search_config_id, scrape_rank, last_scraped_run_id)
+       VALUES (?, ?, ?, ?)
+       ON CONFLICT(listing_id, search_config_id) DO UPDATE SET
+         scrape_rank = excluded.scrape_rank,
+         last_scraped_run_id = excluded.last_scraped_run_id`,
+    ).run(l.id, l.search_config_id, l.scrape_rank ?? null, l.run_id ?? null);
   }
 }
 
@@ -443,39 +450,73 @@ export function getListings({
 } = {}) {
   const conditions = [];
   const params = [];
+  let joinClause = '';
 
-  if (onlyUnseen) conditions.push('is_seen = 0');
-  if (onlyFavorites) conditions.push('is_favorite = 1');
-  if (showBlacklisted) conditions.push('is_blacklisted = 1');
-  else if (!includeBlacklisted && hideBlacklisted) conditions.push('is_blacklisted = 0');
+  // Filter by agent via junction table
+  if (searchConfigId) {
+    joinClause =
+      'INNER JOIN listing_agents la ON la.listing_id = l.id AND la.search_config_id = ?';
+    params.push(searchConfigId);
+  }
+
+  if (onlyUnseen) conditions.push('l.is_seen = 0');
+  if (onlyFavorites) conditions.push('l.is_favorite = 1');
+  if (showBlacklisted) conditions.push('l.is_blacklisted = 1');
+  else if (!includeBlacklisted && hideBlacklisted) conditions.push('l.is_blacklisted = 0');
   if (listingType) {
-    conditions.push('listing_type = ?');
+    conditions.push('l.listing_type = ?');
     params.push(listingType);
   }
   if (provider) {
-    conditions.push('provider = ?');
+    conditions.push('l.provider = ?');
     params.push(provider);
   }
-  if (searchConfigId) {
-    conditions.push('search_config_id = ?');
-    params.push(searchConfigId);
-  }
 
   for (const term of blacklistKeywords) {
     conditions.push(
-      "(LOWER(COALESCE(title,'')) NOT LIKE LOWER(?) AND LOWER(COALESCE(description,'')) NOT LIKE LOWER(?) AND LOWER(COALESCE(publisher,'')) NOT LIKE LOWER(?))",
+      "(LOWER(COALESCE(l.title,'')) NOT LIKE LOWER(?) AND LOWER(COALESCE(l.description,'')) NOT LIKE LOWER(?) AND LOWER(COALESCE(l.publisher,'')) NOT LIKE LOWER(?))",
     );
     params.push(`%${term}%`, `%${term}%`, `%${term}%`);
   }
   const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : '';
-  return db.prepare(`SELECT * FROM listings ${where} ORDER BY first_seen DESC`).all(...params);
+
+  // When filtering by a specific agent, sort by newest run first, then by provider rank within that run
+  let orderBy;
+  if (searchConfigId) {
+    orderBy = 'la.last_scraped_run_id DESC NULLS LAST, la.scrape_rank ASC NULLS LAST';
+  } else {
+    orderBy = 'l.scrape_rank ASC NULLS LAST';
+  }
+
+  const sql = `
+    SELECT l.*,
+      (SELECT GROUP_CONCAT(la2.search_config_id) FROM listing_agents la2 WHERE la2.listing_id = l.id) as agent_ids
+    FROM listings l
+    ${joinClause}
+    ${where}
+    ORDER BY ${orderBy}
+  `;
+  const rows = db.prepare(sql).all(...params);
+  return rows.map((r) => ({
+    ...r,
+    agent_ids: r.agent_ids ? r.agent_ids.split(',').map(Number) : [],
+  }));
 }
 
 export function getListingById(id) {
-  return db.prepare('SELECT * FROM listings WHERE id = ?').get(id);
+  const row = db
+    .prepare(
+      `SELECT l.*,
+        (SELECT GROUP_CONCAT(la.search_config_id) FROM listing_agents la WHERE la.listing_id = l.id) as agent_ids
+       FROM listings l WHERE l.id = ?`,
+    )
+    .get(id);
+  if (!row) return null;
+  return { ...row, agent_ids: row.agent_ids ? row.agent_ids.split(',').map(Number) : [] };
 }
 
 export function resetAll() {
+  db.exec('DELETE FROM listing_agents');
   db.exec('DELETE FROM listings');
   db.exec('DELETE FROM scrape_runs');
   db.exec('DELETE FROM blacklist');
@@ -488,12 +529,44 @@ export function purgeListingsKeepPinned() {
 }
 
 export function purgeListingsByConfig(searchConfigId) {
+  // Remove junction for non-pinned listings of this agent
   db.prepare(
-    'DELETE FROM listings WHERE search_config_id = ? AND is_favorite = 0 AND is_blacklisted = 0',
+    `DELETE FROM listing_agents
+     WHERE search_config_id = ?
+       AND listing_id IN (
+         SELECT id FROM listings WHERE is_favorite = 0 AND is_blacklisted = 0
+       )`,
   ).run(searchConfigId);
+  // Delete listings with no remaining agent associations and not pinned
+  db.prepare(
+    `DELETE FROM listings
+     WHERE is_favorite = 0 AND is_blacklisted = 0
+       AND id NOT IN (SELECT listing_id FROM listing_agents)`,
+  ).run();
 }
 
 export function getExistingIds(provider, listingType, searchConfigId = null) {
+  if (searchConfigId) {
+    // Use junction table for agent-specific lookup
+    const conditions = ['la.search_config_id = ?'];
+    const params = [searchConfigId];
+    if (provider) {
+      conditions.push('l.provider = ?');
+      params.push(provider);
+    }
+    if (listingType) {
+      conditions.push('l.listing_type = ?');
+      params.push(listingType);
+    }
+    return db
+      .prepare(
+        `SELECT l.id FROM listings l
+         INNER JOIN listing_agents la ON la.listing_id = l.id
+         WHERE ${conditions.join(' AND ')}`,
+      )
+      .all(...params)
+      .map((r) => r.id);
+  }
   const conditions = [];
   const params = [];
   if (provider) {
@@ -504,10 +577,6 @@ export function getExistingIds(provider, listingType, searchConfigId = null) {
     conditions.push('listing_type = ?');
     params.push(listingType);
   }
-  if (searchConfigId) {
-    conditions.push('search_config_id = ?');
-    params.push(searchConfigId);
-  }
   const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : '';
   return db
     .prepare(`SELECT id FROM listings ${where}`)
@@ -515,35 +584,41 @@ export function getExistingIds(provider, listingType, searchConfigId = null) {
     .map((r) => r.id);
 }
 
+/** Returns all known listing IDs for a provider (any agent). Used for scrape caching. */
+export function getAllKnownIds(provider) {
+  return db
+    .prepare('SELECT id FROM listings WHERE provider = ?')
+    .all(provider)
+    .map((r) => r.id);
+}
+
 export function getStatsPerConfig() {
   return db
     .prepare(
-      `
-    SELECT
-      search_config_id,
-      SUM(CASE WHEN is_blacklisted = 0 THEN 1 ELSE 0 END) as total,
-      SUM(CASE WHEN is_blacklisted = 0 AND is_seen = 0 THEN 1 ELSE 0 END) as unseen,
-      SUM(CASE WHEN is_favorite = 1 AND is_blacklisted = 0 THEN 1 ELSE 0 END) as favorites,
-      SUM(CASE WHEN is_blacklisted = 1 THEN 1 ELSE 0 END) as blacklisted
-    FROM listings
-    GROUP BY search_config_id
-  `,
+      `SELECT
+        la.search_config_id,
+        SUM(CASE WHEN l.is_blacklisted = 0 THEN 1 ELSE 0 END) as total,
+        SUM(CASE WHEN l.is_blacklisted = 0 AND l.is_seen = 0 THEN 1 ELSE 0 END) as unseen,
+        SUM(CASE WHEN l.is_favorite = 1 AND l.is_blacklisted = 0 THEN 1 ELSE 0 END) as favorites,
+        SUM(CASE WHEN l.is_blacklisted = 1 THEN 1 ELSE 0 END) as blacklisted
+      FROM listing_agents la
+      INNER JOIN listings l ON l.id = la.listing_id
+      GROUP BY la.search_config_id`,
     )
     .all();
 }
 
-// Stats for detached listings (search_config_id IS NULL) – favorites of deleted agents
+// Stats for detached listings (no agent associations) – favorites of deleted agents
 export function getOrphanStats() {
   const row = db
     .prepare(
-      `
-    SELECT
-      COUNT(*) as total,
-      SUM(CASE WHEN is_seen = 0 THEN 1 ELSE 0 END) as unseen,
-      SUM(CASE WHEN is_favorite = 1 THEN 1 ELSE 0 END) as favorites
-    FROM listings
-    WHERE search_config_id IS NULL AND is_blacklisted = 0
-  `,
+      `SELECT
+        COUNT(*) as total,
+        SUM(CASE WHEN is_seen = 0 THEN 1 ELSE 0 END) as unseen,
+        SUM(CASE WHEN is_favorite = 1 THEN 1 ELSE 0 END) as favorites
+      FROM listings
+      WHERE id NOT IN (SELECT listing_id FROM listing_agents)
+        AND is_blacklisted = 0`,
     )
     .get();
   return { total: row?.total ?? 0, unseen: row?.unseen ?? 0, favorites: row?.favorites ?? 0 };
diff --git a/src/providers/immoscout24/index.js b/src/providers/immoscout24/index.js
index 2d2ad7b..2a4bb46 100644
--- a/src/providers/immoscout24/index.js
+++ b/src/providers/immoscout24/index.js
@@ -313,7 +313,7 @@ function collectListingsFromResponse(payload) {
  *
  * @param {string} inputUrl        – Web URL or mobile API URL
  * @param {number} maxPages        – Maximum number of pages
- * @param {{ signal?: AbortSignal, onProgress?: Function }} opts
+ * @param {{ signal?: AbortSignal, onProgress?: Function, knownIds?: Set }} opts
  * @returns {Promise<object[]>}    – Normalized listings
  */
 export async function scrape(inputUrl, maxPages = 10, opts = {}) {
@@ -331,7 +331,8 @@ export async function scrape(inputUrl, maxPages = 10, opts = {}) {
  * @returns {Promise<{ mobileUrl: string, hitCount: number|string, pageCount: number, targetPages: number, pages: Array<{ pageNum: number, listings: object[] }> }>}
  */
 export async function scrapePages(inputUrl, maxPages = 10, opts = {}) {
-  const { signal, onProgress, log = console.log } = opts;
+  const { signal, onProgress, knownIds = new Set(), log = console.log } = opts;
+  const EARLY_STOP_THRESHOLD = 3; // stop after N consecutive pages with no new listings
 
   // Detect if already an API URL
   const isApiEndpoint = inputUrl.includes('api.mobile.immobilienscout24.de');
@@ -362,17 +363,46 @@ export async function scrapePages(inputUrl, maxPages = 10, opts = {}) {
   pages.push({ pageNum: 1, ...collectListingsFromResponse(firstPage) });
   onProgress?.({ pageNum: 1, maxPages: targetPages });
 
+  // Early-stop tracking: count consecutive pages with 0 new listings.
+  let consecutiveKnownPages = 0;
+  if (knownIds.size > 0 && pages[0].listings.length > 0) {
+    const newOnFirst = pages[0].listings.filter((l) => !knownIds.has(l.id)).length;
+    if (newOnFirst === 0) consecutiveKnownPages++;
+  }
+
   for (let p = 2; p <= targetPages; p++) {
     if (signal?.aborted) {
       log(`[immoscout24] Abbruch nach Seite ${p - 1}`);
       break;
     }
 
+    // Early stop: skip remaining pages if N consecutive pages had no new listings
+    if (knownIds.size > 0 && consecutiveKnownPages >= EARLY_STOP_THRESHOLD) {
+      log(
+        `[immoscout24] ⚡ Cache-Stopp: ${EARLY_STOP_THRESHOLD} Seiten ohne neue Inserate – überspringe Seiten ${p}–${targetPages}.`,
+      );
+      break;
+    }
+
     await sleep(600 + Math.random() * 600);
 
     const pageData = await requestPage(apiBase, p, log);
-    pages.push({ pageNum: p, ...collectListingsFromResponse(pageData) });
+    const pageResult = collectListingsFromResponse(pageData);
+    pages.push({ pageNum: p, ...pageResult });
     onProgress?.({ pageNum: p, maxPages: targetPages });
+
+    // Track new-vs-known for early stop
+    if (knownIds.size > 0 && pageResult.listings.length > 0) {
+      const newOnPage = pageResult.listings.filter((l) => !knownIds.has(l.id)).length;
+      if (newOnPage === 0) {
+        consecutiveKnownPages++;
+        log(
+          `[immoscout24] Seite ${p}: 0 neue Inserate (${consecutiveKnownPages}/${EARLY_STOP_THRESHOLD} zum Stopp)`,
+        );
+      } else {
+        consecutiveKnownPages = 0;
+      }
+    }
   }
 
   const results = pages.flatMap((p) => p.listings);
diff --git a/src/scrapers/engine.js b/src/scrapers/engine.js
index d10d5b9..a9592cf 100644
--- a/src/scrapers/engine.js
+++ b/src/scrapers/engine.js
@@ -72,6 +72,8 @@ export async function crawl(sourceConfig, opts = {}) {
   const maxPages = sourceConfig.maxPages ?? 5;
   const signal = opts.signal;
   const onProgress = typeof opts.onProgress === 'function' ? opts.onProgress : null;
+  const knownIds = opts.knownIds || new Set();
+  const EARLY_STOP_THRESHOLD = 3; // stop after N consecutive pages with no new listings
 
   const browser = await chromium.launch({ headless: true });
   const context = await browser.newContext({
@@ -88,6 +90,7 @@ export async function crawl(sourceConfig, opts = {}) {
   // Duplicate detection: track listing IDs from the previous page
   let prevPageIds = new Set();
   let duplicatePageCount = 0;
+  let consecutiveKnownPages = 0;
 
   try {
     let currentUrl = sourceConfig.url;
@@ -162,6 +165,25 @@ export async function crawl(sourceConfig, opts = {}) {
 
       allListings.push(...pageListings);
 
+      // Early-stop: if all listings on this page are already known, count toward threshold.
+      if (knownIds.size > 0 && pageListings.length > 0) {
+        const newOnPage = pageListings.filter((l) => !knownIds.has(l.id)).length;
+        if (newOnPage === 0) {
+          consecutiveKnownPages++;
+          console.log(
+            `[engine] Page ${pageNum}: 0 new listings (${consecutiveKnownPages}/${EARLY_STOP_THRESHOLD} to stop)`,
+          );
+          if (consecutiveKnownPages >= EARLY_STOP_THRESHOLD) {
+            console.log(
+              `[engine] ⚡ Cache stop: ${EARLY_STOP_THRESHOLD} consecutive pages with no new listings – skipping remaining pages.`,
+            );
+            break;
+          }
+        } else {
+          consecutiveKnownPages = 0;
+        }
+      }
+
       // Report progress (completed pages)
       try {
         onProgress && onProgress({ pageNum, maxPages: effectiveMaxPages });
diff --git a/src/services/scraperService.js b/src/services/scraperService.js
index 811deb9..8923589 100644
--- a/src/services/scraperService.js
+++ b/src/services/scraperService.js
@@ -58,20 +58,33 @@ export async function runScrapeForConfig(searchConfig, hooks = {}) {
   const startedAt = Date.now();
 
   try {
+    // Build agent-specific known IDs for early-stop caching and new-listing detection.
+    const knownIds = new Set(
+      getExistingIds(searchConfig.provider, searchConfig.listing_type, searchConfig.id),
+    );
+
     // Use the appropriate engine based on the provider
     const listings = usesCustomScraper
-      ? await provider.scrape(url, maxPages, { signal: hooks.signal, onProgress: hooks.onProgress })
-      : await crawl(crawlConfig, { signal: hooks.signal, onProgress: hooks.onProgress });
+      ? await provider.scrape(url, maxPages, {
+          signal: hooks.signal,
+          onProgress: hooks.onProgress,
+          knownIds,
+        })
+      : await crawl(crawlConfig, {
+          signal: hooks.signal,
+          onProgress: hooks.onProgress,
+          knownIds,
+        });
     console.log(`[scraper] ${listings.length} listings found.`);
 
-    const existingIds = new Set(
-      getExistingIds(searchConfig.provider, searchConfig.listing_type, searchConfig.id),
-    );
+    // Re-use knownIds (built before scraping) – no DB writes happened during scrape
+    const existingIds = knownIds;
     const scrapedIds = new Set();
     let newCount = 0;
     const scrapeTime = now(); // uniform timestamp for all listings in this run
 
-    for (const listing of listings) {
+    for (let i = 0; i < listings.length; i++) {
+      const listing = listings[i];
       const isNew = !existingIds.has(listing.id);
       if (isNew) newCount++;
 
@@ -94,6 +107,8 @@ export async function runScrapeForConfig(searchConfig, hooks = {}) {
         available_from: listing.availableFrom ?? null,
         first_seen: scrapeTime,
         last_seen: scrapeTime,
+        scrape_rank: i + 1,
+        run_id: runId,
       });
 
       scrapedIds.add(listing.id);

From 7a3714aa76b73c2db12f9534541881a69972a65d Mon Sep 17 00:00:00 2001
From: chenkel-data <chenkel.data@gmail.com>
Date: Fri, 3 Apr 2026 16:37:25 +0200
Subject: [PATCH 2/3] fix format

---
 src/db/database.js | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/src/db/database.js b/src/db/database.js
index dea86dc..1932d25 100644
--- a/src/db/database.js
+++ b/src/db/database.js
@@ -122,9 +122,7 @@ safeAlter('ALTER TABLE listing_agents ADD COLUMN last_scraped_run_id INTEGER');
 db.exec('CREATE INDEX IF NOT EXISTS idx_listings_link ON listings(link)');
 
 // Index for fast agent lookups on the junction table
-db.exec(
-  'CREATE INDEX IF NOT EXISTS idx_listing_agents_config ON listing_agents(search_config_id)',
-);
+db.exec('CREATE INDEX IF NOT EXISTS idx_listing_agents_config ON listing_agents(search_config_id)');
 
 // Migration: populate listing_agents from existing search_config_id values
 {
@@ -137,9 +135,7 @@ db.exec(
     )
     .run();
   if (result.changes > 0)
-    console.log(
-      `[db] ${result.changes} listing-agent link(s) migrated to listing_agents table.`,
-    );
+    console.log(`[db] ${result.changes} listing-agent link(s) migrated to listing_agents table.`);
 }
 
 // Migration: set search_config_id = NULL for orphaned listings (deleted agents)
@@ -454,8 +450,7 @@ export function getListings({
 
   // Filter by agent via junction table
   if (searchConfigId) {
-    joinClause =
-      'INNER JOIN listing_agents la ON la.listing_id = l.id AND la.search_config_id = ?';
+    joinClause = 'INNER JOIN listing_agents la ON la.listing_id = l.id AND la.search_config_id = ?';
     params.push(searchConfigId);
   }
 

From e9a3319fd08e0782907231256b34c56d964c3145 Mon Sep 17 00:00:00 2001
From: chenkel-data <chenkel.data@gmail.com>
Date: Fri, 3 Apr 2026 17:00:38 +0200
Subject: [PATCH 3/3] readme

---
 README.md | 91 ++++++++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 74 insertions(+), 17 deletions(-)

diff --git a/README.md b/README.md
index ea58cd6..e806f5e 100644
--- a/README.md
+++ b/README.md
@@ -71,12 +71,44 @@ Keyword blacklist in `config/default.json`:
 
 ## Data Model
 
-`data/listings.db` is created automatically.
+### Table `listings`
+One row per unique listing, deduplicated by provider ID. Stores all scraped content plus user state (`is_seen`, `is_favorite`, `is_blacklisted`). Tracks when a listing was first and last seen and its position in the most recent scrape.
 
-- **`listings`** – listings with price, size, address, timestamps and flags (`is_seen`, `is_favorite`, `is_blacklisted`)
-- **`search_configs`** – search agent configurations
-- **`scrape_runs`** – run history per agent
-- **`blacklist`** – permanent exclusions by ID or URL
+```
+id · source · title · price · size · rooms · address · description · publisher
+link · image · images · provider · listing_type
+is_seen · is_favorite · is_blacklisted · blacklisted_at · favorited_at
+first_seen · last_seen · listed_at · available_from · scrape_rank
+```
+
+### Table `search_configs`
+One row per search agent. Defines provider, listing type, page limit, search URL, and enabled state.
+
+```
+id · name · provider · listing_type · max_pages · extra_params · enabled · created_at
+```
+
+### Table `listing_agents`
+Junction table linking listings to the agents that found them (n:m). Records the rank within that agent's run and which run last actively scraped the listing.
+
+```
+listing_id · search_config_id · scrape_rank · last_scraped_run_id
+```
+
+### Table `scrape_runs`
+Log of every scrape execution with timing, counts, and error info.
+
+```
+id · source · provider · listing_type · search_config_id
+started_at · ended_at · status · new_count · total_count · error
+```
+
+### Table `blacklist`
+Blocked listing IDs or URLs, including keyword-based blocks that have no matching listing row.
+
+```
+id · listing_id · url · created_at
+```
 
 ---
 
@@ -91,11 +123,11 @@ src/                      Express backend
   server.js               Entry point, middleware, routes
   routes/                 listings, scraper, configs
   scrapers/engine.js      Playwright runner + CSS selector config
-  providers/              Adapter registry + Kleinanzeigen implementation
+  providers/              Adapter registry + provider implementations
   services/               Scrape orchestration per agent
   db/database.js          node:sqlite – schema, migrations, upserts
 
-config/default.json       Keyword & neighborhood blacklist
+config/default.json       Global blacklist keyword config
 data/listings.db          SQLite file (auto-created)
 ```
 
@@ -104,16 +136,41 @@ data/listings.db          SQLite file (auto-created)
 ## API
 
 ```
-GET    /api/listings                   Fetch listings (filter via query params)
-PATCH  /api/listings/:id/seen          Mark as seen
-PATCH  /api/listings/:id/favorite      Toggle favorite
-POST   /api/listings/:id/blacklist     Blacklist listing
-DELETE /api/listings/:id/blacklist     Remove from blacklist
-POST   /api/scrape                     Scrape all active agents
-POST   /api/scrape/:configId           Scrape a single agent
-GET    /api/configs                    Get agents
-POST   /api/configs                    Create agent
-GET    /api/providers                  List available providers
+GET    /api/listings                      Fetch listings (filter via query params)
+GET    /api/listings/stats                Aggregate listing counters
+GET    /api/listings/stats/per-config     Listing counters per agent + orphan stats
+GET    /api/listings/runs                 Recent scrape runs
+PATCH  /api/listings/seen-all             Mark all listings as seen
+PATCH  /api/listings/:id/seen             Mark a listing as seen
+PATCH  /api/listings/:id/unseen           Mark a listing as unseen
+PATCH  /api/listings/:id/favorite         Toggle favorite
+POST   /api/listings/:id/blacklist        Blacklist a listing
+DELETE /api/listings/:id/blacklist        Remove listing from blacklist
+DELETE /api/listings/reset                Delete unpinned listings
+DELETE /api/listings/reset/:configId      Delete unpinned listings for one agent
+DELETE /api/listings/clear-favorites      Clear all favorites
+DELETE /api/listings/clear-favorites/:configId
+                                          Clear favorites for one agent
+DELETE /api/listings/clear-blacklist      Clear blacklist flags
+DELETE /api/listings/clear-blacklist/:configId
+                                          Clear blacklist flags for one agent
+GET    /api/listings/:id/images           Fetch or return cached gallery images
+POST   /api/listings/batch-images         Batch image fetch for listing cards
+
+GET    /api/configs                       Get agents
+POST   /api/configs                       Create agent
+PATCH  /api/configs/:id                   Update agent
+DELETE /api/configs/:id                   Delete agent
+POST   /api/configs/infer-url             Infer provider + listing type from URL
+
+POST   /api/scrape                        Scrape all active agents
+POST   /api/scrape/:configId              Scrape a single enabled agent
+POST   /api/scrape/stop                   Cancel a running scrape
+GET    /api/scrape/status                 Current scrape progress
+GET    /api/scrape/config                 Read global scrape config
+PATCH  /api/scrape/config                 Update global scrape config
+
+GET    /api/providers                     List available providers
 ```
 
 ---