feat: add caching-logic, extend data-model for shared listings

README.md

@@ -71,12 +71,44 @@ Keyword blacklist in `config/default.json`:
 
 ## Data Model
 
-`data/listings.db` is created automatically.
+### Table `listings`
+One row per unique listing, deduplicated by provider ID. Stores all scraped content plus user state (`is_seen`, `is_favorite`, `is_blacklisted`). Tracks when a listing was first and last seen and its position in the most recent scrape.
 
-- **`listings`** – listings with price, size, address, timestamps and flags (`is_seen`, `is_favorite`, `is_blacklisted`)
-- **`search_configs`** – search agent configurations
-- **`scrape_runs`** – run history per agent
-- **`blacklist`** – permanent exclusions by ID or URL
+```
+id · source · title · price · size · rooms · address · description · publisher
+link · image · images · provider · listing_type
+is_seen · is_favorite · is_blacklisted · blacklisted_at · favorited_at
+first_seen · last_seen · listed_at · available_from · scrape_rank
+```
+
+### Table `search_configs`
+One row per search agent. Defines provider, listing type, page limit, search URL, and enabled state.
+
+```
+id · name · provider · listing_type · max_pages · extra_params · enabled · created_at
+```
+
+### Table `listing_agents`
+Junction table linking listings to the agents that found them (n:m). Records the rank within that agent's run and which run last actively scraped the listing.
+
+```
+listing_id · search_config_id · scrape_rank · last_scraped_run_id
+```
+
+### Table `scrape_runs`
+Log of every scrape execution with timing, counts, and error info.
+
+```
+id · source · provider · listing_type · search_config_id
+started_at · ended_at · status · new_count · total_count · error
+```
+
+### Table `blacklist`
+Blocked listing IDs or URLs, including keyword-based blocks that have no matching listing row.
+
+```
+id · listing_id · url · created_at
+```
 
 ---
 
@@ -91,11 +123,11 @@ src/                      Express backend
   server.js               Entry point, middleware, routes
   routes/                 listings, scraper, configs
   scrapers/engine.js      Playwright runner + CSS selector config
-  providers/              Adapter registry + Kleinanzeigen implementation
+  providers/              Adapter registry + provider implementations
   services/               Scrape orchestration per agent
   db/database.js          node:sqlite – schema, migrations, upserts
 
-config/default.json       Keyword & neighborhood blacklist
+config/default.json       Global blacklist keyword config
 data/listings.db          SQLite file (auto-created)
 ```
 
@@ -104,16 +136,41 @@ data/listings.db          SQLite file (auto-created)
 ## API
 
 ```
-GET    /api/listings                   Fetch listings (filter via query params)
-PATCH  /api/listings/:id/seen          Mark as seen
-PATCH  /api/listings/:id/favorite      Toggle favorite
-POST   /api/listings/:id/blacklist     Blacklist listing
-DELETE /api/listings/:id/blacklist     Remove from blacklist
-POST   /api/scrape                     Scrape all active agents
-POST   /api/scrape/:configId           Scrape a single agent
-GET    /api/configs                    Get agents
-POST   /api/configs                    Create agent
-GET    /api/providers                  List available providers
+GET    /api/listings                      Fetch listings (filter via query params)
+GET    /api/listings/stats                Aggregate listing counters
+GET    /api/listings/stats/per-config     Listing counters per agent + orphan stats
+GET    /api/listings/runs                 Recent scrape runs
+PATCH  /api/listings/seen-all             Mark all listings as seen
+PATCH  /api/listings/:id/seen             Mark a listing as seen
+PATCH  /api/listings/:id/unseen           Mark a listing as unseen
+PATCH  /api/listings/:id/favorite         Toggle favorite
+POST   /api/listings/:id/blacklist        Blacklist a listing
+DELETE /api/listings/:id/blacklist        Remove listing from blacklist
+DELETE /api/listings/reset                Delete unpinned listings
+DELETE /api/listings/reset/:configId      Delete unpinned listings for one agent
+DELETE /api/listings/clear-favorites      Clear all favorites
+DELETE /api/listings/clear-favorites/:configId
+                                          Clear favorites for one agent
+DELETE /api/listings/clear-blacklist      Clear blacklist flags
+DELETE /api/listings/clear-blacklist/:configId
+                                          Clear blacklist flags for one agent
+GET    /api/listings/:id/images           Fetch or return cached gallery images
+POST   /api/listings/batch-images         Batch image fetch for listing cards
+
+GET    /api/configs                       Get agents
+POST   /api/configs                       Create agent
+PATCH  /api/configs/:id                   Update agent
+DELETE /api/configs/:id                   Delete agent
+POST   /api/configs/infer-url             Infer provider + listing type from URL
+
+POST   /api/scrape                        Scrape all active agents
+POST   /api/scrape/:configId              Scrape a single enabled agent
+POST   /api/scrape/stop                   Cancel a running scrape
+GET    /api/scrape/status                 Current scrape progress
+GET    /api/scrape/config                 Read global scrape config
+PATCH  /api/scrape/config                 Update global scrape config
+
+GET    /api/providers                     List available providers
 ```
 
 ---

client/src/App.jsx

@@ -18,21 +18,6 @@ import { api } from './api.js';
 import { TABS, ITEMS_PER_PAGE, LISTING_TYPE_LABELS, LISTING_TYPE_COLORS, PROVIDER_COLORS, PROVIDER_LABELS } from './constants.js';
 
 const FILTERS_STORAGE_KEY = 'immo.filters.v1';
-const EMPTY_STATS = { total: 0, unseen: 0, favorites: 0, blacklisted: 0 };
-
-function buildStatsFromListings(list) {
-  const stats = { ...EMPTY_STATS };
-  for (const listing of list) {
-    if (listing.is_blacklisted) {
-      stats.blacklisted += 1;
-      continue;
-    }
-    stats.total += 1;
-    if (!listing.is_seen) stats.unseen += 1;
-    if (listing.is_favorite) stats.favorites += 1;
-  }
-  return stats;
-}
 
 function readPersistedFilters() {
   const defaults = {
@@ -69,7 +54,7 @@ export default function App() {
   }, []);
 
   const {
-    listings, loading, orphanStats,
+    listings, loading, stats, orphanStats, configStats,
     loadListings, loadStats, loadConfigStats,
     handleSeen, handleFavorite, handleBlacklist, handleUnblacklist,
     handleMarkAllSeen, handleReset, handleResetConfig,
@@ -142,19 +127,24 @@ export default function App() {
   /* config selection handler – always resets tab to ALL when switching */
   const handleSelectConfig = useCallback((configId) => {
     setActiveConfigId(configId ?? null);
-    currentListingParamsRef.current = { include_blacklisted: true };
+    const params = { include_blacklisted: true };
+    if (configId) params.search_config_id = configId;
+    currentListingParamsRef.current = params;
+    loadListings(params);
     setActiveTab(TABS.ALL);
     setPage(1);
-  }, []);
+  }, [loadListings]);
 
   /* navigate home: deselect agent, reset to unseen tab */
   const handleNavigateHome = useCallback(() => {
     setActiveConfigId(null);
-    currentListingParamsRef.current = { include_blacklisted: true };
+    const params = { include_blacklisted: true };
+    currentListingParamsRef.current = params;
+    loadListings(params);
     setActiveTab(TABS.UNSEEN);
     setPage(1);
     setSearchQuery(''); setMinPrice(''); setMaxPrice(''); setMinSize(''); setMinRooms(''); setListingTypeFilter(''); setProviderFilter(''); setPublisherFilter(''); setMaxAvailableFrom('');
-  }, []);
+  }, [loadListings]);
 
   const isProviderFilterActive = !activeConfigId && activeTab === TABS.ALL;
 
@@ -183,10 +173,9 @@ export default function App() {
   }, [listings, listingTypeFilter, publisherFilter, searchQuery, minPrice, maxPrice, minSize, minRooms, maxAvailableFrom, scrapeConfig]);
 
   const filteredBase = useMemo(() => {
-    if (activeConfigId) return uiFilteredListings.filter(l => l.search_config_id === activeConfigId);
     if (isProviderFilterActive && providerFilter) return uiFilteredListings.filter(l => l.provider === providerFilter);
     return uiFilteredListings;
-  }, [uiFilteredListings, activeConfigId, isProviderFilterActive, providerFilter]);
+  }, [uiFilteredListings, isProviderFilterActive, providerFilter]);
 
   const filtered = useMemo(() => {
     let list = [...filteredBase];
@@ -212,13 +201,17 @@ export default function App() {
   }, [filteredBase, activeTab]);
 
   const tabCounts = useMemo(() => {
+    let base = listings;
+    if (listingTypeFilter) base = base.filter(l => l.listing_type === listingTypeFilter);
+    if (isProviderFilterActive && providerFilter) base = base.filter(l => l.provider === providerFilter);
+    const nonBlacklisted = base.filter(l => !l.is_blacklisted);
     return {
-      [TABS.ALL]: filteredBase.filter(l => !l.is_blacklisted).length,
-      [TABS.UNSEEN]: filteredBase.filter(l => !l.is_blacklisted && !l.is_seen).length,
-      [TABS.FAVORITES]: filteredBase.filter(l => !l.is_blacklisted && l.is_favorite).length,
-      [TABS.BLACKLISTED]: filteredBase.filter(l => l.is_blacklisted).length,
+      [TABS.ALL]: nonBlacklisted.length,
+      [TABS.UNSEEN]: nonBlacklisted.filter(l => !l.is_seen).length,
+      [TABS.FAVORITES]: nonBlacklisted.filter(l => l.is_favorite).length,
+      [TABS.BLACKLISTED]: base.filter(l => l.is_blacklisted).length,
     };
-  }, [filteredBase]);
+  }, [listings, listingTypeFilter, isProviderFilterActive, providerFilter]);
 
   const activeConfigStats = useMemo(() => ({
     total: tabCounts[TABS.ALL],
@@ -227,30 +220,6 @@ export default function App() {
     blacklisted: tabCounts[TABS.BLACKLISTED],
   }), [tabCounts]);
 
-  const sidebarConfigStats = useMemo(() => {
-    const map = {};
-    for (const cfg of configs) map[cfg.id] = { ...EMPTY_STATS };
-    for (const listing of uiFilteredListings) {
-      const configId = listing.search_config_id;
-      if (configId == null) continue;
-      if (!map[configId]) map[configId] = { ...EMPTY_STATS };
-      const bucket = map[configId];
-      if (listing.is_blacklisted) {
-        bucket.blacklisted += 1;
-        continue;
-      }
-      bucket.total += 1;
-      if (!listing.is_seen) bucket.unseen += 1;
-      if (listing.is_favorite) bucket.favorites += 1;
-    }
-    return map;
-  }, [configs, uiFilteredListings]);
-
-  const sidebarGlobalStats = useMemo(() => (
-    buildStatsFromListings(uiFilteredListings)
-  ), [uiFilteredListings]);
-
-
   const pages = Math.max(1, Math.ceil(filtered.length / ITEMS_PER_PAGE));
   const safePage = Math.min(page, pages);
   const paginated = filtered.slice((safePage - 1) * ITEMS_PER_PAGE, safePage * ITEMS_PER_PAGE);
@@ -301,9 +270,9 @@ export default function App() {
           <AgentSidebar
             configs={configs}
             providers={providers}
-            configStats={sidebarConfigStats}
+            configStats={configStats}
             orphanStats={orphanStats}
-            globalStats={sidebarGlobalStats}
+            globalStats={stats}
             activeConfigId={activeConfigId}
             onSelectConfig={handleSelectConfig}
             onAdd={addConfig}
@@ -430,6 +399,7 @@ export default function App() {
               providerFilter={providerFilter}
               providers={providers}
               showProviderFilter={isProviderFilterActive}
+              showListingTypeFilter={!activeConfigId}
               onTabChange={setActiveTab} onListingTypeChange={setListingTypeFilter}
               onSearch={setSearchQuery} onMinPrice={setMinPrice} onMaxPrice={setMaxPrice} onMinSize={setMinSize} onMinRooms={setMinRooms}
               onPublisherFilter={setPublisherFilter}
@@ -457,7 +427,7 @@ export default function App() {
           onClose={() => setSidebarOpen(false)}
           onReset={() => handleReset(currentListingParamsRef.current)}
           showToast={showToast}
-          onSaved={loadScrapeConfig}
+          onSaved={() => { loadScrapeConfig(); reloadAll(); }}
           onClearFavorites={() =>
             askConfirm({
               title: 'Alle Favoriten löschen?',

client/src/components/FilterBar.jsx

@@ -3,12 +3,12 @@ import { TABS, LISTING_TYPE_LABELS, PROVIDER_LABELS } from '../constants.js';
 export default function FilterBar({
   activeTab, stats, listingTypeFilter,
   searchQuery, minPrice, maxPrice, minSize, minRooms,
-  publisherFilter, providerFilter, providers, showProviderFilter,
+  publisherFilter, providerFilter, providers, showProviderFilter, showListingTypeFilter = true,
   tabCounts,
   maxAvailableFrom,
   onTabChange, onListingTypeChange, onSearch, onMinPrice, onMaxPrice, onMinSize, onMinRooms, onPublisherFilter, onProviderFilter, onMaxAvailableFrom, onReset,
 }) {
-  const hasFilters = searchQuery || minPrice || maxPrice || minSize || minRooms || publisherFilter || maxAvailableFrom || (showProviderFilter && providerFilter);
+  const hasFilters = searchQuery || minPrice || maxPrice || minSize || minRooms || publisherFilter || maxAvailableFrom || (showProviderFilter && providerFilter) || (showListingTypeFilter && listingTypeFilter);
 
   const tabs = [
     { id: TABS.ALL, label: 'Alle', count: stats.total, icon: (
@@ -48,9 +48,11 @@ export default function FilterBar({
       </nav>
 
       <div className="filter-controls">
-        <select className="filter-select" value={listingTypeFilter} onChange={(e) => onListingTypeChange(e.target.value)}>
-          {typeOptions.map((t) => <option key={t.id} value={t.id}>{t.label}</option>)}
-        </select>
+        {showListingTypeFilter && (
+          <select className="filter-select" value={listingTypeFilter} onChange={(e) => onListingTypeChange(e.target.value)}>
+            {typeOptions.map((t) => <option key={t.id} value={t.id}>{t.label}</option>)}
+          </select>
+        )}
 
         {showProviderFilter && (
           <select className="filter-select" value={providerFilter} onChange={(e) => onProviderFilter(e.target.value)}>

client/src/components/ListingCard.jsx

@@ -55,6 +55,9 @@ const ListingCard = memo(function ListingCard({ listing: l, onSeen, onFavorite,
   const providerLabel = PROVIDER_LABELS[l.provider] || l.provider;
   const publishedLabel = formatListingDate(l.listed_at);
   const availableFromLabel = formatAvailableFrom(l.available_from);
+  const firstSeenLabel = formatListingDate(l.first_seen);
+  const lastSeenLabel = formatListingDate(l.last_seen);
+  const seenMultipleTimes = l.first_seen && l.last_seen && l.first_seen !== l.last_seen;
 
   return (
     <article className={`card ${isNew ? 'card--new' : ''} ${l.is_seen ? 'card--seen' : ''} ${l.is_blacklisted ? 'card--blacklisted' : ''}`}>
@@ -114,6 +117,9 @@ const ListingCard = memo(function ListingCard({ listing: l, onSeen, onFavorite,
           </div>
         </div>
         <h2 className="card-title">{l.title}</h2>
+        {l.link?.includes('/expose/') && (
+          <p className="card-expose-id">ID: {l.link.split('/expose/')[1]}</p>
+        )}
         {l.address && (
           <p className="card-address">
             <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" className="addr-icon"><path d="M21 10c0 7-9 13-9 13S3 17 3 10a9 9 0 1 1 18 0z"/><circle cx="12" cy="10" r="3"/></svg>
@@ -144,6 +150,20 @@ const ListingCard = memo(function ListingCard({ listing: l, onSeen, onFavorite,
               <span>{l.publisher}</span>
             </span>
           )}
+          {l.first_seen && (
+            <span className="card-date card-date--scraped" title={new Date(l.first_seen).toLocaleString('de-DE')}>
+              <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" className="date-icon"><circle cx="11" cy="11" r="8"/><line x1="21" y1="21" x2="16.65" y2="16.65"/></svg>
+              <span className="card-date-label">Entdeckt:</span>
+              <span>{firstSeenLabel}</span>
+            </span>
+          )}
+          {seenMultipleTimes && (
+            <span className="card-date card-date--scraped" title={new Date(l.last_seen).toLocaleString('de-DE')}>
+              <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" className="date-icon"><polyline points="23 4 23 10 17 10"/><polyline points="1 20 1 14 7 14"/><path d="M3.51 9a9 9 0 0 1 14.85-3.36L23 10M1 14l4.64 4.36A9 9 0 0 0 20.49 15"/></svg>
+              <span className="card-date-label">Zuletzt gesehen:</span>
+              <span>{lastSeenLabel}</span>
+            </span>
+          )}
         </div>
         <a href={l.link} target="_blank" rel="noopener noreferrer" className="card-open-btn" onClick={handleOpen}>
           Öffnen

client/src/hooks/useListings.js

@@ -82,11 +82,16 @@ export function useListings(showToast) {
     setListings((prev) => prev.map((l) => (l.id === id ? { ...l, is_seen: willBeSeen ? 1 : 0 } : l)));
     const delta = willBeSeen ? -1 : 1;
     setStats((prev) => ({ ...prev, unseen: Math.max(0, (prev.unseen ?? 0) + delta) }));
-    if (current.search_config_id != null) {
+    const agentIds = current.agent_ids || [];
+    if (agentIds.length > 0) {
       setConfigStats((prev) => {
-        const existing = prev[current.search_config_id];
-        if (!existing) return prev;
-        return { ...prev, [current.search_config_id]: { ...existing, unseen: Math.max(0, (existing.unseen ?? 0) + delta) } };
+        const next = { ...prev };
+        for (const configId of agentIds) {
+          const existing = next[configId];
+          if (!existing) continue;
+          next[configId] = { ...existing, unseen: Math.max(0, (existing.unseen ?? 0) + delta) };
+        }
+        return next;
       });
     }
   }, [listings]);
@@ -175,7 +180,7 @@ export function useListings(showToast) {
   const handleClearFavoritesByConfig = useCallback(async (configId, loadParams = {}) => {
     await api.listings.clearFavoritesByConfig(configId);
     setListings((prev) => prev.map((l) =>
-      l.search_config_id === configId ? { ...l, is_favorite: 0 } : l
+      (l.agent_ids || []).includes(configId) ? { ...l, is_favorite: 0 } : l
     ));
     await refreshStats();
     showToast?.('Favoriten dieses Agenten entfernt.', 'info');
@@ -191,7 +196,7 @@ export function useListings(showToast) {
   const handleClearBlacklistByConfig = useCallback(async (configId) => {
     await api.listings.clearBlacklistByConfig(configId);
     setListings((prev) => prev.map((l) => (
-      l.search_config_id === configId && l.is_blacklisted
+      (l.agent_ids || []).includes(configId) && l.is_blacklisted
         ? { ...l, is_blacklisted: 0 }
         : l
     )));

client/src/index.css

@@ -563,6 +563,7 @@ a { color: inherit; text-decoration: none; }
 .pill { font-size: .65rem; padding: 2px 7px; border-radius: var(--radius-full); background: var(--gray-100); color: var(--gray-600); font-weight: 600; }
 .card-title { font-size: .875rem; font-weight: 600; line-height: 1.4; display: -webkit-box; line-clamp: 2; -webkit-line-clamp: 2; -webkit-box-orient: vertical; overflow: hidden; margin-bottom: 4px; color: var(--gray-800); }
 .card-address { display: flex; align-items: flex-start; gap: 4px; font-size: .78rem; color: var(--text-muted); margin-bottom: 6px; }
+.card-expose-id { font-size: .72rem; color: var(--text-muted); opacity: .65; margin: -2px 0 6px; font-family: monospace; }
 .addr-icon { width: 13px; height: 13px; flex-shrink: 0; margin-top: 1px; color: var(--gray-400); }
 .card-desc { font-size: .78rem; color: var(--gray-400); line-height: 1.45; display: -webkit-box; line-clamp: 2; -webkit-line-clamp: 2; -webkit-box-orient: vertical; overflow: hidden; flex: 1; }
 .card-footer { display: flex; align-items: center; justify-content: space-between; padding: 10px 16px; border-top: 1px solid var(--gray-100); }