From 3b9bb2eb4f30b4852f5df6927105f8ca893c05a9 Mon Sep 17 00:00:00 2001 From: Joker Date: Thu, 11 Jun 2026 00:10:14 +0000 Subject: [PATCH 1/3] fix(pipeline): deduplicate opportunities in auto-sync route Root cause: auto-sync/route.ts used getOpportunity(parsed.reportId) for dedup, but createOpportunity generates random UUIDs, so the lookup never matched and created duplicates on every GET /api/pipeline/auto-sync call. Fix: use findOpportunityByCompany() for dedup (same as reports-scanner.ts). When match found, UPDATE with max value instead of CREATE. Also adds cleanup script to remove 960 existing duplicates from kanban.db. Bug: 1015 total opportunities, only 55 unique companies (960 duplicates). LawnStarter had 320 identical entries. --- scripts/cleanup-duplicate-opportunities.sh | 118 +++++++++++++++++++++ src/app/api/pipeline/auto-sync/route.ts | 12 ++- 2 files changed, 128 insertions(+), 2 deletions(-) create mode 100755 scripts/cleanup-duplicate-opportunities.sh diff --git a/scripts/cleanup-duplicate-opportunities.sh b/scripts/cleanup-duplicate-opportunities.sh new file mode 100755 index 000000000..ae1b30b96 --- /dev/null +++ b/scripts/cleanup-duplicate-opportunities.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# +# Cleanup script: Remove duplicate opportunities from kanban.db +# +# Dedup strategy: +# - Group by LOWER(TRIM(company)) +# - For each group, keep the OLDEST entry (lowest created_at) +# - Delete all newer duplicates +# +# Usage: +# ./scripts/cleanup-duplicate-opportunities.sh [--dry-run] [--db /path/to/kanban.db] +# +# Options: +# --dry-run Show what would be deleted without deleting +# --db PATH Path to kanban.db (default: data/kanban.db) + +set -euo pipefail + +# Parse args +DRY_RUN=false +DB_PATH="data/kanban.db" + +while [[ $# -gt 0 ]]; do + case "$1" in + --dry-run) DRY_RUN=true; shift ;; + --db) DB_PATH="$2"; shift 2 ;; + *) echo "Unknown option: $1"; exit 1 ;; + esac +done + +if [[ ! -f "$DB_PATH" ]]; then + echo "❌ Database not found: $DB_PATH" + exit 1 +fi + +echo "📦 Database: $DB_PATH" +if $DRY_RUN; then + echo "🔍 DRY RUN — no changes will be made" +else + echo "🗑️ LIVE MODE — duplicates WILL be deleted" +fi +echo "" + +# Get stats +TOTAL=$(sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM opportunities;") +UNIQUE=$(sqlite3 "$DB_PATH" "SELECT COUNT(DISTINCT LOWER(TRIM(company))) FROM opportunities;") +DUPES=$(sqlite3 "$DB_PATH" "SELECT COUNT(*) - COUNT(DISTINCT LOWER(TRIM(company))) FROM opportunities;") + +echo "📊 Total opportunities: $TOTAL" +echo "📈 Unique companies: $UNIQUE" +echo "🔁 Duplicates to remove: $DUPES" +echo "" + +if [[ "$DUPES" -eq 0 ]]; then + echo "✅ No duplicates found. Database is clean." + exit 0 +fi + +# Show top offenders +echo "Top duplicate offenders:" +sqlite3 -header -column "$DB_PATH" " + SELECT company, COUNT(*) as entries, COUNT(*) - 1 as to_remove + FROM opportunities + GROUP BY LOWER(TRIM(company)) + HAVING COUNT(*) > 1 + ORDER BY entries DESC + LIMIT 15; +" +echo "" + +if $DRY_RUN; then + echo "🔍 DRY RUN — showing entries that would be deleted (first 30):" + sqlite3 -header -column "$DB_PATH" " + SELECT o.id, o.company, SUBSTR(o.title, 1, 40) as title, o.stage, o.value, o.created_at + FROM opportunities o + WHERE o.id NOT IN ( + SELECT MIN(o2.id) + FROM opportunities o2 + GROUP BY LOWER(TRIM(o2.company)) + ) + ORDER BY o.company, o.created_at + LIMIT 30; + " + echo "" + echo "✅ DRY RUN complete. No changes made." + echo "Run without --dry-run to actually delete duplicates." +else + # Delete duplicates — keep the oldest (MIN(id)) per company + DELETED=$(sqlite3 "$DB_PATH" " + DELETE FROM opportunities + WHERE id NOT IN ( + SELECT MIN(o.id) + FROM opportunities o + GROUP BY LOWER(TRIM(o.company)) + ); + SELECT changes(); + ") + echo "✅ Deleted $DELETED duplicate opportunities" + + # Verify + REMAINING=$(sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM opportunities;") + echo "📊 Remaining opportunities: $REMAINING" + + # Check for remaining duplicates + STILL_DUPES=$(sqlite3 "$DB_PATH" " + SELECT COUNT(*) FROM ( + SELECT LOWER(TRIM(company)) as key + FROM opportunities + GROUP BY key + HAVING COUNT(*) > 1 + ); + ") + if [[ "$STILL_DUPES" -eq 0 ]]; then + echo "✅ No more duplicates by company." + else + echo "⚠️ Still have $STILL_DUPES company groups with duplicates" + fi +fi diff --git a/src/app/api/pipeline/auto-sync/route.ts b/src/app/api/pipeline/auto-sync/route.ts index 7497f431a..fa8a69b7e 100644 --- a/src/app/api/pipeline/auto-sync/route.ts +++ b/src/app/api/pipeline/auto-sync/route.ts @@ -7,6 +7,7 @@ import { getOpportunity, updateOpportunity, getPipelineKPIs, + findOpportunityByCompany, type CreateOpportunityInput, } from "@/lib/pipeline-db"; import { @@ -139,10 +140,17 @@ function syncReportsToPipeline(): SyncResult { continue; } - const existingOpp = getOpportunity(parsed.reportId); + // Dedup: find existing by company (same company = same deal) + // Previously used getOpportunity(parsed.reportId) which NEVER matched + // because createOpportunity generates a random UUID, not the reportId. + const existingOpp = findOpportunityByCompany(parsed.target); if (existingOpp) { - updateOpportunity(parsed.reportId, { + // UPDATE: keep max value, append notes + const newValue = getEstimatedValue(parsed.serviceType); + const maxValue = Math.max(existingOpp.value, newValue); + updateOpportunity(existingOpp.id, { + value: maxValue, notes: [ `Updated from report: ${file}`, `Confidence: ${parsed.confidence}%`, From cbf8a62e62a0ad65a3d6d08d758bf6b60d34ef64 Mon Sep 17 00:00:00 2001 From: Joker Date: Sat, 13 Jun 2026 01:07:30 +0000 Subject: [PATCH 2/3] feat(journal): auto-generate daily entry from activities - Add POST /api/journal/auto-generate endpoint - Aggregates activities for a given date - Generates narrative summary with success rate, agent activity, errors - Builds highlights array automatically - Skips if entry already exists for date - Add 'Auto-Generar Hoy' button to Journal UI with Sparkles icon - Implements deferred roadmap item 14.7 sub-task --- src/app/(dashboard)/journal/JournalClient.tsx | 52 +++++-- src/app/api/journal/auto-generate/route.ts | 136 ++++++++++++++++++ 2 files changed, 179 insertions(+), 9 deletions(-) create mode 100644 src/app/api/journal/auto-generate/route.ts diff --git a/src/app/(dashboard)/journal/JournalClient.tsx b/src/app/(dashboard)/journal/JournalClient.tsx index c757513cf..60fd244c1 100644 --- a/src/app/(dashboard)/journal/JournalClient.tsx +++ b/src/app/(dashboard)/journal/JournalClient.tsx @@ -1,7 +1,7 @@ "use client"; import { useState, useEffect } from "react"; -import { Plus } from "lucide-react"; +import { Plus, Sparkles } from "lucide-react"; import { JournalTimeline, JournalEntryCard, @@ -23,6 +23,7 @@ export default function JournalClient({ initialData }: { initialData?: JournalIn const [isModalOpen, setIsModalOpen] = useState(false); const [startDate, setStartDate] = useState(""); const [endDate, setEndDate] = useState(""); + const [generating, setGenerating] = useState(false); useEffect(() => { if (!initialData?.entries) { @@ -102,6 +103,28 @@ export default function JournalClient({ initialData }: { initialData?: JournalIn setIsModalOpen(true); }; + const handleAutoGenerate = async () => { + setGenerating(true); + try { + const today = new Date().toISOString().split("T")[0]; + const res = await authFetch("/api/journal/auto-generate", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ date: today }), + }); + const data = await res.json(); + if (data.generated && data.entry) { + setEntries([data.entry, ...entries]); + } else if (data.entry) { + setEntries([data.entry, ...entries.filter((e) => e.id !== data.entry.id)]); + } + } catch { + // ignore + } finally { + setGenerating(false); + } + }; + const handleCloseModal = () => { setIsModalOpen(false); setEditingEntry(null); @@ -128,14 +151,25 @@ export default function JournalClient({ initialData }: { initialData?: JournalIn Registra y revisa tus actividades diarias

- +
+ + +
{/* Filters */} diff --git a/src/app/api/journal/auto-generate/route.ts b/src/app/api/journal/auto-generate/route.ts new file mode 100644 index 000000000..988e86798 --- /dev/null +++ b/src/app/api/journal/auto-generate/route.ts @@ -0,0 +1,136 @@ +import { NextRequest, NextResponse } from "next/server"; +import { getActivities } from "@/lib/activities-db"; +import { createJournalEntry, listJournalEntries } from "@/lib/kanban-db"; + +export const dynamic = "force-dynamic"; + +/** + * POST /api/journal/auto-generate + * Auto-generates a journal entry from the day's activities. + * Body: { date: string (YYYY-MM-DD) } + * + * Aggregates activities for the given date and creates a narrative + * summary with highlights. Skips if entry already exists. + */ +export async function POST(request: NextRequest) { + try { + const body = await request.json(); + const date = body.date; + + if (!date || !/^\d{4}-\d{2}-\d{2}$/.test(date)) { + return NextResponse.json( + { error: "date is required (YYYY-MM-DD format)" }, + { status: 400 } + ); + } + + // Check if entry already exists for this date + const existing = listJournalEntries({ startDate: date, endDate: date }); + if (existing.length > 0) { + return NextResponse.json( + { message: "Journal entry already exists for this date", entry: existing[0] }, + { status: 200 } + ); + } + + // Fetch activities for the date + const result = getActivities({ + startDate: date, + endDate: date, + limit: 500, + sort: "oldest", + }); + + const activities = result.activities; + const total = result.total; + + if (total === 0) { + return NextResponse.json( + { message: "No activities found for this date", entry: null }, + { status: 200 } + ); + } + + // Build narrative from activities + const byType: Record = {}; + const byStatus: Record = {}; + const byAgent: Record = {}; + const errors: string[] = []; + const notableActions: string[] = []; + + for (const act of activities) { + byType[act.type] = (byType[act.type] || 0) + 1; + byStatus[act.status] = (byStatus[act.status] || 0) + 1; + if (act.agent) { + byAgent[act.agent] = (byAgent[act.agent] || 0) + 1; + } + if (act.status === "error") { + errors.push(`[${act.type}] ${act.description}`); + } + // Notable: security, build, or high-duration actions + if (act.type === "security" || act.type === "build" || (act.duration_ms && act.duration_ms > 60000)) { + notableActions.push(`${act.type}: ${act.description}`); + } + } + + const successCount = (byStatus["success"] || 0) + (byStatus["approved"] || 0); + const errorCount = (byStatus["error"] || 0) + (byStatus["rejected"] || 0); + const successRate = total > 0 ? Math.round((successCount / total) * 100) : 100; + const agents = Object.entries(byAgent).sort((a, b) => b[1] - a[1]); + const types = Object.entries(byType).sort((a, b) => b[1] - a[1]); + + // Compose narrative + const lines: string[] = []; + lines.push(`Resumen de operaciones del ${date}: ${total} actividades registradas con ${successRate}% de tasa de éxito.`); + + if (agents.length > 0) { + const agentSummary = agents.map(([name, count]) => `${name} (${count})`).join(", "); + lines.push(`Agentes activos: ${agentSummary}.`); + } + + if (types.length > 0) { + const typeSummary = types.slice(0, 5).map(([t, c]) => `${t}: ${c}`).join(", "); + lines.push(`Distribución: ${typeSummary}.`); + } + + if (errorCount > 0) { + lines.push(`⚠️ ${errorCount} errores detectados.`); + } + + if (notableActions.length > 0) { + lines.push(`Acciones destacadas: ${notableActions.slice(0, 5).join("; ")}.`); + } + + // Total tokens if available + const totalTokens = activities.reduce((sum, a) => sum + (a.tokens_used || 0), 0); + if (totalTokens > 0) { + lines.push(`Tokens consumidos: ${totalTokens.toLocaleString()}.`); + } + + const narrative = lines.join(" "); + + // Build highlights + const highlights: string[] = []; + if (successRate === 100) highlights.push("✅ 100% tasa de éxito"); + if (errorCount > 0) highlights.push(`❌ ${errorCount} errores`); + if (total > 50) highlights.push(`🔥 ${total} actividades (día intenso)`); + if (agents.length > 1) highlights.push(`🤖 ${agents.length} agentes activos`); + if (totalTokens > 10000) highlights.push(`📊 ${Math.round(totalTokens / 1000)}K tokens`); + highlights.push(`📈 ${successRate}% éxito`); + + // Create the entry + const entry = createJournalEntry({ + date, + narrative, + highlights: highlights.slice(0, 10), + }); + + return NextResponse.json({ entry, generated: true }, { status: 201 }); + } catch (error) { + console.error("Failed to auto-generate journal entry:", error); + return NextResponse.json( + { error: "Failed to auto-generate journal entry" }, + { status: 500 } + ); + } +} From a2e14d8b7de7fcb2fae427f0460f19552f103b51 Mon Sep 17 00:00:00 2001 From: Joker Date: Sat, 13 Jun 2026 01:07:48 +0000 Subject: [PATCH 3/3] docs: mark journal auto-generate as completed in roadmap --- ROADMAP.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ROADMAP.md b/ROADMAP.md index 5ac5c67da..8d1c784f9 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -689,7 +689,7 @@ - [x] DB: `operations_journal` table (created in Phase 14.1) - [x] API: `GET/POST/PUT/DELETE /api/journal` -- [ ] **⏳ Lógica: Auto-generar entrada diaria desde activities (deferred)** (est. ~3-4h) +- [x] **Lógica: Auto-generar entrada diaria desde activities** ✅ (commit cbf8a62, 2026-06-13) - [x] UI: Página `/journal` con timeline narrativo - [x] UI: Entry editor para añadir highlights manuales - **Archivos:** `src/app/api/journal/route.ts`, `src/app/api/journal/[id]/route.ts`, `src/app/(dashboard)/journal/page.tsx`, `src/lib/kanban-db.ts`, `src/components/journal/`