From 2c9f78303308752ad13dcf61b0ccf5b4f723ed82 Mon Sep 17 00:00:00 2001 From: Alexandre <44178713+alexbelgium@users.noreply.github.com> Date: Tue, 26 Aug 2025 08:44:00 +0200 Subject: [PATCH 01/36] Add Species Management button and view handling --- homepage/views.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/homepage/views.php b/homepage/views.php index a137c7854..501989eaf 100644 --- a/homepage/views.php +++ b/homepage/views.php @@ -173,6 +173,7 @@ function update_species_list($filename, $species, $add) { + "; } @@ -203,6 +204,10 @@ function update_species_list($filename, $species, $add) { $species_list="whitelist"; include('./scripts/species_list.php'); } + if($_GET['view'] == "Species Management"){ + ensure_authenticated(); + include('scripts/species_tools.php'); + } if($_GET['view'] == "File"){ echo ""; } From cd37822ed0aa729c918a3729a056cf97ee2988ae Mon Sep 17 00:00:00 2001 From: Alexandre <44178713+alexbelgium@users.noreply.github.com> Date: Tue, 26 Aug 2025 08:45:57 +0200 Subject: [PATCH 02/36] Add species_tools.php for species management --- scripts/species_tools.php | 284 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 284 insertions(+) create mode 100644 scripts/species_tools.php diff --git a/scripts/species_tools.php b/scripts/species_tools.php new file mode 100644 index 000000000..ddeec815a --- /dev/null +++ b/scripts/species_tools.php @@ -0,0 +1,284 @@ +busyTimeout(1000); + +/* Paths / lists */ +$base_symlink = $home . '/BirdSongs/Extracted/By_Date'; +$base = realpath($base_symlink); // used only for safety checks + +$exclude_file = __DIR__ . '/exclude_species_list.txt'; +$whitelist_file = __DIR__ . '/whitelist_species_list.txt'; + +$excluded_species = file_exists($exclude_file) ? file($exclude_file, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) : []; +$whitelisted_species = file_exists($whitelist_file) ? file($whitelist_file, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) : []; + +$config = get_config(); +$sf_thresh = isset($config['SF_THRESH']) ? (float)$config['SF_THRESH'] : 0.0; + +/* ---------- helpers (tiny, single-purpose) ---------- */ +function join_path(...$parts): string { + return preg_replace('#/+#', '/', implode('/', $parts)); +} +function can_unlink(string $p): bool { + // unlink-able: symlink (even dangling) or regular file + return is_link($p) || is_file($p); +} +function under_base(string $path, string $base): bool { + if ($base === false) return false; + $baseReal = rtrim(realpath($base) ?: $base, DIRECTORY_SEPARATOR); + + $resolved = realpath($path); + if ($resolved === false) { + $parent = realpath(dirname($path)); + if ($parent === false) return false; + $resolved = $parent . DIRECTORY_SEPARATOR . basename($path); + } + return $resolved === $baseReal || strpos($resolved, $baseReal . DIRECTORY_SEPARATOR) === 0; +} + +/** + * Collect detection count, files to delete (unique), first scientific name, + * and dirs to try rmdir later — for a given species. + */ +function collect_species_targets(SQLite3 $db, string $species, string $home, $base): array { + $stmt = $db->prepare('SELECT Date, Com_Name, Sci_Name, File_Name FROM detections WHERE Com_Name = :name'); + ensure_db_ok($stmt); + $stmt->bindValue(':name', $species, SQLITE3_TEXT); + $res = $stmt->execute(); + + $count = 0; + $files = []; + $dirs = []; + $sci = null; + + while ($row = $res->fetchArray(SQLITE3_ASSOC)) { + $count++; + if ($sci === null) $sci = $row['Sci_Name']; + $dir = str_replace([' ', "'"], ['_', ''], $row['Com_Name']); + + $candidates = [ + join_path($home, 'BirdSongs/Extracted/By_Date', $row['Date'], $dir, $row['File_Name']), + join_path($home, 'BirdSongs/Extracted/By_Date/shifted', $row['Date'], $dir, $row['File_Name']), + ]; + + foreach ($candidates as $c) { + if (can_unlink($c) && under_base($c, $base)) { + $files[$c] = true; $dirs[] = dirname($c); continue; + } + $d = realpath(dirname($c)); + if ($d !== false) { + $alt = $d . DIRECTORY_SEPARATOR . basename($c); + if (can_unlink($alt) && under_base($alt, $base)) { + $files[$alt] = true; $dirs[] = dirname($alt); + } + } + } + } + return [ + 'count' => $count, + 'files' => array_keys($files), + 'dirs' => array_values(array_unique($dirs)), + 'sci' => $sci, + ]; +} + +/* ---------- toggle exclude/whitelist ---------- */ +if (isset($_GET['toggle'], $_GET['species'], $_GET['action'])) { + $list = $_GET['toggle']; + $species = htmlspecialchars_decode($_GET['species'], ENT_QUOTES); + + if ($list === 'exclude') { + $file = $exclude_file; + } elseif ($list === 'whitelist') { + $file = $whitelist_file; + } else { + header('Content-Type: text/plain'); echo 'Invalid list type'; exit; + } + + $lines = file_exists($file) ? file($file, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) : []; + if ($_GET['action'] === 'add') { + if (!in_array($species, $lines, true)) $lines[] = $species; + } else { + $lines = array_values(array_filter($lines, fn($l) => $l !== $species)); + } + file_put_contents($file, implode("\n", $lines) . (empty($lines) ? "" : "\n")); + header('Content-Type: text/plain'); echo 'OK'; exit; +} + +/* ---------- count (keeps your old "getcounts=" API) ---------- */ +if (isset($_GET['getcounts'])) { + header('Content-Type: application/json'); + if ($base === false) { http_response_code(500); exit(json_encode(['error' => 'Base directory not found'])); } + $species = htmlspecialchars_decode($_GET['getcounts'], ENT_QUOTES); + $info = collect_species_targets($db, $species, $home, $base); + echo json_encode(['count' => $info['count'], 'files' => count($info['files'])]); exit; +} + +/* ---------- delete (keeps your old "delete=" API) ---------- */ +if (isset($_GET['delete'])) { + header('Content-Type: application/json'); + if ($base === false) { http_response_code(500); exit(json_encode(['error' => 'Base directory not found'])); } + $species = htmlspecialchars_decode($_GET['delete'], ENT_QUOTES); + $info = collect_species_targets($db, $species, $home, $base); + + $deleted = 0; + foreach ($info['files'] as $fp) { + if (!under_base($fp, $base)) continue; + if (can_unlink($fp) && @unlink($fp)) { + $deleted++; + // thumbnails: "file.wav.png" and "file.png" + foreach ([$fp . '.png', preg_replace('/\.[^.]+$/', '.png', $fp)] as $png) { + if (can_unlink($png)) @unlink($png); + } + } + } + foreach ($info['dirs'] as $dir) { + if (under_base($dir, $base)) @rmdir($dir); // best effort + } + + // DB rows + $del = $db->prepare('DELETE FROM detections WHERE Com_Name = :name'); + ensure_db_ok($del); + $del->bindValue(':name', $species, SQLITE3_TEXT); + $del->execute(); + $lines_deleted = $db->changes(); + + echo json_encode(['lines' => $lines_deleted, 'files' => $deleted]); exit; +} + +/* ---------- page (unchanged semantics; minor tidy) ---------- */ +$result = fetch_species_array('alphabetical'); +?> + + +
+ + + + + + + + + + + + + + +fetchArray(SQLITE3_ASSOC)) { + $common = htmlspecialchars($row['Com_Name'], ENT_QUOTES); + $scient = htmlspecialchars($row['Sci_Name'], ENT_QUOTES); + $count = (int)$row['Count']; + $max_confidence = round((float)$row['MaxConfidence'] * 100, 1); + $identifier = str_replace("'", '', $row['Sci_Name'].'_'.$row['Com_Name']); + + $common_link = "{$common}"; + + $is_excluded = in_array($identifier, $excluded_species, true); + $is_whitelisted = in_array($identifier, $whitelisted_species, true); + + $excl_cell = $is_excluded + ? "" + : ""; + + $white_cell = $is_whitelisted + ? "" + : ""; + + echo "" + . "" + . "" + . "" + . "" + . ""; +} ?> + +
Common NameScientific NameIdentificationsMax ConfidenceThresholdExcludedWhitelistedDelete
{$common_link}{$scient}{$count}{$max_confidence}%0.0000".$excl_cell."".$white_cell."
+
+ + From 50b13ae294749b69684634d17be6bb65b87ec500 Mon Sep 17 00:00:00 2001 From: Alexandre <44178713+alexbelgium@users.noreply.github.com> Date: Tue, 26 Aug 2025 08:51:28 +0200 Subject: [PATCH 03/36] Enhance species management with confirmed list --- scripts/species_tools.php | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/scripts/species_tools.php b/scripts/species_tools.php index ddeec815a..580341ae2 100644 --- a/scripts/species_tools.php +++ b/scripts/species_tools.php @@ -14,9 +14,17 @@ $base_symlink = $home . '/BirdSongs/Extracted/By_Date'; $base = realpath($base_symlink); // used only for safety checks +$confirm_file = __DIR__ . '/confirmed_species_list.txt'; $exclude_file = __DIR__ . '/exclude_species_list.txt'; $whitelist_file = __DIR__ . '/whitelist_species_list.txt'; +foreach ([$confirm_file, $exclude_file, $whitelist_file] as $file) { + if (!file_exists($file)) { + touch($file); + } +} + +$confirmed_species = file_exists($confirm_file) ? file($confirm_file, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) : []; $excluded_species = file_exists($exclude_file) ? file($exclude_file, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) : []; $whitelisted_species = file_exists($whitelist_file) ? file($whitelist_file, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) : []; @@ -90,7 +98,7 @@ function collect_species_targets(SQLite3 $db, string $species, string $home, $ba ]; } -/* ---------- toggle exclude/whitelist ---------- */ +/* ---------- toggle exclude/whitelist/confirmed ---------- */ if (isset($_GET['toggle'], $_GET['species'], $_GET['action'])) { $list = $_GET['toggle']; $species = htmlspecialchars_decode($_GET['species'], ENT_QUOTES); @@ -99,6 +107,8 @@ function collect_species_targets(SQLite3 $db, string $species, string $home, $ba $file = $exclude_file; } elseif ($list === 'whitelist') { $file = $whitelist_file; + } elseif ($list === 'confirmed') { + $file = $confirm_file; } else { header('Content-Type: text/plain'); echo 'Invalid list type'; exit; } @@ -151,6 +161,13 @@ function collect_species_targets(SQLite3 $db, string $species, string $home, $ba $del->execute(); $lines_deleted = $db->changes(); + // Remove from confirmed list + if ($info['sci'] !== null && file_exists($confirm_file)) { + $identifier = str_replace("'", '', $info['sci']); + $lines = array_values(array_filter($confirmed_species, fn($l) => $l !== $identifier)); + file_put_contents($confirm_file, implode("\n", $lines) . (empty($lines) ? "" : "\n")); + } + echo json_encode(['lines' => $lines_deleted, 'files' => $deleted]); exit; } @@ -172,8 +189,9 @@ function collect_species_targets(SQLite3 $db, string $species, string $home, $ba Identifications Max Confidence Threshold - Excluded - Whitelisted + Confirmed + Excluded + Whitelisted Delete @@ -184,13 +202,19 @@ function collect_species_targets(SQLite3 $db, string $species, string $home, $ba $count = (int)$row['Count']; $max_confidence = round((float)$row['MaxConfidence'] * 100, 1); $identifier = str_replace("'", '', $row['Sci_Name'].'_'.$row['Com_Name']); + $identifier_sci = str_replace("'", '', $row['Sci_Name']); $common_link = "{$common}"; + $is_confirmed = in_array($identifier_sci, $confirmed_species, true); $is_excluded = in_array($identifier, $excluded_species, true); $is_whitelisted = in_array($identifier, $whitelisted_species, true); + $confirm_cell = $is_confirmed + ? "" + : ""; + $excl_cell = $is_excluded ? "" : ""; @@ -202,6 +226,7 @@ function collect_species_targets(SQLite3 $db, string $species, string $home, $ba echo "{$common_link}{$scient}{$count}" . "{$max_confidence}%" . "0.0000" + . "".$confirm_cell."" . "".$excl_cell."" . "".$white_cell."" . ""; From 051df1c4760034fb46dafcd6cbb6de49324b4808 Mon Sep 17 00:00:00 2001 From: Alexandre <44178713+alexbelgium@users.noreply.github.com> Date: Tue, 26 Aug 2025 09:22:26 +0200 Subject: [PATCH 04/36] Open db in write only if needed --- scripts/species_tools.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/species_tools.php b/scripts/species_tools.php index 580341ae2..374405aeb 100644 --- a/scripts/species_tools.php +++ b/scripts/species_tools.php @@ -7,7 +7,9 @@ ensure_authenticated(); $home = get_home(); -$db = new SQLite3(__DIR__ . '/birds.db', SQLITE3_OPEN_READWRITE); +// Open database read-only for typical operations; enable writes only for deletions +$flags = isset($_GET['delete']) ? SQLITE3_OPEN_READWRITE : SQLITE3_OPEN_READONLY; +$db = new SQLite3(__DIR__ . '/birds.db', $flags); $db->busyTimeout(1000); /* Paths / lists */ From 0ebf18f9caa08a1c734fae93cc579e7e908c5eb8 Mon Sep 17 00:00:00 2001 From: Alexandre <44178713+alexbelgium@users.noreply.github.com> Date: Tue, 26 Aug 2025 13:54:28 +0200 Subject: [PATCH 05/36] Add search, max confidence --- scripts/species_tools.php | 126 +++++++++++++++++++++++++++++--------- 1 file changed, 96 insertions(+), 30 deletions(-) diff --git a/scripts/species_tools.php b/scripts/species_tools.php index 374405aeb..a8c9ff9f1 100644 --- a/scripts/species_tools.php +++ b/scripts/species_tools.php @@ -21,9 +21,7 @@ $whitelist_file = __DIR__ . '/whitelist_species_list.txt'; foreach ([$confirm_file, $exclude_file, $whitelist_file] as $file) { - if (!file_exists($file)) { - touch($file); - } + if (!file_exists($file)) touch($file); } $confirmed_species = file_exists($confirm_file) ? file($confirm_file, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) : []; @@ -33,12 +31,11 @@ $config = get_config(); $sf_thresh = isset($config['SF_THRESH']) ? (float)$config['SF_THRESH'] : 0.0; -/* ---------- helpers (tiny, single-purpose) ---------- */ +/* ---------- helpers ---------- */ function join_path(...$parts): string { return preg_replace('#/+#', '/', implode('/', $parts)); } function can_unlink(string $p): bool { - // unlink-able: symlink (even dangling) or regular file return is_link($p) || is_file($p); } function under_base(string $path, string $base): bool { @@ -57,17 +54,17 @@ function under_base(string $path, string $base): bool { /** * Collect detection count, files to delete (unique), first scientific name, * and dirs to try rmdir later — for a given species. + * NOTE: Row-wise enumeration (no aggregates) so we gather every file. */ function collect_species_targets(SQLite3 $db, string $species, string $home, $base): array { - $stmt = $db->prepare('SELECT Date, Com_Name, Sci_Name, File_Name FROM detections WHERE Com_Name = :name'); + $stmt = $db->prepare('SELECT Date, Com_Name, Sci_Name, File_Name + FROM detections + WHERE Com_Name = :name'); ensure_db_ok($stmt); $stmt->bindValue(':name', $species, SQLITE3_TEXT); $res = $stmt->execute(); - $count = 0; - $files = []; - $dirs = []; - $sci = null; + $count = 0; $files = []; $dirs = []; $sci = null; while ($row = $res->fetchArray(SQLITE3_ASSOC)) { $count++; @@ -75,7 +72,7 @@ function collect_species_targets(SQLite3 $db, string $species, string $home, $ba $dir = str_replace([' ', "'"], ['_', ''], $row['Com_Name']); $candidates = [ - join_path($home, 'BirdSongs/Extracted/By_Date', $row['Date'], $dir, $row['File_Name']), + join_path($home, 'BirdSongs/Extracted/By_Date', $row['Date'], $dir, $row['File_Name']), join_path($home, 'BirdSongs/Extracted/By_Date/shifted', $row['Date'], $dir, $row['File_Name']), ]; @@ -104,16 +101,11 @@ function collect_species_targets(SQLite3 $db, string $species, string $home, $ba if (isset($_GET['toggle'], $_GET['species'], $_GET['action'])) { $list = $_GET['toggle']; $species = htmlspecialchars_decode($_GET['species'], ENT_QUOTES); - - if ($list === 'exclude') { - $file = $exclude_file; - } elseif ($list === 'whitelist') { - $file = $whitelist_file; - } elseif ($list === 'confirmed') { - $file = $confirm_file; - } else { - header('Content-Type: text/plain'); echo 'Invalid list type'; exit; - } + + if ($list === 'exclude') { $file = $exclude_file; } + elseif ($list === 'whitelist') { $file = $whitelist_file; } + elseif ($list === 'confirmed') { $file = $confirm_file; } + else { header('Content-Type: text/plain'); echo 'Invalid list type'; exit; } $lines = file_exists($file) ? file($file, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) : []; if ($_GET['action'] === 'add') { @@ -173,16 +165,36 @@ function collect_species_targets(SQLite3 $db, string $species, string $home, $ba echo json_encode(['lines' => $lines_deleted, 'files' => $deleted]); exit; } -/* ---------- page (unchanged semantics; minor tidy) ---------- */ -$result = fetch_species_array('alphabetical'); +/* ---------- query species aggregates ---------- */ +$sql = <<query($sql); ?>
+ +
+ + +
+ @@ -190,10 +202,11 @@ function collect_species_targets(SQLite3 $db, string $species, string $home, $ba - - - - + + + + + @@ -206,6 +219,10 @@ function collect_species_targets(SQLite3 $db, string $species, string $home, $ba $identifier = str_replace("'", '', $row['Sci_Name'].'_'.$row['Com_Name']); $identifier_sci = str_replace("'", '', $row['Sci_Name']); + $lastSeen = $row['LastSeen'] ?? ''; + $lastSeenSort = $lastSeen ? (strtotime($lastSeen) ?: 0) : 0; + $lastSeenDisplay = htmlspecialchars($lastSeen, ENT_QUOTES); + $common_link = "{$common}"; @@ -225,13 +242,18 @@ function collect_species_targets(SQLite3 $db, string $species, string $home, $ba ? "" : ""; - echo "" - . "" + echo "" + . "" + . "" + . "" + . "" /* Max Confidence */ + . "" /* Last Seen */ . "" . "" . "" . "" - . ""; + . "" + . ""; } ?>
Scientific Name Identifications Max ConfidenceThresholdConfirmedExcludedWhitelistedLast SeenProbabilityConfirmedExcludedWhitelisted Delete
{$common_link}{$scient}{$count}{$max_confidence}%
{$common_link}{$scient}{$count}{$max_confidence}%{$lastSeenDisplay}0.0000".$confirm_cell."".$excl_cell."".$white_cell."
@@ -244,6 +266,7 @@ function collect_species_targets(SQLite3 $db, string $species, string $home, $ba // tiny fetch helper const get = (url) => fetch(url, {cache:'no-store'}).then(r => r.text()); +// ---------- load thresholds and colorize ---------- function loadThresholds() { get(scriptsBase + 'config.php?threshold=0').then(text => { const lines = (text || '').split(/\r?\n/); @@ -274,6 +297,7 @@ function loadThresholds() { } document.addEventListener('DOMContentLoaded', loadThresholds); +// ---------- toggles / delete ---------- function toggleSpecies(list, species, action) { get(scriptsBase + 'species_tools.php?toggle=' + list + '&species=' + encodeURIComponent(species) + '&action=' + action) .then(t => { if (t.trim() === 'OK') location.reload(); }); @@ -293,6 +317,7 @@ function deleteSpecies(species) { }); } +// ---------- Sorting with persistence ---------- function sortTable(n) { const table = document.getElementById('speciesTable'); const tbody = table.tBodies[0]; @@ -307,5 +332,46 @@ function sortTable(n) { }); rows.forEach(r => tbody.appendChild(r)); table.setAttribute('data-sort-' + n, asc ? 'asc' : 'desc'); + + try { + localStorage.setItem('speciesSortCol', String(n)); + localStorage.setItem('speciesSortAsc', asc ? '1' : '0'); + } catch(e){} } + +function applySavedSort() { + const table = document.getElementById('speciesTable'); + const col = parseInt(localStorage.getItem('speciesSortCol') || '', 10); + const asc = localStorage.getItem('speciesSortAsc'); + if (!Number.isFinite(col)) return; + sortTable(col); + const isAscNow = table.getAttribute('data-sort-' + col) === 'asc'; + if ((asc === '1') !== isAscNow) sortTable(col); +} + +// ---------- Search with persistence ---------- +const q = document.getElementById('q'); +const matchCount = document.getElementById('matchCount'); + +function applyFilter() { + const needle = (q.value || '').trim().toLowerCase(); + let shown = 0, total = 0; + document.querySelectorAll('#speciesTable tbody tr').forEach(tr => { + total++; + const txt = tr.innerText.toLowerCase(); + const vis = txt.includes(needle); + tr.style.display = vis ? '' : 'none'; + if (vis) shown++; + }); + matchCount.textContent = total ? `${shown} / ${total}` : ''; + try { localStorage.setItem('speciesFilter', q.value); } catch(e){} +} + +q.addEventListener('input', applyFilter); + +document.addEventListener('DOMContentLoaded', () => { + try { const saved = localStorage.getItem('speciesFilter'); if (saved !== null) q.value = saved; } catch(e){} + applyFilter(); + applySavedSort(); +}); From f9ebf36c2d81de023b6ef8bb6b04b42178c79ae6 Mon Sep 17 00:00:00 2001 From: Alexandre <44178713+alexbelgium@users.noreply.github.com> Date: Tue, 26 Aug 2025 14:01:28 +0200 Subject: [PATCH 06/36] Clarify what is deleted --- scripts/species_tools.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/species_tools.php b/scripts/species_tools.php index a8c9ff9f1..cb483ac36 100644 --- a/scripts/species_tools.php +++ b/scripts/species_tools.php @@ -306,7 +306,7 @@ function toggleSpecies(list, species, action) { function deleteSpecies(species) { get(scriptsBase + 'species_tools.php?getcounts=' + encodeURIComponent(species)).then(t => { let info; try { info = JSON.parse(t); } catch { alert('Could not parse count response'); return; } - if (!confirm('Delete ' + info.count + ' detections and ' + info.files + ' files for ' + species + '?')) return; + if (!confirm('Delete ' + info.count + ' detections and local audio and png files for ' + species + '?')) return; get(scriptsBase + 'species_tools.php?delete=' + encodeURIComponent(species)).then(t2 => { try { const res = JSON.parse(t2); From c77323e0176c31eb325e35c8473ae909642f8101 Mon Sep 17 00:00:00 2001 From: Alexandre <44178713+alexbelgium@users.noreply.github.com> Date: Wed, 27 Aug 2025 09:58:13 +0200 Subject: [PATCH 07/36] Lazy load local files and threshold --- scripts/species_tools.php | 209 +++++++++++++++++++++----------------- 1 file changed, 114 insertions(+), 95 deletions(-) diff --git a/scripts/species_tools.php b/scripts/species_tools.php index cb483ac36..cf1d39b95 100644 --- a/scripts/species_tools.php +++ b/scripts/species_tools.php @@ -7,14 +7,44 @@ ensure_authenticated(); $home = get_home(); -// Open database read-only for typical operations; enable writes only for deletions + +/* ---------- disk species counts (AJAX endpoint) ---------- */ +if (isset($_GET['diskcounts'])) { + header('Content-Type: application/json'); + $script = __DIR__ . '/disk_species_count.sh'; + $cmd = 'HOME=' . escapeshellarg($home) . ' bash ' . escapeshellarg($script) . ' 2>&1'; + $output = @shell_exec($cmd); + $counts = []; + if ($output !== null) { + foreach (preg_split('/\\r?\\n/', $output) as $line) { + $line = trim($line); + if ($line === '') continue; + if (preg_match('/^([0-9]+(?:\\.[0-9]+)?)(k?)\\s*:\\s*(.+)$/i', $line, $m)) { + $num = (float)$m[1]; + if (strtolower($m[2]) === 'k') $num *= 1000; + $counts[$m[3]] = (int)round($num); + } + } + } + echo json_encode($counts, JSON_UNESCAPED_UNICODE); + exit; +} + +/* ---------- DB open (RO unless deleting) ---------- */ $flags = isset($_GET['delete']) ? SQLITE3_OPEN_READWRITE : SQLITE3_OPEN_READONLY; $db = new SQLite3(__DIR__ . '/birds.db', $flags); $db->busyTimeout(1000); +$db->exec(" + PRAGMA journal_mode=WAL; -- safe read concurrency + PRAGMA synchronous=NORMAL; -- cheaper fsyncs (read-mostly) + PRAGMA temp_store=MEMORY; -- temp data stays in RAM + PRAGMA cache_size=-80000; -- ~80MB page cache (tune to your RAM) + PRAGMA mmap_size=268435456; -- 256MB mmap (set 0 if kernel disallows) +"); /* Paths / lists */ $base_symlink = $home . '/BirdSongs/Extracted/By_Date'; -$base = realpath($base_symlink); // used only for safety checks +$base = realpath($base_symlink); // safety checks $confirm_file = __DIR__ . '/confirmed_species_list.txt'; $exclude_file = __DIR__ . '/exclude_species_list.txt'; @@ -32,16 +62,11 @@ $sf_thresh = isset($config['SF_THRESH']) ? (float)$config['SF_THRESH'] : 0.0; /* ---------- helpers ---------- */ -function join_path(...$parts): string { - return preg_replace('#/+#', '/', implode('/', $parts)); -} -function can_unlink(string $p): bool { - return is_link($p) || is_file($p); -} +function join_path(...$parts): string { return preg_replace('#/+#', '/', implode('/', $parts)); } +function can_unlink(string $p): bool { return is_link($p) || is_file($p); } function under_base(string $path, string $base): bool { if ($base === false) return false; $baseReal = rtrim(realpath($base) ?: $base, DIRECTORY_SEPARATOR); - $resolved = realpath($path); if ($resolved === false) { $parent = realpath(dirname($path)); @@ -51,50 +76,31 @@ function under_base(string $path, string $base): bool { return $resolved === $baseReal || strpos($resolved, $baseReal . DIRECTORY_SEPARATOR) === 0; } -/** - * Collect detection count, files to delete (unique), first scientific name, - * and dirs to try rmdir later — for a given species. - * NOTE: Row-wise enumeration (no aggregates) so we gather every file. - */ +/* Collect files/dirs for a species */ function collect_species_targets(SQLite3 $db, string $species, string $home, $base): array { - $stmt = $db->prepare('SELECT Date, Com_Name, Sci_Name, File_Name - FROM detections - WHERE Com_Name = :name'); + $stmt = $db->prepare('SELECT Date, Com_Name, Sci_Name, File_Name FROM detections WHERE Com_Name = :name'); ensure_db_ok($stmt); $stmt->bindValue(':name', $species, SQLITE3_TEXT); $res = $stmt->execute(); $count = 0; $files = []; $dirs = []; $sci = null; - while ($row = $res->fetchArray(SQLITE3_ASSOC)) { - $count++; - if ($sci === null) $sci = $row['Sci_Name']; + $count++; if ($sci === null) $sci = $row['Sci_Name']; $dir = str_replace([' ', "'"], ['_', ''], $row['Com_Name']); - $candidates = [ join_path($home, 'BirdSongs/Extracted/By_Date', $row['Date'], $dir, $row['File_Name']), join_path($home, 'BirdSongs/Extracted/By_Date/shifted', $row['Date'], $dir, $row['File_Name']), ]; - foreach ($candidates as $c) { - if (can_unlink($c) && under_base($c, $base)) { - $files[$c] = true; $dirs[] = dirname($c); continue; - } + if (can_unlink($c) && under_base($c, $base)) { $files[$c] = true; $dirs[] = dirname($c); continue; } $d = realpath(dirname($c)); if ($d !== false) { $alt = $d . DIRECTORY_SEPARATOR . basename($c); - if (can_unlink($alt) && under_base($alt, $base)) { - $files[$alt] = true; $dirs[] = dirname($alt); - } + if (can_unlink($alt) && under_base($alt, $base)) { $files[$alt] = true; $dirs[] = dirname($alt); } } } } - return [ - 'count' => $count, - 'files' => array_keys($files), - 'dirs' => array_values(array_unique($dirs)), - 'sci' => $sci, - ]; + return ['count'=>$count, 'files'=>array_keys($files), 'dirs'=>array_values(array_unique($dirs)), 'sci'=>$sci]; } /* ---------- toggle exclude/whitelist/confirmed ---------- */ @@ -117,7 +123,7 @@ function collect_species_targets(SQLite3 $db, string $species, string $home, $ba header('Content-Type: text/plain'); echo 'OK'; exit; } -/* ---------- count (keeps your old "getcounts=" API) ---------- */ +/* ---------- count ---------- */ if (isset($_GET['getcounts'])) { header('Content-Type: application/json'); if ($base === false) { http_response_code(500); exit(json_encode(['error' => 'Base directory not found'])); } @@ -126,7 +132,7 @@ function collect_species_targets(SQLite3 $db, string $species, string $home, $ba echo json_encode(['count' => $info['count'], 'files' => count($info['files'])]); exit; } -/* ---------- delete (keeps your old "delete=" API) ---------- */ +/* ---------- delete ---------- */ if (isset($_GET['delete'])) { header('Content-Type: application/json'); if ($base === false) { http_response_code(500); exit(json_encode(['error' => 'Base directory not found'])); } @@ -138,24 +144,19 @@ function collect_species_targets(SQLite3 $db, string $species, string $home, $ba if (!under_base($fp, $base)) continue; if (can_unlink($fp) && @unlink($fp)) { $deleted++; - // thumbnails: "file.wav.png" and "file.png" foreach ([$fp . '.png', preg_replace('/\.[^.]+$/', '.png', $fp)] as $png) { if (can_unlink($png)) @unlink($png); } } } - foreach ($info['dirs'] as $dir) { - if (under_base($dir, $base)) @rmdir($dir); // best effort - } + foreach ($info['dirs'] as $dir) { if (under_base($dir, $base)) @rmdir($dir); } - // DB rows $del = $db->prepare('DELETE FROM detections WHERE Com_Name = :name'); ensure_db_ok($del); $del->bindValue(':name', $species, SQLITE3_TEXT); $del->execute(); $lines_deleted = $db->changes(); - // Remove from confirmed list if ($info['sci'] !== null && file_exists($confirm_file)) { $identifier = str_replace("'", '', $info['sci']); $lines = array_values(array_filter($confirmed_species, fn($l) => $l !== $identifier)); @@ -167,15 +168,9 @@ function collect_species_targets(SQLite3 $db, string $species, string $home, $ba /* ---------- query species aggregates ---------- */ $sql = <<query($sql); ?> @@ -190,27 +185,26 @@ function collect_species_targets(SQLite3 $db, string $species, string $home, $ba
- +
- - - - - - - - - - - - - - - - +
Common NameScientific NameIdentificationsMax ConfidenceLast SeenProbabilityConfirmedExcludedWhitelistedDelete
+ + + + + + + + + + + + + + + fetchArray(SQLITE3_ASSOC)) { $common = htmlspecialchars($row['Com_Name'], ENT_QUOTES); $scient = htmlspecialchars($row['Sci_Name'], ENT_QUOTES); @@ -223,8 +217,7 @@ function collect_species_targets(SQLite3 $db, string $species, string $home, $ba $lastSeenSort = $lastSeen ? (strtotime($lastSeen) ?: 0) : 0; $lastSeenDisplay = htmlspecialchars($lastSeen, ENT_QUOTES); - $common_link = "{$common}"; + $common_link = "{$common}"; $is_confirmed = in_array($identifier_sci, $confirmed_species, true); $is_excluded = in_array($identifier, $excluded_species, true); @@ -246,8 +239,8 @@ function collect_species_targets(SQLite3 $db, string $species, string $home, $ba . "" . "" . "" - . "" /* Max Confidence */ - . "" /* Last Seen */ + . "" + . "" . "" . "" . "" @@ -255,20 +248,18 @@ function collect_species_targets(SQLite3 $db, string $species, string $home, $ba . "" . ""; } ?> - -
Common NameScientific NameIdentificationsMax ConfidenceLast SeenProbabilityConfirmedExcludedWhitelistedDelete
{$common_link}{$scient}{$count}{$max_confidence}%{$lastSeenDisplay}{$max_confidence}%{$lastSeenDisplay}0.0000".$confirm_cell."".$excl_cell."
+ +
From 44ac630c317e26c20f828f8c2b2115a8170507a7 Mon Sep 17 00:00:00 2001 From: Alexandre <44178713+alexbelgium@users.noreply.github.com> Date: Thu, 28 Aug 2025 09:44:24 +0200 Subject: [PATCH 08/36] Add improved mini charts --- scripts/species_tools.php | 139 +++++++++++++++++++++++++++++++++++--- 1 file changed, 131 insertions(+), 8 deletions(-) diff --git a/scripts/species_tools.php b/scripts/species_tools.php index cf1d39b95..c034ffcc2 100644 --- a/scripts/species_tools.php +++ b/scripts/species_tools.php @@ -194,13 +194,14 @@ function collect_species_targets(SQLite3 $db, string $species, string $home, $ba Common Name Scientific Name - Identifications - Max Confidence - Last Seen - Probability - Confirmed - Excluded - Whitelisted + Stats + Identifications + Max Confidence + Last Seen + Probability + Confirmed + Excluded + Whitelisted Delete @@ -223,6 +224,9 @@ function collect_species_targets(SQLite3 $db, string $species, string $home, $ba $is_excluded = in_array($identifier, $excluded_species, true); $is_whitelisted = in_array($identifier, $whitelisted_species, true); + $comnamegraph = str_replace("'", "\'", $row['Com_Name']); + $chart_cell = sprintf("", $comnamegraph); + $confirm_cell = $is_confirmed ? "" : ""; @@ -238,6 +242,7 @@ function collect_species_targets(SQLite3 $db, string $species, string $home, $ba echo "" . "{$common_link}" . "{$scient}" + . "{$chart_cell}" . "{$count}" . "{$max_confidence}%" . "{$lastSeenDisplay}" @@ -251,7 +256,7 @@ function collect_species_targets(SQLite3 $db, string $species, string $home, $ba
- + + + +