From 6c0cb54f3228316c052a458167cecc7a35e7fffd Mon Sep 17 00:00:00 2001 From: Alexandre <44178713+alexbelgium@users.noreply.github.com> Date: Sun, 19 Jan 2025 14:12:03 +0100 Subject: [PATCH 01/11] Align disk cleaning with disk_species_clean.sh --- scripts/disk_check.sh | 39 ++++++++------------------------------- 1 file changed, 8 insertions(+), 31 deletions(-) diff --git a/scripts/disk_check.sh b/scripts/disk_check.sh index c6c98d8fe..f8aa50184 100755 --- a/scripts/disk_check.sh +++ b/scripts/disk_check.sh @@ -7,37 +7,14 @@ used="$(df -h ${EXTRACTED} | tail -n1 | awk '{print $5}')" if [ "${used//%}" -ge 95 ]; then case $FULL_DISK in - purge) echo "Removing oldest data" - cd ${EXTRACTED}/By_Date/ - curl localhost/views.php?view=Species%20Stats &>/dev/null - if ! grep -qxFe \#\#start $HOME/BirdNET-Pi/scripts/disk_check_exclude.txt; then - exit - fi - filestodelete=$(($(find ${EXTRACTED}/By_Date/* -type f | wc -l) / $(find ${EXTRACTED}/By_Date/* -maxdepth 0 -type d | wc -l))) - iter=0 - for i in */*/*; do - if [ $iter -ge $filestodelete ]; then - break - fi - if ! grep -qxFe "$i" $HOME/BirdNET-Pi/scripts/disk_check_exclude.txt; then - rm "$i" - fi - ((iter++)) - done - find ~/BirdSongs/ -type d -empty -mtime +90 -delete - find ${EXTRACTED}/By_Date/ -empty -type d -delete;; - - #rm -drfv "$(find ${EXTRACTED}/By_Date/* -maxdepth 1 -type d -prune \ - # | sort -r | tail -n1)";; - keep) echo "Stopping Core Services" - /usr/local/bin/stop_core_services.sh;; - esac -fi -sleep 1 -if [ "${used//%}" -ge 95 ]; then - case $FULL_DISK in - purge) echo "Removing more data" - rm -rfv ${PROCESSED}/*;; + purge) echo "Removing data to stay below threshold" + max_files_species="1000" + safe_purge_threshold="$((95 * 9 / 10))" + while [ "$(df -h "${EXTRACTED}" | tail -n1 | awk '{print $5}' | tr -d '%')" -ge "$safe_purge_threshold" ]; do + ./disk_species_clean.sh "$max_files_species" + max_files_species=$((max_files_species * 9 / 10)) + done;; + keep) echo "Stopping Core Services" /usr/local/bin/stop_core_services.sh;; esac From 86debe7ccf854a49cc9ce2b3ac8993886390bcd7 Mon Sep 17 00:00:00 2001 From: Alexandre <44178713+alexbelgium@users.noreply.github.com> Date: Sun, 19 Jan 2025 14:14:44 +0100 Subject: [PATCH 02/11] Accept argument --- scripts/disk_species_clean.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/disk_species_clean.sh b/scripts/disk_species_clean.sh index 2526ea1f8..70b5f51ef 100755 --- a/scripts/disk_species_clean.sh +++ b/scripts/disk_species_clean.sh @@ -7,6 +7,11 @@ base_dir="$HOME/BirdSongs/Extracted/By_Date" max_files_species="${MAX_FILES_SPECIES:-1000}" cd "$base_dir" || true +# Check if the first argument is a valid integer +if [[ "$1" =~ ^[0-9]+$ ]]; then + max_files_species="$1" +fi + # If max_files_species is not higher than 1, exit if [[ "$max_files_species" -lt 1 ]]; then exit 0 From 1dcb00ff23ab7304c3a4fb3778d9a708d657727d Mon Sep 17 00:00:00 2001 From: Alexandre <44178713+alexbelgium@users.noreply.github.com> Date: Sat, 25 Jan 2025 20:49:42 +0100 Subject: [PATCH 03/11] Readd logic --- scripts/disk_check.sh | 40 +++++++++++++--------------------------- 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/scripts/disk_check.sh b/scripts/disk_check.sh index 9a1c78f35..37057d4df 100644 --- a/scripts/disk_check.sh +++ b/scripts/disk_check.sh @@ -8,36 +8,22 @@ purge_threshold="${PURGE_THRESHOLD:-95}" if [ "${used//%}" -ge "$purge_threshold" ]; then case $FULL_DISK in - purge) echo "Removing oldest data" - cd ${EXTRACTED}/By_Date/ - curl localhost/views.php?view=Species%20Stats &>/dev/null - if ! grep -qxFe \#\#start $HOME/BirdNET-Pi/scripts/disk_check_exclude.txt; then - exit - fi - filestodelete=$(($(find ${EXTRACTED}/By_Date/* -type f | wc -l) / $(find ${EXTRACTED}/By_Date/* -maxdepth 0 -type d | wc -l))) - iter=0 - for i in */*/*; do - if [ $iter -ge $filestodelete ]; then - break + purge) echo "Removing data to stay below threshold" + max_files_species="1000" + safe_files_species="50" + safe_purge_threshold="$((95 * 9 / 10))" + while [ "$(df -h "${EXTRACTED}" | tail -n1 | awk '{print $5}' | tr -d '%')" -ge "$safe_purge_threshold" ]; do + ./disk_species_clean.sh "$max_files_species" + max_files_species=$((max_files_species * 9 / 10)) + if [ "$max_files_species" -lt "$safe_files_species" ]; then + echo "ERROR : safeguard initiated at $safe_file_species files remaining to make sure that we do not delete too many files. Is there an issue with the path of $EXTRACTED? Stopping core services." + /usr/local/bin/stop_core_services.sh + break fi - if ! grep -qxFe "$i" $HOME/BirdNET-Pi/scripts/disk_check_exclude.txt; then - rm "$i" - fi - ((iter++)) - done - find ~/BirdSongs/ -type d -empty -mtime +90 -delete - find ${EXTRACTED}/By_Date/ -empty -type d -delete;; + sleep 5 + done;; keep) echo "Stopping Core Services" /usr/local/bin/stop_core_services.sh;; esac fi -sleep 1 -if [ "${used//%}" -ge "$purge_threshold" ]; then - case $FULL_DISK in - purge) echo "Removing more data" - rm -rfv ${PROCESSED}/*;; - keep) echo "Stopping Core Services" - /usr/local/bin/stop_core_services.sh;; - esac -fi From 897a787f76739e8c5f3ece2805ec2498bf123db6 Mon Sep 17 00:00:00 2001 From: alexbelgium Date: Sun, 18 May 2025 10:23:49 +0200 Subject: [PATCH 04/11] exec --- scripts/disk_check.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/disk_check.sh diff --git a/scripts/disk_check.sh b/scripts/disk_check.sh old mode 100644 new mode 100755 From 3f04d9ad09941bc9fa06c2128ae6255f6651ce70 Mon Sep 17 00:00:00 2001 From: Alexandre <44178713+alexbelgium@users.noreply.github.com> Date: Mon, 8 Sep 2025 11:54:28 +0200 Subject: [PATCH 05/11] Keep at least 30 files per species --- scripts/disk_check.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/disk_check.sh b/scripts/disk_check.sh index 37057d4df..419bc3651 100755 --- a/scripts/disk_check.sh +++ b/scripts/disk_check.sh @@ -10,7 +10,7 @@ if [ "${used//%}" -ge "$purge_threshold" ]; then case $FULL_DISK in purge) echo "Removing data to stay below threshold" max_files_species="1000" - safe_files_species="50" + safe_files_species="30" safe_purge_threshold="$((95 * 9 / 10))" while [ "$(df -h "${EXTRACTED}" | tail -n1 | awk '{print $5}' | tr -d '%')" -ge "$safe_purge_threshold" ]; do ./disk_species_clean.sh "$max_files_species" From 808e9b75044461508450b296229b279e985eb97a Mon Sep 17 00:00:00 2001 From: Alexandre <44178713+alexbelgium@users.noreply.github.com> Date: Mon, 8 Sep 2025 11:56:42 +0200 Subject: [PATCH 06/11] Fix typo in error message for safeguard initiation --- scripts/disk_check.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/disk_check.sh b/scripts/disk_check.sh index 419bc3651..1e3de6982 100755 --- a/scripts/disk_check.sh +++ b/scripts/disk_check.sh @@ -16,7 +16,7 @@ if [ "${used//%}" -ge "$purge_threshold" ]; then ./disk_species_clean.sh "$max_files_species" max_files_species=$((max_files_species * 9 / 10)) if [ "$max_files_species" -lt "$safe_files_species" ]; then - echo "ERROR : safeguard initiated at $safe_file_species files remaining to make sure that we do not delete too many files. Is there an issue with the path of $EXTRACTED? Stopping core services." + echo "ERROR : safeguard initiated at $safe_files_species files remaining to make sure that we do not delete too many files. Is there an issue with the path of $EXTRACTED? Stopping core services." /usr/local/bin/stop_core_services.sh break fi @@ -24,6 +24,9 @@ if [ "${used//%}" -ge "$purge_threshold" ]; then done;; keep) echo "Stopping Core Services" - /usr/local/bin/stop_core_services.sh;; + /usr/local/bin/stop_core_services.sh;; + + *) echo "Unknown FULL_DISK value: $FULL_DISK" + exit 1;; esac fi From 49569d29313773d0dab289605858412fe896533b Mon Sep 17 00:00:00 2001 From: Alexandre <44178713+alexbelgium@users.noreply.github.com> Date: Mon, 8 Sep 2025 11:57:45 +0200 Subject: [PATCH 07/11] First check if above safe_files_species before removing --- scripts/disk_check.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/disk_check.sh b/scripts/disk_check.sh index 1e3de6982..50549dd24 100755 --- a/scripts/disk_check.sh +++ b/scripts/disk_check.sh @@ -13,13 +13,13 @@ if [ "${used//%}" -ge "$purge_threshold" ]; then safe_files_species="30" safe_purge_threshold="$((95 * 9 / 10))" while [ "$(df -h "${EXTRACTED}" | tail -n1 | awk '{print $5}' | tr -d '%')" -ge "$safe_purge_threshold" ]; do - ./disk_species_clean.sh "$max_files_species" max_files_species=$((max_files_species * 9 / 10)) if [ "$max_files_species" -lt "$safe_files_species" ]; then echo "ERROR : safeguard initiated at $safe_files_species files remaining to make sure that we do not delete too many files. Is there an issue with the path of $EXTRACTED? Stopping core services." /usr/local/bin/stop_core_services.sh break fi + ./disk_species_clean.sh "$max_files_species" sleep 5 done;; From 48ce96e05db45e6bd958c76406bd079d0f6bc387 Mon Sep 17 00:00:00 2001 From: Alexandre <44178713+alexbelgium@users.noreply.github.com> Date: Mon, 15 Sep 2025 08:09:39 +0200 Subject: [PATCH 08/11] Resolve symlinks if needed --- scripts/disk_species_clean.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/disk_species_clean.sh b/scripts/disk_species_clean.sh index 70b5f51ef..04d503b74 100755 --- a/scripts/disk_species_clean.sh +++ b/scripts/disk_species_clean.sh @@ -4,6 +4,7 @@ source /etc/birdnet/birdnet.conf base_dir="$HOME/BirdSongs/Extracted/By_Date" +base_dir="$(realpath -e "$base_dir")" max_files_species="${MAX_FILES_SPECIES:-1000}" cd "$base_dir" || true From a6dfe388ddd33f303fcef04c0787d79db818a39c Mon Sep 17 00:00:00 2001 From: Alexandre <44178713+alexbelgium@users.noreply.github.com> Date: Mon, 15 Sep 2025 08:14:05 +0200 Subject: [PATCH 09/11] Fix base_dir assignment to use realpath correctly --- scripts/disk_species_clean.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/disk_species_clean.sh b/scripts/disk_species_clean.sh index 04d503b74..1805e4c87 100755 --- a/scripts/disk_species_clean.sh +++ b/scripts/disk_species_clean.sh @@ -3,8 +3,7 @@ # KEEP ONLY THE NUMBER OF FILES PER SPECIES DEFINED IN THE OPTIONS source /etc/birdnet/birdnet.conf -base_dir="$HOME/BirdSongs/Extracted/By_Date" -base_dir="$(realpath -e "$base_dir")" +base_dir="$(realpath -e "$HOME/BirdSongs/Extracted/By_Date")" max_files_species="${MAX_FILES_SPECIES:-1000}" cd "$base_dir" || true From 30fe7920168eddb62d2989a21c7102a4c9ff598a Mon Sep 17 00:00:00 2001 From: Alexandre <44178713+alexbelgium@users.noreply.github.com> Date: Mon, 15 Sep 2025 08:49:29 +0200 Subject: [PATCH 10/11] Clean code, add comments, robust failure modes --- scripts/disk_check.sh | 66 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 52 insertions(+), 14 deletions(-) diff --git a/scripts/disk_check.sh b/scripts/disk_check.sh index 50549dd24..d61405bd3 100755 --- a/scripts/disk_check.sh +++ b/scripts/disk_check.sh @@ -1,29 +1,67 @@ #!/usr/bin/env bash set -x +# ============================================================================= +# BirdNET-Pi Disk Space Management Script +# +# PURPOSE: +# This script monitors the disk space used by BirdNET-Pi recordings and +# automatically purges old files if the used space exceeds a configurable +# threshold. It is designed to prevent disk full errors and ensure smooth +# operation of BirdNET-Pi services. +# +# USAGE: +# - The script is intended to be run automatically (e.g., via cron). +# - It sources configuration from `/etc/birdnet/birdnet.conf`. +# - It respects the `FULL_DISK` environment variable to determine whether to +# purge files or stop services when the disk is full. +# - Always leaves at least 30 recordings per species. +# - Deletes files in steps of 5% of MAX_FILES_SPECIES (minimum 5 files per step). +# - Stops core services if disk remains full after purging. +# +# ============================================================================= + source /etc/birdnet/birdnet.conf -used="$(df -h ${EXTRACTED} | tail -n1 | awk '{print $5}')" -purge_threshold="${PURGE_THRESHOLD:-95}" -if [ "${used//%}" -ge "$purge_threshold" ]; then +# Get the variables +max_files_species="${MAX_FILES_SPECIES:-1000}" +purge_threshold="${PURGE_THRESHOLD:-95}" +# Remove 10% below the threshold to avoid running the script too often, ensure integer +purge_threshold_target=$(($purge_threshold * 9 / 10)) +# Deletion step corresponding to 5% of max_files_species, with at least 5 files, ensure integer +safe_purge_step=$(($max_files_species * 5 / 100)) +(( safe_purge_step >= 5 )) || safe_purge_step=5 +# Always leave at least 30 recordings +safe_files_species="30" +# Ensure integers +purge_threshold="${purge_threshold%%[.,]*}" +purge_threshold_target="${purge_threshold_target%%[.,]*}" +safe_purge_step="${safe_purge_step%%[.,]*}" +# Get the disk space in % +base_dir="$(realpath -e "$HOME/BirdSongs/Extracted/By_Date")" +used_disk() { + df -P "$base_dir" | awk 'END { gsub(/%/,"",$5); print $5 }' +} +# If the used space is above the threshold +if [ "$(used_disk)" -ge "$purge_threshold" ]; then case $FULL_DISK in - purge) echo "Removing data to stay below threshold" - max_files_species="1000" - safe_files_species="30" - safe_purge_threshold="$((95 * 9 / 10))" - while [ "$(df -h "${EXTRACTED}" | tail -n1 | awk '{print $5}' | tr -d '%')" -ge "$safe_purge_threshold" ]; do - max_files_species=$((max_files_species * 9 / 10)) - if [ "$max_files_species" -lt "$safe_files_species" ]; then - echo "ERROR : safeguard initiated at $safe_files_species files remaining to make sure that we do not delete too many files. Is there an issue with the path of $EXTRACTED? Stopping core services." + purge) echo "Removing data to stay below threshold of $purge_threshold" + # Loop until the value is below 10% of the threshold, to avoid running the script too often + max_files_species_loop="$max_files_species" + while [ "$(used_disk)" -ge "$purge_threshold_target" ]; do + max_files_species_loop=$(( max_files_species_loop - safe_purge_step )) + if [ "$max_files_species_loop" -gt "$safe_files_species" ]; then + bash "$HOME/BirdNET-Pi/scripts/disk_species_clean.sh" "$max_files_species_loop" + sleep 5 + else + echo "ERROR : safeguard initiated at $safe_files_species files remaining to make sure that we do not delete too many files. Is there an issue with the path of $base_dir? Stopping core services." /usr/local/bin/stop_core_services.sh break fi - ./disk_species_clean.sh "$max_files_species" - sleep 5 done;; - keep) echo "Stopping Core Services" + keep) echo "FULL_DISK=keep → Stopping Core Services." /usr/local/bin/stop_core_services.sh;; *) echo "Unknown FULL_DISK value: $FULL_DISK" From 339fa39d7117264dc212f633606fac7b25561c61 Mon Sep 17 00:00:00 2001 From: Alexandre <44178713+alexbelgium@users.noreply.github.com> Date: Tue, 16 Dec 2025 14:55:02 +0100 Subject: [PATCH 11/11] Enhance disk_check.sh with better error handling Refactor disk space management script for improved error handling and integer sanitization. --- scripts/disk_check.sh | 93 ++++++++++++++++++++++++++++--------------- 1 file changed, 62 insertions(+), 31 deletions(-) diff --git a/scripts/disk_check.sh b/scripts/disk_check.sh index d61405bd3..0ea36943a 100755 --- a/scripts/disk_check.sh +++ b/scripts/disk_check.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -x +set -Eeuo pipefail # ============================================================================= # BirdNET-Pi Disk Space Management Script @@ -23,48 +23,79 @@ set -x source /etc/birdnet/birdnet.conf +# --- tiny helper: sanitize "95", "95.0", "95,0", " 95 " -> integer (or default) --- +int() { + local v="${1:-}" d="${2:-0}" + v="${v//[[:space:]]/}" + v="${v%%[.,]*}" + [[ "$v" =~ ^[0-9]+$ ]] && printf '%s' "$v" || printf '%s' "$d" +} + # Get the variables -max_files_species="${MAX_FILES_SPECIES:-1000}" -purge_threshold="${PURGE_THRESHOLD:-95}" +max_files_species="$(int "${MAX_FILES_SPECIES:-1000}" 1000)" +purge_threshold="$(int "${PURGE_THRESHOLD:-95}" 95)" + # Remove 10% below the threshold to avoid running the script too often, ensure integer -purge_threshold_target=$(($purge_threshold * 9 / 10)) +purge_threshold_target=$(( purge_threshold * 9 / 10 )) + # Deletion step corresponding to 5% of max_files_species, with at least 5 files, ensure integer -safe_purge_step=$(($max_files_species * 5 / 100)) +safe_purge_step=$(( max_files_species * 5 / 100 )) (( safe_purge_step >= 5 )) || safe_purge_step=5 + # Always leave at least 30 recordings safe_files_species="30" -# Ensure integers -purge_threshold="${purge_threshold%%[.,]*}" -purge_threshold_target="${purge_threshold_target%%[.,]*}" -safe_purge_step="${safe_purge_step%%[.,]*}" + # Get the disk space in % -base_dir="$(realpath -e "$HOME/BirdSongs/Extracted/By_Date")" +base_dir="$(realpath -e "${BIRDNET_BASE_DIR:-$HOME/BirdSongs/Extracted/By_Date}" 2>/dev/null || true)" +if [[ -z "$base_dir" ]]; then + echo "ERROR: base_dir not found/resolvable: ${BIRDNET_BASE_DIR:-$HOME/BirdSongs/Extracted/By_Date}. Stopping core services." + /usr/local/bin/stop_core_services.sh || true + exit 1 +fi + used_disk() { - df -P "$base_dir" | awk 'END { gsub(/%/,"",$5); print $5 }' + df -P "$base_dir" | awk 'NR==2 { gsub(/%/,"",$5); print $5 }' } +# Prevent concurrent runs (cron overlap) +lockfile="${LOCK_FILE:-/tmp/birdnet_disk_check.lock}" +exec 9>"$lockfile" +flock -n 9 || exit 0 + # If the used space is above the threshold -if [ "$(used_disk)" -ge "$purge_threshold" ]; then - case $FULL_DISK in - purge) echo "Removing data to stay below threshold of $purge_threshold" - # Loop until the value is below 10% of the threshold, to avoid running the script too often - max_files_species_loop="$max_files_species" - while [ "$(used_disk)" -ge "$purge_threshold_target" ]; do - max_files_species_loop=$(( max_files_species_loop - safe_purge_step )) - if [ "$max_files_species_loop" -gt "$safe_files_species" ]; then - bash "$HOME/BirdNET-Pi/scripts/disk_species_clean.sh" "$max_files_species_loop" - sleep 5 - else - echo "ERROR : safeguard initiated at $safe_files_species files remaining to make sure that we do not delete too many files. Is there an issue with the path of $base_dir? Stopping core services." - /usr/local/bin/stop_core_services.sh - break - fi - done;; +if (( $(used_disk) >= purge_threshold )); then + case "${FULL_DISK:-purge}" in + purge) + echo "Removing data to stay below threshold of $purge_threshold" + # Loop until the value is below 10% of the threshold, to avoid running the script too often + max_files_species_loop="$max_files_species" + while (( $(used_disk) >= purge_threshold_target )); do + max_files_species_loop=$(( max_files_species_loop - safe_purge_step )) + if (( max_files_species_loop > safe_files_species )); then + bash "$HOME/BirdNET-Pi/scripts/disk_species_clean.sh" "$max_files_species_loop" + sleep 5 + else + echo "ERROR : safeguard initiated at $safe_files_species files remaining to make sure that we do not delete too many files. Is there an issue with the path of $base_dir? Stopping core services." + /usr/local/bin/stop_core_services.sh || true + break + fi + done + + # If still full after purge attempts, stop core services + if (( $(used_disk) >= purge_threshold )); then + echo "Disk remains >= threshold after purging. Stopping core services." + /usr/local/bin/stop_core_services.sh || true + fi + ;; - keep) echo "FULL_DISK=keep → Stopping Core Services." - /usr/local/bin/stop_core_services.sh;; + keep) + echo "FULL_DISK=keep → Stopping Core Services." + /usr/local/bin/stop_core_services.sh + ;; - *) echo "Unknown FULL_DISK value: $FULL_DISK" - exit 1;; + *) + echo "Unknown FULL_DISK value: ${FULL_DISK:-}" + exit 1 + ;; esac fi