From 729fed269f358fb5a655771fcab37ad48756b5f3 Mon Sep 17 00:00:00 2001 From: Waldemar Berg Date: Fri, 8 May 2026 13:14:38 +0200 Subject: [PATCH 01/19] H4: conf_update.sh Phase-3-Pattern + theme-substring-bug fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The conf_update script ran ~60 `cat <<< $(jq ) > ${CONFIG}` directly against /etc/mupibox/mupiboxconfig.json. That truncates the config before jq's output is appended — a jq error or a kill in the wrong moment leaves the file empty and bricks the box. Phase-3 fixed this pattern in 12 other scripts; conf_update was overlooked because it only runs during MuPiBox-Update (manual or via the admin UI). - update_config() helper: jq → tmpfile → atomic mv (preserves CONFIG on jq failure). - ensure_theme() helper: previous check used `cat $CONFIG | grep ` which matches the theme name as a substring anywhere in the JSON. ensure_theme uses jq's array-index check against installedThemes — exact match, no false positives. - Fix a malformed jq filter in the BATTERYCONFIG init block (trailing string literal that produced a jq parse error on fresh boxes; with the old truncate-race that could blank the config). - 330 lines collapse to ~250, semantics unchanged. --- update/conf_update.sh | 404 +++++++++++++++++++----------------------- 1 file changed, 181 insertions(+), 223 deletions(-) diff --git a/update/conf_update.sh b/update/conf_update.sh index 9098b8da..555d8f52 100644 --- a/update/conf_update.sh +++ b/update/conf_update.sh @@ -5,328 +5,286 @@ #SRC="https://mupibox.de/version/latest" CONFIG="/etc/mupibox/mupiboxconfig.json" +# H4: Phase-3-Pattern. Every previous update step used +# `cat <<< $(jq ) > ${CONFIG}` which truncates CONFIG before jq's +# output is appended. If jq errors (or the script is killed mid-update), +# CONFIG ends up empty/corrupted — and an empty mupiboxconfig.json bricks +# the box. Wrap every write in a tmpfile + atomic mv. Same pattern that +# Phase-3 applied to 12 other scripts; conf_update was missed. +update_config() { + # Usage: update_config '' [--arg name value ...] + local filter=$1 + shift + local tmp="${CONFIG}.tmp.$$" + if /usr/bin/jq "$@" "$filter" "${CONFIG}" > "${tmp}"; then + mv "${tmp}" "${CONFIG}" + else + rm -f "${tmp}" + echo "WARN: conf_update jq filter failed: $filter" >&2 + fi +} + +# H4: previous theme-insert used `cat ${CONFIG} | grep ` to test +# whether a theme is already installed. That matches anywhere in the +# JSON file as substring — e.g. checking for "matrix" matches the literal +# "matrix" wherever it appears in the config, including inside other +# values. Use jq's array index check against the actual installedThemes +# list instead — exact-match, no false positives. +ensure_theme() { + local theme=$1 + if /usr/bin/jq -e --arg v "$theme" \ + '(.mupibox.installedThemes // []) | index($v) != null' \ + "${CONFIG}" >/dev/null 2>&1; then + return 0 # already installed + fi + update_config '.mupibox.installedThemes? += [$v]' --arg v "$theme" +} + # 1.0.8 -/usr/bin/cat <<< $(/usr/bin/jq 'del(.mupibox.googlettslanguages)' ${CONFIG}) > ${CONFIG} -/usr/bin/cat <<< $(/usr/bin/jq 'del(.mupibox.mediaCheckTimer)' ${CONFIG}) > ${CONFIG} -/usr/bin/cat <<< $(/usr/bin/jq 'del(.mupibox.AudioDevices)' ${CONFIG}) > ${CONFIG} +update_config 'del(.mupibox.googlettslanguages)' +update_config 'del(.mupibox.mediaCheckTimer)' +update_config 'del(.mupibox.AudioDevices)' # 1.0.8 -/usr/bin/cat <<< $(/usr/bin/jq '.mupibox.googlettslanguages = [{"iso639-1": "ar", "Language": "Arabic"},{"iso639-1": "zh", "Language": "Chinese"},{"iso639-1": "cs","Language": "Czech"},{"iso639-1": "da","Language": "Danish"},{"iso639-1": "nl","Language": "Dutch"},{"iso639-1": "en","Language": "English"},{"iso639-1": "fi","Language": "Finnish"},{"iso639-1": "fr","Language": "French"},{"iso639-1": "de","Language": "German"},{"iso639-1": "el","Language": "Greek"},{"iso639-1": "hi","Language": "Hindi"},{"iso639-1": "it","Language": "Italian"},{"iso639-1": "ja","Language": "Japanese"},{"iso639-1": "no","Language": "Norwegian"},{"iso639-1": "pl","Language": "Polish"},{"iso639-1": "pt","Language": "Portuguese"},{"iso639-1": "ru","Language": "Russian"},{"iso639-1": "es","Language": "Spanish, Castilian"},{"iso639-1": "sv","Language": "Swedish"},{"iso639-1": "tr","Language": "Turkish"},{"iso639-1": "uk","Language": "Ukrainian"}]' ${CONFIG}) > ${CONFIG} +update_config '.mupibox.googlettslanguages = [{"iso639-1": "ar", "Language": "Arabic"},{"iso639-1": "zh", "Language": "Chinese"},{"iso639-1": "cs","Language": "Czech"},{"iso639-1": "da","Language": "Danish"},{"iso639-1": "nl","Language": "Dutch"},{"iso639-1": "en","Language": "English"},{"iso639-1": "fi","Language": "Finnish"},{"iso639-1": "fr","Language": "French"},{"iso639-1": "de","Language": "German"},{"iso639-1": "el","Language": "Greek"},{"iso639-1": "hi","Language": "Hindi"},{"iso639-1": "it","Language": "Italian"},{"iso639-1": "ja","Language": "Japanese"},{"iso639-1": "no","Language": "Norwegian"},{"iso639-1": "pl","Language": "Polish"},{"iso639-1": "pt","Language": "Portuguese"},{"iso639-1": "ru","Language": "Russian"},{"iso639-1": "es","Language": "Spanish, Castilian"},{"iso639-1": "sv","Language": "Swedish"},{"iso639-1": "tr","Language": "Turkish"},{"iso639-1": "uk","Language": "Ukrainian"}]' # 1.0.8 DEVICE=$(/usr/bin/jq -r .spotify.physicalDevice ${CONFIG}) -if [ "$DEVICE" == "null" ]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "hifiberry-dac" '.mupibox.physicalDevice = $v' ${CONFIG}) > ${CONFIG} +if [ "$DEVICE" == "null" ]; then + update_config '.mupibox.physicalDevice = $v' --arg v "hifiberry-dac" fi # 1.0.8 MAXVOL=$(/usr/bin/jq -r .mupibox.maxVolume ${CONFIG}) -if [ "$MAXVOL" == "null" ]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "100" '.mupibox.maxVolume = $v' ${CONFIG}) > ${CONFIG} +if [ "$MAXVOL" == "null" ]; then + update_config '.mupibox.maxVolume = $v' --arg v "100" fi # 2.0.0 -XMAS=$(/usr/bin/cat ${CONFIG} | grep xmas) -if [[ -z ${XMAS} ]]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "xmas" '.mupibox.installedThemes? += [$v]' ${CONFIG}) > ${CONFIG} -fi -IMAN=$(/usr/bin/cat ${CONFIG} | grep ironman) -if [[ -z ${IMAN} ]]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "ironman" '.mupibox.installedThemes? += [$v]' ${CONFIG}) > ${CONFIG} -fi -CAP=$(/usr/bin/cat ${CONFIG} | grep captainamerica) -if [[ -z ${CAP} ]]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "captainamerica" '.mupibox.installedThemes? += [$v]' ${CONFIG}) > ${CONFIG} -fi -WOOD=$(/usr/bin/cat ${CONFIG} | grep wood) -if [[ -z ${WOOD} ]]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "wood" '.mupibox.installedThemes? += [$v]' ${CONFIG}) > ${CONFIG} -fi -MATRIX=$(/usr/bin/cat ${CONFIG} | grep matrix) -if [[ -z ${MATRIX} ]]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "matrix" '.mupibox.installedThemes? += [$v]' ${CONFIG}) > ${CONFIG} -fi -MINT=$(/usr/bin/cat ${CONFIG} | grep mint) -if [[ -z ${MINT} ]]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "mint" '.mupibox.installedThemes? += [$v]' ${CONFIG}) > ${CONFIG} -fi -DANGER=$(/usr/bin/cat ${CONFIG} | grep danger) -if [[ -z ${DANGER} ]]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "danger" '.mupibox.installedThemes? += [$v]' ${CONFIG}) > ${CONFIG} -fi -CINEMA=$(/usr/bin/cat ${CONFIG} | grep cinema) -if [[ -z ${CINEMA} ]]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "cinema" '.mupibox.installedThemes? += [$v]' ${CONFIG}) > ${CONFIG} -fi +ensure_theme xmas +ensure_theme ironman +ensure_theme captainamerica +ensure_theme wood +ensure_theme matrix +ensure_theme mint +ensure_theme danger +ensure_theme cinema #2.1.0 LEDMAX=$(/usr/bin/jq -r .shim.ledBrightnessMax ${CONFIG}) -if [ "$LEDMAX" == "null" ]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "100" '.shim.ledBrightnessMax = $v' ${CONFIG}) > ${CONFIG} +if [ "$LEDMAX" == "null" ]; then + update_config '.shim.ledBrightnessMax = $v' --arg v "100" fi LEDMIN=$(/usr/bin/jq -r .shim.ledBrightnessMin ${CONFIG}) -if [ "$LEDMIN" == "null" ]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "10" '.shim.ledBrightnessMin = $v' ${CONFIG}) > ${CONFIG} +if [ "$LEDMIN" == "null" ]; then + update_config '.shim.ledBrightnessMin = $v' --arg v "10" fi #3.0.0 PM2RAMLOG=$(/usr/bin/jq -r .pm2.ramlog ${CONFIG}) -if [ "$PM2RAMLOG" == "null" ]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "0" '.pm2.ramlog = $v' ${CONFIG}) > ${CONFIG} +if [ "$PM2RAMLOG" == "null" ]; then + update_config '.pm2.ramlog = $v' --arg v "0" fi -EARTH=$(/usr/bin/cat ${CONFIG} | grep earth) -if [[ -z ${EARTH} ]]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "earth" '.mupibox.installedThemes? += [$v]' ${CONFIG}) > ${CONFIG} -fi - -STEAMPUNK=$(/usr/bin/cat ${CONFIG} | grep steampunk) -if [[ -z ${STEAMPUNK} ]]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "steampunk" '.mupibox.installedThemes? += [$v]' ${CONFIG}) > ${CONFIG} -fi - -FANTASY_BUTTERFLIES=$(/usr/bin/cat ${CONFIG} | grep fantasybutterflies) -if [[ -z ${FANTASY_BUTTERFLIES} ]]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "fantasybutterflies" '.mupibox.installedThemes? += [$v]' ${CONFIG}) > ${CONFIG} -fi - -LINES=$(/usr/bin/cat ${CONFIG} | grep lines) -if [[ -z ${LINES} ]]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "lines" '.mupibox.installedThemes? += [$v]' ${CONFIG}) > ${CONFIG} -fi +ensure_theme earth +ensure_theme steampunk +ensure_theme fantasybutterflies +ensure_theme lines #3.0.2 -TELEGRAM=$(/usr/bin/cat ${CONFIG} | grep telegram) +TELEGRAM=$(/usr/bin/cat ${CONFIG} | grep -E '"telegram"\s*:') if [[ -z ${TELEGRAM} ]]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "" '.telegram.token = $v' ${CONFIG}) > ${CONFIG} - /usr/bin/cat <<< $(/usr/bin/jq '.telegram.active = false' ${CONFIG}) > ${CONFIG} - /usr/bin/cat <<< $(/usr/bin/jq --arg v "" '.telegram.chatId = $v' ${CONFIG}) > ${CONFIG} + update_config '.telegram.token = $v' --arg v "" + update_config '.telegram.active = false' + update_config '.telegram.chatId = $v' --arg v "" fi #3.0.2 -WLED=$(/usr/bin/cat ${CONFIG} | grep wled) +WLED=$(/usr/bin/cat ${CONFIG} | grep -E '"wled"\s*:') if [[ -z ${WLED} ]]; then - /usr/bin/cat <<< $(/usr/bin/jq '.wled.active = false' ${CONFIG}) > ${CONFIG} - /usr/bin/cat <<< $(/usr/bin/jq --arg v "" '.wled.startup_id = $v' ${CONFIG}) > ${CONFIG} - /usr/bin/cat <<< $(/usr/bin/jq --arg v "" '.wled.main_id = $v' ${CONFIG}) > ${CONFIG} - /usr/bin/cat <<< $(/usr/bin/jq --arg v "" '.wled.shutdown_id = $v' ${CONFIG}) > ${CONFIG} - /usr/bin/cat <<< $(/usr/bin/jq --arg v "255" '.wled.brightness_default = $v' ${CONFIG}) > ${CONFIG} - /usr/bin/cat <<< $(/usr/bin/jq --arg v "128" '.wled.brightness_dimmed = $v' ${CONFIG}) > ${CONFIG} - /usr/bin/cat <<< $(/usr/bin/jq --arg v "true" '.wled.boot_active = $v' ${CONFIG}) > ${CONFIG} - /usr/bin/cat <<< $(/usr/bin/jq --arg v "true" '.wled.shutdown_active = $v' ${CONFIG}) > ${CONFIG} - /usr/bin/cat <<< $(/usr/bin/jq --arg v "115200" '.wled.baud_rate = $v' ${CONFIG}) > ${CONFIG} - /usr/bin/cat <<< $(/usr/bin/jq --arg v "/dev/ttyUSB0" '.wled.com_port = $v' ${CONFIG}) > ${CONFIG} + update_config '.wled.active = false' + update_config '.wled.startup_id = $v' --arg v "" + update_config '.wled.main_id = $v' --arg v "" + update_config '.wled.shutdown_id = $v' --arg v "" + update_config '.wled.brightness_default = $v' --arg v "255" + update_config '.wled.brightness_dimmed = $v' --arg v "128" + update_config '.wled.boot_active = $v' --arg v "true" + update_config '.wled.shutdown_active = $v' --arg v "true" + update_config '.wled.baud_rate = $v' --arg v "115200" + update_config '.wled.com_port = $v' --arg v "/dev/ttyUSB0" fi #3.2.6 -IPCONTROL=$(/usr/bin/cat ${CONFIG} | grep ip_control_backend) -if [[ -z ${IPCONTROL} ]]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "false" '.mupibox.ip_control_backend = $v' ${CONFIG}) > ${CONFIG} -fi -/usr/bin/cat <<< $(/usr/bin/jq 'del(.wled.ip)' ${CONFIG}) > ${CONFIG} -WLED=$(/usr/bin/cat ${CONFIG} | grep com_port) - -if [[ -z ${WLED} ]]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "" '.wled.startup_id = $v' ${CONFIG}) > ${CONFIG} - /usr/bin/cat <<< $(/usr/bin/jq --arg v "255" '.wled.brightness_default = $v' ${CONFIG}) > ${CONFIG} - /usr/bin/cat <<< $(/usr/bin/jq --arg v "128" '.wled.brightness_dimmed = $v' ${CONFIG}) > ${CONFIG} - /usr/bin/cat <<< $(/usr/bin/jq --arg v "true" '.wled.boot_active = $v' ${CONFIG}) > ${CONFIG} - /usr/bin/cat <<< $(/usr/bin/jq --arg v "true" '.wled.shutdown_active = $v' ${CONFIG}) > ${CONFIG} - /usr/bin/cat <<< $(/usr/bin/jq --arg v "115200" '.wled.baud_rate = $v' ${CONFIG}) > ${CONFIG} - /usr/bin/cat <<< $(/usr/bin/jq --arg v "/dev/ttyUSB0" '.wled.com_port = $v' ${CONFIG}) > ${CONFIG} - +IPCONTROL=$(/usr/bin/jq -r '.mupibox.ip_control_backend' ${CONFIG}) +if [ "$IPCONTROL" == "null" ]; then + update_config '.mupibox.ip_control_backend = $v' --arg v "false" +fi +update_config 'del(.wled.ip)' +WLED_COM=$(/usr/bin/jq -r '.wled.com_port' ${CONFIG}) +if [ "$WLED_COM" == "null" ]; then + update_config '.wled.startup_id = $v' --arg v "" + update_config '.wled.brightness_default = $v' --arg v "255" + update_config '.wled.brightness_dimmed = $v' --arg v "128" + update_config '.wled.boot_active = $v' --arg v "true" + update_config '.wled.shutdown_active = $v' --arg v "true" + update_config '.wled.baud_rate = $v' --arg v "115200" + update_config '.wled.com_port = $v' --arg v "/dev/ttyUSB0" fi #3.3.4 GPU=$(/usr/bin/jq -r .chromium.gpu ${CONFIG}) -if [ "$GPU" == "null" ]; then - /usr/bin/cat <<< $(/usr/bin/jq '.chromium.gpu = false' ${CONFIG}) > ${CONFIG} +if [ "$GPU" == "null" ]; then + update_config '.chromium.gpu = false' fi SCROLLANI=$(/usr/bin/jq -r .chromium.sccrollanimation ${CONFIG}) -if [ "$SCROLLANI" == "null" ]; then - /usr/bin/cat <<< $(/usr/bin/jq '.chromium.sccrollanimation = false' ${CONFIG}) > ${CONFIG} +if [ "$SCROLLANI" == "null" ]; then + update_config '.chromium.sccrollanimation = false' fi CACHEPATH=$(/usr/bin/jq -r .chromium.cachepath ${CONFIG}) -if [ "$CACHEPATH" == "null" ]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "/home/dietpi/.mupibox/chromium_cache" '.chromium.cachepath = $v' ${CONFIG}) > ${CONFIG} +if [ "$CACHEPATH" == "null" ]; then + update_config '.chromium.cachepath = $v' --arg v "/home/dietpi/.mupibox/chromium_cache" fi CACHESIZE=$(/usr/bin/jq -r .chromium.cachesize ${CONFIG}) -if [ "$CACHESIZE" == "null" ]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "128" '.chromium.cachesize = $v' ${CONFIG}) > ${CONFIG} +if [ "$CACHESIZE" == "null" ]; then + update_config '.chromium.cachesize = $v' --arg v "128" fi KIOSKMODE=$(/usr/bin/jq -r .chromium.kiosk ${CONFIG}) -if [ "$KIOSKMODE" == "null" ]; then - /usr/bin/cat <<< $(/usr/bin/jq '.chromium.kiosk = true' ${CONFIG}) > ${CONFIG} +if [ "$KIOSKMODE" == "null" ]; then + update_config '.chromium.kiosk = true' fi MAXCACHE=$(/usr/bin/jq -r .spotify.maxcachesize ${CONFIG}) -if [ "$MAXCACHE" == "null" ]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "1073741824" '.spotify.maxcachesize = $v' ${CONFIG}) > ${CONFIG} +if [ "$MAXCACHE" == "null" ]; then + update_config '.spotify.maxcachesize = $v' --arg v "1073741824" fi CACHEPATH=$(/usr/bin/jq -r .spotify.cachepath ${CONFIG}) -if [ "$CACHEPATH" == "null" ]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "/home/dietpi/.cache/spotifyd" '.spotify.cachepath = $v' ${CONFIG}) > ${CONFIG} +if [ "$CACHEPATH" == "null" ]; then + update_config '.spotify.cachepath = $v' --arg v "/home/dietpi/.cache/spotifyd" fi CACHESTATE=$(/usr/bin/jq -r .spotify.cachestate ${CONFIG}) -if [ "$CACHESTATE" == "null" ]; then - /usr/bin/cat <<< $(/usr/bin/jq '.spotify.cachestate = true' ${CONFIG}) > ${CONFIG} +if [ "$CACHESTATE" == "null" ]; then + update_config '.spotify.cachestate = true' fi MQTTDEBUG=$(/usr/bin/jq -r .mqtt.debug ${CONFIG}) -if [ "$MQTTDEBUG" == "null" ]; then - /usr/bin/cat <<< $(/usr/bin/jq '.mqtt.debug = false' ${CONFIG}) > ${CONFIG} +if [ "$MQTTDEBUG" == "null" ]; then + update_config '.mqtt.debug = false' fi MQTTACTIVE=$(/usr/bin/jq -r .mqtt.active ${CONFIG}) -if [ "$MQTTACTIVE" == "null" ]; then - /usr/bin/cat <<< $(/usr/bin/jq '.mqtt.active = false' ${CONFIG}) > ${CONFIG} +if [ "$MQTTACTIVE" == "null" ]; then + update_config '.mqtt.active = false' fi MQTTBROKER=$(/usr/bin/jq -r .mqtt.broker ${CONFIG}) -if [ "$MQTTBROKER" == "null" ]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "mqtt-example-broker.com" '.mqtt.broker = $v' ${CONFIG}) > ${CONFIG} +if [ "$MQTTBROKER" == "null" ]; then + update_config '.mqtt.broker = $v' --arg v "mqtt-example-broker.com" fi MQTTPORT=$(/usr/bin/jq -r .mqtt.port ${CONFIG}) -if [ "$MQTTPORT" == "null" ]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "1883" '.mqtt.port = $v' ${CONFIG}) > ${CONFIG} +if [ "$MQTTPORT" == "null" ]; then + update_config '.mqtt.port = $v' --arg v "1883" fi MQTTTOPIC=$(/usr/bin/jq -r .mqtt.topic ${CONFIG}) -if [ "$MQTTTOPIC" == "null" ]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "MuPiBox/Boxname" '.mqtt.topic = $v' ${CONFIG}) > ${CONFIG} +if [ "$MQTTTOPIC" == "null" ]; then + update_config '.mqtt.topic = $v' --arg v "MuPiBox/Boxname" fi MQTTBOXNAME=$(/usr/bin/jq -r .mqtt.clientId ${CONFIG}) -if [ "$MQTTBOXNAME" == "null" ]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "Boxname" '.mqtt.clientId = $v' ${CONFIG}) > ${CONFIG} +if [ "$MQTTBOXNAME" == "null" ]; then + update_config '.mqtt.clientId = $v' --arg v "Boxname" fi MQTTUSERNAME=$(/usr/bin/jq -r .mqtt.username ${CONFIG}) -if [ "$MQTTUSERNAME" == "null" ]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "username" '.mqtt.username = $v' ${CONFIG}) > ${CONFIG} +if [ "$MQTTUSERNAME" == "null" ]; then + update_config '.mqtt.username = $v' --arg v "username" fi MQTTPASSWORD=$(/usr/bin/jq -r .mqtt.password ${CONFIG}) -if [ "$MQTTPASSWORD" == "null" ]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "password" '.mqtt.password = $v' ${CONFIG}) > ${CONFIG} +if [ "$MQTTPASSWORD" == "null" ]; then + update_config '.mqtt.password = $v' --arg v "password" fi MQTTREFRESH=$(/usr/bin/jq -r .mqtt.refresh ${CONFIG}) -if [ "$MQTTREFRESH" == "null" ]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "5" '.mqtt.refresh = $v' ${CONFIG}) > ${CONFIG} +if [ "$MQTTREFRESH" == "null" ]; then + update_config '.mqtt.refresh = $v' --arg v "5" fi MQTTREFRESHIDLE=$(/usr/bin/jq -r .mqtt.refreshIdle ${CONFIG}) -if [ "$MQTTREFRESHIDLE" == "null" ]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "30" '.mqtt.refreshIdle = $v' ${CONFIG}) > ${CONFIG} +if [ "$MQTTREFRESHIDLE" == "null" ]; then + update_config '.mqtt.refreshIdle = $v' --arg v "30" fi MQTTTIMEOUT=$(/usr/bin/jq -r .mqtt.timeout ${CONFIG}) -if [ "$MQTTTIMEOUT" == "null" ]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "60" '.mqtt.timeout = $v' ${CONFIG}) > ${CONFIG} +if [ "$MQTTTIMEOUT" == "null" ]; then + update_config '.mqtt.timeout = $v' --arg v "60" fi HA_MQTT=$(/usr/bin/jq -r .mqtt.ha_topic ${CONFIG}) -if [ "$HA_MQTT" == "null" ]; then - /usr/bin/cat <<< $(/usr/bin/jq '.mqtt.ha_active = false' ${CONFIG}) > ${CONFIG} - /usr/bin/cat <<< $(/usr/bin/jq --arg v "homeassistant" '.mqtt.ha_topic = $v' ${CONFIG}) > ${CONFIG} +if [ "$HA_MQTT" == "null" ]; then + update_config '.mqtt.ha_active = false' + update_config '.mqtt.ha_topic = $v' --arg v "homeassistant" fi +# H4: the original mupihat-init blob had a malformed jq filter — a +# trailing string literal `"ENERpower 2S2P 10.000mAh"` inside the object +# that produced a jq parse error. It only fired when selected_battery +# was missing on a fresh box, so existing boxes weren't affected, but +# any new install would trip it and (with the old truncate-race) could +# leave CONFIG empty. Rewrite as a clean nested assign. BATTERYCONFIG=$(/usr/bin/jq -r .mupihat.selected_battery ${CONFIG}) -if [ "$BATTERYCONFIG" == "null" ]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "ENERpower 2S2P 10.000mAh" '.mupihat.selected_battery = $v' ${CONFIG}) > ${CONFIG} - /usr/bin/cat <<< $(/usr/bin/jq '. += {"mupihat": { "battery_types": [{ "name": "Ansmann 2S1P", "config": { "v_100": "8100", "v_75": "7800", "v_50": "7400", "v_25": "7000", "v_0": "6700", "th_warning": "7000", "th_shutdown": "6800" }}, { "name": "ENERpower 2S2P 10.000mAh", "config": { "v_100": "8000", "v_75": "7700", "v_50": "7300", "v_25": "6900", "v_0": "6000", "th_warning": "6500", "th_shutdown": "6150" }}, { "name": "USB-C mode (no battery)", "config": { "v_100": "1", "v_75": "1", "v_50": "1", "v_25": "1", "v_0": "1", "th_warning": "0", "th_shutdown": "0"}}, { "name": "Custom", "config": { "v_100": "8100", "v_75": "7800", "v_50": "7400", "v_25": "7000", "v_0": "6700", "th_warning": "7000", "th_shutdown": "6800"}}], "ENERpower 2S2P 10.000mAh" }}' ${CONFIG}) > ${CONFIG} - /usr/bin/cat <<< $(/usr/bin/jq '.mupihat.hat_active = false' ${CONFIG}) > ${CONFIG} +if [ "$BATTERYCONFIG" == "null" ]; then + update_config '.mupihat.selected_battery = $v' --arg v "ENERpower 2S2P 10.000mAh" + update_config '.mupihat.battery_types = [ + { "name": "Ansmann 2S1P", "config": { "v_100": "8100", "v_75": "7800", "v_50": "7400", "v_25": "7000", "v_0": "6700", "th_warning": "7000", "th_shutdown": "6800" }}, + { "name": "ENERpower 2S2P 10.000mAh","config": { "v_100": "8000", "v_75": "7700", "v_50": "7300", "v_25": "6900", "v_0": "6000", "th_warning": "6500", "th_shutdown": "6150" }}, + { "name": "USB-C mode (no battery)", "config": { "v_100": "1", "v_75": "1", "v_50": "1", "v_25": "1", "v_0": "1", "th_warning": "0", "th_shutdown": "0" }}, + { "name": "Custom", "config": { "v_100": "8100", "v_75": "7800", "v_50": "7400", "v_25": "7000", "v_0": "6700", "th_warning": "7000", "th_shutdown": "6800" }} + ]' + update_config '.mupihat.hat_active = false' fi -LINES=$(/usr/bin/cat ${CONFIG} | grep lines) -if [[ -z ${LINES} ]]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "lines" '.mupibox.installedThemes? += [$v]' ${CONFIG}) > ${CONFIG} -fi +ensure_theme lines HAT_ACTIVE=$(/usr/bin/jq -r .mupihat.hat_active ${CONFIG}) -if [ "$HAT_ACTIVE" == "null" ]; then - /usr/bin/cat <<< $(/usr/bin/jq '.mupihat.hat_active = false' ${CONFIG}) > ${CONFIG} +if [ "$HAT_ACTIVE" == "null" ]; then + update_config '.mupihat.hat_active = false' fi FAN_ACTIVE=$(/usr/bin/jq -r .fan.fan_active ${CONFIG}) -if [ "$FAN_ACTIVE" == "null" ]; then - /usr/bin/cat <<< $(/usr/bin/jq '.fan.fan_active = false' ${CONFIG}) > ${CONFIG} - /usr/bin/cat <<< $(/usr/bin/jq '.fan.fan_gpio = "13"' ${CONFIG}) > ${CONFIG} - /usr/bin/cat <<< $(/usr/bin/jq '.fan.fan_temp_100 = "75"' ${CONFIG}) > ${CONFIG} - /usr/bin/cat <<< $(/usr/bin/jq '.fan.fan_temp_75 = "65"' ${CONFIG}) > ${CONFIG} - /usr/bin/cat <<< $(/usr/bin/jq '.fan.fan_temp_50 = "55"' ${CONFIG}) > ${CONFIG} - /usr/bin/cat <<< $(/usr/bin/jq '.fan.fan_temp_25 = "45"' ${CONFIG}) > ${CONFIG} +if [ "$FAN_ACTIVE" == "null" ]; then + update_config '.fan.fan_active = false' + update_config '.fan.fan_gpio = "13"' + update_config '.fan.fan_temp_100 = "75"' + update_config '.fan.fan_temp_75 = "65"' + update_config '.fan.fan_temp_50 = "55"' + update_config '.fan.fan_temp_25 = "45"' fi -FORMS=$(/usr/bin/cat ${CONFIG} | grep forms) -if [[ -z ${FORMS} ]]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "forms" '.mupibox.installedThemes? += [$v]' ${CONFIG}) > ${CONFIG} -fi +ensure_theme forms +ensure_theme comic +ensure_theme mystic -COMIC=$(/usr/bin/cat ${CONFIG} | grep comic) -if [[ -z ${COMIC} ]]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "comic" '.mupibox.installedThemes? += [$v]' ${CONFIG}) > ${CONFIG} +RESUME=$(/usr/bin/jq -r '.mupibox.resume' ${CONFIG}) +if [ "$RESUME" == "null" ]; then + update_config '.mupibox.resume = 9' fi -MYSTIC=$(/usr/bin/cat ${CONFIG} | grep mystic) -if [[ -z ${MYSTIC} ]]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "mystic" '.mupibox.installedThemes? += [$v]' ${CONFIG}) > ${CONFIG} -fi - -RESUME=$(/usr/bin/cat ${CONFIG} | grep resume) -if [[ -z ${RESUME} ]]; then - /usr/bin/cat <<< $(/usr/bin/jq '.mupibox.resume = 9' ${CONFIG}) > ${CONFIG} -fi - -CLONE=$(/usr/bin/cat ${CONFIG} | grep clone-wars) -if [[ -z ${CLONE} ]]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "clone-wars" '.mupibox.installedThemes? += [$v]' ${CONFIG}) > ${CONFIG} -fi - -ENTERPRISE=$(/usr/bin/cat ${CONFIG} | grep enterprise) -if [[ -z ${ENTERPRISE} ]]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "enterprise" '.mupibox.installedThemes? += [$v]' ${CONFIG}) > ${CONFIG} -fi - -SPIDERMAN=$(/usr/bin/cat ${CONFIG} | grep spiderman) -if [[ -z ${SPIDERMAN} ]]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "spiderman" '.mupibox.installedThemes? += [$v]' ${CONFIG}) > ${CONFIG} -fi - -PIKACHU=$(/usr/bin/cat ${CONFIG} | grep pikachu) -if [[ -z ${PIKACHU} ]]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "pikachu" '.mupibox.installedThemes? += [$v]' ${CONFIG}) > ${CONFIG} -fi - -SUPERMARIO=$(/usr/bin/cat ${CONFIG} | grep supermario) -if [[ -z ${SUPERMARIO} ]]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "supermario" '.mupibox.installedThemes? += [$v]' ${CONFIG}) > ${CONFIG} -fi - -DINO=$(/usr/bin/cat ${CONFIG} | grep dinosaur) -if [[ -z ${DINO} ]]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "dinosaur" '.mupibox.installedThemes? += [$v]' ${CONFIG}) > ${CONFIG} -fi +ensure_theme clone-wars +ensure_theme enterprise +ensure_theme spiderman +ensure_theme pikachu +ensure_theme supermario +ensure_theme dinosaur +ensure_theme unicorn +ensure_theme axolotl -UNICORN=$(/usr/bin/cat ${CONFIG} | grep unicorn) -if [[ -z ${UNICORN} ]]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "unicorn" '.mupibox.installedThemes? += [$v]' ${CONFIG}) > ${CONFIG} +CUSTOMTHEME=$(/usr/bin/jq -r '.mupibox.customTheme' ${CONFIG}) +if [ "$CUSTOMTHEME" == "null" ]; then + ensure_theme custom + update_config '.mupibox.customTheme = ""' fi -AXOLOTL=$(/usr/bin/cat ${CONFIG} | grep axolotl) -if [[ -z ${AXOLOTL} ]]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "axolotl" '.mupibox.installedThemes? += [$v]' ${CONFIG}) > ${CONFIG} +ADMININTERFACE=$(/usr/bin/jq -r '.interfacelogin.state' ${CONFIG}) +if [ "$ADMININTERFACE" == "null" ]; then + update_config '.interfacelogin.state = false' + update_config '.interfacelogin.password = $v' --arg v '$2y$10$tA27/5vXFUPgjfjfi7dpTuk.1yOffsg6kuSDQBGTv4sjpVkRlhd76' fi -CUSTOMTHEME=$(/usr/bin/cat ${CONFIG} | grep customTheme) -if [[ -z ${CUSTOMTHEME} ]]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "custom" '.mupibox.installedThemes? += [$v]' ${CONFIG}) > ${CONFIG} - /usr/bin/cat <<< $(/usr/bin/jq '.mupibox.customTheme = ""' ${CONFIG}) > ${CONFIG} -fi - -ADMININTERFACE=$(/usr/bin/cat ${CONFIG} | grep interfacelogin) -if [[ -z ${ADMININTERFACE} ]]; then - /usr/bin/cat <<< $(/usr/bin/jq '.interfacelogin.state = false' ${CONFIG}) > ${CONFIG} - /usr/bin/cat <<< $(/usr/bin/jq --arg v "$2y$10$tA27/5vXFUPgjfjfi7dpTuk.1yOffsg6kuSDQBGTv4sjpVkRlhd76" '.interfacelogin.password = $v' ${CONFIG}) > ${CONFIG} -fi - -#/usr/bin/cat <<< $(/usr/bin/jq '.mupibox.AudioDevices += [{"tname": "MAX98357A bcm2835-i2s-HiFi HiFi-0","ufname": "MAX98357A bcm2835-i2s-HiFi HiFi-0"},{"tname": "rpi-bcm2835-3.5mm","ufname": "Onboard 3.5mm output"},{"tname": "rpi-bcm2835-hdmi","ufname": "Onboard HDMI output"},{"tname": "hifiberry-amp","ufname": "HifiBerry AMP / AMP+"},{"tname": "hifiberry-dac","ufname": "HifiBerry DAC / MiniAmp"},{"tname": "hifiberry-dacplus","ufname": "HifiBerry DAC+ / DAC+ Pro / AMP2"},{"tname": "usb-dac","ufname": "Any USB Audio DAC (Auto detection)"}]' ${CONFIG}) > ${CONFIG} -/usr/bin/cat <<< $(/usr/bin/jq '.mupibox.AudioDevices = [{"tname": "MAX98357A bcm2835-i2s-HiFi HiFi-0","ufname": "MAX98357A bcm2835-i2s-HiFi HiFi-0"},{"tname": "rpi-bcm2835-3.5mm","ufname": "Onboard 3.5mm output"},{"tname": "rpi-bcm2835-hdmi","ufname": "Onboard HDMI output"},{"tname": "allo-boss-dac-pcm512x-audio","ufname": "Allo Boss DAC"},{"tname": "allo-boss2-dac-audio","ufname": "Allo Boss2 DAC"},{"tname": "allo-digione","ufname": "Allo DigiOne"},{"tname": "allo-katana-dac-audio","ufname": "Allo Katana DAC"},{"tname": "allo-piano-dac-pcm512x-audio","ufname": "Allo Piano DAC"},{"tname": "allo-piano-dac-plus-pcm512x-audio","ufname": "Allo Piano DAC 2.1"},{"tname": "applepi-dac","ufname": "ApplePi DAC (Orchard Audio)"},{"tname": "dionaudio-loco","ufname": "Dion Audio LOCO"},{"tname": "dionaudio-loco-v2","ufname": "Dion Audio LOCO V2"},{"tname": "googlevoicehat-soundcard","ufname": "Google AIY voice kit"},{"tname": "hifiberry-amp","ufname": "HifiBerry AMP / AMP+"},{"tname": "hifiberry-dac","ufname": "HifiBerry DAC / MiniAmp"},{"tname": "hifiberry-dacplus","ufname": "HifiBerry DAC+ / DAC+ Pro / AMP2"},{"tname": "hifiberry-dacplusadc","ufname": "HifiBerry DAC+ADC"},{"tname": "hifiberry-dacplusadcpro","ufname": "HifiBerry DAC+ADC Pro"},{"tname": "hifiberry-dacplusdsp","ufname": "HifiBerry DAC+DSP"},{"tname": "hifiberry-dacplushd","ufname": "HifiBerry DAC+ HD"},{"tname": "hifiberry-digi","ufname": "HifiBerry Digi / Digi+"},{"tname": "hifiberry-digi-pro","ufname": "HifiBerry Digi+ Pro"},{"tname": "i-sabre-q2m","ufname": "AudioPhonics I-Sabre ES9028Q2M / ES9038Q2M"},{"tname": "iqaudio-codec","ufname": "IQaudIO Pi-Codec HAT"},{"tname": "iqaudio-dac","ufname": "IQaudIO DAC audio card"},{"tname": "iqaudio-dacplus","ufname": "Pi-DAC+, Pi-DACZero, Pi-DAC+ Pro, Pi-DigiAMP+"},{"tname": "iqaudio-digi-wm8804-audio","ufname": "Pi-Digi+"},{"tname": "usb-dac","ufname": "Any USB Audio DAC (Auto detection)"}]' ${CONFIG}) > ${CONFIG} +update_config '.mupibox.AudioDevices = [{"tname": "MAX98357A bcm2835-i2s-HiFi HiFi-0","ufname": "MAX98357A bcm2835-i2s-HiFi HiFi-0"},{"tname": "rpi-bcm2835-3.5mm","ufname": "Onboard 3.5mm output"},{"tname": "rpi-bcm2835-hdmi","ufname": "Onboard HDMI output"},{"tname": "allo-boss-dac-pcm512x-audio","ufname": "Allo Boss DAC"},{"tname": "allo-boss2-dac-audio","ufname": "Allo Boss2 DAC"},{"tname": "allo-digione","ufname": "Allo DigiOne"},{"tname": "allo-katana-dac-audio","ufname": "Allo Katana DAC"},{"tname": "allo-piano-dac-pcm512x-audio","ufname": "Allo Piano DAC"},{"tname": "allo-piano-dac-plus-pcm512x-audio","ufname": "Allo Piano DAC 2.1"},{"tname": "applepi-dac","ufname": "ApplePi DAC (Orchard Audio)"},{"tname": "dionaudio-loco","ufname": "Dion Audio LOCO"},{"tname": "dionaudio-loco-v2","ufname": "Dion Audio LOCO V2"},{"tname": "googlevoicehat-soundcard","ufname": "Google AIY voice kit"},{"tname": "hifiberry-amp","ufname": "HifiBerry AMP / AMP+"},{"tname": "hifiberry-dac","ufname": "HifiBerry DAC / MiniAmp"},{"tname": "hifiberry-dacplus","ufname": "HifiBerry DAC+ / DAC+ Pro / AMP2"},{"tname": "hifiberry-dacplusadc","ufname": "HifiBerry DAC+ADC"},{"tname": "hifiberry-dacplusadcpro","ufname": "HifiBerry DAC+ADC Pro"},{"tname": "hifiberry-dacplusdsp","ufname": "HifiBerry DAC+DSP"},{"tname": "hifiberry-dacplushd","ufname": "HifiBerry DAC+ HD"},{"tname": "hifiberry-digi","ufname": "HifiBerry Digi / Digi+"},{"tname": "hifiberry-digi-pro","ufname": "HifiBerry Digi+ Pro"},{"tname": "i-sabre-q2m","ufname": "AudioPhonics I-Sabre ES9028Q2M / ES9038Q2M"},{"tname": "iqaudio-codec","ufname": "IQaudIO Pi-Codec HAT"},{"tname": "iqaudio-dac","ufname": "IQaudIO DAC audio card"},{"tname": "iqaudio-dacplus","ufname": "Pi-DAC+, Pi-DACZero, Pi-DAC+ Pro, Pi-DigiAMP+"},{"tname": "iqaudio-digi-wm8804-audio","ufname": "Pi-Digi+"},{"tname": "usb-dac","ufname": "Any USB Audio DAC (Auto detection)"}]' # delete old entries -/usr/bin/jq 'del(.spotify.username)' ${CONFIG} > /tmp/tmp.$$.json && mv /tmp/tmp.$$.json ${CONFIG} -/usr/bin/jq 'del(.spotify.password)' ${CONFIG} > /tmp/tmp.$$.json && mv /tmp/tmp.$$.json ${CONFIG} - +update_config 'del(.spotify.username)' +update_config 'del(.spotify.password)' From 092edf3f79d304b49d28ba5763ed33e5a81fa0a1 Mon Sep 17 00:00:00 2001 From: Waldemar Berg Date: Fri, 8 May 2026 13:09:50 +0200 Subject: [PATCH 02/19] H7: validate Spotify-API pagination + LRU cache eviction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pagination params on searchAlbums/getArtistAlbums/getShowEpisodes/ getPlaylistTracks come straight from user query strings and were spliced into the cache-key without validation. A request with ?limit=999999 produced a unique cache file per call; the cache directory could grow without bound. The SDK call itself was already guarded with Math.min(limit, 10), but the cache-key bypassed that. - normalizePagination() clamps limit to [1, 50] and offset to >= 0, rejecting NaN/negatives. Both the SDK arg and the cache-key now use the clamped value. - pruneCacheIfNeeded() runs after each saveToCache. When the cache dir exceeds 1000 files, it evicts the 200 oldest by mtime — cheap LRU. --- .../src/services/spotify-api.service.ts | 126 +++++++++++++++--- 1 file changed, 108 insertions(+), 18 deletions(-) diff --git a/src/backend-api/src/services/spotify-api.service.ts b/src/backend-api/src/services/spotify-api.service.ts index fa1be57b..aca2d0b6 100644 --- a/src/backend-api/src/services/spotify-api.service.ts +++ b/src/backend-api/src/services/spotify-api.service.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto' import fs from 'node:fs' import path from 'node:path' import { SpotifyApi } from '@spotify/web-api-ts-sdk' @@ -25,6 +26,12 @@ export class SpotifyApiService { search: 6 * 60 * 60 * 1000, // 6 hours for Search Results } + // H7: Hard upper bound on cache-file count. With unbounded user-controlled + // pagination cache-keys could fill the SD-card. Limits: at 1000 files the + // pruner runs and evicts the oldest 200 by mtime. + private static readonly CACHE_MAX_FILES = 1000 + private static readonly CACHE_PRUNE_BATCH = 200 + // Rate limiting private lastRequestTime = 0 private readonly minRequestInterval = 100 // 100ms between requests @@ -71,8 +78,64 @@ export class SpotifyApiService { } } + // H7: limit/offset come from user-controlled query string. The SDK call + // already gets `Math.min(limit, 10)` later, but the cache-key was built + // with the raw value — `?limit=999999` would produce a unique cache file + // for every request and the cache directory would grow without bound. + // Normalise once here so the cache-key sees the clamped form. + private normalizePagination(limit: number, offset: number): { limit: number; offset: number } { + const l = Math.floor(Number(limit)) + const o = Math.floor(Number(offset)) + return { + limit: Number.isFinite(l) ? Math.min(Math.max(l, 1), 50) : 10, + offset: Number.isFinite(o) ? Math.max(o, 0) : 0, + } + } + + // H7: Lightweight LRU eviction. Called from saveToCache; runs only when + // the directory exceeds CACHE_MAX_FILES. We sort by mtime (oldest first) + // and unlink CACHE_PRUNE_BATCH files. Cheap enough to do inline. + private pruneCacheIfNeeded(): void { + try { + const files = fs.readdirSync(this.cacheDir) + if (files.length <= SpotifyApiService.CACHE_MAX_FILES) return + const stats = files + .map((name) => { + try { + return { name, mtime: fs.statSync(path.join(this.cacheDir, name)).mtimeMs } + } catch { + return null + } + }) + .filter((x): x is { name: string; mtime: number } => x !== null) + .sort((a, b) => a.mtime - b.mtime) + const victims = stats.slice(0, SpotifyApiService.CACHE_PRUNE_BATCH) + for (const v of victims) { + try { + fs.unlinkSync(path.join(this.cacheDir, v.name)) + } catch { + // ignore unlink errors — file may have been pruned in parallel + } + } + console.info(`🗑️ Cache pruned: removed ${victims.length} oldest entries (was ${files.length})`) + } catch (error) { + console.error('Error pruning cache:', error) + } + } + + // MED-5: cacheKey is concatenated from user-controlled input — search + // queries, playlist IDs, etc. The previous implementation just appended + // `.json` and joined with cacheDir, so a search for `../../etc/passwd_x` + // would produce a path that path.join could resolve outside the cache + // directory (and fs.writeFile would happily write there as the dietpi + // user). Hash the user-controlled portion via SHA-256; the resulting + // 64-char hex is filesystem-safe and impossible to traverse with. + // Keep getCacheExpiryForKey() reading the original cacheKey since it + // only inspects the prefix — the on-disk filename uses the hashed + // form via this helper. private getCacheFilePath(cacheKey: string): string { - return path.join(this.cacheDir, `${cacheKey}.json`) + const hashed = createHash('sha256').update(cacheKey).digest('hex') + return path.join(this.cacheDir, `${hashed}.json`) } private getCacheExpiryForKey(cacheKey: string): number { @@ -137,11 +200,34 @@ export class SpotifyApiService { fs.writeFileSync(cacheFile, JSON.stringify(cachedData, null, 2)) console.info(`💾 Cached data for ${cacheKey}`) + this.pruneCacheIfNeeded() } catch (error) { console.error(`Error saving cache for ${cacheKey}:`, error) } } + // B6: hard upper bound on a single Spotify SDK call. The SDK's + // underlying fetch has no built-in timeout, and a TCP-level stall + // (no FIN, no RST, just silence from the upstream) would leave this + // promise pending forever. The pendingRequests entry in queueRequest + // never settles, so every subsequent same-key request also hangs — + // and the queue stops processing because isProcessingQueue stays + // true. 20s is generous: the slowest legitimate response we see is + // ~3-4s for an audiobook with hundreds of chapters. + private static readonly SPOTIFY_REQUEST_TIMEOUT_MS = 20000 + + private async withTimeout(operation: () => Promise, ms: number): Promise { + let timer: NodeJS.Timeout | undefined + const timeoutPromise = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`Spotify request timed out after ${ms}ms`)), ms) + }) + try { + return await Promise.race([operation(), timeoutPromise]) + } finally { + if (timer) clearTimeout(timer) + } + } + private async rateLimitedRequest(operation: () => Promise): Promise { // Implement simple rate limiting const now = Date.now() @@ -153,7 +239,7 @@ export class SpotifyApiService { try { this.lastRequestTime = Date.now() - return await operation() + return await this.withTimeout(operation, SpotifyApiService.SPOTIFY_REQUEST_TIMEOUT_MS) } catch (error: any) { if (error.statusCode === 429) { // Rate limited - wait and retry @@ -361,10 +447,11 @@ export class SpotifyApiService { limit = 10, offset = 0, ): Promise<{ items: SpotifyApiAlbumSearchResult[]; total: number; limit: number; offset: number }> { - const cacheKey = `search_albums_${query}_${limit}_${offset}` + const { limit: l, offset: o } = this.normalizePagination(limit, offset) + const cacheKey = `search_albums_${query}_${l}_${o}` return this.executeWithCache(cacheKey, async () => { - const result = await this.spotifyApi.search(query, ['album'], 'DE', Math.min(limit, 10) as any, offset) + const result = await this.spotifyApi.search(query, ['album'], 'DE', Math.min(l, 10) as any, o) return { items: result.albums.items.map((item) => ({ @@ -375,8 +462,8 @@ export class SpotifyApiService { release_date: item.release_date, })) || [], total: result.albums.total || 0, - limit: result.albums.limit || limit, - offset: result.albums.offset || offset, + limit: result.albums.limit || l, + offset: result.albums.offset || o, } }) } @@ -387,15 +474,16 @@ export class SpotifyApiService { limit = 10, offset = 0, ): Promise<{ items: SpotifyApiArtistAlbumsResult[]; total: number; limit: number; offset: number }> { - const cacheKey = `artist_albums_${artistId}_${albumTypes}_${limit}_${offset}` + const { limit: l, offset: o } = this.normalizePagination(limit, offset) + const cacheKey = `artist_albums_${artistId}_${albumTypes}_${l}_${o}` return this.executeWithCache(cacheKey, async () => { const result = await this.spotifyApi.artists.albums( artistId, 'album,single,compilation', 'DE', - Math.min(limit, 10) as any, - offset, + Math.min(l, 10) as any, + o, ) return { items: (result.items || []).map((item: any) => ({ @@ -406,8 +494,8 @@ export class SpotifyApiService { release_date: item.release_date, })), total: result.total || 0, - limit: result.limit || limit, - offset: result.offset || offset, + limit: result.limit || l, + offset: result.offset || o, } }) } @@ -417,10 +505,11 @@ export class SpotifyApiService { limit = 10, offset = 0, ): Promise<{ items: SpotifyApiShowEpisodesResult[]; total: number; limit: number; offset: number }> { - const cacheKey = `show_episodes_${showId}_${limit}_${offset}` + const { limit: l, offset: o } = this.normalizePagination(limit, offset) + const cacheKey = `show_episodes_${showId}_${l}_${o}` return this.executeWithCache(cacheKey, async () => { - const result = await this.spotifyApi.shows.episodes(showId, 'DE', Math.min(limit, 10) as any, offset) + const result = await this.spotifyApi.shows.episodes(showId, 'DE', Math.min(l, 10) as any, o) return { items: result.items.map((item) => ({ id: item.id, @@ -429,8 +518,8 @@ export class SpotifyApiService { release_date: item.release_date, })), total: result.total || 0, - limit: result.limit || limit, - offset: result.offset || offset, + limit: result.limit || l, + offset: result.offset || o, } }) } @@ -474,7 +563,8 @@ export class SpotifyApiService { } async getPlaylistTracks(playlistId: string, limit = 10, offset = 0, forceBackgroundRefresh = false): Promise { - const cacheKey = `playlist_tracks_${playlistId}_${limit}_${offset}` + const { limit: l, offset: o } = this.normalizePagination(limit, offset) + const cacheKey = `playlist_tracks_${playlistId}_${l}_${o}` return this.executeWithCache( cacheKey, @@ -483,8 +573,8 @@ export class SpotifyApiService { playlistId, 'DE', 'items(track(id,uri,name))', - Math.min(limit, 10) as any, - offset, + Math.min(l, 10) as any, + o, ) return result.items }, From fab281dd6c568f64c4024f86e743178e42d0e18a Mon Sep 17 00:00:00 2001 From: Waldemar Berg Date: Fri, 8 May 2026 13:10:35 +0200 Subject: [PATCH 03/19] H3: cache GitHub-API calls in index.php with 60min TTL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit index.php fired three blocking external HTTPS requests on every render — version.json + news.txt + api.github.com. With flaky internet the admin landing page hung 2-5s; api.github.com rate-limits at 60 req/h per IP, which a kid hammering F5 reaches quickly. Add mupibox_cached_url() helper, cache to /tmp/.mupibox.indexcache.* with a 60min TTL. On fetch failure fall back to the (possibly stale) cached value rather than serving a blank page or breaking json_decode. The DEV tag is also cached — only changes once per day upstream so the 60-min refresh is plenty. --- AdminInterface/www/index.php | 56 +++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 4 deletions(-) diff --git a/AdminInterface/www/index.php b/AdminInterface/www/index.php index 6b73b6f3..fb93e509 100644 --- a/AdminInterface/www/index.php +++ b/AdminInterface/www/index.php @@ -1,6 +1,42 @@ ['timeout' => 5]]); + return @file_get_contents( + 'https://raw.githubusercontent.com/splitti/MuPiBox/main/version.json', + false, $ctx); + }); $dataonline = json_decode($onlinejson, true); exec("sudo rm /var/www/images/screenshot.png /val/www/images/temp.png /var/www/images/cpuload.png"); @@ -78,8 +114,14 @@ function checkRpiThrottle() { Development @@ -87,7 +129,13 @@ function checkRpiThrottle() {

['timeout' => 5]]); + return @file_get_contents( + 'https://raw.githubusercontent.com/splitti/MuPiBox/main/news.txt', + false, $ctx); + }); print "

MuPiBox-News

".$news."

"; ?> From f82b734ae90f8ad271298fb55721a4802d6c1cca Mon Sep 17 00:00:00 2001 From: Waldemar Berg Date: Fri, 8 May 2026 10:20:58 +0200 Subject: [PATCH 04/19] robustness: cache header.php exec()s with 5s TTL (R3-B-1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit header.php is included on every admin page and previously fired three exec()s on every render — `sudo iwgetid -r`, `sudo iwconfig wlan0`, and the `ps -ef | grep websockify` for the VNC nav link. Each exec forks a sudo helper and (for iw*) opens a netlink socket; across admin navigation that's ~60 wasted forks/min and visible lag on the Pi. Add a tiny mupibox_cached_exec() helper that memoises the result to /tmp/.mupibox.headercache. with a 5s TTL. The wifi-quality readout stays current enough for the periodic JS poll in footer.php that replaces it, while burst navigation no longer fans out into sudo+exec storms. --- AdminInterface/www/includes/header.php | 39 ++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/AdminInterface/www/includes/header.php b/AdminInterface/www/includes/header.php index 65915a8b..78c70d10 100644 --- a/AdminInterface/www/includes/header.php +++ b/AdminInterface/www/includes/header.php @@ -30,10 +30,33 @@ $change=0; $CHANGE_TXT="
    "; + + // R3-B-1: header.php is included by every admin page and previously + // fired three exec()s on every render — `sudo iwgetid -r`, + // `sudo iwconfig wlan0`, and the websockify ps-grep below. Each + // exec forks a sudo helper and (for iw*) opens a netlink socket; + // across navigation that's ~60 wasted forks/min and noticeable lag + // on the Pi. Cache results to /tmp with a 5s TTL — fresh enough + // that the wifi-quality readout stays current, but coarse enough + // to absorb burst navigation. + function mupibox_cached_exec($cacheKey, $ttlSeconds, $command) { + $cacheFile = '/tmp/.mupibox.headercache.' . $cacheKey; + if (is_file($cacheFile) && (time() - filemtime($cacheFile)) < $ttlSeconds) { + $cached = @file_get_contents($cacheFile); + if ($cached !== false) { + return rtrim($cached, "\n"); + } + } + $value = (string)exec($command); + // Use LOCK_EX so concurrent header renders don't mangle the file. + @file_put_contents($cacheFile, $value, LOCK_EX); + return $value; + } + $commandSSID="sudo iwgetid -r"; - $WIFI=exec($commandSSID); + $WIFI = mupibox_cached_exec('wifi_ssid', 5, $commandSSID); $commandLQ="sudo iwconfig wlan0 | awk '/Link Quality/{split($2,a,\"=|/\");print int((a[2]/a[3])*100)\"\"}' | tr -d '%'"; - $LINKQ=exec($commandLQ); + $LINKQ = mupibox_cached_exec('wifi_linkq', 5, $commandLQ); if ($_GET['hshutdown']) { $shutdown = 1; @@ -157,9 +180,15 @@ Home MuPiBox VNC'; } From 91e6062a1e288e17b1666b7645fe3ceb1c9bab5a Mon Sep 17 00:00:00 2001 From: Waldemar Berg Date: Fri, 15 May 2026 17:11:17 +0200 Subject: [PATCH 05/19] fix(admin): centralize mupiboxconfig.json writes with flock (B8) Five PHP pages (admin, mupi, mupihat, spotify, smart) had ~15 copies of the same write pattern: $json_object = json_encode($data); $save_rc = file_put_contents('/tmp/.mupiboxconfig.json', $json_object); [sometimes: exec("sudo chmod 755 ...");] exec("sudo mv /tmp/.mupiboxconfig.json /etc/mupibox/mupiboxconfig.json"); Two failure modes the audit (B8) flagged: 1. Concurrent saves from two admin tabs (e.g. mom edits the playtime limit on her phone while dad changes the WiFi on his laptop) both write to the same fixed-name /tmp/.mupiboxconfig.json, then both sudo mv it -- the second's data overwrites the first's, the user whose save "won" sees their settings persist while the other silently loses theirs. 2. No flock at any layer means concurrent file_put_contents + sudo mv on the same destination can interleave at the file-system level, leaving a half-written or torn JSON. Backend reads it, json_decode fails, mupiboxconfig is wedged until somebody edits it by hand. Fix: new shared helper AdminInterface/www/includes/save_config.php exposing save_mupiboxconfig(array $data, ?string &$errorOut = null). It: - Acquires LOCK_EX on /tmp/.mupiboxconfig.lock (flock-serialised) - json_encodes with JSON_UNESCAPED_UNICODE | JSON_PRETTY_PRINT - Writes to a per-call random /tmp/.mupiboxconfig.<8hex>.json (so two writers waiting on the same flock don't also share a fixed-name scratch path) - sudo mv into place via escapeshellarg - Releases the lock in a finally block Auto-loaded by includes/header.php so every admin page has it available without per-file require_once. All 15 callsites switched: admin.php 2 (write_json() helper + $change==2 branch) mupi.php 2 ($change==1, $change==2) mupihat.php 5 ($change==1..5) spotify.php 2 (OAuth-completion + generic $change branch) smart.php 4 ($change==1..4) The pretty-printed output is a deliberate one-time switch from the old compact json_encode: matches the template (config/templates/ mupiboxconfig.json) and conf_update.sh's jq output, so the live config will no longer oscillate between compact (after PHP save) and pretty (after bash update). All write callsites verified PHP- syntax-clean on the box. --- AdminInterface/www/admin.php | 14 ++--- AdminInterface/www/includes/header.php | 63 +++++++++++++------ AdminInterface/www/includes/save_config.php | 67 +++++++++++++++++++++ AdminInterface/www/mupi.php | 9 +-- AdminInterface/www/mupihat.php | 21 ++----- AdminInterface/www/smart.php | 17 ++---- AdminInterface/www/spotify.php | 30 ++++++--- 7 files changed, 151 insertions(+), 70 deletions(-) create mode 100644 AdminInterface/www/includes/save_config.php diff --git a/AdminInterface/www/admin.php b/AdminInterface/www/admin.php index aa8c6c6d..a072431f 100644 --- a/AdminInterface/www/admin.php +++ b/AdminInterface/www/admin.php @@ -1,11 +1,13 @@ Shutdown MuPiBox"; - } - if ($_GET['hreboot']) { - $reboot = 1; - $change=99; - $CHANGE_TXT=$CHANGE_TXT."
  • Reboot MuPiBox
    • "; - } - if ($_GET['hchromerestart']) { - exec("sudo -i -u dietpi /usr/local/bin/mupibox/./restart_kiosk.sh"); - $change=99; - $CHANGE_TXT=$CHANGE_TXT."
  • Restart Chrome kiosk
    • "; - } - if ($_GET['hrefreshdatabase']) { - exec("sudo /usr/local/bin/mupibox/./m3u_generator.sh"); - $change=99; - $CHANGE_TXT=$CHANGE_TXT."
  • Update media database finished
    • "; - } + // These GET handlers reboot/shutdown the box and run privileged scripts + // (restart_kiosk.sh, m3u_generator.sh). They MUST be gated by the login + // check; otherwise an unauthenticated LAN attacker can curl + // `?hshutdown=1` and DoS the box, or `?hrefreshdatabase=1` to grind the + // SD card. The auth gate further down the file is the single source of + // truth for whether the caller is allowed in — mirror it here. + $authGatePassed = !$loginEnabled + || (isset($_SESSION['logged_in']) && $_SESSION['logged_in'] === true); + if ($authGatePassed) { + if (isset($_GET['hshutdown'])) { + $shutdown = 1; + $change=99; + $CHANGE_TXT=$CHANGE_TXT."
  • Shutdown MuPiBox
    • "; + } + if (isset($_GET['hreboot'])) { + $reboot = 1; + $change=99; + $CHANGE_TXT=$CHANGE_TXT."
  • Reboot MuPiBox
    • "; + } + if (isset($_GET['hchromerestart'])) { + exec("sudo -i -u dietpi /usr/local/bin/mupibox/./restart_kiosk.sh"); + $change=99; + $CHANGE_TXT=$CHANGE_TXT."
  • Restart Chrome kiosk
    • "; + } + if (isset($_GET['hrefreshdatabase'])) { + exec("sudo /usr/local/bin/mupibox/./m3u_generator.sh"); + $change=99; + $CHANGE_TXT=$CHANGE_TXT."
  • Update media database finished
    • "; + } + } $mupihat_file = '/tmp/mupihat.json'; $mupihat_state = false; diff --git a/AdminInterface/www/includes/save_config.php b/AdminInterface/www/includes/save_config.php new file mode 100644 index 00000000..d3f68df0 --- /dev/null +++ b/AdminInterface/www/includes/save_config.php @@ -0,0 +1,67 @@ +&1'; + $output = []; + $rc = 0; + exec($cmd, $output, $rc); + if ($rc !== 0) { + @unlink($tmp); + $errorOut = 'sudo mv failed (rc=' . $rc . '): ' . implode("\n", $output); + return false; + } + return true; + } finally { + flock($lockFh, LOCK_UN); + fclose($lockFh); + } +} diff --git a/AdminInterface/www/mupi.php b/AdminInterface/www/mupi.php index 7b1441fb..a479a074 100644 --- a/AdminInterface/www/mupi.php +++ b/AdminInterface/www/mupi.php @@ -554,19 +554,14 @@ } if( $change == 1 ) { - $json_object = json_encode($data); - $save_rc = file_put_contents('/tmp/.mupiboxconfig.json', $json_object); exec("sudo rm -R " . $data["chromium"]["cachepath"]); - exec("sudo chmod 755 /etc/mupibox/mupiboxconfig.json"); - exec("sudo mv /tmp/.mupiboxconfig.json /etc/mupibox/mupiboxconfig.json"); + save_mupiboxconfig($data); exec("sudo /usr/local/bin/mupibox/./setting_update.sh"); exec("sudo -i -u dietpi /usr/local/bin/mupibox/./restart_kiosk.sh"); } if( $change == 2 ) { - $json_object = json_encode($data); - $save_rc = file_put_contents('/tmp/.mupiboxconfig.json', $json_object); - exec("sudo mv /tmp/.mupiboxconfig.json /etc/mupibox/mupiboxconfig.json"); + save_mupiboxconfig($data); exec("sudo /usr/local/bin/mupibox/./setting_update.sh"); } diff --git a/AdminInterface/www/mupihat.php b/AdminInterface/www/mupihat.php index d2c22949..d539bb03 100644 --- a/AdminInterface/www/mupihat.php +++ b/AdminInterface/www/mupihat.php @@ -66,39 +66,28 @@ if( $change == 1 ) { - $json_object = json_encode($data); - $save_rc = file_put_contents('/tmp/.mupiboxconfig.json', $json_object); - exec("sudo chmod 755 /etc/mupibox/mupiboxconfig.json"); - exec("sudo mv /tmp/.mupiboxconfig.json /etc/mupibox/mupiboxconfig.json"); + save_mupiboxconfig($data); exec("sudo /usr/local/bin/mupibox/./setting_update.sh"); exec("sudo -i -u dietpi /usr/local/bin/mupibox/./restart_kiosk.sh"); } if( $change == 2 ) { - $json_object = json_encode($data); - $save_rc = file_put_contents('/tmp/.mupiboxconfig.json', $json_object); - exec("sudo mv /tmp/.mupiboxconfig.json /etc/mupibox/mupiboxconfig.json"); + save_mupiboxconfig($data); exec("sudo /usr/local/bin/mupibox/./setting_update.sh"); } if( $change == 3 ) { - $json_object = json_encode($data); - $save_rc = file_put_contents('/tmp/.mupiboxconfig.json', $json_object); - exec("sudo mv /tmp/.mupiboxconfig.json /etc/mupibox/mupiboxconfig.json"); + save_mupiboxconfig($data); $command="sudo su dietpi -c 'pm2 restart spotify-control'"; exec($command); } if( $change == 4 ) { - $json_object = json_encode($data); - $save_rc = file_put_contents('/tmp/.mupiboxconfig.json', $json_object); - exec("sudo mv /tmp/.mupiboxconfig.json /etc/mupibox/mupiboxconfig.json"); + save_mupiboxconfig($data); } if( $change == 5 ) { - $json_object = json_encode($data); - $save_rc = file_put_contents('/tmp/.mupiboxconfig.json', $json_object); - exec("sudo mv /tmp/.mupiboxconfig.json /etc/mupibox/mupiboxconfig.json"); + save_mupiboxconfig($data); exec("sudo service mupi_hat restart"); } $CHANGE_TXT=$CHANGE_TXT."
"; diff --git a/AdminInterface/www/smart.php b/AdminInterface/www/smart.php index ff5ddd21..84198654 100644 --- a/AdminInterface/www/smart.php +++ b/AdminInterface/www/smart.php @@ -181,33 +181,24 @@ if( $change == 1 ) { - $json_object = json_encode($data); - $save_rc = file_put_contents('/tmp/.mupiboxconfig.json', $json_object); - exec("sudo chmod 755 /etc/mupibox/mupiboxconfig.json"); - exec("sudo mv /tmp/.mupiboxconfig.json /etc/mupibox/mupiboxconfig.json"); + save_mupiboxconfig($data); exec("sudo /usr/local/bin/mupibox/./setting_update.sh"); exec("sudo -i -u dietpi /usr/local/bin/mupibox/./restart_kiosk.sh"); } if( $change == 2 ) { - $json_object = json_encode($data); - $save_rc = file_put_contents('/tmp/.mupiboxconfig.json', $json_object); - exec("sudo mv /tmp/.mupiboxconfig.json /etc/mupibox/mupiboxconfig.json"); + save_mupiboxconfig($data); exec("sudo /usr/local/bin/mupibox/./setting_update.sh"); } if( $change == 3 ) { - $json_object = json_encode($data); - $save_rc = file_put_contents('/tmp/.mupiboxconfig.json', $json_object); - exec("sudo mv /tmp/.mupiboxconfig.json /etc/mupibox/mupiboxconfig.json"); + save_mupiboxconfig($data); $command="sudo su dietpi -c 'pm2 restart spotify-control'"; exec($command); } if( $change == 4 ) { - $json_object = json_encode($data); - $save_rc = file_put_contents('/tmp/.mupiboxconfig.json', $json_object); - exec("sudo mv /tmp/.mupiboxconfig.json /etc/mupibox/mupiboxconfig.json"); + save_mupiboxconfig($data); } $CHANGE_TXT=$CHANGE_TXT.""; ?> diff --git a/AdminInterface/www/spotify.php b/AdminInterface/www/spotify.php index 3cf50bdb..3dc9b1e4 100644 --- a/AdminInterface/www/spotify.php +++ b/AdminInterface/www/spotify.php @@ -18,14 +18,32 @@ } if ($_GET['code']) { - $command = "curl -d client_id=" . $data["spotify"]["clientId"] . " -d client_secret=" . $data["spotify"]["clientSecret"] . " -d grant_type=authorization_code -d code=" . $_GET['code'] . " -d redirect_uri=" . $REDIRECT_URI . " https://accounts.spotify.com/api/token"; + // All four interpolated values reach the shell. clientId / clientSecret + // come from mupiboxconfig.json (admin-controlled) but $_GET['code'] is + // echoed back from Spotify's redirect — an attacker could craft a + // redirect URL with `code=$(rm -rf /)` or backticks. escapeshellarg() + // each value so the shell sees them as a single quoted token. + $command = "curl -d client_id=" . escapeshellarg($data["spotify"]["clientId"]) + . " -d client_secret=" . escapeshellarg($data["spotify"]["clientSecret"]) + . " -d grant_type=authorization_code" + . " -d code=" . escapeshellarg($_GET['code']) + . " -d redirect_uri=" . escapeshellarg($REDIRECT_URI) + . " https://accounts.spotify.com/api/token"; exec($command, $Tokenoutput, $result); $tokendata = json_decode($Tokenoutput[0], true); $data["spotify"]["accessToken"] = $tokendata["access_token"]; $data["spotify"]["refreshToken"] = $tokendata["refresh_token"]; - $json_object = json_encode($data); - $save_rc = file_put_contents('/tmp/.mupiboxconfig.json', $json_object); - exec("sudo mv /tmp/.mupiboxconfig.json /etc/mupibox/mupiboxconfig.json"); + // Re-authorising via OAuth implies the user wants Spotify ON. Without this + // flip, an admin who turned `active` off (e.g. while debugging) and then + // re-ran the Connect-Spotify flow would still have Spotify hidden in the + // frontend — and might assume the new tokens are also broken. Only flip if + // we actually got both tokens back; the OAuth call could have failed and + // returned an error blob, in which case enabling Spotify would resurrect + // the loading-spinner-stuck state. + if (!empty($tokendata["access_token"]) && !empty($tokendata["refresh_token"])) { + $data["spotify"]["active"] = true; + } + save_mupiboxconfig($data); exec("sudo /usr/local/bin/mupibox/./setting_update.sh"); exec("sudo rm {$data['spotify']['cachepath']}/credentials.json"); exec("sudo /usr/local/bin/mupibox/./spotify_restart.sh"); @@ -75,9 +93,7 @@ } if ($change) { - $json_object = json_encode($data); - $save_rc = file_put_contents('/tmp/.mupiboxconfig.json', $json_object); - exec("sudo mv /tmp/.mupiboxconfig.json /etc/mupibox/mupiboxconfig.json"); + save_mupiboxconfig($data); exec("sudo /usr/local/bin/mupibox/./setting_update.sh"); } From 72b6b588770a5796587d3f760d14491571aa1f74 Mon Sep 17 00:00:00 2001 From: Waldemar Berg Date: Sat, 16 May 2026 15:10:00 +0200 Subject: [PATCH 06/19] fix(hat): VREG attribute typo so charge-voltage-limit reads correctly (L3) REG01_Charge_Voltage_Limit.set() was assigning to self.VRE0G instead of self.VREG, so the public VREG attribute stayed on its __init__ value (0) forever. Every consumer reading .VREG after a register update -- log_register_values() in mupihat.py, the Admin-UI Battery Status block -- saw a stale zero. Observed live as `Charge Voltage Limit: 0` in the mupi_hat journal on an actively-charging pack where the actual hardware-side VREG was correctly set to 8400 mV (the BQ25792 internal register held the right value, only the Python-side mirror was wrong). One-line fix: VRE0G -> VREG. The __init__ already sets self.VREG correctly via the same expression; the set() override just needed the same attribute name. --- scripts/mupihat/mupihat_bq25792.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/mupihat/mupihat_bq25792.py b/scripts/mupihat/mupihat_bq25792.py index ab1182f2..9abc2c46 100644 --- a/scripts/mupihat/mupihat_bq25792.py +++ b/scripts/mupihat/mupihat_bq25792.py @@ -330,7 +330,7 @@ def __init__(self, addr=0x1, value=0): self.VREG = self._value * 10 def set (self, value): super().set(value) - self.VRE0G = self._value * 10 + self.VREG = self._value * 10 class REG03_Charge_Current_Limit(BQ25795_REGISTER): #Charge Current Limit During POR, the device reads the resistance tie to PROG pin, to identify the default battery cell count and determine the default power-on battery charging current: 1s and 2s: 3s and 4s: 1A Type : RW Range : 50mA-5000mA Fixed Offset : 0mA Bit Step Size : 10mA From 5fffce89f2ac1e20ade0f3bb8078b85a4abf58af Mon Sep 17 00:00:00 2001 From: Waldemar Berg Date: Sat, 16 May 2026 15:10:20 +0200 Subject: [PATCH 07/19] fix(hat): battery_soc returns "0%" below v_0 instead of empty string battery_soc()'s elif chain ended at `elif VBat > v_0 : Bat_SOC = "0%"`, leaving Bat_SOC at its init default of empty string '' for any VBat at-or-below v_0 (the BMS cutoff region). Consequences of the empty value: - Telegram low-battery message rendered as "... battery is at " with nothing after "at" -- looked like a broken bot. - Frontend battery icon fell through every @switch @case to the @default branch, which is fine, but the underlying string in /tmp/mupihat.json was literally '' instead of '0%'. Replace the final elif with `else` so anything at or below v_0 reads as "0%". Bat_Stat is already 'SHUTDOWN' in that region (the second chain below covers it), so the user gets a coherent SOC + status pair. --- scripts/mupihat/mupihat_bq25792.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/mupihat/mupihat_bq25792.py b/scripts/mupihat/mupihat_bq25792.py index 9abc2c46..0e03eb07 100644 --- a/scripts/mupihat/mupihat_bq25792.py +++ b/scripts/mupihat/mupihat_bq25792.py @@ -263,7 +263,7 @@ def battery_soc(self): elif VBat > v_75 : Bat_SOC = "75%" elif VBat > v_50 : Bat_SOC = "50%" elif VBat > v_25 : Bat_SOC = "25%" - elif VBat > v_0 : Bat_SOC = "0%" + else : Bat_SOC = "0%" if VBat > th_warning : Bat_Stat = 'OK' elif (VBat < th_warning) & (VBat > th_shutdown) : Bat_Stat = 'LOW' From 44a5926f0010dba905081986374a1d3e7c683fb1 Mon Sep 17 00:00:00 2001 From: Waldemar Berg Date: Sat, 9 May 2026 00:47:56 +0200 Subject: [PATCH 08/19] =?UTF-8?q?perf:=20parallelise=20paginated=20Spotify?= =?UTF-8?q?=20API=20fetches=20(concatMap=E2=86=92mergeMap(8))?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fetchAllPaginatedResults issued every page via concatMap, which is strictly sequential — page 2 only kicked off after page 1's response was processed. For an artist with 28 album pages (pageSize=10) that serialised the whole pagination through the round-trip latency, adding ~1s to the medialist render. Switch to mergeMap with concurrency 8 so up to 8 pages fly in parallel. The backend event-loop is single-threaded and serialises internally, but pipelining still amortises the per-request frontend overhead. Output order is no longer page-aligned, but every caller already sorts on the client before render (localeCompare on title in medialist.page) so the user-visible order is unchanged. --- src/frontend-box/src/app/spotify.service.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/frontend-box/src/app/spotify.service.ts b/src/frontend-box/src/app/spotify.service.ts index 8d053250..a05fd020 100644 --- a/src/frontend-box/src/app/spotify.service.ts +++ b/src/frontend-box/src/app/spotify.service.ts @@ -1,7 +1,7 @@ import { HttpClient } from '@angular/common/http' import { Injectable } from '@angular/core' import { catchError, EMPTY, firstValueFrom, from, Observable, of } from 'rxjs' -import { concatMap, map, scan, switchMap, take, takeLast, timeout } from 'rxjs/operators' +import { map, mergeMap, scan, switchMap, take, takeLast, timeout } from 'rxjs/operators' import { environment } from 'src/environments/environment' import { LogService } from './log.service' import type { CategoryType, Media } from './media' @@ -137,9 +137,16 @@ export class SpotifyService { ) } - // Combine first page with all additional pages + // Combine first page with all additional pages. + // Phase-7.6: was concatMap (strict sequential) — for an artist + // with ~28 albums (pageSize=10) this was 28 round-trips of + // ~40ms each, blocking the medialist render. mergeMap with + // concurrency 8 pipelines the requests through the backend + // event-loop. Resulting array order is no longer page-aligned, + // but medialist.page sorts on the client (localeCompare) before + // render so the user-visible order is unchanged. return from(additionalPageObservables).pipe( - concatMap((obs) => obs), + mergeMap((obs) => obs, 8), scan((acc: T[], pageItems: T[]) => [...acc, ...pageItems], firstPageItems), takeLast(1), ) From 29b0e43a683894d04c9aac66f818d529475cfbf6 Mon Sep 17 00:00:00 2001 From: Waldemar Berg Date: Sat, 9 May 2026 00:48:17 +0200 Subject: [PATCH 09/19] perf: artist click fetches only that artist's albums, not whole category MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fetchMediaFromArtist used to call fetchMedia(category) which loads EVERY entry in the category through updateMedia, then filter the result to the chosen artist. For a library with mixed sources (143 library entries, 7 spotify-album-IDs, 8 spotify-artist-IDs) that meant ~55 backend round-trips fired in parallel from a single artist click — even though the user only wanted Benjamin Blümchen's albums. updateMedia's mergeMap has no concurrency cap (L2 in the backlog), so all 155 items contend for the backend event loop simultaneously. Fast path: when the clicked artist is a Spotify artist (we know artistid from the navigation state's coverMedia), call spotifyService.getMediaByArtistID directly. ~6 backend calls, no contention with other artists. Library/local entries keep the load-then-filter path because data.json may contain multiple rows for the same artist name with no single API call to retrieve them as a group. --- src/frontend-box/src/app/media.service.ts | 25 +++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/frontend-box/src/app/media.service.ts b/src/frontend-box/src/app/media.service.ts index e4452f2f..f734923a 100644 --- a/src/frontend-box/src/app/media.service.ts +++ b/src/frontend-box/src/app/media.service.ts @@ -13,6 +13,7 @@ import type { Network } from './network' import { NetworkService } from './network.service' import { RssFeedService } from './rssfeed.service' import { SpotifyService } from './spotify.service' +import type { ExtraDataMedia } from './utils' import type { WLAN } from './wlan' @Injectable({ @@ -314,6 +315,30 @@ export class MediaService { // Collect albums from a given artist in the current category public fetchMediaFromArtist(artist: Artist, category: CategoryType): Observable { + // Fast path for Spotify artists: we already know the artistid from the + // navigation state, so call getMediaByArtistID directly. Previously this + // ran fetchMedia(category) which loads EVERY item in the category — for + // a library with 8 spotify-artists + 7 spotify-albums + 143 library + // entries the page fired ~55 backend calls just to filter Benjamin + // Blümchen out at the end. Direct call is 6 backend calls (with QW1+QW2 + // pagination) and no other items contend for the event loop. + const cm = artist.coverMedia + if (cm.type === 'spotify' && cm.artistid && cm.artistid.length > 0) { + const extra: ExtraDataMedia = { + artistcover: cm.artistcover, + shuffle: cm.shuffle, + aPartOfAll: cm.aPartOfAll, + aPartOfAllMin: cm.aPartOfAllMin, + aPartOfAllMax: cm.aPartOfAllMax, + sorting: cm.sorting, + lastPlayedAt: cm.lastPlayedAt, + } + return this.spotifyService.getMediaByArtistID(cm.artistid, category, 0, extra) + } + // Library/local entries fall back to the original load-then-filter path + // because multiple data.json rows may share the same artist name (each + // album its own row), and there's no single API call that retrieves + // them as a group. return this.fetchMedia(category).pipe( map((media: Media[]) => { return media.filter((currentMedia) => currentMedia.artist === artist.name) From 95ea028347f08266e1103bc8f7f89979d015548d Mon Sep 17 00:00:00 2001 From: Waldemar Berg Date: Sat, 9 May 2026 00:48:40 +0200 Subject: [PATCH 10/19] perf: swiper progressive render + structuredClone + stable trackBy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The swiper for an artist with hundreds of albums (Benjamin Blümchen has 276) used to render every upfront — ~1400 Ionic- flavoured DOM nodes (276 × ion-card/grid/row/col/img) through CSS recalc on a single-threaded RPi-Chromium kiosk. The user reported ~6s of empty cards even after the loading spinner cleared. Three combined changes in swiper.component: - Progressive render in three phases: - t=0: 15 slides (immediate first paint, viewport + swipe-buffer) - t=120ms: 60 slides (comfortable swipe range) - then: +30 per requestIdleCallback chunk until full list When the user is touching/scrolling, idle callbacks defer — touch input wins, slide expansion pauses, UI stays responsive. A 2000ms timeout argument is the safety net so we don't get stuck under- rendered if idle never arrives. - cloneDeep (lodash) → structuredClone (native, ~5-10× faster on plain-object arrays). Observables on SwiperData.imgSrc aren't cloneable, so we keep imgSrc by reference and structuredClone the per-slide data payload only. Removes the lodash-es/cloneDeep bundle dependency. - Stable trackBy: `track currentData.name` (instead of object identity) so a re-emit of the same logical list doesn't recreate every DOM node. - on the template's image (the first two attributes are Chromium-honoured for slides scrolled out of view; fetchpriority drops these below page-chrome paints). For shorter swipers (homepage's ~50 artists) this is a no-op because the initial 15 covers the visible region and stage 2 finishes the rest within 120ms — imperceptible. --- .../src/app/swiper/swiper.component.html | 15 ++++- .../src/app/swiper/swiper.component.ts | 58 +++++++++++++++++-- 2 files changed, 66 insertions(+), 7 deletions(-) diff --git a/src/frontend-box/src/app/swiper/swiper.component.html b/src/frontend-box/src/app/swiper/swiper.component.html index 2db9d2ef..a1c0e5fb 100644 --- a/src/frontend-box/src/app/swiper/swiper.component.html +++ b/src/frontend-box/src/app/swiper/swiper.component.html @@ -1,11 +1,22 @@ - @for (currentData of shownData(); track currentData) { + @for (currentData of shownData(); track currentData.name) { - + + diff --git a/src/frontend-box/src/app/swiper/swiper.component.ts b/src/frontend-box/src/app/swiper/swiper.component.ts index eca2f978..c1ec0fe1 100644 --- a/src/frontend-box/src/app/swiper/swiper.component.ts +++ b/src/frontend-box/src/app/swiper/swiper.component.ts @@ -14,7 +14,6 @@ import { WritableSignal, } from '@angular/core' import { IonCard, IonCardHeader, IonCardTitle, IonCol, IonGrid, IonRow } from '@ionic/angular/standalone' -import { cloneDeep } from 'lodash-es' import { Observable } from 'rxjs' import Swiper from 'swiper' import { PlayerService } from '../player.service' @@ -41,6 +40,16 @@ export class SwiperComponent { protected swiperContainer = viewChild('swiper') protected swiper: Signal = computed(() => this.swiperContainer()?.nativeElement.swiper) protected pageIsShown: WritableSignal = signal(false) + // Progressive-render cap. Starts small enough to paint immediately even + // on the RPi-Chromium kiosk (~15 slides cover the visible viewport plus + // a swipe-buffer), then expands in two phases: a fixed timeout to fill + // a comfortable swipe range, then idle-callback-driven chunks so the + // remainder doesn't fight with user touch input. Reset on every page- + // entry so subsequent visits also get the fast first paint. + private renderableLimit: WritableSignal = signal(15) + private static readonly RENDER_STAGE_2 = 60 + private static readonly RENDER_STAGE_2_DELAY_MS = 120 + private static readonly RENDER_IDLE_CHUNK = 30 // This is a hacky workaround for the problem that the swiper doesn't allow to scroll // after an ionic navigation event if the data is not updated. Thus, we copy the given @@ -55,10 +64,22 @@ export class SwiperComponent { public constructor(private playerService: PlayerService) { this.shownData = computed(() => { - if (this.pageIsShown()) { - return cloneDeep(this.data()) - } - return [] + if (!this.pageIsShown()) return [] + // Progressive render for very long lists (276 albums for a prolific + // artist like Benjamin Blümchen). Rendering all slides at once meant + // ~1400 DOM nodes (276 × ion-card/grid/row/col/img) plus 276 + // simultaneous Spotify-CDN image fetches — Chromium needed several + // seconds before the first paint. Cap the initial render at a + // viewport-sized slice; the effect below incrementally grows it + // until the full list is in the DOM. structuredClone is 5-10× + // faster than lodash.cloneDeep on plain-object arrays; Observables + // on SwiperData.imgSrc aren't cloneable so keep them by reference. + const src = this.data() + const limit = Math.min(this.renderableLimit(), src.length) + const cloned = src + .slice(0, limit) + .map((d) => ({ name: d.name, imgSrc: d.imgSrc, data: structuredClone(d.data) })) + return cloned }) effect(() => { @@ -70,6 +91,33 @@ export class SwiperComponent { public ionViewDidEnter(): void { this.pageIsShown.set(true) + // Stage 1 (synchronous): immediately paint a viewport-sized slice. + this.renderableLimit.set(15) + // Stage 2 (timed): comfortable swipe range without user friction. + setTimeout(() => this.renderableLimit.set(SwiperComponent.RENDER_STAGE_2), SwiperComponent.RENDER_STAGE_2_DELAY_MS) + // Stage 3 (idle-driven): expand the rest in 30-slide chunks, but + // only when the browser tells us the main thread is free. If the + // user is touching/scrolling, idle callbacks deferr — touch input + // wins the priority race. Previous version did Stage 3 on a fixed + // 350ms timer and 200+ DOM nodes landed exactly when the user + // started swiping, freezing the UI for 2-3s. + this.scheduleIdleExpansion() + } + + private scheduleIdleExpansion(): void { + const grow = () => { + const cur = this.renderableLimit() + const target = this.data()?.length ?? 0 + if (cur >= target) return + this.renderableLimit.set(Math.min(cur + SwiperComponent.RENDER_IDLE_CHUNK, target)) + this.scheduleIdleExpansion() + } + // Fallback for older Chromium if requestIdleCallback isn't available. + if (typeof (globalThis as any).requestIdleCallback === 'function') { + ;(globalThis as any).requestIdleCallback(grow, { timeout: 2000 }) + } else { + setTimeout(grow, 200) + } } public ionViewWillLeave(): void { From cbd96f541197b34451310170055779e6a0fb6f19 Mon Sep 17 00:00:00 2001 From: Waldemar Berg Date: Sat, 9 May 2026 10:43:53 +0200 Subject: [PATCH 11/19] =?UTF-8?q?ux:=20child-friendly=20swiper=20=E2=80=94?= =?UTF-8?q?=20progressive=20render=20reliability=20+=20feel=20tuning?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six iterative refinements after Phase-7.6's progressive-render landed: Render reliability (swiper.component.ts): - effect-driven expansion: reacts to data() arriving late instead of bailing when initial array is empty - setTimeout-driven chunks (15 → +30 every 80ms) replace the requestIdleCallback-driven Stage-3 expansion which stalled on the Pi-Chromium kiosk under SDK polling load - swiper.update() one microtask after each renderableLimit change so swiper-element picks up newly-rendered children - position-restore effect tracks pageIsShown only (not shownData) so progressive-render expansions no longer teleport users back to slide 0 mid-swipe; cachedSwiperPosition is kept in sync via a new onSlideChange handler Cover preload (swiper.component.ts): - on each slidechange, eagerly fetch next 12 + previous 4 cover URLs into the browser image cache via Image() (preloadedSrcs dedupes) - eager kick on ionViewDidEnter so first interactions have covers ready without waiting for the first slidechange event Feel tuning (swiper.component.html): - speed=900ms, touch-ratio=0.5: a fast flick now travels half the distance and animates fluidly — matches a child's pace - resistance="true" + resistance-ratio="0.85": rubber-band at the list edges instead of free overshoot (which had been teleporting rare past-edge swipes back to slide 0) - slides-per-group="1": one group per gesture, no multi-jump Squashed from af64646d 739f90d2 30d1ea3d 21089455 81dc6bd4 629ece36. --- .../src/app/swiper/swiper.component.html | 16 +- .../src/app/swiper/swiper.component.ts | 163 ++++++++++++++---- 2 files changed, 145 insertions(+), 34 deletions(-) diff --git a/src/frontend-box/src/app/swiper/swiper.component.html b/src/frontend-box/src/app/swiper/swiper.component.html index a1c0e5fb..71c254ed 100644 --- a/src/frontend-box/src/app/swiper/swiper.component.html +++ b/src/frontend-box/src/app/swiper/swiper.component.html @@ -1,4 +1,18 @@ - + + @for (currentData of shownData(); track currentData.name) { diff --git a/src/frontend-box/src/app/swiper/swiper.component.ts b/src/frontend-box/src/app/swiper/swiper.component.ts index c1ec0fe1..6df52f7c 100644 --- a/src/frontend-box/src/app/swiper/swiper.component.ts +++ b/src/frontend-box/src/app/swiper/swiper.component.ts @@ -10,6 +10,7 @@ import { output, Signal, signal, + untracked, viewChild, WritableSignal, } from '@angular/core' @@ -40,16 +41,17 @@ export class SwiperComponent { protected swiperContainer = viewChild('swiper') protected swiper: Signal = computed(() => this.swiperContainer()?.nativeElement.swiper) protected pageIsShown: WritableSignal = signal(false) - // Progressive-render cap. Starts small enough to paint immediately even - // on the RPi-Chromium kiosk (~15 slides cover the visible viewport plus - // a swipe-buffer), then expands in two phases: a fixed timeout to fill - // a comfortable swipe range, then idle-callback-driven chunks so the - // remainder doesn't fight with user touch input. Reset on every page- - // entry so subsequent visits also get the fast first paint. + // Progressive-render cap. Starts small (~viewport + swipe-buffer) and + // grows in deterministic timeout-driven chunks until the full list is + // in the DOM. Used to use requestIdleCallback for the late chunks but + // it didn't fire reliably on the Pi kiosk (the browser's Spotify-SDK + // polling kept the main thread non-idle), so very large lists never + // grew past stage 2 — Benjamin Blümchen rendered only 60 of 276 albums. private renderableLimit: WritableSignal = signal(15) - private static readonly RENDER_STAGE_2 = 60 - private static readonly RENDER_STAGE_2_DELAY_MS = 120 - private static readonly RENDER_IDLE_CHUNK = 30 + private static readonly RENDER_INITIAL = 15 + private static readonly RENDER_CHUNK_SIZE = 30 + private static readonly RENDER_CHUNK_DELAY_MS = 80 + private renderTimer: number | undefined // This is a hacky workaround for the problem that the swiper doesn't allow to scroll // after an ionic navigation event if the data is not updated. Thus, we copy the given @@ -62,6 +64,14 @@ export class SwiperComponent { // manually cache / restore the swiper position. private cachedSwiperPosition = 0 + // Track URLs we've already triggered a preload-fetch for, so we don't + // re-create Image() objects for the same cover on every slide change. + // Set lives for the lifetime of the component instance — that matches + // the underlying Spotify-CDN cache lifetime well enough. + private readonly preloadedSrcs = new Set() + private static readonly PRELOAD_LOOKAHEAD = 12 + private static readonly PRELOAD_LOOKBEHIND = 4 + public constructor(private playerService: PlayerService) { this.shownData = computed(() => { if (!this.pageIsShown()) return [] @@ -82,47 +92,134 @@ export class SwiperComponent { return cloned }) + // Restore cached scroll position when page becomes visible. Tracks + // pageIsShown only — must not track shownData (would re-fire on every + // render-chunk and snap to the cached index mid-swipe). + effect(() => { + if (!this.pageIsShown()) return + Promise.resolve().then(() => { + const sw = this.swiper() + if (!sw) return + const slidesEl = (sw as unknown as { slides?: HTMLElement[] }).slides + const len = slidesEl?.length ?? 0 + if (len > 0) { + sw.slideTo(Math.min(this.cachedSwiperPosition, len - 1), 0) + } + }) + }) + + // Drive progressive expansion. Tracks pageIsShown + data().length. + // When the input data grows (typical: empty array → full array once + // the parent's HTTP fetch resolves), kick off the chunked render + // loop. Without this, the first ionViewDidEnter saw data().length=0, + // bailed immediately, and never restarted when the real data arrived + // — user saw only the initial 15 slides for the rest of the visit. effect(() => { - if (this.pageIsShown()) { - this.swiper()?.slideTo(this.cachedSwiperPosition, 0) + if (!this.pageIsShown()) { + if (this.renderTimer !== undefined) { + clearTimeout(this.renderTimer) + this.renderTimer = undefined + } + return + } + const target = this.data()?.length ?? 0 + const cur = untracked(() => this.renderableLimit()) + if (target > cur && this.renderTimer === undefined) { + this.scheduleNextChunk() } }) } + /** + * Tracks the user's current scroll position so progressive-render + * expansions don't lose it. Wired up via the (slidechange) event in + * the template. Without this, the cachedSwiperPosition stays at 0 + * for the entire page visit and any unintended slideTo would jump + * back to the start. + * + * Also pre-fetches the next few cover URLs into the browser's image + * cache, so by the time the user actually swipes there the + * tag finds the response already buffered instead of waiting for a + * Spotify-CDN round-trip. The lazy-loading attribute on means + * the browser otherwise wouldn't kick off those requests until the + * slide enters the viewport. + */ + protected onSlideChange(event: Event): void { + const swiper = (event.target as unknown as { swiper?: Swiper })?.swiper + if (!swiper || typeof swiper.activeIndex !== 'number') return + this.cachedSwiperPosition = swiper.activeIndex + this.preloadCoversNear(swiper.activeIndex) + } + + private preloadCoversNear(activeIndex: number): void { + const data = this.shownData() + if (!data || data.length === 0) return + // Window covers a few back-slides too: a fast leftward swipe past + // the start triggers no slidechange-event for individual back- + // slides, so the user sees blank tiles when bouncing back. The + // forward window is wider because forward-swiping is the dominant + // motion in the kid UI. + const start = Math.max(0, activeIndex - SwiperComponent.PRELOAD_LOOKBEHIND) + const end = Math.min(activeIndex + 3 + SwiperComponent.PRELOAD_LOOKAHEAD, data.length) + for (let i = start; i < end; i++) { + const item = data[i] + if (!item?.imgSrc) continue + // imgSrc is `of(url)` — a single-emit completing observable, so + // the subscription self-cleans. No takeUntilDestroyed needed. + item.imgSrc.subscribe((url) => { + if (!url || this.preloadedSrcs.has(url)) return + this.preloadedSrcs.add(url) + const img = new Image() + img.src = url + }) + } + } + public ionViewDidEnter(): void { this.pageIsShown.set(true) - // Stage 1 (synchronous): immediately paint a viewport-sized slice. - this.renderableLimit.set(15) - // Stage 2 (timed): comfortable swipe range without user friction. - setTimeout(() => this.renderableLimit.set(SwiperComponent.RENDER_STAGE_2), SwiperComponent.RENDER_STAGE_2_DELAY_MS) - // Stage 3 (idle-driven): expand the rest in 30-slide chunks, but - // only when the browser tells us the main thread is free. If the - // user is touching/scrolling, idle callbacks deferr — touch input - // wins the priority race. Previous version did Stage 3 on a fixed - // 350ms timer and 200+ DOM nodes landed exactly when the user - // started swiping, freezing the UI for 2-3s. - this.scheduleIdleExpansion() + this.renderableLimit.set(SwiperComponent.RENDER_INITIAL) + // Don't kick the render timer here — the effect tracking pageIsShown + + // data().length will start it as soon as data has arrived. + // Eager preload of the initial window so the first few swipes + // don't catch the user with blank tiles. preloadCoversNear is + // safe with empty data (early-returns). + Promise.resolve().then(() => this.preloadCoversNear(0)) } - private scheduleIdleExpansion(): void { - const grow = () => { + private scheduleNextChunk(): void { + // Single in-flight timer guard. The effect calls this whenever + // data grows; we only want one chunked loop running at a time. + if (this.renderTimer !== undefined) return + this.renderTimer = window.setTimeout(() => { + this.renderTimer = undefined + if (!this.pageIsShown()) return const cur = this.renderableLimit() const target = this.data()?.length ?? 0 if (cur >= target) return - this.renderableLimit.set(Math.min(cur + SwiperComponent.RENDER_IDLE_CHUNK, target)) - this.scheduleIdleExpansion() - } - // Fallback for older Chromium if requestIdleCallback isn't available. - if (typeof (globalThis as any).requestIdleCallback === 'function') { - ;(globalThis as any).requestIdleCallback(grow, { timeout: 2000 }) - } else { - setTimeout(grow, 200) - } + this.renderableLimit.set(Math.min(cur + SwiperComponent.RENDER_CHUNK_SIZE, target)) + // Tell the swiper element about its new slides — without an + // explicit update() call the element's internal Swiper instance + // can keep counting only the slides it saw at first init, which + // means navigating past the original visible range silently + // refuses to advance. Defer one tick so Angular has actually + // committed the @for changes to the DOM. + Promise.resolve().then(() => { + const swiper = this.swiper() + if (swiper && typeof (swiper as unknown as { update?: () => void }).update === 'function') { + ;(swiper as unknown as { update: () => void }).update() + } + }) + this.scheduleNextChunk() + }, SwiperComponent.RENDER_CHUNK_DELAY_MS) as unknown as number } public ionViewWillLeave(): void { this.cachedSwiperPosition = this.swiper()?.activeIndex ?? 0 this.pageIsShown.set(false) + if (this.renderTimer !== undefined) { + clearTimeout(this.renderTimer) + this.renderTimer = undefined + } } public resetSwiperPosition(): void { From a4d7eec736a1a0068c9f09966d9ae0fdf47e9d16 Mon Sep 17 00:00:00 2001 From: Waldemar Berg Date: Fri, 15 May 2026 19:55:02 +0200 Subject: [PATCH 12/19] perf(api): async cache writes + in-memory LRU on top of SD cache (M3, M4) M3 -- saveToCache used fs.writeFileSync, blocking the event loop 5-30 ms per write on the SD card. During an artist-browse burst that populates dozens of cache files in a row, the cumulative pause was seconds of unresponsive frontend. Switched to fs/promises.writeFile; saveToCache is already async, so the change is a single `await` and zero ripple. M4 -- getFromCache used to do existsSync + statSync + readFileSync + JSON.parse on every hit, ~5-30 ms of SD time per call. With a typical artist browse hitting the cache hundreds of times, that's the dominant bottleneck. New in-memory layer in front of the SD cache: - memCache: Map - memCacheCap: auto-sized to 5% of os.freemem() / ~10KB-per-entry, clamped to [50, 500]. On Pi 4 with 3 GB free this saturates at 500 entries (~5 MB). On a Pi 3 with 100 MB free it caps around 50 entries -- same code stays safe on either platform. - LRU via Map insertion order: on get/set we delete-then-reinsert so the touched entry moves to the tail; when size > cap, we evict the first key (oldest). getFromCache now checks memCache first -- if hit, no syscall at all. On miss, the SD read is also async (M3 sibling change) so the event loop stays free, and the result is populated into memCache for the next hit. saveToCache also populates memCache after the write so fresh writes are immediately mem-cached. Hit/miss counters (memHits, memMisses) on the service instance for future observability. No log on mem-hit to keep journal volume sane; only stale-mem and actual SD reads log. tsc --noEmit clean. --- .../src/services/spotify-api.service.ts | 80 +++++++++++++++++-- 1 file changed, 74 insertions(+), 6 deletions(-) diff --git a/src/backend-api/src/services/spotify-api.service.ts b/src/backend-api/src/services/spotify-api.service.ts index aca2d0b6..ec485893 100644 --- a/src/backend-api/src/services/spotify-api.service.ts +++ b/src/backend-api/src/services/spotify-api.service.ts @@ -1,5 +1,7 @@ import { createHash } from 'node:crypto' import fs from 'node:fs' +import fsPromises from 'node:fs/promises' +import os from 'node:os' import path from 'node:path' import { SpotifyApi } from '@spotify/web-api-ts-sdk' import type { ServerConfig } from '../models/server.model' @@ -32,6 +34,26 @@ export class SpotifyApiService { private static readonly CACHE_MAX_FILES = 1000 private static readonly CACHE_PRUNE_BATCH = 200 + // M4: In-memory LRU layer sitting in front of the SD-backed JSON cache. + // getFromCache used to cost 3 sync syscalls (existsSync + statSync + + // readFileSync + JSON.parse) on every hit -- 5-30 ms of event-loop block + // on SD per call, hundreds of calls during a single artist click. + // The Map preserves insertion order; on every get/set we delete-and- + // reinsert to keep the most-recent at the tail, so eviction (delete first + // key) drops the least-recently-used. memCacheCap is auto-sized to 5% of + // available RAM (capped at 500 entries) so the same code stays safe on a + // Pi 3 (~100 MB free -> ~50 entries) and a Pi 4 (~3 GB free -> 500). + private memCache = new Map() + private memCacheCap: number = Math.max( + 50, + Math.min( + 500, + Math.floor((os.freemem() * 0.05) / (10 * 1024)), // estimate ~10KB per cached entry + ), + ) + private memHits = 0 + private memMisses = 0 + // Rate limiting private lastRequestTime = 0 private readonly minRequestInterval = 100 // 100ms between requests @@ -160,16 +182,52 @@ export class SpotifyApiService { return this.cacheExpiry.dynamic // Fallback } + // M4: LRU touch -- delete then re-insert so the entry moves to the tail. + // Map iteration order is insertion order in V8, so the first key is the + // least-recently-used and evictable. + private memCacheTouch(cacheKey: string, value: CachedSpotifyData): void { + if (this.memCache.has(cacheKey)) { + this.memCache.delete(cacheKey) + } + this.memCache.set(cacheKey, value) + while (this.memCache.size > this.memCacheCap) { + const oldestKey = this.memCache.keys().next().value + if (oldestKey === undefined) break + this.memCache.delete(oldestKey) + } + } + private async getFromCache(cacheKey: string): Promise<{ data: any | null; isStale: boolean }> { + // M4: in-memory hit first. Touch-on-get keeps the LRU ordering correct. + const memEntry = this.memCache.get(cacheKey) + if (memEntry !== undefined) { + this.memCache.delete(cacheKey) + this.memCache.set(cacheKey, memEntry) + this.memHits++ + const isStale = Date.now() > (memEntry.expiresAt || Date.now()) + if (isStale) { + console.info(`📦 Cache stale (mem) for ${cacheKey}, will update in background`) + } + // No "Fresh cache hit" log on mem-hit to keep the log volume sane — + // mem-hits are the common path; only stale-mem and SD reads log. + return { data: memEntry.data, isStale } + } + this.memMisses++ try { const cacheFile = this.getCacheFilePath(cacheKey) - if (!fs.existsSync(cacheFile)) { - return { data: null, isStale: false } + // M3: async read so the event loop stays free while the SD seeks. + // ENOENT is the "no cache yet" path -- swallow it and report as miss. + let raw: string + try { + raw = await fsPromises.readFile(cacheFile, 'utf8') + } catch (readErr) { + if ((readErr as NodeJS.ErrnoException).code === 'ENOENT') { + return { data: null, isStale: false } + } + throw readErr } - - const _stats = fs.statSync(cacheFile) - const cachedData: CachedSpotifyData = JSON.parse(fs.readFileSync(cacheFile, 'utf8')) + const cachedData: CachedSpotifyData = JSON.parse(raw) const isStale = Date.now() > (cachedData.expiresAt || Date.now()) @@ -179,6 +237,10 @@ export class SpotifyApiService { console.info(`✅ Fresh cache hit for ${cacheKey}`) } + // M4: populate the in-mem layer so the next hit of the same key skips + // the SD round-trip entirely. + this.memCacheTouch(cacheKey, cachedData) + return { data: cachedData.data, isStale } } catch (error) { console.error(`Error reading cache for ${cacheKey}:`, error) @@ -198,7 +260,13 @@ export class SpotifyApiService { expiresAt: Date.now() + expiryTime, } - fs.writeFileSync(cacheFile, JSON.stringify(cachedData, null, 2)) + // M3: async write -- the previous writeFileSync blocked the event loop + // 5-30 ms per save on SD, which during a "fetch all albums of an artist" + // burst added up to seconds of stutter under high cache-miss load. + // M4: also populate the in-memory layer so the next read of the same + // key skips the SD round-trip. + await fsPromises.writeFile(cacheFile, JSON.stringify(cachedData, null, 2)) + this.memCacheTouch(cacheKey, cachedData) console.info(`💾 Cached data for ${cacheKey}`) this.pruneCacheIfNeeded() } catch (error) { From c210a57ec789a8b9c5b4355eef22b535600a6c25 Mon Sep 17 00:00:00 2001 From: Waldemar Berg Date: Fri, 15 May 2026 19:57:27 +0200 Subject: [PATCH 13/19] perf(admin): request-scoped mupibox_config() helper (M5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit admin.php had 6 separate file_get_contents+json_decode pairs reading /etc/mupibox/mupiboxconfig.json across one HTTP request, plus another in header.php. On an SD card that's 35-140 ms of pure re-parsing the same bytes per admin page load. On closer inspection only 1 of those 6 in admin.php is genuinely redundant -- the pre-header auth-gate (line 24). The other 5 read after an external mutation (sudo unzip restore, sudo conf_update.sh, sudo start_mupibox_update.sh) and need fresh state. header.php's read is the redundant counterpart to the auth-gate. New mupibox_config(bool $forceReread = false): array in includes/save_config.php (now misnamed but fits -- it lives next to save_mupiboxconfig). Static $cache in the function persists per request; $forceReread = true is an explicit opt-out for the post- mutation call sites that need fresh data. Call sites converted: - admin.php line 24-27 -- pre-header auth gate, default cache mode - header.php line 39-42 -- routes through mupibox_config() so the shared static avoids a second file_get_contents per request - admin.php 5× post-mutation reads -- mupibox_config(true) to force re-read after the external command finished In the common case (no admin POST action, just page navigation), this drops from 2 file_get_contents to 1 + memory hit. In the action case where mutations chain, each chain link still does exactly one read, no redundant ones. All three files php-syntax-clean on the box. --- AdminInterface/www/admin.php | 114 +++++++++++++++----- AdminInterface/www/includes/header.php | 10 +- AdminInterface/www/includes/save_config.php | 18 ++++ 3 files changed, 114 insertions(+), 28 deletions(-) diff --git a/AdminInterface/www/admin.php b/AdminInterface/www/admin.php index a072431f..072a261c 100644 --- a/AdminInterface/www/admin.php +++ b/AdminInterface/www/admin.php @@ -12,37 +12,102 @@ function write_json($data) exec("sudo -i -u dietpi /usr/local/bin/mupibox/./restart_kiosk.sh"); } + // Narrow header-only auth gate for the submitfile upload handler below. + // The handler lives ABOVE `include 'includes/header.php'`, so without + // this an unauthenticated LAN POST can drop a crafted zip and have it + // extracted to / via `unzip -d /`. All other POST handlers in this + // file run AFTER the include — header.php's own auth gate already + // blocks them on unauth, so we explicitly do NOT block other POSTs + // here. In particular the login POST (password=...) must flow through + // to header.php so the user can authenticate in the first place. + session_start(); + // M5: route the pre-header auth gate through the shared reader so + // header.php's later read hits the same static cache rather than + // doing a second file_get_contents + json_decode round. + require_once __DIR__ . '/includes/save_config.php'; + $__authCfg = mupibox_config(); + $__loginRequired = !empty($__authCfg['interfacelogin']['state']); + $__loggedIn = isset($_SESSION['logged_in']) && $_SESSION['logged_in'] === true; + if ($__loginRequired && !$__loggedIn && !empty($_POST['submitfile'])) { + http_response_code(403); + exit('Authentication required'); + } + $shutdown=0; $reboot=0; - if( $_POST['submitfile'] ) + if( !empty($_POST['submitfile']) ) { $target_dir = "/tmp/"; - $target_file = $target_dir . basename($_FILES["fileToUpload"]["name"]); + // Strip any directory components from the user-controlled filename. + // The filename is later interpolated into a shell command, so even + // after escapeshellarg() we want the basename so the file lands in + // /tmp/ and not somewhere else via a relative path inside the name. + $rawName = basename($_FILES["fileToUpload"]["name"]); + // Conservative whitelist on filename: letters, digits, dot, dash, + // underscore. Anything else (spaces, quotes, semicolons, …) is + // rejected outright. Backup zips produced by backup.php/fullbackup.php + // match this pattern. $uploadOk = 1; - $FileType = strtolower(pathinfo($target_file,PATHINFO_EXTENSION)); - // Allow zip file format - if($FileType != "zip" ) - { + if (!preg_match('/^[A-Za-z0-9._-]+\.zip$/', $rawName)) { $uploadOk = 0; - } + } + $target_file = $target_dir . $rawName; // Check if $uploadOk is set to 0 by an error if ($uploadOk == 0) { - $CHANGE_TXT=$CHANGE_TXT."
  • WARNING: Please upload a .zip-File!
    • "; + $CHANGE_TXT=$CHANGE_TXT."
  • WARNING: Please upload a .zip-File! (only A-Z, 0-9, ._- allowed in filename)
    • "; $change=0; } else { if (move_uploaded_file($_FILES["fileToUpload"]["tmp_name"], $target_file)) { - $string = file_get_contents('/etc/mupibox/mupiboxconfig.json', true); - $data = json_decode($string, true); + // ZIP-Slip / arbitrary-path defence. backup.php and + // fullbackup.php only ever pack files under three roots — + // reject any zip entry that escapes them. Without this, + // `unzip -o -a -d /` happily writes anywhere on disk. + $allowedPrefixes = [ + 'home/dietpi/MuPiBox/media/', + 'etc/mupibox/mupiboxconfig.json', + 'home/dietpi/.mupibox/Sonos-Kids-Controller-master/server/config/data.json', + ]; + $zip = new ZipArchive(); + $zipOk = false; + $badEntry = ''; + if ($zip->open($target_file) === true) { + $zipOk = true; + for ($i = 0; $i < $zip->numFiles; $i++) { + $entry = $zip->getNameIndex($i); + // Normalise: strip leading slash, forbid `..` + $norm = ltrim($entry, '/'); + if (strpos($norm, '..') !== false) { + $zipOk = false; + $badEntry = $entry; + break; + } + $matched = false; + foreach ($allowedPrefixes as $p) { + if (strpos($norm, $p) === 0) { $matched = true; break; } + } + if (!$matched) { + $zipOk = false; + $badEntry = $entry; + break; + } + } + $zip->close(); + } + if (!$zipOk) { + exec("sudo rm " . escapeshellarg($target_file)); + $CHANGE_TXT=$CHANGE_TXT."
  • ERROR: Backup rejected (entry outside whitelist: ".htmlspecialchars($badEntry).")
    • "; + $change=0; + } else { + // M5: external command above just mutated the config -- force fresh re-read. + $data = mupibox_config(true); $old_version = $data["mupibox"]["version"]; - $command = "sudo unzip -o -a '".$target_file."' -d / >> /tmp/restore.log"; - #$command = "sudo su - -c \"unzip -o -a '".$target_file."' -d / >> /tmp/restore.log && sleep 1\""; - #$command = "sudo su - -c 'tar xvzf ".$target_file." >> /tmp/restore.log'"; + $command = "sudo unzip -o -a " . escapeshellarg($target_file) . " -d / >> /tmp/restore.log"; exec($command, $output, $result ); exec("sudo chown root:www-data /etc/mupibox/mupiboxconfig.json"); exec("sudo chmod 644 /etc/mupibox/mupiboxconfig.json"); @@ -52,22 +117,23 @@ function write_json($data) $command = "cd; curl -L https://raw.githubusercontent.com/splitti/MuPiBox/main/update/conf_update.sh | sudo bash"; exec($command, $output, $result ); - $string = file_get_contents('/etc/mupibox/mupiboxconfig.json', true); - $data = json_decode($string, true); + // M5: external command above just mutated the config -- force fresh re-read. + $data = mupibox_config(true); $data["mupibox"]["version"] = $old_version; write_json($data); - $command = "sudo /boot/dietpi/func/change_hostname " . $data["mupibox"]["host"]; + $command = "sudo /boot/dietpi/func/change_hostname " . escapeshellarg($data["mupibox"]["host"]); $change_hostname = exec($command, $output, $change_hostname ); $command = "sudo su dietpi -c '/usr/local/bin/mupibox/./set_hostname.sh'"; exec($command); - - $command = "sudo rm '".$target_file."'"; + + $command = "sudo rm " . escapeshellarg($target_file); exec($command, $output, $result ); $change=99; $CHANGE_TXT=$CHANGE_TXT."
  • Backup-File restored! The MuPiBox will reboot now!
    • "; $reboot=1; } + } else { $CHANGE_TXT=$CHANGE_TXT."
  • ERROR: Error on uploading Backup-File!
    • "; @@ -138,8 +204,8 @@ function write_json($data) { $command = "cd; curl -L https://raw.githubusercontent.com/splitti/MuPiBox/main/update/start_mupibox_update.sh | sudo bash -s -- stable"; exec($command, $output, $result ); - $string = file_get_contents('/etc/mupibox/mupiboxconfig.json', true); - $data = json_decode($string, true); + // M5: external command above just mutated the config -- force fresh re-read. + $data = mupibox_config(true); $change=3; $reboot=1; $CHANGE_TXT=$CHANGE_TXT."
  • Update complete to Version ".$data["mupibox"]["version"]."
    • "; @@ -148,8 +214,8 @@ function write_json($data) { $command = "cd; curl -L https://raw.githubusercontent.com/splitti/MuPiBox/main/update/start_mupibox_update.sh | sudo bash -s -- beta"; exec($command, $output, $result ); - $string = file_get_contents('/etc/mupibox/mupiboxconfig.json', true); - $data = json_decode($string, true); + // M5: external command above just mutated the config -- force fresh re-read. + $data = mupibox_config(true); $change=1; $reboot=1; $data["mupibox"]["version"]=$data["mupibox"]["version"]." BETA"; @@ -160,8 +226,8 @@ function write_json($data) $command = "cd; curl -L https://raw.githubusercontent.com/splitti/MuPiBox/main/update/start_mupibox_update.sh | sudo bash -s -- dev"; exec($command, $output, $result ); - $string = file_get_contents('/etc/mupibox/mupiboxconfig.json', true); - $data = json_decode($string, true); + // M5: external command above just mutated the config -- force fresh re-read. + $data = mupibox_config(true); $change=1; $reboot=1; $data["mupibox"]["version"]=$data["mupibox"]["version"]." DEVELOPMENT"; diff --git a/AdminInterface/www/includes/header.php b/AdminInterface/www/includes/header.php index 01b796f0..ae6893b3 100644 --- a/AdminInterface/www/includes/header.php +++ b/AdminInterface/www/includes/header.php @@ -36,10 +36,12 @@ $_SESSION['last_activity'] = time(); // Zeit aktualisieren } - $string = file_get_contents('/etc/mupibox/mupiboxconfig.json', true); - $data = json_decode($string, true); - $loginEnabled = $data['interfacelogin']['state']; - $hashedPassword = $data['interfacelogin']['password']; + // M5: route through the request-scoped reader so the same file isn't + // re-parsed 7× per admin page load. $data stays exposed as a local for + // downstream PHP that reads $data directly. + $data = mupibox_config(); + $loginEnabled = $data['interfacelogin']['state'] ?? false; + $hashedPassword = $data['interfacelogin']['password'] ?? ''; $change=0; $CHANGE_TXT="
      "; diff --git a/AdminInterface/www/includes/save_config.php b/AdminInterface/www/includes/save_config.php index d3f68df0..9d969bbd 100644 --- a/AdminInterface/www/includes/save_config.php +++ b/AdminInterface/www/includes/save_config.php @@ -1,4 +1,22 @@ Date: Fri, 15 May 2026 19:59:16 +0200 Subject: [PATCH 14/19] perf(ops): reduce sudo+fork volume from save_rrd and get_monitor (M7) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two scripts dominated the box's sudo audit.log: save_rrd.sh -- crontab fired it 4×/minute (every 15s), each invocation issued 3 sudo+rrdtool forks. That's ~17,000 sudo/day for a database the user looks at via Admin-UI graphs at second-to-minute resolution, not 15s. crontab.template now invokes it once per minute -- 75% reduction with no user-visible loss of graph fidelity. get_monitor.sh -- ran a `while true; sleep 1; sudo vcgencmd display_power; jq+mv` loop in a systemd service. ~86,400 sudo+fork + ~86,400 file rewrites per day, almost all of them writing the SAME state ("On" / "Off") that was already on disk. Two changes: - sleep 1 -> sleep 5. 80% fewer iterations; the kid never notices a few-second delay before touch is re-enabled after the screen actually blanks. - In-process LAST_STATE tracking + write_state() helper. monitor.json only gets rewritten when the polled state changes from the previously-written one. In typical daily use ("On" for hours, "Off" overnight) that drops file rewrites to a handful per day, not 86k. Also tightened existing seed/init paths: switched to consistent quoting of "${VAR}", added a fallback for /sys/class/backlight read when vcgencmd returns -1 (was unquoted before, would error if any backlight glob failed). Combined audit.log impact: ~83% fewer sudo entries from these two sources, freeing ~90k log lines/day plus the corresponding CPU. bash -n syntax-clean. --- config/templates/crontab.template | 8 ++-- scripts/mupibox/get_monitor.sh | 68 +++++++++++++++++++++++-------- 2 files changed, 55 insertions(+), 21 deletions(-) diff --git a/config/templates/crontab.template b/config/templates/crontab.template index 495fda04..aa216d78 100644 --- a/config/templates/crontab.template +++ b/config/templates/crontab.template @@ -23,9 +23,11 @@ # m h dom mon dow command #@reboot /usr/local/bin/mupibox/./get_network.sh #@reboot sleep 30; /usr/local/bin/mupibox/./get_network.sh +# M7: dropped from 4 invocations/minute to 1. The RRD database serves +# cputemp/cpuusage/RAM graphs that the user looks at via the Admin-UI on +# the order of seconds-to-minutes, not 15s resolution. Each invocation +# forks rrdtool 3× under sudo (~17k sudo/day at 4/min) -- the audit.log +# was full of these, and the baseline CPU time was non-trivial on the Pi. * * * * * /usr/local/bin/mupibox/./save_rrd.sh -* * * * * sleep 15; /usr/local/bin/mupibox/./save_rrd.sh -* * * * * sleep 30; /usr/local/bin/mupibox/./save_rrd.sh -* * * * * sleep 45; /usr/local/bin/mupibox/./save_rrd.sh * * * * * /usr/local/bin/mupibox/./get_network.sh * * * * * sleep 30; /usr/local/bin/mupibox/./get_network.sh diff --git a/scripts/mupibox/get_monitor.sh b/scripts/mupibox/get_monitor.sh index 19728868..96422676 100644 --- a/scripts/mupibox/get_monitor.sh +++ b/scripts/mupibox/get_monitor.sh @@ -5,34 +5,66 @@ MONITOR_FILE="/home/dietpi/.mupibox/Sonos-Kids-Controller-master/server/config/monitor.json" minimumsize=18 +# M7: track the last persisted state in-process so we only write monitor.json +# when the polled state actually changed. Previously the script did a full +# jq + mv cycle every second regardless -- ~86k file rewrites/day, all of +# them identical content. Skip-if-unchanged drops that to a handful per day. +# Plus the polling cadence went from 1s to 5s -- the kid never notices a +# 4-second delay before the touch is re-enabled after the screen blanks. +LAST_STATE="" + +write_state() { + local NEW_STATE="$1" + if [ "${NEW_STATE}" = "${LAST_STATE}" ]; then + return + fi + local _TMP="${MONITOR_FILE}.tmp.$$" + if [ -f "${MONITOR_FILE}" ]; then + /usr/bin/jq --arg v "${NEW_STATE}" '.monitor = $v' "${MONITOR_FILE}" > "${_TMP}" \ + && mv "${_TMP}" "${MONITOR_FILE}" \ + || rm -f "${_TMP}" + else + /usr/bin/jq -n --arg v "${NEW_STATE}" '.monitor = $v' > "${_TMP}" \ + && mv "${_TMP}" "${MONITOR_FILE}" \ + || rm -f "${_TMP}" + fi + LAST_STATE="${NEW_STATE}" +} + +seed_file() { + # HIGH-14 (Phase-3) + Phase-5 follow-up: drop the sudo -- + # MONITOR_FILE lives under /home/dietpi/.../config/ and the + # script runs as dietpi, so direct write works. + rm -f "${MONITOR_FILE}" + echo -n "{}" > "${MONITOR_FILE}" + LAST_STATE="" # force write_state to write the next polled value +} + while true do - actualsize=$(wc -c <"${MONITOR_FILE}") - - if [ ! -f ${MONITOR_FILE} ]; then - sudo echo -n "{}" ${MONITOR_FILE} - sudo chown dietpi:dietpi ${MONITOR_FILE} - /usr/bin/cat <<< $(/usr/bin/jq -n --arg v "On" '.monitor = $v' ${MONITOR_FILE}) > ${MONITOR_FILE} - elif [ $actualsize -le $minimumsize ]; then - sudo rm ${MONITOR_FILE} - sudo echo -n "{}" ${MONITOR_FILE} - sudo chown dietpi:dietpi ${MONITOR_FILE} - /usr/bin/cat <<< $(/usr/bin/jq -n --arg v "On" '.monitor = $v' ${MONITOR_FILE}) > ${MONITOR_FILE} + actualsize=$(wc -c <"${MONITOR_FILE}" 2>/dev/null || echo 0) + + if [ ! -f "${MONITOR_FILE}" ]; then + seed_file + write_state "On" + elif [ "$actualsize" -le "$minimumsize" ]; then + seed_file + write_state "On" else MONITOR=$(sudo -H -u root bash -c "vcgencmd display_power") MONITOR=(${MONITOR##*=}) POWER=-1 - if [ ${MONITOR} == "-1" ]; then - POWER=$(cat /sys/class/backlight/*/bl_power) + if [ "${MONITOR}" = "-1" ]; then + POWER=$(cat /sys/class/backlight/*/bl_power 2>/dev/null || echo -1) fi - if [ ${MONITOR} == "0" ] || [ ${POWER} == "4" ]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "Off" '.monitor = $v' ${MONITOR_FILE}) > ${MONITOR_FILE} - elif [ ${MONITOR} == "1" ] || [ ${POWER} == "0" ]; then - /usr/bin/cat <<< $(/usr/bin/jq --arg v "On" '.monitor = $v' ${MONITOR_FILE}) > ${MONITOR_FILE} + if [ "${MONITOR}" = "0" ] || [ "${POWER}" = "4" ]; then + write_state "Off" + elif [ "${MONITOR}" = "1" ] || [ "${POWER}" = "0" ]; then + write_state "On" fi fi - sleep 1 + sleep 5 done From a94ba9163f58a31a0e62be6e603bf1def9ffc66d Mon Sep 17 00:00:00 2001 From: Waldemar Berg Date: Fri, 15 May 2026 20:01:31 +0200 Subject: [PATCH 15/19] perf(frontend): cap mergeMap concurrency in updateMedia pipeline (L2) updateMedia's second mergeMap (the one that flattens Observable> coming out of the per-item iif() chain) had no concurrency parameter -- default unlimited. For a library of N items, all N inner Observables subscribed in parallel, each kicking off a Spotify/RSS HTTP fetch + cache lookup. With N=100+ items the result was a thundering herd against the backend plus a synchronous cache-write storm on SD. mergeMap((items) => from(items), 5) caps the concurrent inner subscriptions to 5. The Spotify SDK service has a 100 ms minRequestInterval, so 5 concurrent gives ~50 req/s peak which is well under the Spotify quota. The staggering also smooths the SD write pressure from saveToCache (now async after M3, but still sequential per-request). The first mergeMap (line 520) is intentionally NOT capped because its upstream emits only a single Observable from the http.get, so concurrency is irrelevant there -- only one inner subscription ever. tsc --noEmit clean. --- src/frontend-box/src/app/media.service.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/frontend-box/src/app/media.service.ts b/src/frontend-box/src/app/media.service.ts index f734923a..4334229b 100644 --- a/src/frontend-box/src/app/media.service.ts +++ b/src/frontend-box/src/app/media.service.ts @@ -589,7 +589,14 @@ export class MediaService { ), ), ), - mergeMap((items) => from(items)), // seperate arrays to single observables + // L2: cap concurrent inner subscriptions. Upstream is Observable> + // -- one inner Observable per library item, each doing a Spotify/RSS HTTP fetch. + // Without a cap, 100+ inner Observables subscribe in parallel and the backend + // sees a thundering herd of HTTP requests + Spotify SDK calls. 5 keeps the + // Spotify rate-limiter (100 ms minRequestInterval) comfortable -- ~50 req/s + // peak, well under quota, and the staggering smooths the cache-write storm + // on SD. + mergeMap((items) => from(items), 5), mergeAll(), // merge everything together toArray(), // convert to array map((media) => { From ad75d9d4f19dc0ea3a4c8cb2384d46d02c13effa Mon Sep 17 00:00:00 2001 From: Waldemar Berg Date: Fri, 15 May 2026 21:28:16 +0200 Subject: [PATCH 16/19] =?UTF-8?q?feat(cover-cache):=20SD+RAM-LRU=20f=C3=BC?= =?UTF-8?q?r=20Spotify-Cover=20(Phase=2011)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cover-Latenz-Optimierung. Nach Phase 7.6 brauchte ein Klick auf einen Spotify-Artist mit vielen Alben (Benjamin Blümchen ~276) noch ~3 s, dominiert von i.scdn.co-Cover-Roundtrips: Chromium limitiert HTTP/1.1-Verbindungen auf 6 parallel pro Host, 276 Cover × 50-150 ms ÷ 6 = ~2.5 s pures CDN-Warten. Backend (src/backend-api): - Neuer CoverCacheService mit drei Layern: 1. In-Memory-LRU (Map; Cap dynamisch via os.freemem(), clamped auf [512 KB, 10 MB]; LRU via Map-Insertion-Order) 2. SD-LRU (cache/covers/.jpg; Cap 2000 Files ~150 MB; Pruning 400 ältester per mtime bei MAX_FILES Überschreitung) 3. Upstream-CDN (https://i.scdn.co/image/ mit 5 s AbortSignal-Timeout) - Robustheits-Layer: * isValidImageId() /^[A-Za-z0-9]{32,80}$/ wehrt Path-Traversal ab * 5-Minuten-Negative-Cache verhindert hammering bei broken Images * pendingFetches dedupliziert concurrent same-key requests (ein Artist-Klick = 1 CDN-Hit pro imageId, nicht 200×) * SD-Hit touched mtime async für sauberes LRU-by-mtime - GET /api/spotify/cover/:imageId Endpoint mit Cache-Control: public, max-age=31536000, immutable (Spotify image-IDs sind content-addressed, Browser darf aggressiv cachen) Frontend (src/frontend-box): - utils.ts: neuer localizeCoverUrl() Helper * https://i.scdn.co/image/ → /api/spotify/cover/ * undefined/null → '../assets/images/nocover_mupi.png' * Alle anderen URLs pass-through (lokale /cover/*, RSS-URLs) - spotify.service.ts: 11 callsites umgestellt * 9× album/audiobook/episode/playlist cover * 2× artistcover / showcover Verifikation post-deploy: Cold-Miss (CDN-Fetch + SD-Write): HTTP 200, 189 KB, 169 ms Hot-RAM (gleiche ID): HTTP 200, 189 KB, 5.5 ms (30× schneller) MD5-identisch zwischen den beiden Antworten Path-Traversal-Test: HTTP 404 (validator greift) Robust gegen Chromium-Cache-Wipes, Box-Reboots, Spotify-CDN-Aussetzer für bereits gecachte Bilder. Cache überlebt pm2 restart (SD-persistent); nur das In-Memory-Layer regeneriert sich beim Backend-Start. Squashed from 5170e387 + 6e69caef (Backup-Tag pre-squashes-backup). --- src/backend-api/src/server.ts | 26 +++ .../src/services/cover-cache.service.ts | 217 ++++++++++++++++++ src/frontend-box/src/app/spotify.service.ts | 20 +- src/frontend-box/src/app/utils.ts | 17 ++ 4 files changed, 270 insertions(+), 10 deletions(-) create mode 100644 src/backend-api/src/services/cover-cache.service.ts diff --git a/src/backend-api/src/server.ts b/src/backend-api/src/server.ts index 51399c7d..a69dcdb3 100644 --- a/src/backend-api/src/server.ts +++ b/src/backend-api/src/server.ts @@ -12,6 +12,7 @@ import { LogRequest, LogResponse } from './models/log.model' import type { MupiboxConfig } from './models/mupibox-config.model' import { ServerConfig } from './models/server.model' import type { SpotifyValidationRequest, SpotifyValidationResponse } from './models/spotify-api.model' +import { CoverCacheService } from './services/cover-cache.service' import { SpotifyApiService } from './services/spotify-api.service' import { SpotifyMediaInfo } from './services/spotify-media-info.service' @@ -50,6 +51,12 @@ readJsonFile(`${configBasePath}/config.json`).then((configFile) => { console.warn('No Spotify configuration found, Spotify API service will not be available') } }) +// Phase-X: SD-backed LRU cache for Spotify cover images. Frontend rewrites +// i.scdn.co URLs to /api/spotify/cover/:imageId so the box serves covers +// from its own SD+RAM after the first miss -- caps Chromium's parallel +// connection limit to one host and saves repeat CDN round-trips. +const coverCacheService = new CoverCacheService(path.join(process.cwd(), 'cache')) + const mupiboxConfigPath = '/etc/mupibox/mupiboxconfig.json' const mupiboxConfigDir = path.dirname(mupiboxConfigPath) const mupiboxConfigFile = path.basename(mupiboxConfigPath) @@ -149,6 +156,25 @@ app.get('/api/resume', (_req, res) => { } }) +// Phase-X cover proxy. Frontend converts every i.scdn.co/image/ URL to +// /api/spotify/cover/; we serve from the SD+RAM cache after first miss. +// imageId is validated as alphanumeric inside CoverCacheService -- any +// crafted path traversal attempt returns 400. +app.get('/api/spotify/cover/:imageId', async (req, res) => { + const buf = await coverCacheService.get(req.params.imageId) + if (!buf) { + res.status(404).type('text/plain').send('cover not available') + return + } + // Long max-age + immutable since Spotify image IDs are content-addressed: + // a given imageId always resolves to the same bytes, so the browser can + // cache aggressively. ETag echoes the imageId for conditional requests. + res.set('Content-Type', 'image/jpeg') + res.set('Cache-Control', 'public, max-age=31536000, immutable') + res.set('ETag', `"${req.params.imageId}"`) + res.send(buf) +}) + app.get('/api/mupihat', (_req, res) => { if (fs.existsSync(mupihat)) { jsonfile.readFile(mupihat, (error, data) => { diff --git a/src/backend-api/src/services/cover-cache.service.ts b/src/backend-api/src/services/cover-cache.service.ts new file mode 100644 index 00000000..1e5945b6 --- /dev/null +++ b/src/backend-api/src/services/cover-cache.service.ts @@ -0,0 +1,217 @@ +import { existsSync, mkdirSync } from 'node:fs' +import fsPromises from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' + +/** + * SD-backed LRU cache for Spotify cover images. + * + * Phase-7.6 took the artist-click latency from 8-10 s down to ~3 s. + * The remaining ~3 s comes from Spotify-CDN image round-trips: Chromium + * limits parallel connections to i.scdn.co to ~6 (HTTP/1.1), and the + * Pi sees 50-150 ms per cover. With 276 albums for Benjamin Blümchen, + * the cover load is the dominant cost after the API responses are + * cached. + * + * This service serves /api/spotify/cover/:imageId from the box's own + * SD + RAM, with these properties: + * - First request fetches from i.scdn.co/image/, persists to + * /.jpg, returns the bytes. + * - Subsequent requests on the same imageId serve from RAM (hot) or + * SD (warm) -- no CDN round-trip, no Chromium parallelism cap. + * - LRU eviction: at MAX_FILES (2000, ~100-150 MB) prune the oldest + * PRUNE_BATCH (400) by mtime. mem-LRU caps at MEM_CACHE_MAX_BYTES + * (auto-sized to 5% of os.freemem(), max 10 MB). + * - Concurrent same-key fetches de-duplicate via pendingFetches: + * when 200 covers are requested at once, each unique imageId + * still hits i.scdn.co only once. + * - HTTP 404 from i.scdn.co is short-circuited via negativeCache so + * a non-existent image doesn't get re-fetched 6× per page. + * + * SD-Wear: writes happen only on first-miss per imageId. After the + * cache is warm, hits do reads + occasional mtime touches for LRU. + * Reads are free for SD lifetime; writes are the cost. With 2000 + * covers × ~50 KB = ~100 MB ever-written, sub-trivial. + */ +export class CoverCacheService { + private cacheDir: string + private static readonly MAX_FILES = 2000 + private static readonly PRUNE_BATCH = 400 + private static readonly UPSTREAM_TIMEOUT_MS = 5000 + private static readonly NEGATIVE_CACHE_TTL_MS = 5 * 60 * 1000 + + private memCache = new Map() + private memCacheBytes = 0 + private memCacheMaxBytes: number + + // De-dup map: when N requests for the same imageId arrive while a + // fetch is in flight, the second through Nth await the same Promise + // rather than firing N concurrent fetches. + private pendingFetches = new Map>() + + // Track recent 404s so we don't re-hammer i.scdn.co for known-missing + // images. Key -> expiry-epoch. + private negativeCache = new Map() + + // Observability (no UI yet). + public stats = { memHits: 0, sdHits: 0, cdnHits: 0, negativeHits: 0, fetchErrors: 0 } + + constructor(baseDir: string) { + this.cacheDir = path.join(baseDir, 'covers') + if (!existsSync(this.cacheDir)) { + mkdirSync(this.cacheDir, { recursive: true }) + } + this.memCacheMaxBytes = Math.max( + 512 * 1024, // 512 KB floor — at least handful of covers on a Pi 3 + Math.min( + 10 * 1024 * 1024, // 10 MB ceiling — even big covers fit comfortably on Pi 4 + Math.floor(os.freemem() * 0.05), + ), + ) + } + + /** + * Validate that the requested ID is a Spotify image identifier. + * Spotify CDN paths after /image/ are alphanumeric; observed lengths + * are 40-60 chars. Reject anything else to prevent path traversal / + * SSRF via crafted URLs. + */ + private static isValidImageId(id: string): boolean { + return /^[A-Za-z0-9]{32,80}$/.test(id) + } + + async get(imageId: string): Promise { + if (!CoverCacheService.isValidImageId(imageId)) return null + + // RAM + const mem = this.memCache.get(imageId) + if (mem !== undefined) { + // Touch LRU position + this.memCache.delete(imageId) + this.memCache.set(imageId, mem) + this.stats.memHits++ + return mem + } + + // Negative cache short-circuit + const negExp = this.negativeCache.get(imageId) + if (negExp !== undefined) { + if (Date.now() < negExp) { + this.stats.negativeHits++ + return null + } + this.negativeCache.delete(imageId) + } + + // SD + const filePath = path.join(this.cacheDir, `${imageId}.jpg`) + try { + const buf = await fsPromises.readFile(filePath) + // Touch mtime asynchronously for LRU-by-mtime ordering, don't wait + const now = new Date() + fsPromises.utimes(filePath, now, now).catch(() => {}) + this.memCachePut(imageId, buf) + this.stats.sdHits++ + return buf + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== 'ENOENT') { + console.error(`cover-cache SD read error for ${imageId}:`, err) + } + } + + // Miss — fetch from i.scdn.co. Dedup concurrent requests. + const inflight = this.pendingFetches.get(imageId) + if (inflight) return inflight + + const fetchPromise = this.fetchFromCdn(imageId, filePath).finally(() => { + this.pendingFetches.delete(imageId) + }) + this.pendingFetches.set(imageId, fetchPromise) + return fetchPromise + } + + private async fetchFromCdn(imageId: string, filePath: string): Promise { + const url = `https://i.scdn.co/image/${imageId}` + try { + const response = await fetch(url, { + signal: AbortSignal.timeout(CoverCacheService.UPSTREAM_TIMEOUT_MS), + }) + if (response.status === 404) { + this.negativeCache.set(imageId, Date.now() + CoverCacheService.NEGATIVE_CACHE_TTL_MS) + return null + } + if (!response.ok) { + console.warn(`cover-cache upstream non-OK for ${imageId}: HTTP ${response.status}`) + this.stats.fetchErrors++ + return null + } + const arrayBuf = await response.arrayBuffer() + const buf = Buffer.from(arrayBuf) + + // Persist to SD (fire-and-forget — the response can already start + // serving from RAM while the write completes). + fsPromises.writeFile(filePath, buf).catch((err) => { + console.error(`cover-cache SD write error for ${imageId}:`, err) + }) + this.memCachePut(imageId, buf) + this.stats.cdnHits++ + // Prune in background -- no need to block the response. + this.pruneIfNeeded().catch(() => {}) + return buf + } catch (err) { + console.error(`cover-cache fetch failed for ${imageId}:`, err) + this.stats.fetchErrors++ + return null + } + } + + private memCachePut(key: string, buf: Buffer): void { + if (this.memCache.has(key)) { + const old = this.memCache.get(key) as Buffer + this.memCacheBytes -= old.length + this.memCache.delete(key) + } + this.memCache.set(key, buf) + this.memCacheBytes += buf.length + + while (this.memCacheBytes > this.memCacheMaxBytes) { + const oldest = this.memCache.keys().next().value + if (oldest === undefined) break + const oldBuf = this.memCache.get(oldest) as Buffer + this.memCacheBytes -= oldBuf.length + this.memCache.delete(oldest) + } + } + + private async pruneIfNeeded(): Promise { + try { + const files = await fsPromises.readdir(this.cacheDir) + if (files.length <= CoverCacheService.MAX_FILES) return + + const stats = await Promise.all( + files.map(async (name) => { + try { + const s = await fsPromises.stat(path.join(this.cacheDir, name)) + return { name, mtime: s.mtimeMs } + } catch { + return null + } + }), + ) + const valid = stats.filter((s): s is { name: string; mtime: number } => s !== null) + valid.sort((a, b) => a.mtime - b.mtime) + const victims = valid.slice(0, CoverCacheService.PRUNE_BATCH) + + for (const v of victims) { + try { + await fsPromises.unlink(path.join(this.cacheDir, v.name)) + } catch { + // ignore — file may have been pruned in parallel + } + } + console.info(`🗑️ Cover-cache pruned: removed ${victims.length} of ${files.length}`) + } catch (err) { + console.error('cover-cache prune error:', err) + } + } +} diff --git a/src/frontend-box/src/app/spotify.service.ts b/src/frontend-box/src/app/spotify.service.ts index a05fd020..bca8d1a2 100644 --- a/src/frontend-box/src/app/spotify.service.ts +++ b/src/frontend-box/src/app/spotify.service.ts @@ -7,7 +7,7 @@ import { LogService } from './log.service' import type { CategoryType, Media } from './media' import { SpotifyConfig } from './spotify' import { SpotifyPlayerService } from './spotify-player.service' -import { ExtraDataMedia, Utils } from './utils' +import { ExtraDataMedia, localizeCoverUrl, Utils } from './utils' @Injectable({ providedIn: 'root', @@ -177,7 +177,7 @@ export class SpotifyService { id: album.id, artist: album.artists?.[0]?.name || 'Unknown Artist', title: album.name, - cover: album.images?.[0]?.url || '../assets/images/nocover_mupi.png', + cover: localizeCoverUrl(album.images?.[0]?.url), release_date: album.release_date, type: 'spotify', category, @@ -208,7 +208,7 @@ export class SpotifyService { return this.http.get(artistUrl).pipe( switchMap((artist) => { - const artistcover = artist.images?.[0]?.url || '../assets/images/nocover_mupi.png' + const artistcover = localizeCoverUrl(artist.images?.[0]?.url) return this.fetchAllPaginatedResults(artistAlbumsUrl, {}).pipe( map((albums) => { @@ -217,7 +217,7 @@ export class SpotifyService { id: album.id, artist: album.artists?.[0]?.name || 'Unknown Artist', title: album.name, - cover: album.images?.[0]?.url || '../assets/images/nocover_mupi.png', + cover: localizeCoverUrl(album.images?.[0]?.url), artistcover: artistcover, release_date: album.release_date, type: 'spotify', @@ -252,7 +252,7 @@ export class SpotifyService { return this.http.get(showUrl).pipe( switchMap((show) => { const showName = show.name || 'Unknown Show' - const showcover = show.images?.[0]?.url || '../assets/images/nocover_mupi.png' + const showcover = localizeCoverUrl(show.images?.[0]?.url) return this.fetchAllPaginatedResults(showEpisodesUrl, {}).pipe( map((episodes) => { @@ -263,7 +263,7 @@ export class SpotifyService { showid: episode.id, artist: showName, title: episode.name, - cover: episode.images?.[0]?.url || showcover, + cover: episode.images?.[0]?.url ? localizeCoverUrl(episode.images[0].url) : showcover, artistcover: showcover, type: 'spotify', category, @@ -304,7 +304,7 @@ export class SpotifyService { id: album.id, artist: album.artists?.[0]?.name || 'Unknown Artist', title: album.name, - cover: album.images?.[0]?.url || '../assets/images/nocover_mupi.png', + cover: localizeCoverUrl(album.images?.[0]?.url), type: 'spotify', release_date: album.release_date, category, @@ -356,7 +356,7 @@ export class SpotifyService { audiobookid: audiobook.id, artist: audiobook.authors?.[0]?.name || 'Unknown Author', title: audiobook.name, - cover: audiobook.images?.[0]?.url || '../assets/images/nocover_mupi.png', + cover: localizeCoverUrl(audiobook.images?.[0]?.url), type: 'spotify', category, index, @@ -407,7 +407,7 @@ export class SpotifyService { showid: episode.id, artist: episode.show?.[0]?.name || 'Unknown Show', title: episode.name, - cover: episode.images?.[0]?.url || '../assets/images/nocover_mupi.png', + cover: localizeCoverUrl(episode.images?.[0]?.url), type: 'spotify', release_date: episode.release_date, category, @@ -467,7 +467,7 @@ export class SpotifyService { const media: Media = { playlistid: isFromBackend ? id : response.id, title: isFromBackend ? response.playlist.name : response.name, - cover: isFromBackend ? response.playlist.images?.[0]?.url : response?.images?.[0]?.url, + cover: localizeCoverUrl(isFromBackend ? response.playlist.images?.[0]?.url : response?.images?.[0]?.url), type: 'spotify', category, index, diff --git a/src/frontend-box/src/app/utils.ts b/src/frontend-box/src/app/utils.ts index b6c6fcd4..8f609d1f 100644 --- a/src/frontend-box/src/app/utils.ts +++ b/src/frontend-box/src/app/utils.ts @@ -5,6 +5,23 @@ export type ExtraDataMedia = Pick< 'artistcover' | 'shuffle' | 'aPartOfAll' | 'aPartOfAllMin' | 'aPartOfAllMax' | 'sorting' > +/** + * Phase-X cover-cache localizer. If {@link url} is a Spotify CDN image URL + * (https://i.scdn.co/image/), rewrite it to /api/spotify/cover/ so + * the backend serves the bytes from its SD+RAM cache after the first miss. + * All other URLs (local /cover/* paths, RSS-feed image URLs, the no-cover + * fallback asset) pass through unchanged. An empty/undefined input returns + * the no-cover asset. + */ +const NO_COVER_FALLBACK = '../assets/images/nocover_mupi.png' +const SPOTIFY_CDN_RE = /^https:\/\/i\.scdn\.co\/image\/([A-Za-z0-9]+)$/ + +export function localizeCoverUrl(url: string | undefined | null): string { + if (!url) return NO_COVER_FALLBACK + const match = url.match(SPOTIFY_CDN_RE) + return match ? `/api/spotify/cover/${match[1]}` : url +} + export namespace Utils { /** * Copies the properties of {@link ExtraDataMedia} from {@link source} to {@link target}. From a6a2eafd9dbdb1e469f919017118801714b03ccc Mon Sep 17 00:00:00 2001 From: Waldemar Berg Date: Sat, 16 May 2026 15:04:38 +0200 Subject: [PATCH 17/19] feat(hat): granular 5%-step battery percent display (Phase 12) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a percent number stacked under the existing 4-bucket battery icon so users can tell 80% from 75% from 70%, instead of the icon jumping straight from "75%" to "25%" with nothing in between. Why 5% steps and not 1%: The Li-Ion discharge curve in the plateau zone (~3.7-3.9 V/cell) drops only 2-3 mV per 1% energy -- smaller than the BQ25792 ADC's noise floor (~10 mV LSB) and tiny compared to the 30-100 mV load sag the box's audio + display produce. A 1% display would zap-zap-zap constantly between adjacent values for purely electrical reasons. 5% steps map to ~90 mV per step which sits cleanly above all that noise. Backend (mupihat_bq25792.py): - new record_vbat_sample() / smoothed_vbat() helpers maintaining an 8-element rolling buffer of recent VBAT readings (~32 s window at the 4 s polling cycle). Without smoothing the granular percent would visibly flap when audio loads spike. - new battery_percent_granular() returns (int 0-100 rounded to 5%, source: "voltage" | "charging"). Piecewise-linear interpolation across the same v_100..v_0 thresholds the legacy battery_soc() already uses, so existing per-pack config drives both displays. - to_json() exposes Bat_Percent + Bat_PercentSource alongside the legacy Bat_SOC string. SOC string stays for Telegram bot messages and any other text consumer that wants a label. - battery_soc() `>` -> `>=` so a pack sitting exactly at v_100 renders as "100%" instead of falling one bucket down to "75%". Without this fix, a freshly-finished CV charge at VBat=8200 shows up as "75%" everywhere the legacy string is consumed. USB-C mode (sentinel v_100=1 from the "no battery" battery type) returns (0, "voltage") and the frontend skips the percent text entirely -- the existing plug icon already conveys it. Backend (mupihat.py): periodic_json_dump now calls hat.record_vbat_sample(read_Vbat()) inside the i2c_lock right after read_all_register() so the smoothing buffer is fed once per cycle from the freshly refreshed register state. Frontend (mupihat.ts): Interface gains Bat_Percent (number) and Bat_PercentSource ("voltage" | "charging"). Frontend (mupihat-icon.component): - Icon bucket now driven by Bat_Percent (>=95 / >=70 / >=45 / >=20 / else) instead of the legacy Bat_SOC string. Both icon and percent text now come from the same source so they can never disagree at threshold edges. Backward-compat: if Bat_Percent is undefined (older backend), falls back to the legacy 4-case Bat_SOC switch. - Layout stacks ion-icon ABOVE the percent text (column flex) instead of side-by-side. Reason: home.page reserves a fixed 70x70 px ion-button for status icons, holding BOTH the cloud-online/offline indicator AND mupihat-icon. A horizontal " 78%" expansion pushed the cloud icon out of the 70 px width. Vertical stacking keeps the horizontal footprint at icon-only size, so the cloud icon retains its space, and the percent label sits legibly under the bucket icon. - :host inline-flex+centered so the component element behaves as a single inline cell inside ion-button's children area. - During active CC/Taper charge the percent is prefixed with a ⚡ symbol — the voltage-based estimate is systematically optimistic during charge (pack held above its rest curve by the charge current), so the indicator tells the user "this number is in-flight, not a settled reading". Verified post-deploy on the box: cold Pack at VBat=8188 mV correctly shows Bat_Percent=100 via piecewise interpolation + smoothing, icon renders as Battery100.svg, percent text reads "100%". Cloud icon visible alongside as before. Squashed from 7ce1cb76 + c214943a + 14fe18f4 (Backup-Tag pre-phase12-squash-backup). --- scripts/mupihat/mupihat.py | 64 +++++++--- scripts/mupihat/mupihat_bq25792.py | 120 +++++++++++++++++- .../mupihat-icon/mupihat-icon.component.html | 61 ++++++--- .../mupihat-icon/mupihat-icon.component.scss | 38 ++++++ src/frontend-box/src/app/mupihat.ts | 5 + 5 files changed, 252 insertions(+), 36 deletions(-) diff --git a/scripts/mupihat/mupihat.py b/scripts/mupihat/mupihat.py index 07d6e568..77f8aa97 100644 --- a/scripts/mupihat/mupihat.py +++ b/scripts/mupihat/mupihat.py @@ -35,13 +35,19 @@ import argparse from datetime import datetime from flask import Flask, render_template, jsonify -from threading import Thread +from threading import Thread, Lock from mupihat_bq25792 import bq25792 app = Flask(__name__) # Global variables hat = None +# AR5-5: Flask is threaded by default — without serialising access to the +# shared BQ25792 driver, the periodic_json_dump background thread and the +# Flask request workers can interleave their I2C transactions on the same +# smbus2.SMBus handle, producing "Remote I/O error" (errno 121) on the +# Pi's i2c bus. All hat.* calls below run under this lock. +i2c_lock = Lock() log_flag = False json_flag = False json_file = "/tmp/mupihat.json" @@ -84,7 +90,9 @@ def log_register_values(): def index(): """Flask route to display register values.""" try: - return render_template("index.html", registers=hat.to_json_registers()) + with i2c_lock: + registers = hat.to_json_registers() + return render_template("index.html", registers=registers) except Exception as e: return f"Error reading registers: {str(e)}", 500 @@ -92,8 +100,10 @@ def index(): @app.route("/api/registers") def api_registers(): """Flask API endpoint to return register values as JSON.""" - try: - return jsonify(hat.to_json_registers()) + try: + with i2c_lock: + registers = hat.to_json_registers() + return jsonify(registers) except Exception as e: return jsonify({"error": str(e)}), 500 @@ -102,19 +112,36 @@ def periodic_json_dump(): """Periodically writes the register values to a JSON file.""" global json_flag, json_file while True: - hat.watchdog_reset() - time.sleep(0.1) # Allow time for the watchdog reset - hat.read_all_register() - time.sleep(1) # Allow time for the registers to be updated - if json_flag: + # AR5-5: serialise I2C work via i2c_lock so Flask request workers + # don't interleave bus transactions with this thread. Sleeps stay + # OUTSIDE the lock so request workers can fit between phases. + snapshot = None + try: + with i2c_lock: + hat.watchdog_reset() + time.sleep(0.1) # Allow time for the watchdog reset + with i2c_lock: + hat.read_all_register() + # Phase-12: append the current VBAT to the smoothing ring so + # battery_percent_granular() sees a moving-average of the + # last ~32s instead of a single sample. Cheap, done inside + # the lock so we use the value the bulk-read just refreshed. + hat.record_vbat_sample(hat.read_Vbat()) + time.sleep(1) # Allow time for the registers to be updated + with i2c_lock: + if json_flag: + snapshot = hat.to_json() + if log_flag: + log_register_values() + except Exception as e: + logging.error("periodic_json_dump I2C cycle failed: %s", str(e)) + if snapshot is not None: try: with open(json_file, "w") as outfile: - json.dump(hat.to_json(), outfile, indent=4) + json.dump(snapshot, outfile, indent=4) except Exception as e: logging.error("Failed to write JSON dump: %s", str(e)) - if log_flag: - log_register_values() - time.sleep(3.9) # Run every 4 seconds + time.sleep(3.9) # Run every ~4 seconds def parse_arguments(): @@ -172,9 +199,16 @@ def main(): json_thread = Thread(target=periodic_json_dump, daemon=True) json_thread.start() - # Flask web server + # AR5-20: bind the debug Flask server to loopback only. No consumer on + # the box hits port 5000 — all consumers (frontend, admin-ui, server.ts, + # telegram_*.py, mqtt.py, .bashrc, fan_control.py) read /tmp/mupihat.json + # directly. Port 5000 is the BQ25792 register inspector at "/" and + # "/api/registers". With 0.0.0.0 the raw charger registers were + # unauthenticated for every client on the LAN; harmless on a home + # WLAN, problematic on guest/school networks. For remote debugging + # use `ssh -L 5000:127.0.0.1:5000 mupibox`. try: - app.run(host="0.0.0.0", port=5000, debug=False) + app.run(host="127.0.0.1", port=5000, debug=False) except KeyboardInterrupt: print("MuPiHAT stopped by Keyboard Interrupt") sys.exit(0) diff --git a/scripts/mupihat/mupihat_bq25792.py b/scripts/mupihat/mupihat_bq25792.py index 0e03eb07..e95d13d1 100644 --- a/scripts/mupihat/mupihat_bq25792.py +++ b/scripts/mupihat/mupihat_bq25792.py @@ -139,6 +139,13 @@ def __init__(self, i2c_device=1, i2c_addr=0x6b, busWS_ms=10, exit_on_error = Fal 'th_warning': 7000, 'th_shutdown': 6800 } self._exit_on_error = exit_on_error + # Phase-12: VBAT history for granular-percent smoothing (5%-step display). + # Voltage swings 30-100 mV under load (Bass-pumping, Display wake); + # without smoothing the 5%-display would visibly flap. 8 samples at + # 4s cycle = ~32s window, well below the timescale of real SoC + # change but long enough to absorb the load-sag transients. + self._vbat_history: list[int] = [] + self._vbat_history_max = 8 self.i2c_device = i2c_device self.i2c_addr = i2c_addr self.busWS_ms = busWS_ms @@ -259,10 +266,16 @@ def battery_soc(self): VBat = self.read_Vbat() - if VBat > v_100 : Bat_SOC = "100%" - elif VBat > v_75 : Bat_SOC = "75%" - elif VBat > v_50 : Bat_SOC = "50%" - elif VBat > v_25 : Bat_SOC = "25%" + # Phase-12 follow-up: changed > to >= so a pack sitting exactly at + # the v_100 threshold (8200 mV for a freshly-finished CV charge on + # the 2S3P pack) renders as "100%" / Battery100.svg instead of + # falling through to "75%" / Battery70.svg. Without this the icon + # bucket disagrees with the new granular Bat_Percent text at the + # exact-voll-Fall (icon: 3/4-strich, text: 100%) which looks broken. + if VBat >= v_100 : Bat_SOC = "100%" + elif VBat >= v_75 : Bat_SOC = "75%" + elif VBat >= v_50 : Bat_SOC = "50%" + elif VBat >= v_25 : Bat_SOC = "25%" else : Bat_SOC = "0%" if VBat > th_warning : Bat_Stat = 'OK' @@ -271,6 +284,97 @@ def battery_soc(self): return Bat_SOC, Bat_Stat + def record_vbat_sample(self, vbat_mv: int): + ''' + Phase-12: append a VBAT reading to the smoothing ring buffer. + Called from mupihat.py's periodic_json_dump after read_all_register(). + Drops the oldest sample once the buffer is full so the average always + reflects the most-recent ~32s window. + ''' + if vbat_mv is None or vbat_mv <= 0: + return # ignore invalid reads (post-reopen, etc.) + self._vbat_history.append(int(vbat_mv)) + if len(self._vbat_history) > self._vbat_history_max: + self._vbat_history.pop(0) + + def smoothed_vbat(self) -> int: + ''' + Return the moving-average VBAT over the recorded samples. Falls back + to the current single read if the buffer is empty (cold start). + ''' + if not self._vbat_history: + return self.read_Vbat() + return sum(self._vbat_history) // len(self._vbat_history) + + def battery_percent_granular(self): + ''' + Phase-12: piecewise-linear interpolation between the 5 thresholds for + a 0-100 % SoC value, rounded to 5 % steps. Returns a tuple: + (percent: int, source: str) + where source is "charging" while CC/Taper charge is running (the + voltage-based estimate is systematically optimistic during active + charge because the pack is held above its rest curve), otherwise + "voltage". The frontend can show a ⚡ next to the number when + source == "charging" to signal the disclaimer. + + Uses smoothed_vbat() to absorb sub-second voltage swings -- without + smoothing the 5%-display would flap visibly under audio load. + + Why 5% steps and not 1%: the discharge curve in the plateau region + (~3.7-3.9 V/cell) gives ~2-3 mV per 1 %, smaller than the BQ25792 + ADC's noise floor (~10 mV) and far smaller than load-sag transients + (30-60 mV under typical box load). 1 % steps would zap-zap-zap + constantly; 5 % steps map cleanly to ~90 mV per step which IS + resolvable above the noise. + ''' + try: + v_100 = int(self.battery_conf['v_100']) + v_75 = int(self.battery_conf['v_75']) + v_50 = int(self.battery_conf['v_50']) + v_25 = int(self.battery_conf['v_25']) + v_0 = int(self.battery_conf['v_0']) + except (KeyError, ValueError, TypeError): + return (0, "voltage") # config not loaded yet — safe default + + # USB-C mode profile has v_*=1 (sentinel for "no battery"). Don't + # report any percent in that case — the icon will fall back to its + # plug-symbol mode. + if v_100 <= 10: + return (0, "voltage") + + v = self.smoothed_vbat() + + # Piecewise linear: each segment maps [v_lower, v_upper] to a + # 25-percentage-point range linearly. + if v >= v_100: + pct = 100.0 + elif v >= v_75: + pct = 75.0 + 25.0 * (v - v_75) / max(1, v_100 - v_75) + elif v >= v_50: + pct = 50.0 + 25.0 * (v - v_50) / max(1, v_75 - v_50) + elif v >= v_25: + pct = 25.0 + 25.0 * (v - v_25) / max(1, v_50 - v_25) + elif v >= v_0: + pct = 0.0 + 25.0 * (v - v_0) / max(1, v_25 - v_0) + else: + pct = 0.0 + + # Round to 5 % steps -- see docstring above for why not 1 %. + rounded = int(round(pct / 5.0) * 5) + rounded = max(0, min(100, rounded)) + + # During active charging the voltage-based estimate is too optimistic + # (pack held above rest curve by CC current). Flag the source so the + # frontend can render a charging indicator (⚡) instead of treating + # the number as a settled SoC reading. + try: + _, chg_str = self.read_ChargerStatus() + source = "charging" if 'Charge' in chg_str and 'Done' not in chg_str else "voltage" + except Exception: + source = "voltage" + + return (rounded, source) + # BQ25795 Register class BQ25795_REGISTER: def __init__(self, addr, value=0): @@ -5724,6 +5828,7 @@ def to_json(self): Input Current Limit obtained from ICO or ILIM_HIZ pin setting ''' bat_SOC, bat_Stat = self.battery_soc() + bat_Percent, bat_PercentSource = self.battery_percent_granular() return { 'Charger_Status': self.read_ChargerStatus(), 'Vbat': self.read_Vbat(), @@ -5735,7 +5840,12 @@ def to_json(self): 'Bat_SOC' : bat_SOC, 'Bat_Stat' : bat_Stat, 'Bat_Type' : self.battery_conf['battery_type'], - 'Input_Current_Limit' : self.read_InputCurrentLimit() + 'Input_Current_Limit' : self.read_InputCurrentLimit(), + # Phase-12: granular 5%-step percent + charging-state hint for + # the frontend. Bat_SOC stays for backwards-compat (Telegram bot + # messages, legacy Admin-UI bits). + 'Bat_Percent' : bat_Percent, + 'Bat_PercentSource' : bat_PercentSource, } def to_json_registers(self): diff --git a/src/frontend-box/src/app/mupihat-icon/mupihat-icon.component.html b/src/frontend-box/src/app/mupihat-icon/mupihat-icon.component.html index 2d58fab3..2d485b22 100644 --- a/src/frontend-box/src/app/mupihat-icon/mupihat-icon.component.html +++ b/src/frontend-box/src/app/mupihat-icon/mupihat-icon.component.html @@ -1,22 +1,51 @@ @if (hat_active && mupihat() !== undefined) { @if (mupihat().BatteryConnected === 1) { @let svgBaseName = mupihat().IBus > 0 ? 'Charging' : 'Battery'; - @switch (mupihat().Bat_SOC) { - @case ('100%') { - + + @let pct = mupihat().Bat_Percent; + + @if (pct !== undefined && pct !== null) { + @if (pct >= 95) { + + } @else if (pct >= 70) { + + } @else if (pct >= 45) { + + } @else if (pct >= 20) { + + } @else { + + } + + @if (mupihat().Bat_PercentSource === 'charging') {⚡}{{ pct }}% + + } @else { + + @switch (mupihat().Bat_SOC) { + @case ('100%') { + + } + @case ('75%') { + + } + @case ('50%') { + + } + @case ('25%') { + + } + @default { + + } + } } - @case ('75%') { - - } - @case ('50%') { - - } - @case ('25%') { - - } - @default { - - } - } + } } diff --git a/src/frontend-box/src/app/mupihat-icon/mupihat-icon.component.scss b/src/frontend-box/src/app/mupihat-icon/mupihat-icon.component.scss index e69de29b..58e6cc3f 100644 --- a/src/frontend-box/src/app/mupihat-icon/mupihat-icon.component.scss +++ b/src/frontend-box/src/app/mupihat-icon/mupihat-icon.component.scss @@ -0,0 +1,38 @@ +// Phase-12 follow-up: stack icon over percent text instead of side-by-side. +// Reason: the home.page toolbar slot is a fixed 70x70 px ion-button that +// already holds the cloud-on/cloud-off internet icon + this component. A +// horizontal " " expansion of mupihat-icon pushed the cloud +// icon off the 70 px width. Vertical stacking keeps the whole mupihat +// component inside its old horizontal footprint while still showing the +// granular percent below the bucket icon. +// +// :host is set to inline-flex too so the mupihat-icon component element +// itself participates in its parent's flex layout (e.g. the ion-button +// children area) as a single inline cell, no extra wrapping needed at +// the call sites. +:host { + display: inline-flex; + align-items: center; + justify-content: center; + line-height: 1; +} + +.mupihat-icon-row { + display: inline-flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 1px; + line-height: 1; +} + +.bat-percent { + font-size: 0.55rem; + font-weight: 500; + line-height: 1; + white-space: nowrap; + // Tiny -1px margin pulls the percent label snugly under the icon's + // base — without it the line-height of the ion-icon's baseline plus + // the percent's own line-height left a visible gap. + margin-top: -2px; +} diff --git a/src/frontend-box/src/app/mupihat.ts b/src/frontend-box/src/app/mupihat.ts index 52b7b43b..20223dfc 100644 --- a/src/frontend-box/src/app/mupihat.ts +++ b/src/frontend-box/src/app/mupihat.ts @@ -9,4 +9,9 @@ export interface Mupihat { Bat_SOC?: string Bat_Stat?: string Bat_Type?: string + // Phase-12: granular 5%-step SoC + charging-state hint. Backend computes + // these via piecewise-linear interpolation over the same v_100..v_0 config + // thresholds, smoothed over a ~32 s VBAT window. + Bat_Percent?: number + Bat_PercentSource?: 'voltage' | 'charging' } From d778ce556859d658a66bd977c64d0f720890f7e4 Mon Sep 17 00:00:00 2001 From: Waldemar Berg Date: Fri, 15 May 2026 17:20:50 +0200 Subject: [PATCH 18/19] fix(frontend): enable noImplicitAny in tsconfig + fix the 37 it caught (AR5-19) frontend-box's tsconfig.json had no strict-mode flags at all, while backend-api has full "strict": true. That gap is what enabled the "(component as any).x.set(...)" pattern across the specs (Backlog L6) plus a handful of silently-untyped function parameters that were never caught. Full "strict": true would have unleashed hundreds of strictNullChecks errors that require a much larger refactor. noImplicitAny alone is the high-value subset: it forbids untyped parameters / object indexes and is what closes the (x as any) loophole, without forcing a big- bang nullability migration. 37 errors surfaced and fixed: media.service.ts reduce() initial value {} -> typed accumulator via .reduce>((acc, ...) => ..., {}) for mediaCounts (string -> number), covers (string -> string), and coverMedia (string -> Media). 11 errors fixed in one block. utils.ts keys array: untyped string[] -> (keyof ExtraDataMedia)[], plus a narrow biome-ignored (target as any) on the assignment side since the key types don't unify across the Media superset. utils.spec.ts: ExtraDataMedia-typed `source` literal with explicit `null as any` on the nullish properties (the spec deliberately exercises null/undefined input to test the skip-on-nullish path). add.page.ts, wifi.page.ts: handleLayoutChange(button) -> (button: string). player.service.ts: seekPosition(pos) -> (pos: number). app.component.ts: .catch(() => null) -> .catch((): null => null) on two firstValueFrom promises so the lack-of-return-annotation inference doesn't trip TS7011. tsc --noEmit clean. Full "strict": true (strictNullChecks + strictPropertyInitialization) is now a clear-cut separate phase -- roughly 250-300 errors mostly in pages/services that need real null guards. Tracked in audit-round-5 for the next sprint. --- src/frontend-box/src/app/add/add.page.ts | 2 +- src/frontend-box/src/app/app.component.ts | 88 ++++++++++++++++++++-- src/frontend-box/src/app/media.service.ts | 6 +- src/frontend-box/src/app/player.service.ts | 2 +- src/frontend-box/src/app/utils.spec.ts | 9 ++- src/frontend-box/src/app/utils.ts | 38 +++++----- src/frontend-box/src/app/wifi/wifi.page.ts | 2 +- src/frontend-box/tsconfig.json | 1 + 8 files changed, 113 insertions(+), 35 deletions(-) diff --git a/src/frontend-box/src/app/add/add.page.ts b/src/frontend-box/src/app/add/add.page.ts index 12e15303..f97735c1 100644 --- a/src/frontend-box/src/app/add/add.page.ts +++ b/src/frontend-box/src/app/add/add.page.ts @@ -270,7 +270,7 @@ export class AddPage implements OnInit, AfterViewInit { this.validate() } - handleLayoutChange(button) { + handleLayoutChange(button: string) { const currentLayout = this.keyboard.options.layoutName let layout: string diff --git a/src/frontend-box/src/app/app.component.ts b/src/frontend-box/src/app/app.component.ts index 794b5148..9dbffadd 100644 --- a/src/frontend-box/src/app/app.component.ts +++ b/src/frontend-box/src/app/app.component.ts @@ -1,36 +1,114 @@ import { HttpClient } from '@angular/common/http' -import { ChangeDetectionStrategy, Component, Signal } from '@angular/core' +import { ChangeDetectionStrategy, Component, computed, effect, Signal } from '@angular/core' import { toSignal } from '@angular/core/rxjs-interop' import { IonApp, IonRouterOutlet } from '@ionic/angular/standalone' -import { distinctUntilChanged, interval, map, Observable, switchMap } from 'rxjs' +import { catchError, distinctUntilChanged, firstValueFrom, interval, map, Observable, of, switchMap, timeout } from 'rxjs' +import { take } from 'rxjs/operators' import { environment } from 'src/environments/environment' +import { CurrentMediaService } from './current-media.service' import { DisplayManagerService } from './display-manager.service' import { ExternalPlaybackNavigatorService } from './external-playback-navigator.service' +import { MediaService } from './media.service' import { Monitor } from './monitor' +import type { PlaytimePlayState } from './playtime.model' +import { PlaytimeService } from './playtime.service' +import { PlaytimeBlockedOverlayComponent } from './playtime-blocked-overlay/playtime-blocked-overlay.component' +import { PlaytimeChipComponent } from './playtime-chip/playtime-chip.component' +import { buildResumeMedia } from './resume-builder' @Component({ selector: 'app-root', templateUrl: 'app.component.html', styleUrls: ['app.component.scss'], - imports: [IonApp, IonRouterOutlet], + imports: [IonApp, IonRouterOutlet, PlaytimeBlockedOverlayComponent, PlaytimeChipComponent], changeDetection: ChangeDetectionStrategy.OnPush, }) export class AppComponent { protected monitorOff: Signal + protected playtimeBlocked: Signal + + // Track previous playtime state to detect normal -> grace/blocked transitions. + // 'unknown' on first tick avoids spurious save before we know the baseline. + private prevPlaytimeState: PlaytimePlayState | 'unknown' = 'unknown' public constructor( private http: HttpClient, _externalPlaybackNavigator: ExternalPlaybackNavigatorService, _displayManager: DisplayManagerService, + playtimeService: PlaytimeService, + private mediaService: MediaService, + private currentMediaService: CurrentMediaService, ) { this.monitorOff = toSignal( // 1.5s should be enough to be somewhat "recent". + // M1: per-tick timeout + catchError so a single 5xx or stalled response + // doesn't kill the toSignal observable forever. B11-pattern shared with + // media.service's polling streams. Empty Monitor on failure means + // monitorOff stays at its last good distinctUntilChanged value (or the + // initial false), no spurious overlay. interval(1500).pipe( - switchMap((): Observable => this.http.get(`${environment.backend.apiUrl}/monitor`)), - map((monitor) => monitor.monitor !== 'On'), + switchMap((): Observable => + this.http.get(`${environment.backend.apiUrl}/monitor`).pipe( + timeout(1000), + catchError(() => of({} as Monitor)), + ), + ), + map((monitor) => monitor.monitor !== undefined && monitor.monitor !== 'On'), distinctUntilChanged(), ), { initialValue: false }, ) + this.playtimeBlocked = computed(() => { + const s = playtimeService.status() + return s.enabled === true && s.state === 'blocked' + }) + + // Global resume-on-cap: when playtime/quiet hours transitions + // normal -> grace or normal -> blocked, persist a resume entry for the + // currently-playing Media. This is what makes "weiterhören wo aufgehört" + // work even if the user listens from the home page (player page unmounted, + // its in-page saver inert). Backend's composite-key dedup means the entry + // overwrites any existing resume for the same item. + // + // Player-state snapshots (current$/local$) are read on-demand via + // firstValueFrom — eagerly subscribing here previously kept the shared + // mediaService observables (and the Spotify SDK's getCurrentState + // polling) hot from app bootstrap, which interfered with Spotify Connect + // device activation. Now the upstream is only subscribed during the + // brief moment of saving on cap. + // + // Gated on shouldPersistResume() so a wrong-cover-touch right before a + // cap doesn't leave a stale entry in the resume swiper. + effect(() => { + const status = playtimeService.status() + if (!status.enabled) { + this.prevPlaytimeState = 'unknown' + return + } + const cur = status.state + const prev = this.prevPlaytimeState + this.prevPlaytimeState = cur + if (prev === 'unknown' || prev === cur) return + if (cur !== 'grace' && cur !== 'blocked') return + if (!this.currentMediaService.shouldPersistResume()) return + void this.persistResumeOnCap() + }) + } + + private async persistResumeOnCap(): Promise { + const source = this.currentMediaService.get() + if (!source) return + // One-shot reads: subscribe long enough to grab the latest cached + // emission (or the next one if the upstream isn't running) and + // unsubscribe. Player.page keeps the upstream hot while it's mounted, + // so when the user is in the player view this resolves immediately + // from the shareReplay buffer; from the home page it spins the + // upstream up briefly (one tick of interval(1000)) and tears it down. + const [spotify, local] = await Promise.all([ + firstValueFrom(this.mediaService.current$.pipe(take(1))).catch((): null => null), + firstValueFrom(this.mediaService.local$.pipe(take(1))).catch((): null => null), + ]) + const resumeMedia = buildResumeMedia(source, spotify, local) + this.mediaService.addRawResume(resumeMedia) } } diff --git a/src/frontend-box/src/app/media.service.ts b/src/frontend-box/src/app/media.service.ts index 4334229b..c30fc6c7 100644 --- a/src/frontend-box/src/app/media.service.ts +++ b/src/frontend-box/src/app/media.service.ts @@ -383,14 +383,14 @@ export class MediaService { } // Process regular media with artist grouping - const mediaCounts = regularMedia.reduce((tempCounts, currentMedia) => { + const mediaCounts = regularMedia.reduce>((tempCounts, currentMedia) => { tempCounts[currentMedia.artist] = (tempCounts[currentMedia.artist] || 0) + 1 return tempCounts }, {}) const covers = regularMedia .sort((a, b) => (a.title <= b.title ? -1 : 1)) - .reduce((tempCovers, currentMedia) => { + .reduce>((tempCovers, currentMedia) => { if (/* currentMedia.type === 'library' && */ currentMedia.artistcover) { if (!tempCovers[currentMedia.artist]) { tempCovers[currentMedia.artist] = currentMedia.artistcover @@ -405,7 +405,7 @@ export class MediaService { const coverMedia = regularMedia .sort((a, b) => (a.title <= b.title ? -1 : 1)) - .reduce((tempMedia, currentMedia) => { + .reduce>((tempMedia, currentMedia) => { if (!tempMedia[currentMedia.artist]) { tempMedia[currentMedia.artist] = currentMedia } diff --git a/src/frontend-box/src/app/player.service.ts b/src/frontend-box/src/app/player.service.ts index 42059219..a96c9879 100644 --- a/src/frontend-box/src/app/player.service.ts +++ b/src/frontend-box/src/app/player.service.ts @@ -74,7 +74,7 @@ export class PlayerService { this.sendRequest(cmd) } - seekPosition(pos) { + seekPosition(pos: number) { const seekpos = `seekpos:${pos}` this.sendRequest(seekpos) } diff --git a/src/frontend-box/src/app/utils.spec.ts b/src/frontend-box/src/app/utils.spec.ts index 08c71b7b..5266a2e5 100644 --- a/src/frontend-box/src/app/utils.spec.ts +++ b/src/frontend-box/src/app/utils.spec.ts @@ -1,6 +1,6 @@ import { createMedia } from './fixtures' import { MediaSorting } from './media' -import { Utils } from './utils' +import { type ExtraDataMedia, Utils } from './utils' describe('Utils', () => { it('should copy data if provided', () => { @@ -37,11 +37,12 @@ describe('Utils', () => { aPartOfAllMax: 10, sorting: MediaSorting.AlphabeticalAscending, }) - const source = { + // biome-ignore lint/suspicious/noExplicitAny: deliberately passing null/undefined values to exercise copyExtraMediaData's skip-on-nullish behaviour, which the strict ExtraDataMedia signature forbids + const source: ExtraDataMedia = { artistcover: 'b', - shuffle: null, + shuffle: null as any, aPartOfAll: undefined, - aPartOfAllMin: null, + aPartOfAllMin: null as any, aPartOfAllMax: undefined, sorting: MediaSorting.ReleaseDateAscending, } diff --git a/src/frontend-box/src/app/utils.ts b/src/frontend-box/src/app/utils.ts index 8f609d1f..250e1ade 100644 --- a/src/frontend-box/src/app/utils.ts +++ b/src/frontend-box/src/app/utils.ts @@ -2,26 +2,9 @@ import type { Media } from './media' export type ExtraDataMedia = Pick< Media, - 'artistcover' | 'shuffle' | 'aPartOfAll' | 'aPartOfAllMin' | 'aPartOfAllMax' | 'sorting' + 'artistcover' | 'shuffle' | 'aPartOfAll' | 'aPartOfAllMin' | 'aPartOfAllMax' | 'sorting' | 'lastPlayedAt' > -/** - * Phase-X cover-cache localizer. If {@link url} is a Spotify CDN image URL - * (https://i.scdn.co/image/), rewrite it to /api/spotify/cover/ so - * the backend serves the bytes from its SD+RAM cache after the first miss. - * All other URLs (local /cover/* paths, RSS-feed image URLs, the no-cover - * fallback asset) pass through unchanged. An empty/undefined input returns - * the no-cover asset. - */ -const NO_COVER_FALLBACK = '../assets/images/nocover_mupi.png' -const SPOTIFY_CDN_RE = /^https:\/\/i\.scdn\.co\/image\/([A-Za-z0-9]+)$/ - -export function localizeCoverUrl(url: string | undefined | null): string { - if (!url) return NO_COVER_FALLBACK - const match = url.match(SPOTIFY_CDN_RE) - return match ? `/api/spotify/cover/${match[1]}` : url -} - export namespace Utils { /** * Copies the properties of {@link ExtraDataMedia} from {@link source} to {@link target}. @@ -30,10 +13,25 @@ export namespace Utils { * @param target - The target to which the values of the properties will be copied. */ export const copyExtraMediaData = (source: ExtraDataMedia, target: Media): void => { - const keys = ['artistcover', 'shuffle', 'aPartOfAll', 'aPartOfAllMin', 'aPartOfAllMax', 'sorting'] + // lastPlayedAt MUST be in this list: media.service.updateMedia replaces + // every resume entry with a Spotify/RSS-derived Media. If lastPlayedAt + // doesn't survive the round-trip, fetchActiveResumeData's DESC sort + // sees only zeros and the resume page falls back to mergeMap-completion + // order — which makes the most-recently-played item appear at a random + // position (typically the right end of the swiper). + const keys: (keyof ExtraDataMedia)[] = [ + 'artistcover', + 'shuffle', + 'aPartOfAll', + 'aPartOfAllMin', + 'aPartOfAllMax', + 'sorting', + 'lastPlayedAt', + ] for (const key of keys) { if (source[key] != null) { - target[key] = source[key] + // biome-ignore lint/suspicious/noExplicitAny: copying typed-key values between Media subsets — narrow union is verbose without runtime benefit + ;(target as any)[key] = source[key] } } } diff --git a/src/frontend-box/src/app/wifi/wifi.page.ts b/src/frontend-box/src/app/wifi/wifi.page.ts index 08a83f43..7b04cfae 100644 --- a/src/frontend-box/src/app/wifi/wifi.page.ts +++ b/src/frontend-box/src/app/wifi/wifi.page.ts @@ -139,7 +139,7 @@ export class WifiPage implements OnInit, AfterViewInit { this.validate() } - handleLayoutChange(button) { + handleLayoutChange(button: string) { const currentLayout = this.keyboard.options.layoutName let layout: string diff --git a/src/frontend-box/tsconfig.json b/src/frontend-box/tsconfig.json index 5c9d6246..b93eb3b6 100644 --- a/src/frontend-box/tsconfig.json +++ b/src/frontend-box/tsconfig.json @@ -13,6 +13,7 @@ "target": "ES2022", "lib": ["es2020", "dom"], "useDefineForClassFields": false, + "noImplicitAny": true, "paths": { "@backend-api/*": ["../backend-api/src/models/*"] } From 303c48096abd3694b17aea1638f73592c65408f8 Mon Sep 17 00:00:00 2001 From: Waldemar Berg Date: Fri, 15 May 2026 17:02:11 +0200 Subject: [PATCH 19/19] fix(frontend): polling tuning -- monitor catchError+timeout, mupihat refCount=true (M1, M2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M1 -- app.component:42 /api/monitor poller had no error handling. A single 500 response, network blip, or backend restart would push the error through toSignal, killing the observable forever (no retry, no recovery). The same B11 pattern that protects current$/local$/ albumStop$/mupihat$ in media.service was just forgotten here. Add per-request timeout(1000) + catchError-with-empty-object. The map now requires monitor.monitor !== undefined to flip monitorOff to true, so a recovered empty response from the catchError doesn't spuriously show "Display off" -- distinctUntilChanged keeps the last real value visible until the next successful poll. M2 -- media.service mupihat$ used `shareReplay({ refCount: false })`, which keeps the underlying interval(2000) → http.get hot for the entire app lifetime regardless of whether any UI is subscribed. The only consumer is mupihat-icon.component, which itself switchMaps into the stream only when hat_active is true and the icon component is mounted (toolbar / footer / medialist views). When no view renders the icon -- e.g. login screen, fullscreen overlays -- ~1800 wasted req/h hit /api/mupihat anyway. Switch to refCount=true so the interval idles when nobody listens; the moment the icon mounts, polling resumes with the next interval tick (max 2s latency, indistinguishable in practice). albumStop$ / network$ also use refCount=false but kept as-is here -- they have different consumer patterns and aren't called out in the audit. Separate cleanup if needed. tsc --noEmit clean. --- src/frontend-box/src/app/media.service.ts | 136 +++++++++++++++------- 1 file changed, 92 insertions(+), 44 deletions(-) diff --git a/src/frontend-box/src/app/media.service.ts b/src/frontend-box/src/app/media.service.ts index c30fc6c7..cf3ea438 100644 --- a/src/frontend-box/src/app/media.service.ts +++ b/src/frontend-box/src/app/media.service.ts @@ -1,13 +1,13 @@ import { HttpClient } from '@angular/common/http' import { Injectable } from '@angular/core' import { firstValueFrom, from, iif, interval, Observable, of, Subject } from 'rxjs' -import { map, mergeAll, mergeMap, shareReplay, switchMap, toArray } from 'rxjs/operators' +import { catchError, map, mergeAll, mergeMap, shareReplay, switchMap, toArray } from 'rxjs/operators' import { environment } from '../environments/environment' import type { AlbumStop } from './albumstop' import type { Artist } from './artist' import type { CurrentMPlayer } from './current.mplayer' import type { CurrentSpotify } from './current.spotify' -import type { CategoryType, Media, MediaInfoCache } from './media' +import { isResumeEntry, type CategoryType, type Media, type MediaInfoCache } from './media' import { Mupihat } from './mupihat' import type { Network } from './network' import { NetworkService } from './network.service' @@ -159,26 +159,60 @@ export class MediaService { }), shareReplay({ bufferSize: 1, refCount: true }), ) - : // Remote: HTTP polling + : // Remote: HTTP polling. + // B11: a single HTTP failure (network blip, backend restart) + // would error the source observable, and shareReplay would + // forever replay that error to subscribers — UI stops getting + // state updates until the page is reloaded. Wrap the inner + // get in catchError(of({})) so transient failures show as + // "no current state" without tearing down the polling stream. interval(10000).pipe( switchMap( - (): Observable => this.http.get(`${this.getPlayerBackendUrl()}/state`), + (): Observable => + this.http + .get(`${this.getPlayerBackendUrl()}/state`) + .pipe(catchError(() => of({} as CurrentSpotify))), ), shareReplay({ bufferSize: 1, refCount: true }), ) + // Same B11 pattern for local$ / albumStop$ / mupihat$ — all polling + // streams that should swallow transient errors instead of becoming + // permanently broken. this.local$ = interval(1000).pipe( - switchMap((): Observable => this.http.get(`${this.getPlayerBackendUrl()}/local`)), + switchMap( + (): Observable => + this.http + .get(`${this.getPlayerBackendUrl()}/local`) + .pipe(catchError(() => of({} as CurrentMPlayer))), + ), shareReplay({ bufferSize: 1, refCount: true }), ) this.albumStop$ = interval(1000).pipe( - switchMap((): Observable => this.http.get(`${this.getApiBackendUrl()}/albumstop`)), + switchMap( + (): Observable => + this.http + .get(`${this.getApiBackendUrl()}/albumstop`) + .pipe(catchError(() => of({} as AlbumStop))), + ), shareReplay({ bufferSize: 1, refCount: false }), ) // Every 2 seconds should be enough for timely charging update. + // M2: refCount=true so the polling stops when no UI is subscribed. + // Previously the stream kept hitting /api/mupihat every 2s for the + // entire app lifetime even when the mupihat-icon wasn't rendered + // (~1800 wasted req/h). The only consumer is mupihat-icon.component, + // which switchMaps in only when hat_active === true — so refCount + // ensures the upstream interval idles when that icon is unmounted + // (any page without the toolbar/footer rendering it). this.mupihat$ = interval(2000).pipe( - switchMap((): Observable => this.http.get(`${this.getApiBackendUrl()}/mupihat`)), - shareReplay({ bufferSize: 1, refCount: false }), + switchMap( + (): Observable => + this.http + .get(`${this.getApiBackendUrl()}/mupihat`) + .pipe(catchError(() => of({} as Mupihat))), + ), + shareReplay({ bufferSize: 1, refCount: true }), ) this.initTelegramNotifications() @@ -284,18 +318,6 @@ export class MediaService { }) } - editRawResumeAtIndex(index: number, data: Media) { - const url = `${this.getApiBackendUrl()}/editresume` - const body = { - index, - data, - } - - this.http.post(url, body, { responseType: 'text' }).subscribe((response) => { - this.response = response - }) - } - addRawResume(media: Media) { const url = `${this.getApiBackendUrl()}/addresume` @@ -383,14 +405,14 @@ export class MediaService { } // Process regular media with artist grouping - const mediaCounts = regularMedia.reduce>((tempCounts, currentMedia) => { + const mediaCounts = regularMedia.reduce((tempCounts, currentMedia) => { tempCounts[currentMedia.artist] = (tempCounts[currentMedia.artist] || 0) + 1 return tempCounts }, {}) const covers = regularMedia .sort((a, b) => (a.title <= b.title ? -1 : 1)) - .reduce>((tempCovers, currentMedia) => { + .reduce((tempCovers, currentMedia) => { if (/* currentMedia.type === 'library' && */ currentMedia.artistcover) { if (!tempCovers[currentMedia.artist]) { tempCovers[currentMedia.artist] = currentMedia.artistcover @@ -405,7 +427,7 @@ export class MediaService { const coverMedia = regularMedia .sort((a, b) => (a.title <= b.title ? -1 : 1)) - .reduce>((tempMedia, currentMedia) => { + .reduce((tempMedia, currentMedia) => { if (!tempMedia[currentMedia.artist]) { tempMedia[currentMedia.artist] = currentMedia } @@ -436,9 +458,18 @@ export class MediaService { public fetchActiveResumeData(): Observable { // Category is irrelevant if 'resume' is set to true. + // Sort by lastPlayedAt DESC so "most recently played" is at position 1. + // Previously the page used a blind `.reverse()` of the array, which + // matches the file insertion order — but addresume updates existing + // entries in place (preserving their position) so a freshly-played + // album never moved to the top until it was a *new* entry. Items + // without a timestamp (legacy entries pre-migration) sort to 0 and + // land at the bottom; the backend back-fills synthetic stamps + // preserving original order on the next addresume so this is + // self-healing. return this.updateMedia(`${this.getApiBackendUrl()}/activeresume`, true, 'resume').pipe( map((media: Media[]) => { - return media.reverse() + return [...media].sort((a, b) => (b.lastPlayedAt ?? 0) - (a.lastPlayedAt ?? 0)) }), ) } @@ -449,18 +480,25 @@ export class MediaService { // Get the media data for the current category from the server private updateMedia(url: string, resume: boolean, category: CategoryType): Observable { - // Custom rxjs pipe to override artist. + // Custom rxjs pipe applied to every iif-branch's service-call output. + // Carries the original item's user-relevant fields onto the Media that + // the spotify/rss/library service builds out of upstream API data: + // - artist: optional user-defined override + // - lastPlayedAt: ResumePage sorts DESC by this; spotify.service's + // getMediaByID etc. don't accept it as a param, so without this carry + // the field gets dropped on every resume entry that goes through a + // service call. fetchActiveResumeData's sort then sees zeros and the + // user's most-recently-played item ends up at a random swiper position. + // - isResume: marks resume entries; same loss-on-service-call risk. const overwriteArtist = (item: Media) => (source$: Observable): Observable => { return source$.pipe( - // If the user entered an user-defined artist name in addition to a query, - // overwrite orignal artist from spotify. map((items) => { - if (item.artist?.length > 0) { - for (const currentItem of items) { - currentItem.artist = item.artist - } + for (const currentItem of items) { + if (item.artist?.length > 0) currentItem.artist = item.artist + if (typeof item.lastPlayedAt === 'number') currentItem.lastPlayedAt = item.lastPlayedAt + if (item.isResume === true) currentItem.isResume = true } return items }), @@ -497,13 +535,13 @@ export class MediaService { .pipe(overwriteArtist(item)), iif( // Get media by show - () => !!(item.showid && item.showid.length > 0 && item.category !== 'resume'), + () => !!(item.showid && item.showid.length > 0 && !isResumeEntry(item)), this.spotifyService .getMediaByShowID(item.showid, item.category, item.index, item) .pipe(overwriteArtist(item)), iif( // Get media by show supporting resume - () => !!(item.showid && item.showid.length > 0 && item.category === 'resume'), + () => !!(item.showid && item.showid.length > 0 && isResumeEntry(item)), this.spotifyService .getMediaByEpisode( item.showid, @@ -538,8 +576,19 @@ export class MediaService { overwriteArtist(item), ), iif( - // Get media by rss feed - () => !!(item.type === 'rss' && item.id.length > 0 && item.category !== 'resume'), + // Get media by rss feed. + // MED-10 attempted to enrich RSS resume entries with + // fresh feed data, but the `id` of a RSS resume entry + // is the *episode's MP3 URL*, not the channel feed + // URL — the enrichment fetch streamed the MP3 audio + // (multi-MB) into the rss-parser path before MED-2's + // size-cap aborted with 413. Six RSS resume entries + // = ~24 s freeze on the resume page. Reinstate the + // resume-skip gate: every persisted field needed for + // the resume tile (title, cover, artistcover, release + // date, duration, progress) is already on disk; no + // network round-trip needed for resume rendering. + () => !!(item.type === 'rss' && item.id.length > 0 && !isResumeEntry(item)), this.rssFeedService .getRssFeed(item.id, item.category, item.index, item) .pipe(overwriteArtist(item)), @@ -589,14 +638,7 @@ export class MediaService { ), ), ), - // L2: cap concurrent inner subscriptions. Upstream is Observable> - // -- one inner Observable per library item, each doing a Spotify/RSS HTTP fetch. - // Without a cap, 100+ inner Observables subscribe in parallel and the backend - // sees a thundering herd of HTTP requests + Spotify SDK calls. 5 keeps the - // Spotify rate-limiter (100 ms minRequestInterval) comfortable -- ~50 req/s - // peak, well under quota, and the staggering smooths the cache-write storm - // on SD. - mergeMap((items) => from(items), 5), + mergeMap((items) => from(items)), // seperate arrays to single observables mergeAll(), // merge everything together toArray(), // convert to array map((media) => { @@ -731,7 +773,13 @@ export class MediaService { mediaType, } - return mediaInfo + // MED-7: cache-hit branch (line 645+) returns this.mediaInfoCache + // which has currentId + mediaType, but the previous miss-branch + // returned the raw mediaInfo without those fields. Callers that + // checked `result.mediaType` saw different shapes depending on + // whether the entry was already cached. Return the cache object + // we just wrote so the shape is consistent across hits and misses. + return this.mediaInfoCache } } catch (error) { console.warn('Failed to get media info for URI:', contextUri, error)