From e35e5e85b03ad487497bead98e830c41b1992349 Mon Sep 17 00:00:00 2001 From: Max Engel Date: Sat, 25 Jul 2026 00:43:18 +0000 Subject: [PATCH] rclone: safer restores, RSYNCRMDIR, drop legacy mount code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Correctness and safety cleanup of the cloud backup/restore tools: - Restore can no longer delete local data: --delete-excluded is stripped during restore option normalization. The filter rules are an allowlist, so on a RESTOREMETHOD="sync" restore that flag would have deleted everything local that isn't a save/state/screenshot (ROMs, BIOS, artwork). Backup keeps it, where it only tidies the remote. Also removes the dead RESTORE_RCLONEOPTS assignment. - Implement RSYNCRMDIR: it shipped enabled and documented but no script ever read it. After a successful game-saves backup, cloud_backup now prunes empty remote directories via rclone rmdirs --leave-root. - Report rclone exit code 9 as "no files transferred" instead of a retry failure, add --stats-one-line to the game-saves flows, and remove controller-input regex branches that could never match (quoted patterns were compared literally). - Apply the capability-checked --progress-terminal-width flag in all four rclone flows instead of only the system-backup restore. - cloud_sync_helper: create the initial config with sed so multi-line backslash-continued values like RCLONEOPTS survive intact; the old line-based loop truncated them. - Remove the pre-cloud_sync legacy path: the rclonectl FUSE-mount wrapper and its rsync.conf/rsync-rules.conf (reachable only via manual SSH, no UI entry) plus the post-update block that re-seeded them on every OS update, and 001-setup's boot-time reset of the now-removed cloud-mount toggle state. Existing copies on user storage are left untouched. 💘 Generated with Crush Assisted-by: Crush:anthropic/claude-fable-5 --- .../packages/network/rclone/package.mk | 3 - .../network/rclone/sources/cloud_backup | 28 +++++- .../network/rclone/sources/cloud_restore | 22 ++++- .../network/rclone/sources/cloud_sync_helper | 12 +-- .../packages/network/rclone/sources/rclonectl | 95 ------------------- .../network/rclone/sources/rsync-rules.conf | 22 ----- .../network/rclone/sources/rsync.conf | 23 ----- .../packages/rocknix/autostart/001-setup | 4 - .../packages/rocknix/sources/post-update | 5 - 9 files changed, 43 insertions(+), 171 deletions(-) delete mode 100755 projects/ROCKNIX/packages/network/rclone/sources/rclonectl delete mode 100644 projects/ROCKNIX/packages/network/rclone/sources/rsync-rules.conf delete mode 100644 projects/ROCKNIX/packages/network/rclone/sources/rsync.conf diff --git a/projects/ROCKNIX/packages/network/rclone/package.mk b/projects/ROCKNIX/packages/network/rclone/package.mk index ddd1c603037..5d941eefaa9 100644 --- a/projects/ROCKNIX/packages/network/rclone/package.mk +++ b/projects/ROCKNIX/packages/network/rclone/package.mk @@ -35,15 +35,12 @@ unpack() { makeinstall_target() { mkdir -p ${INSTALL}/usr/bin/ mkdir -p ${INSTALL}/usr/config/ - cp rclonectl ${INSTALL}/usr/bin/ cp cloud_backup ${INSTALL}/usr/bin/ cp cloud_restore ${INSTALL}/usr/bin/ cp cloud_sync_helper ${INSTALL}/usr/bin/ cp cloud_sync_cleanup_duplicates.sh ${INSTALL}/usr/bin/ cp ${PKG_BUILD}/${PKG_RCLONE} ${INSTALL}/usr/bin/ chmod 0755 ${INSTALL}/usr/bin/* - cp rsync-rules.conf ${INSTALL}/usr/config/ - cp rsync.conf ${INSTALL}/usr/config/ cp cloud_sync-rules.txt ${INSTALL}/usr/config/ cp cloud_sync.conf ${INSTALL}/usr/config/ cp cloud_sync.conf.defaults ${INSTALL}/usr/config/ diff --git a/projects/ROCKNIX/packages/network/rclone/sources/cloud_backup b/projects/ROCKNIX/packages/network/rclone/sources/cloud_backup index fd88d7a999c..6877a8446ae 100755 --- a/projects/ROCKNIX/packages/network/rclone/sources/cloud_backup +++ b/projects/ROCKNIX/packages/network/rclone/sources/cloud_backup @@ -98,9 +98,9 @@ read_controller_input() { echo "left" elif [[ "$event_output" =~ BTN_DPAD_RIGHT.*value[[:space:]]+1 ]] || [[ "$event_output" =~ ABS_HAT0X.*value[[:space:]]+1 ]]; then echo "right" - elif [[ "$event_output" =~ BTN_SOUTH.*value[[:space:]]+1 ]] || [[ "$event_output" =~ BTN_A.*value[[:space:]]+1 ]] || [[ "$event_output" =~ "code 0.*value[[:space:]]+1" ]] || [[ "$event_output" =~ BTN_EAST.*value[[:space:]]+1 ]]; then + elif [[ "$event_output" =~ BTN_SOUTH.*value[[:space:]]+1 ]] || [[ "$event_output" =~ BTN_A.*value[[:space:]]+1 ]] || [[ "$event_output" =~ BTN_EAST.*value[[:space:]]+1 ]]; then echo "confirm" - elif [[ "$event_output" =~ BTN_B.*value[[:space:]]+1 ]] || [[ "$event_output" =~ "code 1.*value[[:space:]]+1" ]]; then + elif [[ "$event_output" =~ BTN_B.*value[[:space:]]+1 ]]; then echo "cancel" elif [[ "$event_output" =~ BTN_START.*value[[:space:]]+1 ]]; then echo "start" @@ -306,7 +306,7 @@ report_rclone_error() { 6) log_message "${operation} failed: Less serious errors (e.g. partly transferred)" "true" "WARN" ;; 7) log_message "${operation} failed: Fatal error - rclone giving up" "true" "ERROR" ;; 8) log_message "${operation} failed: Transfer exceeded - limits exceeded" "true" "ERROR" ;; - 9) log_message "${operation} failed: Operation failed - no retry" "true" "ERROR" ;; + 9) log_message "${operation} completed: no files transferred" "true" ;; *) log_message "${operation} failed with unknown error code: $exit_code" "true" "ERROR" ;; esac } @@ -379,7 +379,7 @@ backup_game_saves() { log_message "Ensuring remote sync path exists: ${REMOTENAME}${SYNCPATH}" "false" rclone mkdir "${REMOTENAME}${SYNCPATH}" 2>/dev/null - # Set log level to DEBUG when INFO is selected for enhanced logging + # Pass an explicit --log-level INFO when LOG_LEVEL is INFO local rclone_debug="" local filtered_opts=("${RCLONE_OPTS_ARRAY[@]}") @@ -407,12 +407,18 @@ backup_game_saves() { # Add exclusions all_opts+=( + "--stats-one-line" "--exclude=${BACKUPFOLDER}/**" "--exclude=backups/**" "--exclude=bios/**" "--exclude=*.zip" ) + # Check if the rclone version supports the terminal width flag + if rclone help | grep -q progress-terminal-width; then + all_opts+=("--progress-terminal-width=80") + fi + # Execute rclone with enhanced error handling execute_rclone_with_error_handling \ "${BACKUPMETHOD}" \ @@ -425,6 +431,15 @@ backup_game_saves() { # Report the result with detailed error information report_rclone_error $BACKUP_STATUS "Game saves backup" + # RSYNCRMDIR: after a successful backup, remove orphaned empty directories on + # the remote (--leave-root keeps the sync root itself). Exit code 9 means + # "success, no files transferred", so tidy on that too. + if [ "${RSYNCRMDIR}" == "yes" ] && { [ $BACKUP_STATUS -eq 0 ] || [ $BACKUP_STATUS -eq 9 ]; }; then + log_message "Removing orphaned empty directories on remote: ${REMOTENAME}${SYNCPATH}" "false" + rclone rmdirs "${REMOTENAME}${SYNCPATH}" --leave-root --log-file /var/log/cloud_sync.log 2>/dev/null \ + || log_message "Empty-directory cleanup failed (non-fatal)" "true" "WARN" + fi + return $BACKUP_STATUS } @@ -470,6 +485,11 @@ backup_system_files() { "--include=*.zip" "--stats-one-line" ) + + # Check if the rclone version supports the terminal width flag + if rclone help | grep -q progress-terminal-width; then + backup_opts+=("--progress-terminal-width=80") + fi # Add any additional options from RESTORE_RCLONEOPTS if [ -n "$RESTORE_RCLONEOPTS" ]; then diff --git a/projects/ROCKNIX/packages/network/rclone/sources/cloud_restore b/projects/ROCKNIX/packages/network/rclone/sources/cloud_restore index 6533fc89d04..7b412078d15 100755 --- a/projects/ROCKNIX/packages/network/rclone/sources/cloud_restore +++ b/projects/ROCKNIX/packages/network/rclone/sources/cloud_restore @@ -94,9 +94,9 @@ read_controller_input() { echo "left" elif [[ "$event_output" =~ BTN_DPAD_RIGHT.*value[[:space:]]+1 ]] || [[ "$event_output" =~ ABS_HAT0X.*value[[:space:]]+1 ]]; then echo "right" - elif [[ "$event_output" =~ BTN_SOUTH.*value[[:space:]]+1 ]] || [[ "$event_output" =~ BTN_A.*value[[:space:]]+1 ]] || [[ "$event_output" =~ "code 0.*value[[:space:]]+1" ]] || [[ "$event_output" =~ BTN_EAST.*value[[:space:]]+1 ]]; then + elif [[ "$event_output" =~ BTN_SOUTH.*value[[:space:]]+1 ]] || [[ "$event_output" =~ BTN_A.*value[[:space:]]+1 ]] || [[ "$event_output" =~ BTN_EAST.*value[[:space:]]+1 ]]; then echo "confirm" - elif [[ "$event_output" =~ BTN_B.*value[[:space:]]+1 ]] || [[ "$event_output" =~ "code 1.*value[[:space:]]+1" ]]; then + elif [[ "$event_output" =~ BTN_B.*value[[:space:]]+1 ]]; then echo "cancel" elif [[ "$event_output" =~ BTN_START.*value[[:space:]]+1 ]]; then echo "start" @@ -342,6 +342,13 @@ load_config() { RCLONEOPTS=$(echo "${RCLONEOPTS}" | sed 's/--verbose//g' | sed 's/-v / /g') log_message "After removing verbose flags: ${RCLONEOPTS}" "false" + # Restore writes to local storage. The filter rules are an allowlist, so with + # RESTOREMETHOD="sync" a --delete-excluded would delete everything local that + # is not a save/state/screenshot (ROMs, BIOS, artwork). Strip it on restore; + # backup keeps it, where it only tidies the remote. + RCLONEOPTS=$(echo "${RCLONEOPTS}" | sed 's/--delete-excluded//g') + log_message "After removing restore-unsafe delete flags: ${RCLONEOPTS}" "false" + RCLONEOPTS=$(echo "${RCLONEOPTS}" | sed 's/--/ --/g' | sed 's/^ //') log_message "After spacing fix: ${RCLONEOPTS}" "false" @@ -354,7 +361,6 @@ load_config() { RCLONE_OPTS_ARRAY+=("$opt") done log_message "Options array has ${#RCLONE_OPTS_ARRAY[@]} elements" "false" - RESTORE_RCLONEOPTS=$(echo "${RCLONEOPTS}" | sed 's/--delete-excluded//') log_message "Configuration loaded successfully" "false" } @@ -373,7 +379,7 @@ report_rclone_error() { 6) log_message "${operation} failed: Less serious errors (e.g. partly transferred)" "true" "WARN" ;; 7) log_message "${operation} failed: Fatal error - rclone giving up" "true" "ERROR" ;; 8) log_message "${operation} failed: Transfer exceeded - limits exceeded" "true" "ERROR" ;; - 9) log_message "${operation} failed: Operation failed - no retry" "true" "ERROR" ;; + 9) log_message "${operation} completed: no files transferred" "true" ;; *) log_message "${operation} failed with unknown error code: $exit_code" "true" "ERROR" ;; esac } @@ -435,7 +441,7 @@ restore_game_saves() { mkdir -p "${RESTOREPATH}" fi - # Set log level to DEBUG when INFO is selected for enhanced logging + # Pass an explicit --log-level INFO when LOG_LEVEL is INFO local rclone_debug="" local filtered_opts=("${RCLONE_OPTS_ARRAY[@]}") @@ -461,10 +467,16 @@ restore_game_saves() { all_opts+=("$rclone_debug" "INFO") fi # Add exclusions all_opts+=( + "--stats-one-line" "--exclude=${BACKUPFOLDER}/**" "--exclude=backups/**" "--exclude=bios/**" ) + + # Check if the rclone version supports the terminal width flag + if rclone help | grep -q progress-terminal-width; then + all_opts+=("--progress-terminal-width=80") + fi # Execute rclone with enhanced error handling execute_rclone_with_error_handling \ diff --git a/projects/ROCKNIX/packages/network/rclone/sources/cloud_sync_helper b/projects/ROCKNIX/packages/network/rclone/sources/cloud_sync_helper index 42d5d90d8b3..40bc23faf7a 100644 --- a/projects/ROCKNIX/packages/network/rclone/sources/cloud_sync_helper +++ b/projects/ROCKNIX/packages/network/rclone/sources/cloud_sync_helper @@ -131,16 +131,8 @@ update_cloud_sync_config() { # No existing config file, create one with defaults log_to_file "${log_file}" "No existing config file, creating from template" - # Create a basic user config from defaults - touch /storage/.config/cloud_sync.conf - - # Process each default variable - while IFS='=' read -r key value; do - if [[ "$key" == DEFAULT_* ]]; then - var_name="${key#DEFAULT_}" - echo "${var_name}=${value}" >> /storage/.config/cloud_sync.conf - fi - done < /usr/config/cloud_sync.conf.defaults + # Strip the DEFAULT_ prefix; sed preserves multi-line backslash-continued values + sed 's/^DEFAULT_//' /usr/config/cloud_sync.conf.defaults > /storage/.config/cloud_sync.conf fi return 0 diff --git a/projects/ROCKNIX/packages/network/rclone/sources/rclonectl b/projects/ROCKNIX/packages/network/rclone/sources/rclonectl deleted file mode 100755 index e6ec6ee880e..00000000000 --- a/projects/ROCKNIX/packages/network/rclone/sources/rclonectl +++ /dev/null @@ -1,95 +0,0 @@ -#!/bin/bash -# SPDX-License-Identifier: GPL-2.0 -# Copyright (C) 2024 ROCKNIX (https://github.com/ROCKNIX) -# -# rclonectl - Command line utility for managing rclone cloud mounts -# This script provides a simple interface to mount/unmount cloud storage - -# Configuration path -CONFIGPATH="/storage/.config" - -# Determine mount path: either from command line, config file, or default -if [ -z "$2" ] -then - if [ -e "${CONFIGPATH}/rsync.conf" ] - then - source ${CONFIGPATH}/rsync.conf - else - # Default values if no config found - MOUNTPATH="/storage/cloud" - SYNCPATH="GAMES" - fi -else - MOUNTPATH=${2} -fi - -# Copy default configuration files if they don't exist -for CONFIG in rsync.conf rsync-rules.conf -do - if [ ! -e "${CONFIGPATH}/${CONFIG}" ] - then - cp -rf /usr/config/${CONFIG} ${CONFIGPATH}/${CONFIG} - fi -done - -# Verify rclone is configured and mount point exists -# -# This function checks if: -# 1. rclone has a valid configuration file -# 2. The specified mount point directory exists (creates it if not) -# -# Exits with error code 1 if rclone is not configured -function checkconfig() { - if [ ! -e "${CONFIGPATH}/rclone/rclone.conf" ] - then - echo "You must configure rclone before using this tool. Run \`rclone config\` to get started." - exit 1 - fi - if [ ! -d "${MOUNTPATH}" ] - then - mkdir -p ${MOUNTPATH} - fi -} - -# Mount a cloud storage remote using rclone -# Parameters: $1 - Mount point path -function mount() { - TARGET=$(rclone listremotes | awk '{printf $1; exit}') - rclone mount ${TARGET} ${1} \ - --vfs-cache-mode writes \ - --vfs-cache-max-size 100M \ - --vfs-read-chunk-size-limit 128M \ - --allow-non-empty \ - --dir-cache-time 48h \ - --log-file /var/log/rclone.log \ - --daemon -} - -# Unmount a previously mounted cloud storage -# Parameters: $1 - Mount point path -function unmount() { - fusermount -u ${1} -} - -# Command processing -case "${1}" in - "mount") - checkconfig - mount ${MOUNTPATH} - ;; - "unmount") - checkconfig - unmount ${MOUNTPATH} - ;; - # Show help message if command is unknown or no arguments were provided - "$*") - cat <>${LOG} -rsync --ignore-existing /usr/config/rsync-rules.conf /storage/.config/ -rsync --ignore-existing /usr/config/rsync.conf /storage/.config/ - ### Add cloud sync-related files if missing echo "Update cloud sync files..." >>${LOG} if [ -x "/usr/bin/cloud_sync_helper" ]; then