diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f2681e73..666f12d0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,14 +1,68 @@ -name: "build_image" +name: build_image on: push: - paths: - - "**" - - "!README.md" + branches: + - main + - develop + - 'feat/**' + tags: + - 'v*' + paths-ignore: + - 'README.md' + - 'CHANGELOG.md' + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} jobs: build: - #uses: tiredofit/github_actions/.github/workflows/default_amd64_armv7_arm64.yml@main - #uses: tiredofit/github_actions/.github/workflows/default_amd64.yml@main - uses: tiredofit/github_actions/.github/workflows/default_amd64_arm64.yml@main - secrets: inherit + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=tag + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=raw,value=latest,enable={{is_default_branch}} + type=sha,format=short + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/manual.yml b/.github/workflows/manual.yml index 2b4ebfab..3fb005b2 100644 --- a/.github/workflows/manual.yml +++ b/.github/workflows/manual.yml @@ -1,15 +1,61 @@ -name: "manual_build_image" +name: manual_build_image on: workflow_dispatch: inputs: - Manual Build: - description: 'Manual Build' + tag: + description: 'Optional extra tag to apply (e.g. test-build)' required: false + default: '' + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} jobs: build: - #uses: tiredofit/github_actions/.github/workflows/default_amd64_armv7_arm64.yml@main - #uses: tiredofit/github_actions/.github/workflows/default_amd64.yml@main - uses: tiredofit/github_actions/.github/workflows/default_amd64_arm64.yml@main - secrets: inherit + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=sha,format=short + type=raw,value=${{ inputs.tag }},enable=${{ inputs.tag != '' }} + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..e0b68d2e --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +# IDE +.vscode/ +.idea/ + +# Claude Code +.claude/ + +# OS +.DS_Store +Thumbs.db + +# Environment +.env +.env.* diff --git a/Dockerfile b/Dockerfile index dda0f6b2..fc041604 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ ARG DISTRO=alpine ARG DISTRO_VARIANT=3.21-7.10.31 FROM docker.io/tiredofit/${DISTRO}:${DISTRO_VARIANT} -LABEL maintainer="Dave Conroy (github.com/tiredofit)" +LABEL maintainer="nfrastack (github.com/nfrastack)" ENV INFLUX1_CLIENT_VERSION=1.8.0 \ INFLUX2_CLIENT_VERSION=2.7.5 \ @@ -10,12 +10,13 @@ ENV INFLUX1_CLIENT_VERSION=1.8.0 \ MSSQL_VERSION=18.6.1.1-1 \ MYSQL_VERSION=mysql-8.4.8 \ MYSQL_REPO_URL=https://github.com/mysql/mysql-server \ + NEO4J_VERSION=5.26.0 \ AWS_CLI_VERSION=1.44.56 \ POSTGRES_VERSION=18.3 \ CONTAINER_ENABLE_MESSAGING=TRUE \ CONTAINER_ENABLE_MONITORING=TRUE \ - IMAGE_NAME="tiredofit/db-backup" \ - IMAGE_REPO_URL="https://github.com/tiredofit/docker-db-backup/" + IMAGE_NAME="nfrastack/container-db-backup" \ + IMAGE_REPO_URL="https://github.com/nfrastack/container-db-backup/" RUN source /assets/functions/00-container && \ set -ex && \ @@ -71,8 +72,6 @@ RUN source /assets/functions/00-container && \ awk '$1 == "#define" && $2 == "DEFAULT_PGSOCKET_DIR" && $3 == "\"/tmp\"" { $3 = "\"/var/run/postgresql\""; print; next } { print }' src/include/pg_config_manual.h > src/include/pg_config_manual.h.new && \ grep '/var/run/postgresql' src/include/pg_config_manual.h.new && \ mv src/include/pg_config_manual.h.new src/include/pg_config_manual.h && \ - wget -O config/config.guess 'https://git.savannah.gnu.org/cgit/config.git/plain/config.guess?id=7d3d27baf8107b630586c962c057e22149653deb' && \ - wget -O config/config.sub 'https://git.savannah.gnu.org/cgit/config.git/plain/config.sub?id=7d3d27baf8107b630586c962c057e22149653deb' && \ export LLVM_CONFIG="/usr/lib/llvm19/bin/llvm-config" && \ export CLANG=clang-19 && \ ./configure \ @@ -157,12 +156,14 @@ RUN source /assets/functions/00-container && \ gpg \ gpg-agent \ groff \ + jq \ libarchive \ libtirpc \ mariadb-client \ mariadb-connector-c \ mongodb-tools \ ncurses \ + openjdk17-jre-headless \ openssl \ pigz \ pixz \ @@ -211,6 +212,14 @@ RUN source /assets/functions/00-container && \ echo >&2 "Unable to build Influx 2 on this system" ; \ fi ; \ \ + mkdir -p /usr/src/neo4j && \ + curl -sSL https://dist.neo4j.org/neo4j-community-${NEO4J_VERSION}-unix.tar.gz | tar xzf - --strip-components=1 -C /usr/src/neo4j && \ + mkdir -p /opt/neo4j/bin /opt/neo4j/lib && \ + cp /usr/src/neo4j/bin/cypher-shell /opt/neo4j/bin/ && \ + cp /usr/src/neo4j/lib/*.jar /opt/neo4j/lib/ && \ + chmod +x /opt/neo4j/bin/cypher-shell && \ + ln -sf /opt/neo4j/bin/cypher-shell /usr/local/bin/cypher-shell && \ + \ clone_git_repo https://github.com/influxdata/influxdb "${INFLUX1_CLIENT_VERSION}" && \ go build -o /usr/sbin/influxd ./cmd/influxd && \ strip /usr/sbin/influxd && \ diff --git a/README.md b/README.md index 5feda898..7e012e89 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ This will build a container for backing up multiple types of DB Servers -Backs up CouchDB, InfluxDB, MySQL/MariaDB, Microsoft SQL, MongoDB, Postgres, Redis servers. +Backs up CouchDB, InfluxDB, MySQL/MariaDB, Microsoft SQL, MongoDB, Neo4J, Postgres, Redis servers. - dump to local filesystem or backup to S3 Compatible services, and Azure. - multiple backup job support @@ -64,6 +64,7 @@ Backs up CouchDB, InfluxDB, MySQL/MariaDB, Microsoft SQL, MongoDB, Postgres, Red - [MariaDB/MySQL](#mariadbmysql) - [Microsoft SQL](#microsoft-sql) - [MongoDB](#mongodb) + - [Neo4J](#neo4j) - [Postgresql](#postgresql) - [Redis](#redis) - [Default Storage Options](#default-storage-options) @@ -84,6 +85,7 @@ Backs up CouchDB, InfluxDB, MySQL/MariaDB, Microsoft SQL, MongoDB, Postgres, Red - [MariaDB/MySQL](#mariadbmysql-1) - [Microsoft SQL](#microsoft-sql-1) - [MongoDB](#mongodb-1) + - [Neo4J](#neo4j-1) - [Postgresql](#postgresql-1) - [Redis](#redis-1) - [SQLite](#sqlite) @@ -105,6 +107,8 @@ Backs up CouchDB, InfluxDB, MySQL/MariaDB, Microsoft SQL, MongoDB, Postgres, Red - [Shell Access](#shell-access) - [Manual Backups](#manual-backups) - [Restoring Databases](#restoring-databases) + - [Dedicated Restore Session](#dedicated-restore-session) + - [Redis Restore](#redis-restore) - [Support](#support) - [Usage](#usage) - [Bugfixes](#bugfixes) @@ -248,9 +252,12 @@ Encryption occurs after compression and the encrypted filename will have a `.gpg ###### CouchDB -| Variable | Description | Default | `_FILE` | -| -------------- | ------------ | ------- | ------- | -| `DEFAULT_PORT` | CouchDB Port | `5984` | x | +| Variable | Description | Default | `_FILE` | +| -------------- | ---------------------------------------------------------------------------------------- | ------- | ------- | +| `DEFAULT_PORT` | CouchDB Port | `5984` | x | +| `DEFAULT_NAME` | Database name or comma-separated list. Use `ALL` to back up all non-system databases | | | + +> System databases (prefixed with `_`) are automatically excluded when using `ALL`. Use `DB_NAME_EXCLUDE` to exclude additional databases. ###### InfluxDB @@ -299,6 +306,38 @@ Encryption occurs after compression and the encrypted filename will have a `.gpg | | This environment variable will be parsed and populate the `DB_NAME` and `DB_HOST` variables to properly build your backup filenames. | | | | | You can override them by making your own entries | | | +###### Neo4J + +| Variable | Description | Default | `_FILE` | +| -------------- | ----------- | ------- | ------- | +| `DEFAULT_PORT` | Neo4J Bolt Port | `7687` | x | + +> Requires the [APOC](https://neo4j.com/docs/apoc/current/) plugin to be installed on the Neo4J server. +> `DB_NAME` should be set to the target database name (default Neo4J database is named `neo4j`). +> Uses the Bolt protocol via `cypher-shell` to export all data as Cypher statements. +> +> **Installing APOC on your Neo4J server:** +> +> *Docker (environment variable):* +> ```bash +> docker run \ +> -e NEO4J_PLUGINS='["apoc"]' \ +> -e NEO4J_apoc_export_file_enabled=true \ +> -e NEO4J_dbms_security_procedures__unrestricted='apoc.*' \ +> neo4j:latest +> ``` +> +> *Docker Compose:* +> ```yaml +> environment: +> - NEO4J_PLUGINS=["apoc"] +> - NEO4J_apoc_export_file_enabled=true +> - NEO4J_dbms_security_procedures__unrestricted=apoc.* +> ``` +> +> *Manual installation:* +> Download the APOC JAR from [Neo4J APOC Releases](https://github.com/neo4j/apoc/releases) and place it in your Neo4J `plugins/` directory. + ###### Postgresql | Variable | Description | Default | `_FILE` | @@ -514,9 +553,13 @@ Encryption will occur after compression and the resulting filename will have a ` ###### CouchDB -| Variable | Description | Default | `_FILE` | -| ----------- | ------------ | ------- | ------- | -| `DB01_PORT` | CouchDB Port | `5984` | x | +| Variable | Description | Default | `_FILE` | +| ------------------ | ---------------------------------------------------------------------------------------- | ------- | ------- | +| `DB01_PORT` | CouchDB Port | `5984` | x | +| `DB01_NAME` | Database name or comma-separated list. Use `ALL` to back up all non-system databases | | | +| `DB01_NAME_EXCLUDE`| Comma-separated list of databases to exclude when using `ALL` | | | + +> System databases (prefixed with `_`) are automatically excluded when using `ALL`. Backups are saved as JSON files using the `_all_docs` API. Restore uses `_bulk_docs` to re-import documents. ###### InfluxDB @@ -573,6 +616,18 @@ Encryption will occur after compression and the resulting filename will have a ` | | This environment variable will be parsed and populate the `DB_NAME` and `DB_HOST` variables to properly build your backup filenames. | | | | | You can override them by making your own entries | | | +###### Neo4J + +| Variable | Description | Default | `_FILE` | +| ------------------------ | --------------------------------------------------------------------------------------------------------- | ------- | ------- | +| `DB01_EXTRA_OPTS` | Pass extra arguments to the backup and database enumeration command, add them here e.g. `--extra-command` | | | +| `DB01_EXTRA_BACKUP_OPTS` | Pass extra arguments to the backup command only, add them here e.g. `--extra-command` | | | +| `DB01_NAME` | Database name e.g. `neo4j` | | x | +| `DB01_PORT` | Neo4J Bolt Port | `7687` | x | + +> Requires the [APOC](https://neo4j.com/docs/apoc/current/) plugin to be installed on the Neo4J server. +> Uses `cypher-shell` via the Bolt protocol to export the entire database as Cypher statements. + ###### Postgresql | Variable | Description | Default | `_FILE` | @@ -815,7 +870,7 @@ Manual Backups can be performed by entering the container and typing `backup-now ### Restoring Databases -Entering in the container and executing `restore` will execute a menu based script to restore your backups - MariaDB, Postgres, and Mongo supported. +Entering in the container and executing `restore` will execute a menu based script to restore your backups - MariaDB, Postgres, Mongo, Neo4j, CouchDB, and Redis supported. You will be presented with a series of menus allowing you to choose: @@ -836,7 +891,52 @@ The script can also be executed skipping the interactive mode by using the follo If you only enter some of the arguments you will be prompted to fill them in. +#### Dedicated Restore Session + +When performing a restore, you may want to prevent scheduled backups from running (e.g. to avoid backing up still-empty databases and overwriting good backup files). Set `MODE=MANUAL` and `MANUAL_RUN_FOREVER=TRUE` in your environment to disable all backup schedulers while keeping the container running. Then exec into the container to run `restore` interactively: + +```bash +docker exec -it restore +``` + +Remember to switch back to `MODE=AUTO` (or remove the override) once the restore is complete so that scheduled backups resume. + +#### Redis Restore + +Redis backups are binary `.rdb` files. The restore writes the RDB file directly into Redis's data directory and sends `DEBUG RELOAD NOSAVE`, which instructs the running Redis server to load the file without a prior dump. This approach works with every RDB version the Redis server itself supports, and does not depend on any third-party RDB parser. + +**CLI invocation:** + +```bash +restore redis +``` + +The `db_name` positional argument is accepted for consistency with other DB types but is not used. The `db_user` argument is used when Redis is configured with ACL users; pass the ACL username or an empty string for `requirepass`-only setups. + +**Prerequisites:** + +- **Shared data volume.** The restore script uses `CONFIG GET dir` to discover where Redis stores its RDB file, then copies the backup there. This only works if the Redis data volume is mounted into the backup container **at the same path Redis uses** (typically `/data`). + + Example Compose snippet: + ```yaml + services: + redis: + image: redis + volumes: + - redis-data:/data + backup-db: + image: nfrastack/container-db-backup + volumes: + - redis-data:/data # same named volume, same mount path + ``` + +- **DEBUG command enabled.** The restore sends `DEBUG RELOAD NOSAVE`. If the Redis server has renamed or disabled the `DEBUG` command (e.g. `rename-command DEBUG ""`), the reload step will fail. Re-enable it for the duration of the restore or use the `ACL` system to grant access. + +**Caveats:** +- **FLUSHALL prompt — default is YES.** The restore always prompts whether to `FLUSHALL` the target before importing. Answering `N` or `No` merges/overwrites keys into the existing keyspace. Any other input — including pressing Enter — triggers `FLUSHALL`. This prompt appears even when all arguments are supplied on the command line. +- **Post-restore verification.** After reload the script issues `DBSIZE` against the target. If the key count is 0 or non-numeric (e.g. an auth error string) the restore is flagged as failed (exit code 4). A positive integer count is reported as a successful verify; it does not confirm every key round-tripped correctly. +- **Compressed backups.** Variants with compression suffixes (`.rdb.gz`, `.rdb.zst`, `.rdb.bz2`, etc.) are decompressed to a temporary file before the copy step. ## Support diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index 63aec237..0062eac1 100644 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -370,9 +370,10 @@ EOF case "${backup_job_db_type,,}" in couch* ) dbtype=couch - backup_job_backup_job_db_port=${backup_job_db_port:-5984} + backup_job_db_port=${backup_job_db_port:-5984} check_var backup_job_db_user DB"${v_instance}"_USER "database username" check_var backup_job_db_pass DB"${v_instance}"_PASS "database password" + check_var backup_job_db_name DB"${v_instance}"_NAME "database name. Seperate multiple with commas or use 'ALL'" ;; influx* ) dbtype=influx @@ -481,6 +482,13 @@ EOF dbtype=mssql backup_job_db_port=${backup_job_db_port:-1433} ;; + neo4j* ) + dbtype=neo4j + backup_job_db_port=${backup_job_db_port:-7687} + check_var backup_job_db_name DB"${v_instance}"_NAME "database name" + check_var backup_job_db_user DB"${v_instance}"_USER "database username" + check_var backup_job_db_pass DB"${v_instance}"_PASS "database password" + ;; postgres* | "pgsql" ) dbtype=pgsql backup_job_db_port=${backup_job_db_port:-5432} @@ -529,19 +537,62 @@ EOF } backup_couch() { - prepare_dbbackup - backup_job_filename=couch_${backup_job_db_name}_${backup_job_db_host#*//}_${now}.txt - backup_job_filename_base=couch_${backup_job_db_name}_${backup_job_db_host#*//} - compression + if var_true "${DEBUG_BACKUP_COUCH}" ; then debug on; fi + + ## Build auth string + if [ -n "${backup_job_db_user}" ] && [ -n "${backup_job_db_pass}" ]; then + _couch_auth="-u ${backup_job_db_user}:${backup_job_db_pass}" + fi + + ## Enumerate databases + if [ "${backup_job_db_name,,}" = "all" ] ; then + write_log debug "[backup_couch] Preparing to back up all databases except system databases" + db_names=$(run_as_user curl -sSL ${_couch_auth} ${backup_job_db_host}:${backup_job_db_port}/_all_dbs | tr -d '[]"' | tr ',' '\n' | grep -v '^_') + if [ -n "${backup_job_db_name_exclude}" ] ; then + db_names_exclusions=$(echo "${backup_job_db_name_exclude}" | tr ',' '\n') + for db_exclude in ${db_names_exclusions} ; do + write_log debug "Excluding '${db_exclude}' from ALL DB_NAME backups" + db_names=$(echo "$db_names" | sed "/${db_exclude}/d") + done + fi + else + db_names=$(echo "${backup_job_db_name}" | tr ',' '\n') + fi if var_true "${DEBUG_BACKUP_COUCH}" ; then debug off; fi - check_exit_code backup "${backup_job_filename}" - timer backup finish - file_encryption - generate_checksum - move_dbbackup - check_exit_code move "${backup_job_filename}" - post_dbbackup ${backup_job_db_name} - cleanup_old_data + write_log debug "Databases Found: $(echo ${db_names} | xargs | tr ' ' ',')" + + for db in ${db_names} ; do + if [ -z "${db}" ] || [ "${db}" = "undefined" ] || [ "${db}" = "null" ] ; then + write_log warn "[backup_couch] Skipping invalid database name: '${db}'" + continue + fi + prepare_dbbackup + if var_true "${DEBUG_BACKUP_COUCH}" ; then debug on; fi + backup_job_filename=couch_${db}_${backup_job_db_host#*//}_${now}.json + backup_job_filename_base=couch_${db}_${backup_job_db_host#*//} + compression + if var_true "${DEBUG_BACKUP_COUCH}" ; then debug off; fi + pre_dbbackup "${db}" + write_log notice "Dumping CouchDB database: '${db}' ${compression_string}" + if var_true "${DEBUG_BACKUP_COUCH}" ; then debug on; fi + ## Export in _bulk_docs format: {"docs":[...]} with _rev stripped for clean restore + run_as_user ${play_fair} curl -sSL ${_couch_auth} \ + "${backup_job_db_host}:${backup_job_db_port}/${db}/_all_docs?include_docs=true" \ + ${backup_job_extra_opts} \ + ${backup_job_extra_backup_opts} \ + | jq -c '{docs: [.rows[].doc | del(._rev, ._revisions)]}' \ + | ${compress_cmd} | run_as_user tee "${temporary_directory}"/"${backup_job_filename}" > /dev/null + exit_code=$((PIPESTATUS[0] + PIPESTATUS[1] + PIPESTATUS[2] + PIPESTATUS[3])) + if var_true "${DEBUG_BACKUP_COUCH}" ; then debug off; fi + check_exit_code backup "${backup_job_filename}" + timer backup finish + file_encryption + generate_checksum + move_dbbackup + check_exit_code move "${backup_job_filename}" + post_dbbackup "${db}" + cleanup_old_data + done } backup_influx() { @@ -699,6 +750,46 @@ backup_mssql() { esac } +backup_neo4j() { + if var_true "${DEBUG_BACKUP_NEO4J}" ; then debug on; fi + prepare_dbbackup + backup_job_filename=neo4j_${backup_job_db_name}_${backup_job_db_host,,}_${now}.cypher + backup_job_filename_base=neo4j_${backup_job_db_name}_${backup_job_db_host,,} + compression + pre_dbbackup "${backup_job_db_name}" + write_log notice "Dumping Neo4J database: '${backup_job_db_name}' ${compression_string}" + if var_true "${DEBUG_BACKUP_NEO4J}" ; then debug on; fi + ## Pipeline: cypher-shell streams APOC output as a JSON-encoded string. + ## Post-processing strips the encoding so the file contains clean, directly-importable Cypher: + ## tail -n +2 : remove 'cypherStatements' column header + ## sed (first) : strip surrounding double quotes and unescape internal \" + ## awk : convert literal \n sequences to actual newlines + run_as_user ${play_fair} cypher-shell \ + -a "neo4j://${backup_job_db_host}:${backup_job_db_port}" \ + -u "${backup_job_db_user}" \ + -p "${backup_job_db_pass}" \ + -d "${backup_job_db_name}" \ + --format plain \ + ${backup_job_extra_opts} \ + ${backup_job_extra_backup_opts} \ + "CALL apoc.export.cypher.all(null, {stream:true, format:'cypher-shell'}) YIELD cypherStatements RETURN cypherStatements" \ + | tail -n +2 \ + | sed 's/^"//;s/"$//;s/\\"/"/g' \ + | awk '{gsub(/\\n/,"\n")}1' \ + | ${compress_cmd} | run_as_user tee "${temporary_directory}"/"${backup_job_filename}" > /dev/null + exit_code=$((PIPESTATUS[0] + PIPESTATUS[1] + PIPESTATUS[2] + PIPESTATUS[3] + PIPESTATUS[4])) + if var_true "${DEBUG_BACKUP_NEO4J}" ; then debug off; fi + check_exit_code backup "${backup_job_filename}" + timer backup finish + file_encryption + generate_checksum + move_dbbackup + check_exit_code move "${backup_job_filename}" + post_dbbackup "${backup_job_db_name}" + cleanup_old_data + if var_true "${DEBUG_BACKUP_NEO4J}" ; then debug off; fi +} + backup_mysql() { if var_true "${DEBUG_BACKUP_MYSQL}" ; then debug on; fi if var_true "${backup_job_mysql_events}" ; then @@ -1014,6 +1105,14 @@ check_availability() { write_log warn "MSSQL Host '${backup_job_db_host}' is not accessible, retrying.. ($counter seconds so far)" done ;; + "neo4j" ) + counter=0 + while ! (run_as_user cypher-shell -a "neo4j://${backup_job_db_host}:${backup_job_db_port}" -u "${backup_job_db_user}" -p "${backup_job_db_pass}" -d "${backup_job_db_name}" "RETURN 1;" > /dev/null 2>&1) ; do + sleep 5 + (( counter+=5 )) + write_log warn "Neo4J Host '${backup_job_db_host}' is not accessible, retrying.. ($counter seconds so far)" + done + ;; "pgsql" ) counter=0 until run_as_user pg_isready --host=${backup_job_db_host} --port=${backup_job_db_port} -q @@ -1234,16 +1333,20 @@ create_schedulers() { if var_true "${DEBUG_CREATE_SCHEDULERS}" ; then debug on; fi backup() { - local backup_instances=$(set -o posix ; set | grep -Pc "^(DB[0-9]._HOST=|.*MONGO_CUSTOM_URI=)") - print_debug "[create_schedulers] Found '${backup_instances}' DB_HOST instances" - - if [ -n "${DB_HOST}" ] && [ "${backup_instances}" ]; then - backup_instances=1; + # Build the instance list from the actual configured DB_HOST / DB_MONGO_CUSTOM_URI + # numbers rather than a count, so non-contiguous numbering (e.g. DB01 + DB04 with DB02/DB03 + # unset) doesn't silently drop or misassign jobs. + local backup_instance_list + backup_instance_list=$(set -o posix ; set | grep -Po "^DB[0-9]+_(HOST|MONGO_CUSTOM_URI)=" | grep -Po "[0-9]+" | sort -un) + print_debug "[create_schedulers] Found DB_HOST instances: '${backup_instance_list}'" + + if [ -n "${DB_HOST}" ]; then + backup_instance_list=1 print_debug "[create_schedulers] Detected using old DB_ variables" fi - for (( instance = 01; instance <= backup_instances; )) ; do - instance=$(printf "%02d" $instance) + for instance_number in ${backup_instance_list} ; do + instance=$(printf "%02d" "${instance_number}") cp -R /assets/dbbackup/template-dbbackup /etc/services.available/dbbackup-"${instance}" sed -i "s|{{BACKUP_NUMBER}}|${instance}|g" /etc/services.available/dbbackup-"${instance}"/run if [ "${MODE,,}" = "manual" ] ; then service_stop dbbackup-"${instance}" ; fi @@ -1269,8 +1372,6 @@ EOF else echo "/usr/bin/backup${instance}-now now" > /usr/bin/backup-now fi - - instance=$(echo "${instance} +1" | bc) done } diff --git a/install/usr/local/bin/restore b/install/usr/local/bin/restore index 7a31994c..75341634 100755 --- a/install/usr/local/bin/restore +++ b/install/usr/local/bin/restore @@ -619,6 +619,21 @@ get_dbtype() { DEFAULT_PORT=${DEFAULT_PORT:-"5432"} print_debug "Parsed DBType: Postgresql" ;; + couch* ) + parsed_type=true + DEFAULT_PORT=${DEFAULT_PORT:-"5984"} + print_debug "Parsed DBType: CouchDB" + ;; + neo4j* ) + parsed_type=true + DEFAULT_PORT=${DEFAULT_PORT:-"7687"} + print_debug "Parsed DBType: Neo4j" + ;; + redis* ) + parsed_type=true + DEFAULT_PORT=${DEFAULT_PORT:-"6379"} + print_debug "Parsed DBType: Redis" + ;; * ) print_debug "Parsed DBType: UNKNOWN" ;; @@ -669,9 +684,12 @@ EOF What Database Type are you looking to restore? ${q_dbtype_menu} + C ) CouchDB M ) MySQL / MariaDB + N ) Neo4j O ) MongoDB P ) Postgresql + R ) Redis Q ) Quit EOF @@ -679,13 +697,23 @@ EOF case "${q_dbtype_variant}" in 1 ) while true; do - read -p "$(echo -e ${clg}** ${cdgy}Enter Value \(${cwh}M${cdgy}\) \| \(${cwh}O${cdgy}\) \| \(${cwh}P${cdgy}\) : ${cwh}${coff}) " q_dbtype + read -p "$(echo -e ${clg}** ${cdgy}Enter Value \(${cwh}C${cdgy}\) \| \(${cwh}M${cdgy}\) \| \(${cwh}N${cdgy}\) \| \(${cwh}O${cdgy}\) \| \(${cwh}P${cdgy}\) \| \(${cwh}R${cdgy}\) : ${cwh}${coff}) " q_dbtype case "${q_dbtype,,}" in + c* ) + r_dbtype=couch + DEFAULT_PORT=${DEFAULT_PORT:-"5984"} + break + ;; m* ) r_dbtype=mysql DEFAULT_PORT=${DEFAULT_PORT:-"3306"} break ;; + n* ) + r_dbtype=neo4j + DEFAULT_PORT=${DEFAULT_PORT:-"7687"} + break + ;; o* ) r_dbtype=mongo DEFAULT_PORT=${DEFAULT_PORT:-"27017"} @@ -696,6 +724,11 @@ EOF DEFAULT_PORT=${DEFAULT_PORT:-"5432"} break ;; + r* ) + r_dbtype=redis + DEFAULT_PORT=${DEFAULT_PORT:-"6379"} + break + ;; q* ) print_info "Quitting Script" exit 1 @@ -705,17 +738,27 @@ EOF ;; 2 ) while true; do - read -p "$(echo -e ${clg}** ${cdgy}Enter Value \(${cwh}E${cdgy}\) \| \(${cwh}M${cdgy}\) \| \(${cwh}O${cdgy}\) \| \(${cwh}P${cdgy}\) : ${cwh}${coff}) " q_dbtype + read -p "$(echo -e ${clg}** ${cdgy}Enter Value \(${cwh}E${cdgy}\) \| \(${cwh}C${cdgy}\) \| \(${cwh}M${cdgy}\) \| \(${cwh}N${cdgy}\) \| \(${cwh}O${cdgy}\) \| \(${cwh}P${cdgy}\) \| \(${cwh}R${cdgy}\) : ${cwh}${coff}) " q_dbtype case "${q_dbtype,,}" in e* | "" ) r_dbtype=${DB_TYPE} break ;; + c* ) + r_dbtype=couch + DEFAULT_PORT=${DEFAULT_PORT:-"5984"} + break + ;; m* ) r_dbtype=mysql DEFAULT_PORT=${DEFAULT_PORT:-"3306"} break ;; + n* ) + r_dbtype=neo4j + DEFAULT_PORT=${DEFAULT_PORT:-"7687"} + break + ;; o* ) r_dbtype=mongo DEFAULT_PORT=${DEFAULT_PORT:-"27017"} @@ -726,6 +769,11 @@ EOF DEFAULT_PORT=${DEFAULT_PORT:-"5432"} break ;; + r* ) + r_dbtype=redis + DEFAULT_PORT=${DEFAULT_PORT:-"6379"} + break + ;; q* ) print_info "Quitting Script" exit 1 @@ -735,28 +783,44 @@ EOF ;; 3 ) while true; do - read -p "$(echo -e ${clg}** ${cdgy}Enter Value \(${cwh}F${cdgy}\) \(Default\) \| \(${cwh}M${cdgy}\) \| \(${cwh}O${cdgy}\) \| \(${cwh}P${cdgy}\) : ${cwh}${coff}) " q_dbtype + read -p "$(echo -e ${clg}** ${cdgy}Enter Value \(${cwh}F${cdgy}\) \(Default\) \| \(${cwh}C${cdgy}\) \| \(${cwh}M${cdgy}\) \| \(${cwh}N${cdgy}\) \| \(${cwh}O${cdgy}\) \| \(${cwh}P${cdgy}\) \| \(${cwh}R${cdgy}\) : ${cwh}${coff}) " q_dbtype case "${q_dbtype,,}" in f* | "" ) r_dbtype=${p_dbtype} case "${r_dbtype}" in + couch* ) + DEFAULT_PORT=${DEFAULT_PORT:-"5984"} + ;; mongo ) DEFAULT_PORT=${DEFAULT_PORT:-"27017"} ;; mysql ) DEFAULT_PORT=${DEFAULT_PORT:-"3306"} ;; + neo4j ) + DEFAULT_PORT=${DEFAULT_PORT:-"7687"} + ;; pgsql ) DEFAULT_PORT=${DEFAULT_PORT:-"5432"} ;; esac break ;; + c* ) + r_dbtype=couch + DEFAULT_PORT=${DEFAULT_PORT:-"5984"} + break + ;; m* ) r_dbtype=mysql DEFAULT_PORT=${DEFAULT_PORT:-"3306"} break ;; + n* ) + r_dbtype=neo4j + DEFAULT_PORT=${DEFAULT_PORT:-"7687"} + break + ;; o* ) r_dbtype=mongo DEFAULT_PORT=${DEFAULT_PORT:-"27017"} @@ -767,6 +831,11 @@ EOF DEFAULT_PORT=${DEFAULT_PORT:-"5432"} break ;; + r* ) + r_dbtype=redis + DEFAULT_PORT=${DEFAULT_PORT:-"6379"} + break + ;; q* ) print_info "Quitting Script" exit 1 @@ -777,7 +846,7 @@ EOF 4 ) while true; do - read -p "$(echo -e ${clg}** ${cdgy}Enter Value \(${cwh}E${cdgy}\) \| \(${cwh}F${cdgy}\) \| \(${cwh}M${cdgy}\) \| \(${cwh}O${cdgy}\) \| \(${cwh}P${cdgy}\) : ${cwh}${coff}) " q_dbtype + read -p "$(echo -e ${clg}** ${cdgy}Enter Value \(${cwh}E${cdgy}\) \| \(${cwh}F${cdgy}\) \| \(${cwh}C${cdgy}\) \| \(${cwh}M${cdgy}\) \| \(${cwh}N${cdgy}\) \| \(${cwh}O${cdgy}\) \| \(${cwh}P${cdgy}\) \| \(${cwh}R${cdgy}\) : ${cwh}${coff}) " q_dbtype case "${q_dbtype,,}" in e* | "" ) r_dbtype=${DB_TYPE} @@ -787,11 +856,21 @@ EOF r_dbtype=${p_dbtype} break ;; + c* ) + r_dbtype=couch + DEFAULT_PORT=${DEFAULT_PORT:-"5984"} + break + ;; m* ) r_dbtype=mysql DEFAULT_PORT=${DEFAULT_PORT:-"3306"} break ;; + n* ) + r_dbtype=neo4j + DEFAULT_PORT=${DEFAULT_PORT:-"7687"} + break + ;; o* ) r_dbtype=mongo DEFAULT_PORT=${DEFAULT_PORT:-"27017"} @@ -802,6 +881,11 @@ EOF DEFAULT_PORT=${DEFAULT_PORT:-"5432"} break ;; + r* ) + r_dbtype=redis + DEFAULT_PORT=${DEFAULT_PORT:-"6379"} + break + ;; q* ) print_info "Quitting Script" exit 1 @@ -888,7 +972,11 @@ EOF get_filename() { COLUMNS=12 prompt="Please select a file to restore:" - options=( $(find "${DEFAULT_FILESYSTEM_PATH}" -type f -maxdepth 2 -not -name '*.md5' -not -name '*.sha1' -not -name '*.gpg' -print0 | sort -z | xargs -0) ) + ## Include symlinks (-type l) so 'latest-*' pointers appear. Surface latest-* first, + ## then the rest newest-first (filenames embed YYYYMMDD_HHMMSS, so reverse-sort == newest-first). + latest_files=( $(find "${DEFAULT_FILESYSTEM_PATH}" -maxdepth 2 \( -type f -o -type l \) -name 'latest-*' -not -name '*.md5' -not -name '*.sha1' -not -name '*.gpg' -print0 | sort -z | xargs -0) ) + other_files=( $(find "${DEFAULT_FILESYSTEM_PATH}" -maxdepth 2 \( -type f -o -type l \) -not -name 'latest-*' -not -name '*.md5' -not -name '*.sha1' -not -name '*.gpg' -print0 | sort -zr | xargs -0) ) + options=( "${latest_files[@]}" "${other_files[@]}" ) PS3="$prompt " select opt in "${options[@]}" "Custom" "Quit" ; do if (( REPLY == 2 + ${#options[@]} )) ; then @@ -1076,7 +1164,12 @@ fi ## Parse Extension -case "${r_filename##*.}" in +## Resolve symlinks so latest-* aliases pick up the real file's extension. +_r_filename_for_ext="${r_filename}" +if [ -L "${r_filename}" ] ; then + _r_filename_for_ext=$(readlink -f "${r_filename}") +fi +case "${_r_filename_for_ext##*.}" in bz* ) decompress_cmd='bz' print_debug "Detected 'bzip2' compression" @@ -1093,11 +1186,32 @@ case "${r_filename##*.}" in decompress_cmd='zstd' print_debug "Detected 'zstd' compression" ;; - sql ) + sql | cypher | json ) print_debug "Detected No compression" ;; * ) - print_debug "Cannot tell what the extension is for compression" + _magic=$(od -A n -N 4 -t x1 "${r_filename}" 2>/dev/null | tr -d ' \n') + case "${_magic}" in + 1f8b* ) + decompress_cmd="z" + print_debug "Detected 'gzip' compression (via magic bytes)" + ;; + 425a68* ) + decompress_cmd="bz" + print_debug "Detected 'bzip2' compression (via magic bytes)" + ;; + fd377a58* ) + decompress_cmd="xz" + print_debug "Detected 'xzip' compression (via magic bytes)" + ;; + 28b52ffd* ) + decompress_cmd="zstd" + print_debug "Detected 'zstd' compression (via magic bytes)" + ;; + * ) + print_debug "Cannot tell what the extension is for compression" + ;; + esac ;; esac @@ -1116,8 +1230,14 @@ case "${r_dbtype}" in pgsql | postgres* ) print_info "Restoring '${r_filename}' into '${r_dbhost}'/'${r_dbname}'" export PGPASSWORD=${r_dbpass} - pv ${r_filename} | ${decompress_cmd}cat | psql -d ${r_dbname} -h ${r_dbhost} -p ${r_dbport} -U ${r_dbuser} + pv ${r_filename} | ${decompress_cmd}cat | psql -d ${r_dbname} -h ${r_dbhost} -p ${r_dbport} -U ${r_dbuser} exit_code=$? + if [ "${exit_code}" = 0 ] ; then + psql -d ${r_dbname} -h ${r_dbhost} -p ${r_dbport} -U ${r_dbuser} -c "ANALYZE;" > /dev/null 2>&1 + pgsql_table_count=$(psql -d ${r_dbname} -h ${r_dbhost} -p ${r_dbport} -U ${r_dbuser} -t -c "SELECT COUNT(*) FROM pg_stat_user_tables;" 2>/dev/null | tr -d ' \n') + pgsql_row_count=$(psql -d ${r_dbname} -h ${r_dbhost} -p ${r_dbport} -U ${r_dbuser} -t -c "SELECT COALESCE(SUM(n_live_tup),0) FROM pg_stat_user_tables;" 2>/dev/null | tr -d ' \n') + print_info "Restore summary — ${pgsql_table_count:-0} tables, ~${pgsql_row_count:-0} rows restored" + fi ;; mongo ) cat << EOF @@ -1161,6 +1281,258 @@ EOF mongorestore ${mongo_compression} -d=${r_dbname} -h=${r_dbhost} --port=${r_dbport} ${mongo_dropdb} ${mongo_user} ${mongo_pass} --archive=${r_filename} ${mongo_auth_database} exit_code=$? ;; + neo4j ) + cat << EOF + +Do you wish to clear the existing database before restoring? + Y ) Yes - Delete all nodes, relationships, user indexes, and constraints + N ) No - Merge/add to existing data + Q ) Quit + +EOF + + echo -e "${coff}" + read -p "$(echo -e ${clg}** ${cdgy}Enter Value \(${cwh}Y${cdgy}\) \| \(${cwh}N${cdgy}\) \| \(${cwh}Q${cdgy}\) : ${cwh}${coff})" q_menu_neo4j_clear + case "${q_menu_neo4j_clear,,}" in + "n" | "no" ) + unset neo4j_clear + ;; + "q" | "exit" ) + print_info "Quitting Script" + exit 1 + ;; + "y" | "yes" | * ) + neo4j_clear=true + ;; + esac + + print_info "Restoring '${r_filename}' into '${r_dbhost}'/'${r_dbname}'" + + ## Clear database if requested — delete nodes, then drop every user index and every + ## constraint so a previously-failed import cannot leave behind 'UNIQUE_IMPORT_NAME' or + ## 'search' artifacts that would block a retry with "equivalent index already exists". + if [ "${neo4j_clear}" = "true" ] ; then + print_info "Clearing nodes and relationships from database '${r_dbname}'" + cypher-shell \ + -a "neo4j://${r_dbhost}:${r_dbport}" \ + -u "${r_dbuser}" \ + -p "${r_dbpass}" \ + -d "${r_dbname}" \ + "MATCH (n) DETACH DELETE n;" + + print_info "Dropping user-defined indexes from database '${r_dbname}'" + neo4j_indexes=$(cypher-shell \ + -a "neo4j://${r_dbhost}:${r_dbport}" \ + -u "${r_dbuser}" \ + -p "${r_dbpass}" \ + -d "${r_dbname}" \ + --format plain \ + "SHOW INDEXES YIELD name, type WHERE type <> 'LOOKUP' RETURN name;" 2>/dev/null \ + | tail -n +2 | tr -d '"') + for idx in ${neo4j_indexes} ; do + cypher-shell \ + -a "neo4j://${r_dbhost}:${r_dbport}" \ + -u "${r_dbuser}" \ + -p "${r_dbpass}" \ + -d "${r_dbname}" \ + "DROP INDEX \`${idx}\` IF EXISTS;" >/dev/null 2>&1 + done + + print_info "Dropping constraints from database '${r_dbname}'" + neo4j_constraints=$(cypher-shell \ + -a "neo4j://${r_dbhost}:${r_dbport}" \ + -u "${r_dbuser}" \ + -p "${r_dbpass}" \ + -d "${r_dbname}" \ + --format plain \ + "SHOW CONSTRAINTS YIELD name RETURN name;" 2>/dev/null \ + | tail -n +2 | tr -d '"') + for c in ${neo4j_constraints} ; do + cypher-shell \ + -a "neo4j://${r_dbhost}:${r_dbport}" \ + -u "${r_dbuser}" \ + -p "${r_dbpass}" \ + -d "${r_dbname}" \ + "DROP CONSTRAINT \`${c}\` IF EXISTS;" >/dev/null 2>&1 + done + fi + + ## Restore: pipe Cypher into cypher-shell. + ## Handle both clean Cypher (current backup format) and raw APOC stream output + ## (older backups where the 'cypherStatements' header and JSON-encoding were not stripped). + pv "${r_filename}" | ${decompress_cmd}cat \ + | awk 'NR==1&&$0=="cypherStatements"{raw=1;next}raw{l=$0;sub(/^"/,"",l);sub(/"$/,"",l);gsub(/\\"/,"\"",l);gsub(/\\n/,"\n",l);print l;next}{print}' \ + | cypher-shell \ + -a "neo4j://${r_dbhost}:${r_dbport}" \ + -u "${r_dbuser}" \ + -p "${r_dbpass}" \ + -d "${r_dbname}" \ + --format plain + exit_code=$? + + ## Verify — count nodes; a silent zero-node "success" must fail loudly. + print_info "Verifying restore — counting nodes in '${r_dbname}'" + neo4j_node_count=$(cypher-shell \ + -a "neo4j://${r_dbhost}:${r_dbport}" \ + -u "${r_dbuser}" \ + -p "${r_dbpass}" \ + -d "${r_dbname}" \ + --format plain \ + "MATCH (n) RETURN count(n) AS node_count;" 2>/dev/null \ + | tail -n +2 | tr -d '[:space:]"') + if [ -z "${neo4j_node_count}" ] || [ "${neo4j_node_count}" = "0" ] ; then + print_error "VERIFY FAILED — Neo4j database '${r_dbname}' contains 0 nodes after restore" + restore_verify_failed="neo4j:${r_dbname}" + exit_code=4 + else + print_info "Verify passed — Neo4j database '${r_dbname}' contains ${neo4j_node_count} nodes" + fi + ;; + couch* ) + if [ -n "${r_dbuser}" ] && [ -n "${r_dbpass}" ]; then + _couch_auth="-u ${r_dbuser}:${r_dbpass}" + fi + + cat << EOF + +Do you wish to drop any existing data before restoring? + Y ) Yes - Delete and recreate the database + N ) No - Merge/overwrite documents + Q ) Quit + +EOF + + echo -e "${coff}" + read -p "$(echo -e ${clg}** ${cdgy}Enter Value \(${cwh}Y${cdgy}\) \| \(${cwh}N${cdgy}\) \| \(${cwh}Q${cdgy}\) : ${cwh}${coff})" q_menu_couch_dropdb + case "${q_menu_couch_dropdb,,}" in + "n" | "update" ) + unset couch_dropdb + ;; + "q" | "exit" ) + print_info "Quitting Script" + exit 1 + ;; + "y" | "yes" | * ) + couch_dropdb=true + ;; + esac + + print_info "Restoring '${r_filename}' into '${r_dbhost}'/'${r_dbname}'" + + ## Drop and recreate if requested + if [ "${couch_dropdb}" = "true" ] ; then + print_info "Dropping database '${r_dbname}'" + curl -sSL ${_couch_auth} -X DELETE "${r_dbhost}:${r_dbport}/${r_dbname}" 2>/dev/null + fi + + ## Create database (ignore error if exists) + curl -sSL ${_couch_auth} -X PUT "${r_dbhost}:${r_dbport}/${r_dbname}" 2>/dev/null + + ## Restore: handle both new format ({"docs":[...]}) and old format ({"rows":[...]}) + pv ${r_filename} | ${decompress_cmd}cat | \ + jq -c 'if .docs then . else {docs: [.rows[].doc | del(._rev)]} end' | \ + curl -sSL ${_couch_auth} -X POST "${r_dbhost}:${r_dbport}/${r_dbname}/_bulk_docs" \ + -H "Content-Type: application/json" -d @- + exit_code=$? + + ## Verify — count docs; a silent zero-doc "success" must fail loudly. + print_info "Verifying restore — counting documents in '${r_dbname}'" + couch_doc_count=$(curl -sSL ${_couch_auth} "${r_dbhost}:${r_dbport}/${r_dbname}" 2>/dev/null | jq -r '.doc_count // 0') + if [ -z "${couch_doc_count}" ] || [ "${couch_doc_count}" = "0" ] || [ "${couch_doc_count}" = "null" ] ; then + print_error "VERIFY FAILED — CouchDB database '${r_dbname}' contains 0 documents after restore" + restore_verify_failed="${restore_verify_failed:+${restore_verify_failed},}couch:${r_dbname}" + exit_code=4 + else + print_info "Verify passed — CouchDB database '${r_dbname}' contains ${couch_doc_count} documents" + fi + ;; + redis ) + cat << EOF + +Do you wish to flush the target Redis before restoring? + Y ) Yes - FLUSHALL, then import + N ) No - Merge/overwrite into existing keyspace + Q ) Quit + +EOF + + echo -e "${coff}" + read -p "$(echo -e ${clg}** ${cdgy}Enter Value \(${cwh}Y${cdgy}\) \| \(${cwh}N${cdgy}\) \| \(${cwh}Q${cdgy}\) : ${cwh}${coff})" q_menu_redis_flush + case "${q_menu_redis_flush,,}" in + "n" | "no" ) + unset redis_flush + ;; + "q" | "exit" ) + print_info "Quitting Script" + exit 1 + ;; + "y" | "yes" | * ) + redis_flush=true + ;; + esac + + print_info "Restoring '${r_filename}' into '${r_dbhost}:${r_dbport}'" + + redis_auth_args=() + if [ -n "${r_dbpass}" ] ; then + if [ -n "${r_dbuser}" ] ; then + redis_auth_args+=(--user "${r_dbuser}" --pass "${r_dbpass}" --no-auth-warning) + else + redis_auth_args+=(-a "${r_dbpass}" --no-auth-warning) + fi + fi + + ## rdbtools needs a seekable file (checksum trailer). Decompress first, work on a temp copy. + redis_tmp_rdb=$(mktemp --suffix=.rdb) + if [ -n "${decompress_cmd}" ] ; then + ${decompress_cmd}cat "${r_filename}" > "${redis_tmp_rdb}" + else + cp "${r_filename}" "${redis_tmp_rdb}" + fi + + if [ "${redis_flush}" = "true" ] ; then + print_info "FLUSHALL on '${r_dbhost}:${r_dbport}'" + redis-cli -h "${r_dbhost}" -p "${r_dbport}" "${redis_auth_args[@]}" FLUSHALL + fi + + ## Restore via direct RDB file placement + DEBUG RELOAD NOSAVE. + ## CONFIG GET dir returns the data path as Redis sees it; if that path is mounted into + ## this container at the same location the copy succeeds and the reload is instant. + ## Works with every RDB version the Redis server itself supports. + _redis_data_dir=$(redis-cli -h "${r_dbhost}" -p "${r_dbport}" "${redis_auth_args[@]}" CONFIG GET dir 2>/dev/null | awk 'NR==2{print;exit}') + _redis_dbfile=$(redis-cli -h "${r_dbhost}" -p "${r_dbport}" "${redis_auth_args[@]}" CONFIG GET dbfilename 2>/dev/null | awk 'NR==2{print;exit}') + _redis_dbfile="${_redis_dbfile:-dump.rdb}" + + if [ -z "${_redis_data_dir}" ] || [ ! -d "${_redis_data_dir}" ] ; then + print_error "Redis data directory '${_redis_data_dir:-}' is not accessible from this container." + print_error "Mount the Redis data volume into this container at the same path the Redis server uses (commonly /data)." + print_error "Example (Compose): add 'volumes: - redis-data:/data' to the backup service." + exit_code=1 + else + print_info "Writing RDB to '${_redis_data_dir}/${_redis_dbfile}'" + cp "${redis_tmp_rdb}" "${_redis_data_dir}/${_redis_dbfile}" + _reload_result=$(redis-cli -h "${r_dbhost}" -p "${r_dbport}" "${redis_auth_args[@]}" DEBUG RELOAD NOSAVE 2>&1) + if echo "${_reload_result}" | grep -qi "^ok$" ; then + exit_code=0 + else + print_error "DEBUG RELOAD NOSAVE failed: ${_reload_result}" + print_error "Ensure the Redis DEBUG command is not disabled (rename-command DEBUG \"\")." + exit_code=1 + fi + fi + rm -f "${redis_tmp_rdb}" + + ## Verify — DBSIZE > 0. + print_info "Verifying restore — counting keys in '${r_dbhost}:${r_dbport}'" + redis_key_count=$(redis-cli -h "${r_dbhost}" -p "${r_dbport}" "${redis_auth_args[@]}" DBSIZE 2>/dev/null | tr -d '[:space:]') + if [ -z "${redis_key_count}" ] || [ "${redis_key_count}" = "0" ] || ! [[ "${redis_key_count}" =~ ^[0-9]+$ ]] ; then + print_error "VERIFY FAILED — Redis at '${r_dbhost}:${r_dbport}' contains 0 keys after restore (DBSIZE returned: '${redis_key_count}')" + restore_verify_failed="${restore_verify_failed:+${restore_verify_failed},}redis:${r_dbhost}" + exit_code=4 + else + print_info "Verify passed — Redis at '${r_dbhost}:${r_dbport}' contains ${redis_key_count} keys" + fi + ;; * ) print_info "Unable to restore DB of type '${r_dbtype}'" exit_code=3 @@ -1168,7 +1540,10 @@ EOF esac print_debug "Exit code: ${exit_code}" -if [ "${exit_code}" = 0 ] ; then +if [ -n "${restore_verify_failed}" ] ; then + print_error "RESTORE VERIFY FAILED for: ${restore_verify_failed}" + print_error "A restore that imports zero rows is NOT a successful restore. Investigate before relying on this database." +elif [ "${exit_code}" = 0 ] ; then print_info "Restore complete!" else print_error "Restore reported errors"