Db rotation

db/init-repology.sql

@@ -1,13 +1,118 @@
--- Create database and user
-CREATE DATABASE repology;
+-- ============================================================================
+-- Repology Database Initialization with Rotation Support
+-- ============================================================================
+
+-- Create repology user
 CREATE USER repology WITH PASSWORD 'repology';
+
+-- Create control database for metadata and rotation management
+CREATE DATABASE repology_control;
+
+-- Create two Repology databases for rotation
+CREATE DATABASE repology;
+CREATE DATABASE repology_replica;
+
+-- Grant permissions on all databases
+GRANT ALL ON DATABASE repology_control TO repology;
 GRANT ALL ON DATABASE repology TO repology;
+GRANT ALL ON DATABASE repology_replica TO repology;
 
+-- ============================================================================
+-- Setup Main Database
+-- ============================================================================
 \connect repology
 
 -- Schema permissions
 GRANT CREATE ON SCHEMA public TO PUBLIC;
 
 -- Enable extensions
-CREATE EXTENSION pg_trgm;
-CREATE EXTENSION libversion;
+CREATE EXTENSION IF NOT EXISTS pg_trgm;
+CREATE EXTENSION IF NOT EXISTS libversion;
+
+-- ============================================================================
+-- Setup Replica Database
+-- ============================================================================
+\connect repology_replica
+
+-- Schema permissions
+GRANT CREATE ON SCHEMA public TO PUBLIC;
+
+-- Enable extensions
+CREATE EXTENSION IF NOT EXISTS pg_trgm;
+CREATE EXTENSION IF NOT EXISTS libversion;
+
+-- ============================================================================
+-- Setup Control Database
+-- ============================================================================
+\connect repology_control
+
+-- Create metadata table to track active database
+CREATE TABLE IF NOT EXISTS repology_control (
+    id SERIAL PRIMARY KEY,
+    active_database VARCHAR(50) NOT NULL,
+    last_updated TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    last_rotation TIMESTAMP WITH TIME ZONE,
+    CONSTRAINT check_database_name CHECK (active_database IN ('repology', 'repology_replica'))
+);
+
+-- Grant permissions on control table
+GRANT SELECT, INSERT, UPDATE ON repology_control TO repology;
+GRANT USAGE, SELECT ON SEQUENCE repology_control_id_seq TO repology;
+
+-- Initialize with main database as default active
+INSERT INTO repology_control (active_database)
+VALUES ('repology')
+ON CONFLICT DO NOTHING;
+
+-- Create view for easy active database lookup
+CREATE OR REPLACE VIEW repology_active_config AS
+SELECT 
+    active_database,
+    last_updated,
+    last_rotation
+FROM repology_control
+ORDER BY id DESC
+LIMIT 1;
+
+-- Grant view access
+GRANT SELECT ON repology_active_config TO repology;
+
+-- Create function to get active database name
+CREATE OR REPLACE FUNCTION get_active_repology_database()
+RETURNS VARCHAR(50) AS $$
+DECLARE
+    db_name VARCHAR(50);
+BEGIN
+    SELECT active_database INTO db_name
+    FROM repology_active_config;
+    RETURN db_name;
+END;
+$$ LANGUAGE plpgsql SECURITY DEFINER;
+
+-- Grant execute permission
+GRANT EXECUTE ON FUNCTION get_active_repology_database() TO repology;
+
+-- Create table to track rotation history
+CREATE TABLE IF NOT EXISTS repology_rotation_history (
+    id SERIAL PRIMARY KEY,
+    from_database VARCHAR(50),
+    to_database VARCHAR(50),
+    rotation_timestamp TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    success BOOLEAN DEFAULT true,
+    validation_status TEXT,
+    error_message TEXT,
+    duration_seconds INTEGER
+);
+
+-- Grant permissions on history table
+GRANT SELECT, INSERT ON repology_rotation_history TO repology;
+GRANT USAGE, SELECT ON SEQUENCE repology_rotation_history_id_seq TO repology;
+
+-- ============================================================================
+-- Verification
+-- ============================================================================
+
+-- Display configuration
+SELECT 'Database initialization complete!' AS status;
+SELECT 'Active database: ' || active_database AS configuration
+FROM repology_active_config;

db/load-repology.sh

@@ -1,57 +1,252 @@
 #!/bin/bash
 set -euo pipefail
 
+# ============================================================================
+# Repology Database Load Script with Rotation
+# ============================================================================
+
 # Database configuration from environment variables with defaults
 DB_HOST="${POSTGRES_HOST:-localhost}"
 DB_PORT="${POSTGRES_PORT:-5432}"
-DB_NAME="${POSTGRES_DB:-repology}"
 DB_USER="${POSTGRES_USER:-repology}"
 DB_PASSWORD="${POSTGRES_PASSWORD:-}"
+CONTROL_DB="${CONTROL_DB:-repology_control}"
 
 # Proxy configuration (optional)
 PROXY_URL="${PROXY_URL:-}"
 
 # Dump URL configuration (optional)
 DUMP_URL="${DUMP_URL:-https://dumps.repology.org/repology-database-dump-latest.sql.zst}"
 
-echo "Downloading Repology dump..."
-echo "Source: $DUMP_URL"
+# Start time for duration tracking
+START_TIME=$(date +%s)
+
+# ============================================================================
+# Helper Functions
+# ============================================================================
+
+log() {
+    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"
+}
+
+error() {
+    echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: $*" >&2
+}
+
+# Execute SQL query and return result
+execute_sql() {
+    local database=$1
+    local query=$2
+
+    if [ -n "$DB_PASSWORD" ]; then
+        export PGPASSWORD="$DB_PASSWORD"
+    fi
+
+    psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$database" -t -A -c "$query" 2>/dev/null || echo ""
+
+    unset PGPASSWORD
+}
+
+# Get active database from control table
+get_active_database() {
+    local active_db=$(execute_sql "$CONTROL_DB" "SELECT active_database FROM repology_active_config;")
+
+    if [ -z "$active_db" ]; then
+        error "Failed to query active database from control table"
+        exit 1
+    fi
+
+    echo "$active_db"
+}
+
+# Get inactive database (the one to load data into)
+get_inactive_database() {
+    local active_db=$1
+
+    if [ "$active_db" = "repology" ]; then
+        echo "repology_replica"
+    else
+        echo "repology"
+    fi
+}
+
+# Switch active database in control table
+switch_active_database() {
+    local from_db=$1
+    local to_db=$2
+    local validation_status=$3
+
+    log "Switching active database: $from_db -> $to_db"
+
+    if [ -n "$DB_PASSWORD" ]; then
+        export PGPASSWORD="$DB_PASSWORD"
+    fi
+
+    # Calculate duration
+    local end_time=$(date +%s)
+    local duration=$((end_time - START_TIME))
+
+    # Update control table
+    psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$CONTROL_DB" -v ON_ERROR_STOP=1 <<EOF
+-- Update active database
+UPDATE repology_control 
+SET active_database = '$to_db',
+    last_updated = CURRENT_TIMESTAMP,
+    last_rotation = CURRENT_TIMESTAMP
+WHERE id = (SELECT id FROM repology_control ORDER BY id DESC LIMIT 1);
+
+-- Record rotation history
+INSERT INTO repology_rotation_history 
+    (from_database, to_database, success, validation_status, duration_seconds)
+VALUES 
+    ('$from_db', '$to_db', true, '$validation_status', $duration);
+EOF
+
+    unset PGPASSWORD
+
+    log "✅ Successfully switched to $to_db"
+}
+
+# Record failed rotation
+record_failed_rotation() {
+    local from_db=$1
+    local to_db=$2
+    local error_msg=$3
+
+    if [ -n "$DB_PASSWORD" ]; then
+        export PGPASSWORD="$DB_PASSWORD"
+    fi
+
+    local end_time=$(date +%s)
+    local duration=$((end_time - START_TIME))
+
+    # Escape single quotes in error message
+    error_msg=$(echo "$error_msg" | sed "s/'/''/g")
+
+    psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$CONTROL_DB" -v ON_ERROR_STOP=1 <<EOF
+INSERT INTO repology_rotation_history 
+    (from_database, to_database, success, validation_status, error_message, duration_seconds)
+VALUES 
+    ('$from_db', '$to_db', false, 'FAILED', '$error_msg', $duration);
+EOF
+
+    unset PGPASSWORD
+}
+
+# ============================================================================
+# Main Script
+# ============================================================================
+
+log "=========================================="
+log "Starting Repology Database Rotation"
+log "=========================================="
+
+# Query which database is currently active
+log "Querying active database..."
+ACTIVE_DB=$(get_active_database)
+log "Current active database: $ACTIVE_DB"
+
+# Determine which database to load into (the inactive one)
+TARGET_DB=$(get_inactive_database "$ACTIVE_DB")
+log "Target database for loading: $TARGET_DB"
+
+# Download dump
+log "Downloading Repology dump..."
+log "Source: $DUMP_URL"
 
 DUMP_FILE="/tmp/repology-dump.sql.zst"
 SQL_FILE="/tmp/repology-dump.sql"
 
 # Build curl command with optional proxy
 CURL_CMD="curl -fSL"
 if [ -n "$PROXY_URL" ]; then
-    echo "Using proxy: $PROXY_URL"
+    log "Using proxy: $PROXY_URL"
     CURL_CMD="$CURL_CMD -x $PROXY_URL"
 fi
 
 # Download to file
-$CURL_CMD "$DUMP_URL" -o "$DUMP_FILE"
-
-echo "Verifying dump..."
-# Optional: quick zstd integrity check
-zstd -t "$DUMP_FILE"
+if ! $CURL_CMD "$DUMP_URL" -o "$DUMP_FILE"; then
+    error "Failed to download dump"
+    record_failed_rotation "$ACTIVE_DB" "$TARGET_DB" "Download failed"
+    exit 1
+fi
 
-echo "Decompressing..."
-zstd -d -o "$SQL_FILE" "$DUMP_FILE"
+log "Verifying dump integrity..."
+if ! zstd -t "$DUMP_FILE"; then
+    error "Dump file integrity check failed"
+    rm -f "$DUMP_FILE"
+    record_failed_rotation "$ACTIVE_DB" "$TARGET_DB" "Dump integrity check failed"
+    exit 1
+fi
 
-echo "Loading into Postgres..."
-# Build psql connection string
-PSQL_CMD="psql -h $DB_HOST -p $DB_PORT -U $DB_USER -d $DB_NAME -v ON_ERROR_STOP=1 -f $SQL_FILE"
+log "Decompressing dump..."
+if ! zstd -d -f -o "$SQL_FILE" "$DUMP_FILE"; then
+    error "Failed to decompress dump"
+    rm -f "$DUMP_FILE" "$SQL_FILE"
+    record_failed_rotation "$ACTIVE_DB" "$TARGET_DB" "Decompression failed"
+    exit 1
+fi
 
-# Set password if provided
+# Set password for psql commands
 if [ -n "$DB_PASSWORD" ]; then
     export PGPASSWORD="$DB_PASSWORD"
 fi
 
-$PSQL_CMD
+# Drop and recreate target database to ensure clean state
+log "Preparing target database: $TARGET_DB"
+psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d postgres <<EOF
+-- Terminate existing connections
+SELECT pg_terminate_backend(pg_stat_activity.pid)
+FROM pg_stat_activity
+WHERE pg_stat_activity.datname = '$TARGET_DB'
+  AND pid <> pg_backend_pid();
+
+-- Drop and recreate database
+DROP DATABASE IF EXISTS $TARGET_DB;
+CREATE DATABASE $TARGET_DB;
+GRANT ALL ON DATABASE $TARGET_DB TO $DB_USER;
+EOF
+
+# Connect to target database and enable extensions
+log "Setting up extensions in $TARGET_DB..."
+psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$TARGET_DB" <<EOF
+GRANT CREATE ON SCHEMA public TO PUBLIC;
+CREATE EXTENSION IF NOT EXISTS pg_trgm;
+CREATE EXTENSION IF NOT EXISTS libversion;
+EOF
+
+# Load dump into target database
+log "Loading dump into $TARGET_DB..."
+log "This may take several minutes..."
+
+if ! psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$TARGET_DB" -v ON_ERROR_STOP=1 -f "$SQL_FILE"; then
+    error "Failed to load dump into $TARGET_DB"
+    unset PGPASSWORD
+    rm -f "$DUMP_FILE" "$SQL_FILE"
+    record_failed_rotation "$ACTIVE_DB" "$TARGET_DB" "Database load failed"
+    exit 1
+fi
+
+log "Dump loaded successfully"
 
 # Clear password from environment
 unset PGPASSWORD
 
-echo "Cleanup..."
+# Switch active database
+switch_active_database "$ACTIVE_DB" "$TARGET_DB" "SQL execution successful"
+
+# Cleanup
+log "Cleaning up temporary files..."
 rm -f "$DUMP_FILE" "$SQL_FILE"
 
-echo "✅ Repology dump imported successfully."
+# Calculate total duration
+END_TIME=$(date +%s)
+TOTAL_DURATION=$((END_TIME - START_TIME))
+log "Total duration: ${TOTAL_DURATION}s"
+
+log "=========================================="
+log "✅ Database rotation completed successfully"
+log "✅ New active database: $TARGET_DB"
+log "=========================================="
+
+exit 0