From fa80aa4ea16add55d43490a244e05b98d6e7d99c Mon Sep 17 00:00:00 2001 From: Philip McClarence Date: Fri, 26 Jan 2018 11:11:52 +0000 Subject: [PATCH 01/12] test --- test.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 test.txt diff --git a/test.txt b/test.txt new file mode 100644 index 0000000..e69de29 From 4e93df5da3bdde72e92355be39f0c2238686ab19 Mon Sep 17 00:00:00 2001 From: Philip McClarence Date: Fri, 26 Jan 2018 11:12:32 +0000 Subject: [PATCH 02/12] test --- test.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 test.txt diff --git a/test.txt b/test.txt deleted file mode 100644 index e69de29..0000000 From dce3a8bd1367a13995f097d63219be2617777825 Mon Sep 17 00:00:00 2001 From: Philip McClarence Date: Fri, 26 Jan 2018 11:15:53 +0000 Subject: [PATCH 03/12] adding RDS backup code option --- bin/pgbackman_dump | 410 +++++++++++++++++++++++---------------------- 1 file changed, 206 insertions(+), 204 deletions(-) diff --git a/bin/pgbackman_dump b/bin/pgbackman_dump index 34b9ca9..9d91b18 100755 --- a/bin/pgbackman_dump +++ b/bin/pgbackman_dump @@ -31,7 +31,7 @@ import signal import argparse from pgbackman.logs import * -from pgbackman.database import * +from pgbackman.database import * from pgbackman.config import * ''' @@ -46,10 +46,10 @@ pgsql_node_cache_data = {} # ############################################ # Function pg_dumpall() # ############################################ - + def pg_dumpall(db): '''Used to take backups with code CLUSTER''' - + global global_parameters # @@ -63,7 +63,7 @@ def pg_dumpall(db): ' -p ' + global_parameters['pgsql_node_port'] + \ ' -U ' + global_parameters['pgsql_node_admin_user'] + \ ' ' + global_parameters['extra_backup_parameters'] + \ - ' | /bin/gzip > ' + global_parameters['cluster_dump_file'] + ' | /bin/gzip > ' + global_parameters['cluster_dump_file'] else: pg_dumpall_command = global_parameters['backup_server_pgsql_bin_dir'] + '/pg_dumpall' + \ ' -h ' + global_parameters['pgsql_node_fqdn'] + \ @@ -74,7 +74,7 @@ def pg_dumpall(db): try: with open(global_parameters['cluster_log_file'],'w') as cluster_log_file: - + cluster_log_file.write('------------------------------------\n') cluster_log_file.write('Timestamp:' + str(datetime.datetime.now()) + '\n') cluster_log_file.write('Command: ' + pg_dumpall_command + '\n') @@ -84,41 +84,41 @@ def pg_dumpall(db): proc = subprocess.Popen([pg_dumpall_command],stdout=cluster_log_file,stderr=subprocess.STDOUT,shell=True) proc.wait() - + if proc.returncode == 0: logs.logger.info('Cluster dump file created - %s',global_parameters['cluster_dump_file']) cluster_log_file.write('[OK] Cluster dump file created - ' + global_parameters['cluster_dump_file'] + '\n') - + global_parameters['execution_status'] = 'SUCCEEDED' else: logs.logger.critical('Cluster dump file could not be created. Return code = %s. Check log file: %s',proc.returncode,global_parameters['cluster_log_file']) cluster_log_file.write('[ERROR] Cluster dump file could not be created. Return code = ' + str(proc.returncode) + '. Check log file: ' + global_parameters['cluster_log_file'] + '\n') - + global_parameters['execution_status'] = 'ERROR' global_parameters['error_message'] = 'pgdumpall returncode: ' + str(proc.returncode) + '. Check log file.' register_backup_catalog(db) sys.exit(1) - + except Exception as e: logs.logger.critical('Could not generate the final cluster dump file %s - %s',global_parameters['cluster_dump_file'],e) - + global_parameters['execution_status'] = 'ERROR' global_parameters['error_message'] = e - register_backup_catalog(db) + register_backup_catalog(db) sys.exit(1) - + # ############################################ # Function pg_dump() # ############################################ - + def pg_dump(db): '''Used to take database backups with codes FULL, SCHEMA and DATA''' global global_parameters if global_parameters['backup_code'] == 'FULL': - + pg_dump_command = global_parameters['backup_server_pgsql_bin_dir'] + '/pg_dump' + \ ' -h ' + global_parameters['pgsql_node_fqdn'] + \ ' -p ' + global_parameters['pgsql_node_port'] + \ @@ -128,10 +128,10 @@ def pg_dump(db): ' --blobs' + \ ' --verbose' + \ ' ' + global_parameters['extra_backup_parameters'] + \ - ' ' + global_parameters['dbname'] - + ' ' + global_parameters['dbname'] + elif global_parameters['backup_code'] == 'SCHEMA': - + pg_dump_command = global_parameters['backup_server_pgsql_bin_dir'] + '/pg_dump' + \ ' -h ' + global_parameters['pgsql_node_fqdn'] + \ ' -p ' + global_parameters['pgsql_node_port'] + \ @@ -142,7 +142,7 @@ def pg_dump(db): ' --verbose' + \ ' ' + global_parameters['extra_backup_parameters'] + \ ' ' + global_parameters['dbname'] - + elif global_parameters['backup_code'] == 'DATA': pg_dump_command = global_parameters['backup_server_pgsql_bin_dir'] + '/pg_dump' + \ @@ -156,30 +156,30 @@ def pg_dump(db): ' --verbose' + \ ' ' + global_parameters['extra_backup_parameters'] + \ ' ' + global_parameters['dbname'] - + try: with open(global_parameters['database_log_file'],'w') as database_log_file: - + database_log_file.write('------------------------------------\n') database_log_file.write('Timestamp:' + str(datetime.datetime.now()) + '\n') database_log_file.write('Command: ' + pg_dump_command + '\n') database_log_file.write('------------------------------------\n\n') - + database_log_file.flush() proc = subprocess.Popen([pg_dump_command],stdout=database_log_file,stderr=subprocess.STDOUT,shell=True) proc.wait() - + if proc.returncode != 0: logs.logger.critical('Database dump file could not be created. Return code = %s. Check log file: %s', proc.returncode, global_parameters['database_log_file']) - + global_parameters['execution_status'] = 'ERROR' global_parameters['error_message'] = 'pg_dump returncode: ' + str(proc.returncode) + '. Check log file.' - register_backup_catalog(db) + register_backup_catalog(db) sys.exit(1) - + logs.logger.info('Database dump file created - %s',global_parameters['database_dump_file']) global_parameters['execution_status'] = 'SUCCEEDED' @@ -189,17 +189,17 @@ def pg_dump(db): global_parameters['execution_status'] = 'ERROR' global_parameters['error_message'] = e - register_backup_catalog(db) + register_backup_catalog(db) sys.exit(1) - + # ############################################ # Function pg_dump_users() # ############################################ - + def pg_dump_users(db): '''Used to backup roles information associated with a database.''' - + global global_parameters roles = [] grant_granted_by_roles= [] @@ -221,10 +221,10 @@ def pg_dump_users(db): # with huge sql dumps # - # + # # Extracting roles from the schema dump of the database # - + try: # @@ -246,43 +246,43 @@ def pg_dump_users(db): ' ' + global_parameters['dbname'] with open(global_parameters['roles_log_file'],'w') as roles_log_file: - + roles_log_file.write('------------------------------------\n') roles_log_file.write('Timestamp:' + str(datetime.datetime.now()) + '\n') roles_log_file.write('Command: ' + pg_dump_command + '\n') roles_log_file.write('------------------------------------\n\n') - + roles_log_file.flush() proc = subprocess.Popen([pg_dump_command],stdout=roles_log_file,stderr=subprocess.STDOUT,shell=True) proc.wait() - + if proc.returncode != 0: logs.logger.critical('The command used to generate the tmp schema dump of the database has a return value != 0') global_parameters['execution_status'] = 'ERROR' - global_parameters['error_message'] = 'pg_dump returncode: ' + str(proc.returncode) + '. Check log file.' - register_backup_catalog(db) + global_parameters['error_message'] = 'pg_dump returncode: ' + str(proc.returncode) + '. Check log file.' + register_backup_catalog(db) sys.exit(1) - + with open(pg_dump_schema_temp_file.name, 'r') as sqldump: for line in sqldump: - + if 'OWNER TO' in line: role = line.split(' OWNER TO ') - + if len(role) == 2: if role[1].replace(';\n','').find(' ') < 0: roles.append(role[1].replace(';\n','')) - + if 'GRANT' in line: role = line.split(' TO ') - + if len(role) == 2: if role[1].replace(';\n','').find(' ') < 0: if role[1].replace(';\n','') != 'PUBLIC': roles.append(role[1].replace(';\n','')) - + # # Find out all roles that grant privileges to some of the @@ -304,45 +304,45 @@ def pg_dump_users(db): ' --file=' + pg_dumpall_schema_temp_file.name with open(global_parameters['roles_log_file'],'a') as roles_log_file: - + proc = subprocess.Popen([pg_dumpall_command],stdout=roles_log_file,stderr=subprocess.STDOUT,shell=True) proc.wait() - + if proc.returncode != 0: logs.logger.critical('The command used to generate the tmp role dumpall of the database has a return value != 0') global_parameters['execution_status'] = 'ERROR' - global_parameters['error_message'] = 'pg_dumpall returncode: ' + str(proc.returncode) + '. Check log file.' - register_backup_catalog(db) + global_parameters['error_message'] = 'pg_dumpall returncode: ' + str(proc.returncode) + '. Check log file.' + register_backup_catalog(db) sys.exit(1) - + with open(pg_dumpall_schema_temp_file.name, 'r') as sqldump: for line in sqldump: for role in roles_unique_tmp: if 'GRANT ' in line: if ' TO ' + role + ' GRANTED BY ' in line: roles.append(line.split(' TO' )[0].replace('GRANT ','')) - + unique_role_list = set(roles) logs.logger.debug('The list of roles we need to restore the database has been generated - %s',unique_role_list) - + except Exception as e: logs.logger.critical('pg_dump schema temp file could not be created in directory %s - %s',global_parameters['tmp_dir'],e) - + global_parameters['execution_status'] = 'ERROR' global_parameters['error_message'] = e - register_backup_catalog(db) + register_backup_catalog(db) sys.exit(1) - + # # Extracting sql statements for our roles from the pg_dumpall -r # dump of the cluster - # - + # + try: pg_dumpall_roles_temp_file = tempfile.NamedTemporaryFile(delete=True,dir=global_parameters['tmp_dir']) logs.logger.debug('pg_dumpall roles temp file created %s',pg_dumpall_roles_temp_file.name) - + pg_dumpall_command = global_parameters['backup_server_pgsql_bin_dir'] + '/pg_dumpall' + \ ' -r' + \ ' -h ' + global_parameters['pgsql_node_fqdn'] + \ @@ -356,18 +356,18 @@ def pg_dump_users(db): roles_log_file.write('Timestamp:' + str(datetime.datetime.now()) + '\n') roles_log_file.write('Command: ' + pg_dumpall_command + '\n') roles_log_file.write('------------------------------------\n\n') - + roles_log_file.flush() - + proc = subprocess.Popen([pg_dumpall_command],stdout=roles_log_file,stderr=subprocess.STDOUT,shell=True) proc.wait() - + if proc.returncode != 0: logs.logger.critical('The command used to generate the role dump for the PgSQL node running the database has a return value != 0') - + global_parameters['execution_status'] = 'ERROR' global_parameters['error_message'] = 'pg_dumpall returncode: ' + str(proc.returncode) + '. Check log file.' - register_backup_catalog(db) + register_backup_catalog(db) sys.exit(1) with open(global_parameters['roles_dump_file'],'w') as roles_dump_file: @@ -383,19 +383,19 @@ def pg_dump_users(db): # for line in sqldump_roles: for role in unique_role_list: - + # # CREATE ROLE statements # if ' ROLE ' + role + ';' in line: roles_dump_file.write(line) - + # # ALTER ROLE statements # if ' ROLE ' + role + ' ' in line: roles_dump_file.write(line) - + # # Role memberships statements, GRANT ... TO ... GRANTED BY .. # @@ -404,10 +404,10 @@ def pg_dump_users(db): grant_granted_by_statements.append(line) # - # GRANT roleX TO role GRANTED BY ... We need the list - # of roleX roles. If they do not own something or have privileges in + # GRANT roleX TO role GRANTED BY ... We need the list + # of roleX roles. If they do not own something or have privileges in # the database we are backing up, this is the only place - # where we have information of them. + # where we have information of them. # # We build a new role list with these roles so we can get # the CREATE ROLE and ALTER ROLE needed by them. @@ -416,18 +416,18 @@ def pg_dump_users(db): grant_granted_by_roles.append(role[0].replace('GRANT ','')) - unique_grant_granted_by_roles_list = set(grant_granted_by_roles) - + unique_grant_granted_by_roles_list = set(grant_granted_by_roles) + # # Get all statements for the roles not owning anything or without - # privileges in the database we are backing up, but needed by + # privileges in the database we are backing up, but needed by # GRANT ... TO ... GRANTED BY ... statements # sqldump_roles.seek(0, 0) for line in sqldump_roles: for role in unique_grant_granted_by_roles_list - unique_role_list: - + print "#" + line print "#" + role @@ -436,46 +436,46 @@ def pg_dump_users(db): # if ' ROLE ' + role + ';' in line: roles_dump_file.write(line) - + # # ALTER ROLE statements # if ' ROLE ' + role + ' ' in line: roles_dump_file.write(line) - - + + # # Write to disk all GRANT ... TO ... GRANTED BY ... statements # for line in set(grant_granted_by_statements): roles_dump_file.write(line) - + logs.logger.debug('The list of role statements we need from pg_dumpall has been generated') roles_dump_file.write('\nCOMMIT;\n\n') roles_dump_file.write('-- \n-- PgBackMan roles dump completed\n--\n\n') - + logs.logger.info('Final roles dump file created - %s',global_parameters['roles_dump_file']) global_parameters['execution_status'] = 'SUCCEEDED' unique_role_list_tmp = list(unique_role_list) unique_role_list_tmp.sort() global_parameters['role_list'] = unique_role_list_tmp - + except Exception as e: logs.logger.critical('Could not generate the final roles dump file %s - %s',global_parameters['roles_dump_file'],e) - + global_parameters['execution_status'] = 'ERROR' global_parameters['error_message'] = e - register_backup_catalog(db) + register_backup_catalog(db) sys.exit(1) - + # ############################################ # Function pg_dump_database_config() # ############################################ - + def pg_dump_database_config(db): '''Used to backup database configuration information associated with a database''' @@ -484,15 +484,15 @@ def pg_dump_database_config(db): # # Extracting sql statements for our database from the pg_dumpall # -s dump of the cluster - # + # # We extract all the CREATE DATABASE, ALTER DATABASE and # GRANT/REVOKE ... ON DATABASE statements for our database # - + try: pg_dumpall_dbconfig_temp_file = tempfile.NamedTemporaryFile(delete=True,dir=global_parameters['tmp_dir']) logs.logger.debug('pg_dumpall dbconfig temp file created %s',pg_dumpall_dbconfig_temp_file.name) - + pg_dumpall_command = global_parameters['backup_server_pgsql_bin_dir'] + '/pg_dumpall' + \ ' -s' + \ ' -h ' + global_parameters['pgsql_node_fqdn'] + \ @@ -506,20 +506,20 @@ def pg_dump_database_config(db): dbconfig_log_file.write('Timestamp:' + str(datetime.datetime.now()) + '\n') dbconfig_log_file.write('Command: ' + pg_dumpall_command + '\n') dbconfig_log_file.write('------------------------------------\n\n') - + dbconfig_log_file.flush() - + proc = subprocess.Popen([pg_dumpall_command], stdout=dbconfig_log_file,stderr=subprocess.STDOUT,shell=True) proc.wait() if proc.returncode != 0: logs.logger.critical('The command used to generate the dbconfig dump for the PgSQL node running the database has a return value != 0') - + global_parameters['execution_status'] = 'ERROR' global_parameters['error_message'] = 'pg_dumpall returncode: ' + str(proc.returncode) + '. Check log file.' - register_backup_catalog(db) + register_backup_catalog(db) sys.exit(1) - + with open(global_parameters['dbconfig_dump_file'],'w') as dbconfig_dump_file: dbconfig_dump_file.write('-- \n-- PgBackMan \n-- \n-- Database attributes needed by the database: \n-- ' + global_parameters['dbname'] + '@' + global_parameters['pgsql_node_fqdn'] + ' \n--\n\n') @@ -533,41 +533,41 @@ def pg_dump_database_config(db): # if 'CREATE DATABASE ' + global_parameters['dbname'] + ' ' in line: dbconfig_dump_file.write(line) - + # # ALTER DATABASE statements # if 'ALTER DATABASE ' + global_parameters['dbname'] + ' SET ' in line: dbconfig_dump_file.write(line) - + # # GRANT/REVOKE ... ON DATABASE statements # if ' ON DATABASE ' + global_parameters['dbname'] + ' ' in line: dbconfig_dump_file.write(line) - + logs.logger.debug('The list of dbconfig statements we need from pg_dumpall has been generated') dbconfig_dump_file.write('\nCOMMIT;\n\n') dbconfig_dump_file.write('-- \n-- PgBackMan dbconfig dump completed\n--\n\n') - + logs.logger.info('Final dbconfig dump file created - %s',global_parameters['dbconfig_dump_file']) global_parameters['execution_status'] = 'SUCCEEDED' - + except Exception as e: logs.logger.critical('Could not generate the final dbconfig dump file %s - %s',global_parameters['dbconfig_dump_file'],e) - + global_parameters['execution_status'] = 'ERROR' global_parameters['error_message'] = e - register_backup_catalog(db) + register_backup_catalog(db) sys.exit(1) # ############################################ # Function get_pgsql_node_dsn() # ############################################ - + def get_pgsql_node_dsn(): '''Get the DSN values needed to connect to a PgSQL node''' @@ -577,15 +577,15 @@ def get_pgsql_node_dsn(): logs.logger.debug('DSN value for PgSQL node is %s',dsn_value) return dsn_value - + # ############################################ # Function get_pgsql_node_release() # ############################################ - + def get_pgsql_node_release(db,db_pgnode): '''Get the postgreSQL release version a PgSQL node is running''' - + pgsql_node_version = '' try: @@ -594,7 +594,7 @@ def get_pgsql_node_release(db,db_pgnode): db_pgnode.pg_close() except Exception as e: logs.logger.critical('Problems getting the postgreSQL version running on Pgsql node - %s',e) - + logs.logger.debug('PgSQL node version: %s',pgsql_node_version) if pgsql_node_version == '100': @@ -626,12 +626,12 @@ def get_pgsql_node_release(db,db_pgnode): register_backup_catalog(db) sys.exit(1) - + # ############################################ # Function get_pg_dump_release() # ############################################ - + def get_pg_dump_release(db): ''' @@ -649,7 +649,7 @@ def get_pg_dump_release(db): [1] http://www.postgresql.org/docs/current/static/upgrading.html ''' - + pg_dump_release = '' try: @@ -660,7 +660,7 @@ def get_pg_dump_release(db): # if global_parameters['pg_dump_release'] == '': - pg_dump_release = global_parameters['pgsql_node_release'] + pg_dump_release = global_parameters['pgsql_node_release'] # @@ -669,7 +669,7 @@ def get_pg_dump_release(db): # elif global_parameters['pg_dump_release'] != '': - pg_dump_release = global_parameters['pg_dump_release'] + pg_dump_release = global_parameters['pg_dump_release'] # # pg_dump release defined by the user is lower than the @@ -678,7 +678,7 @@ def get_pg_dump_release(db): # if int(pg_dump_release.replace('_','')) < int(global_parameters['pgsql_node_release'].replace('_','')): - + logs.logger.critical('The pg_dump/all release defined in the snapshot definition is lower than the release running in the pgSQL node: (%s < %s)',global_parameters['pg_dump_release'].replace('_','.'),global_parameters['pgsql_node_release'].replace('_','.')) global_parameters['execution_status'] = 'ERROR' global_parameters['error_message'] = 'pg_dump release defined is too old' @@ -693,11 +693,11 @@ def get_pg_dump_release(db): sys.exit(1) - + # ############################################ # Function get_backup_server_pgsql_bin_dir() # ############################################ - + def get_backup_server_pgsql_bin_dir(db): '''Get the directory with postgreSQL binaries to use''' @@ -706,24 +706,24 @@ def get_backup_server_pgsql_bin_dir(db): try: pgsql_bin_dir = db.get_backup_server_config_value(global_parameters['backup_server_id'],'pgsql_bin_' + global_parameters['pg_dump_release']) logs.logger.debug('pgsql bin directory to use: %s',pgsql_bin_dir) - + return pgsql_bin_dir except Exception as e: pgsql_bin_dir = backup_server_cache_data['pgsql_bin_' + global_parameters['pg_dump_release']] logs.logger.debug('pgsql bin directory to use: %s',pgsql_bin_dir) - + return pgsql_bin_dir - + # ############################################ # Function get_filename_id() # ############################################ - + def get_filename_id(dump_type,file_type): '''Generate the filename used for the backup and log files of a backup job''' - + global global_parameters timestamp = datetime.datetime.now().strftime('%Y%m%dT%H%M%S') @@ -737,7 +737,7 @@ def get_filename_id(dump_type,file_type): if dump_type == 'CLUSTER': filename_id = global_parameters['pgsql_node_backup_dir'] + '/' + file_type + '/' + dump_type + '-' + global_parameters['pgsql_node_fqdn'] + '-v' + global_parameters['pg_dump_release'] + '-defid' + global_parameters['def_id'] + '-c' + global_parameters['backup_code'] + '-' + timestamp else: - filename_id = global_parameters['pgsql_node_backup_dir'] + '/' + file_type + '/' + global_parameters['dbname'] + '-' + global_parameters['pgsql_node_fqdn'] + '-v' + global_parameters['pg_dump_release'] + '-defid' + global_parameters['def_id'] + '-c' + global_parameters['backup_code'] + '-' + timestamp + '-' + dump_type + filename_id = global_parameters['pgsql_node_backup_dir'] + '/' + file_type + '/' + global_parameters['dbname'] + '-' + global_parameters['pgsql_node_fqdn'] + '-v' + global_parameters['pg_dump_release'] + '-defid' + global_parameters['def_id'] + '-c' + global_parameters['backup_code'] + '-' + timestamp + '-' + dump_type # # Dump generated by a CRON job @@ -748,7 +748,7 @@ def get_filename_id(dump_type,file_type): if dump_type == 'CLUSTER': filename_id = global_parameters['pgsql_node_backup_dir'] + '/' + file_type + '/' + dump_type + '-' + global_parameters['pgsql_node_fqdn'] + '-v' + global_parameters['pg_dump_release'] + '-snapid' + global_parameters['snapshot_id'] + '-c' + global_parameters['backup_code'] + '-' + timestamp else: - filename_id = global_parameters['pgsql_node_backup_dir'] + '/' + file_type + '/' + global_parameters['dbname'] + '-' + global_parameters['pgsql_node_fqdn'] + '-v' + global_parameters['pg_dump_release'] + '-snapid' + global_parameters['snapshot_id'] + '-c' + global_parameters['backup_code'] + '-' + timestamp + '-' + dump_type + filename_id = global_parameters['pgsql_node_backup_dir'] + '/' + file_type + '/' + global_parameters['dbname'] + '-' + global_parameters['pgsql_node_fqdn'] + '-v' + global_parameters['pg_dump_release'] + '-snapid' + global_parameters['snapshot_id'] + '-c' + global_parameters['backup_code'] + '-' + timestamp + '-' + dump_type return filename_id @@ -758,7 +758,7 @@ def get_filename_id(dump_type,file_type): # ############################################ # Function register_backup_catalog() # ############################################ - + def register_backup_catalog(db): '''Update the backup catalog information in the database''' @@ -773,14 +773,14 @@ def register_backup_catalog(db): if global_parameters['def_id'] == None: global_parameters['execution_method'] = 'AT' - + elif global_parameters['snapshot_id'] == None: global_parameters['execution_method'] = 'CRON' # # pg_dump_file information # - + pg_dump_file_size = 0 if os.path.exists(global_parameters['cluster_dump_file']): @@ -802,7 +802,7 @@ def register_backup_catalog(db): # >= 1.1.0. We have to find out all files in the directory # to get the total size of the database dump. # - + if os.path.isdir(pg_dump_file): for files in os.listdir(pg_dump_file): pg_dump_file_size += os.path.getsize(pg_dump_file + '/' + files) @@ -812,7 +812,7 @@ def register_backup_catalog(db): else: pg_dump_file = 'None' - + if os.path.exists(global_parameters['cluster_log_file']): pg_dump_log_file = global_parameters['cluster_log_file'] elif os.path.exists(global_parameters['database_log_file']): @@ -836,7 +836,7 @@ def register_backup_catalog(db): logs.logger.error('Could not get size of pg_dump roles file: %s - %s',pg_dump_roles_file,e) else: pg_dump_roles_file = 'None' - + if os.path.exists(global_parameters['roles_log_file']): pg_dump_roles_log_file = global_parameters['roles_log_file'] else: @@ -858,7 +858,7 @@ def register_backup_catalog(db): logs.logger.error('Could not get size of pg_dump dbconfig file: %s - %s',pg_dump_dbconfig_file,e) else: pg_dump_dbconfig_file = 'None' - + if os.path.exists(global_parameters['dbconfig_log_file']): pg_dump_dbconfig_log_file = global_parameters['dbconfig_log_file'] else: @@ -872,7 +872,7 @@ def register_backup_catalog(db): procpid = os.getpid() db.register_backup_catalog(global_parameters['def_id'], procpid, - global_parameters['backup_server_id'], + global_parameters['backup_server_id'], global_parameters['pgsql_node_id'], global_parameters['dbname'], global_parameters['backup_start'], @@ -896,62 +896,62 @@ def register_backup_catalog(db): global_parameters['pgsql_node_release'].replace('_','.'), global_parameters['pg_dump_release'].replace('_','.') ) - + logs.logger.info('Backup job catalog for DefID: %s or SnapshotID: %s updated in the database',str(global_parameters['def_id']),str(global_parameters['snapshot_id'])) - + except Exception as e: # # We create a pending log file if we can not update the # database. This file will be processed by # pgbackman_maintenence later. - # + # logs.logger.warning('Problems updating the backup job catalog for DefID: %s or SnapshotID: %s in the database - %s',str(global_parameters['def_id']),str(global_parameters['snapshot_id']),e) pending_log_file = '' - + try: procpid = os.getpid() pending_log_file = global_parameters['backup_server_pending_registration_dir'] + '/backup_jobs_pending_log_updates_nodeid_' + str(global_parameters['pgsql_node_id']) + '_' + str(procpid) + '.log' if global_parameters['def_id'] == None: global_parameters['def_id'] = '' - + elif global_parameters['snapshot_id'] == None: global_parameters['snapshot_id'] = '' - + with open(pending_log_file,'w+') as catalog_pending: catalog_pending.write(str(global_parameters['def_id']) + '::' + str(procpid) + '::' + - str(global_parameters['backup_server_id']) + '::' + - str(global_parameters['pgsql_node_id']) + '::' + - global_parameters['dbname'] + '::' + - str(global_parameters['backup_start']) + '::' + - str(global_parameters['backup_stop']) + '::' + - str(duration) + '::' + - pg_dump_file + '::' + - str(pg_dump_file_size) + '::' + - pg_dump_log_file + '::' + - pg_dump_roles_file + '::' + - str(pg_dump_roles_file_size) + '::' + - pg_dump_roles_log_file + '::' + - pg_dump_dbconfig_file + '::' + - str(pg_dump_dbconfig_file_size) + '::' + - pg_dump_dbconfig_log_file + '::' + - global_parameters['global_log_file'] + '::' + + str(global_parameters['backup_server_id']) + '::' + + str(global_parameters['pgsql_node_id']) + '::' + + global_parameters['dbname'] + '::' + + str(global_parameters['backup_start']) + '::' + + str(global_parameters['backup_stop']) + '::' + + str(duration) + '::' + + pg_dump_file + '::' + + str(pg_dump_file_size) + '::' + + pg_dump_log_file + '::' + + pg_dump_roles_file + '::' + + str(pg_dump_roles_file_size) + '::' + + pg_dump_roles_log_file + '::' + + pg_dump_dbconfig_file + '::' + + str(pg_dump_dbconfig_file_size) + '::' + + pg_dump_dbconfig_log_file + '::' + + global_parameters['global_log_file'] + '::' + global_parameters['execution_status'] + '::' + global_parameters['execution_method'] + '::' + global_parameters['error_message'] + '::' + - global_parameters['snapshot_id'] + '::' + + global_parameters['snapshot_id'] + '::' + ' '.join(global_parameters['role_list']) + '::' + - global_parameters['pgsql_node_release'].replace('_','.') + '::' + + global_parameters['pgsql_node_release'].replace('_','.') + '::' + global_parameters['pg_dump_release'].replace('_','.') + '\n') - + logs.logger.info('Catalog pending log file: %s created',pending_log_file) - + except Exception as e: logs.logger.error('Could not generate the catalog pending log file: %s - %s',pending_log_file,e) @@ -960,7 +960,7 @@ def register_backup_catalog(db): # Function get_backup_server_parameters_from_cache() # ################################################## -def get_backup_server_parameters_from_cache(db,backup_server_fqdn): +def get_backup_server_parameters_from_cache(db,backup_server_fqdn): '''Get backup server parameters from cache file''' global backup_server_cache_data @@ -968,12 +968,12 @@ def get_backup_server_parameters_from_cache(db,backup_server_fqdn): try: backup_server_cache_file = global_parameters['root_backup_dir'] + '/cache_dir/backup_server_' + backup_server_fqdn + '.cache' - + with open(backup_server_cache_file,'r') as backup_server_cache: for line in backup_server_cache: (key, val) = line.split('::') backup_server_cache_data[key] = val.replace('\n','') - + except Exception as e: logs.logger.error('Could not read the cache file for the backup server: %s - %s',backup_server_fqdn,e) global_parameters['execution_status'] = 'ERROR' @@ -987,7 +987,7 @@ def get_backup_server_parameters_from_cache(db,backup_server_fqdn): # Function get_pgsql_node_parameters_from_cache() # ############################################### -def get_pgsql_node_parameters_from_cache(db): +def get_pgsql_node_parameters_from_cache(db): '''Get pgsql_node parameters from cache file''' global pgsql_node_cache_data @@ -1000,7 +1000,7 @@ def get_pgsql_node_parameters_from_cache(db): for line in pgsql_node_cache: (key, val) = line.split('::') pgsql_node_cache_data[key] = val.replace('\n','') - + except Exception as e: logs.logger.critical('Could not read the cache file for the PgSQL node - %s',e) global_parameters['execution_status'] = 'ERROR' @@ -1013,7 +1013,7 @@ def get_pgsql_node_parameters_from_cache(db): # ############################################## # Function check_pgbackman_database_connection() # ############################################## - + def check_pgbackman_database_connection(db): '''Check if we can connect to the pgbackman database''' @@ -1022,14 +1022,14 @@ def check_pgbackman_database_connection(db): db.pg_close() return True - except Exception as e: + except Exception as e: logs.logger.error('The pgbackman database is not available - %s',e) return False # ############################################### # Function check_pgsql_node_database_connection() # ############################################### - + def check_pgsql_node_database_connection(db_pgnode): '''Check if we can connect to the pgsql node database''' @@ -1038,7 +1038,7 @@ def check_pgsql_node_database_connection(db_pgnode): db_pgnode.pg_close() return True - except Exception as e: + except Exception as e: logs.logger.critical('The PgSQL node is not available - %s',e) return False @@ -1046,56 +1046,56 @@ def check_pgsql_node_database_connection(db_pgnode): # ############################################## # Function pause_pg_recovery() # ############################################## - + def pause_pg_recovery(db_pgnode): '''Pause the postgres recovery in the PgSQL node''' try: - + # # Check if the PgSQL node is in recovery modus # if db_pgnode.pg_recovery_in_progress() == True: logs.logger.info('(Pause recovery) PgSQL node is a slave node in recovery modus.') - + # # Check if the postgres recovery is paused in this PgSQL # node. If it is running, pause it. # if db_pgnode.pg_recovery_paused() == False: - + try: db_pgnode.pause_pg_recovery() logs.logger.info('Recovery process paused on PgSQL node') - except Exception as e: + except Exception as e: logs.logger.error('Problems pausing the recovery process on PgSQL node - %s',e) - + else: logs.logger.info('The recovery process is already paused on PgSQL node') - + else: logs.logger.debug('PgSQL node is not a slave node in recovery modus (pause).') - except Exception as e: + except Exception as e: logs.logger.error('Problems getting recovery modus information on PgSQL node - %s',e) # ############################################## # Function resume_pg_recovery() # ############################################## - + def resume_pg_recovery(db_pgnode): '''Resume the postgres recovery in the PgSQL node''' try: - + # # Check if the PgSQL node is in recovery modus # if db_pgnode.pg_recovery_in_progress() == True: logs.logger.info('(Resume recovery) PgSQL node is a slave node in recovery modus.') - + # # Check if the postgres recovery is paused in this PgSQL # node. If it is paused and no other pgbackman_dump @@ -1111,13 +1111,13 @@ def resume_pg_recovery(db_pgnode): pgbackman_dump_process_count = db_pgnode.get_pgbackman_dump_count() - if db_pgnode.pg_recovery_paused() == True and pgbackman_dump_process_count == 1: + if db_pgnode.pg_recovery_paused() == True and pgbackman_dump_process_count == 1: try: db_pgnode.resume_pg_recovery() logs.logger.info('Recovery process resumed on PgSQL node') - except Exception as e: + except Exception as e: logs.logger.error('Problems resuming the recovery process on PgSQL node - %s',e) else: @@ -1127,14 +1127,14 @@ def resume_pg_recovery(db_pgnode): else: logs.logger.debug('PgSQL node is not a slave node in recovery modus (resume).') - except Exception as e: + except Exception as e: logs.logger.error('Problems getting recovery modus information on PgSQL node - %s',e) # ############################################ # Function signal_handler() # ############################################ - + def signal_handler(signum, frame): logs.logger.info('**** pgbackman_dump stopped. ****') sys.exit(0) @@ -1143,10 +1143,10 @@ def signal_handler(signum, frame): # ############################################ # Function Main() # ############################################ - + def main(): '''Main function''' - + global global_parameters conf = PgbackmanConfiguration() @@ -1164,21 +1164,21 @@ def main(): global_parameters['roles_dump_file'] = '' global_parameters['roles_log_file'] = '' - + global_parameters['dbconfig_dump_file'] = '' global_parameters['dbconfig_log_file'] = '' global_parameters['error_message'] = '' - global_parameters['role_list'] = [] + global_parameters['role_list'] = [] global_parameters['pgsql_node_release'] = '' - + db = PgbackmanDB(pgbackman_dsn, 'pgbackman_dump') - + pgsql_node_dsn = get_pgsql_node_dsn() db_pgnode = PgbackmanDB(pgsql_node_dsn, 'pgbackman_dump') - # + # # The backup server FQDN to be used can be defined in the # pgbackman configuration file. If the configuration parameter # 'backup_server' is not defined, the return value of @@ -1220,27 +1220,27 @@ def main(): # necessary parameters from the cache files saved in the backup # server # - + check_pgnode_db = check_pgsql_node_database_connection(db_pgnode) if not check_pgnode_db: logs.logger.critical('The database is not available. Shutting down the backup job with DefID: %s',str(global_parameters['def_id'])) - + global_parameters['execution_status'] = 'ERROR' global_parameters['error_message'] = 'Database to backup not available' register_backup_catalog(db) - sys.exit(1) + sys.exit(1) try: global_parameters['pgsql_node_backup_dir'] = db.get_pgsql_node_config_value(global_parameters['pgsql_node_id'],'pgnode_backup_partition') except psycopg2.Error as e: global_parameters['pgsql_node_backup_dir'] = pgsql_node_cache_data['pgnode_backup_partition'] - + global_parameters['pgsql_node_release'] = get_pgsql_node_release(db,db_pgnode) global_parameters['pg_dump_release'] = get_pg_dump_release(db) global_parameters['backup_server_pgsql_bin_dir'] = get_backup_server_pgsql_bin_dir(db) - + if os.path.exists('/bin/gzip') == True: global_parameters['cluster_dump_file'] = get_filename_id('CLUSTER','dump') + '.sql.gz' else: @@ -1253,7 +1253,7 @@ def main(): global_parameters['roles_dump_file'] = get_filename_id('USERS','dump') + '.sql' global_parameters['roles_log_file'] = get_filename_id('USERS','log') + '.log' - + global_parameters['dbconfig_dump_file'] = get_filename_id('DBCONFIG','dump') + '.sql' global_parameters['dbconfig_log_file'] = get_filename_id('DBCONFIG','log') + '.log' @@ -1269,15 +1269,15 @@ def main(): # By default postgreSQL will kill any backup running against a # slave node if the recovery process is running and the backup is # taking too long to finish because the database is large. - # + # if conf.pause_recovery_process_on_slave == 'ON': - + try: db_pgnode.pg_connect() logs.logger.info('Opening a persistant connection to the PgSQL node') - except Exception as e: + except Exception as e: logs.logger.error('Problems opening a persistant connection to the PgSQL node') # @@ -1285,16 +1285,18 @@ def main(): # will stop the recovery process if necessary while pgbackman_dump # is running. # - + if conf.pause_recovery_process_on_slave == 'ON': pause_pg_recovery(db_pgnode) - + # # Running the backups # if global_parameters['backup_code'] == 'CLUSTER': pg_dumpall(db) + elif global_parameters['backup_code'] == 'RDSDATAONLY': + pg_dump(db) else: pg_dump(db) pg_dump_users(db) @@ -1303,9 +1305,9 @@ def main(): # # If the PgSQL node is a slave node in a replication system, we # will resume the recovery process if no other pgbackman_dump - # processes are running. + # processes are running. # - + if conf.pause_recovery_process_on_slave == 'ON': resume_pg_recovery(db_pgnode) @@ -1320,7 +1322,7 @@ def main(): db_pgnode.pg_connect() logs.logger.info('Closing a persistant connection to the PgSQL node') - except Exception as e: + except Exception as e: logs.logger.error('Problems closing a persistant connection to the PgSQL node') # @@ -1331,7 +1333,7 @@ def main(): # ############################################ -# +# # ############################################ if __name__ == '__main__': @@ -1347,26 +1349,26 @@ if __name__ == '__main__': parser.add_argument('--def-id', metavar='JOBID', required=False, help='Backup job ID', dest='def_id') parser.add_argument('--snapshot-id', metavar='SNAPSHOT-ID', required=False, help='snapshot ID', dest='snapshot_id') parser.add_argument('--dbname', metavar='DBNAME', required=False, help='Database name', dest='dbname') - parser.add_argument('--backup-code', metavar='[FULL|SCHEMA|DATA|CLUSTER]', choices=['FULL', 'SCHEMA', 'DATA', 'CLUSTER'], required=True, help='Backup code', dest='backup_code') + parser.add_argument('--backup-code', metavar='[FULL|SCHEMA|DATA|CLUSTER|RDSDATAONLY]', choices=['FULL', 'SCHEMA', 'DATA', 'CLUSTER', 'RDSDATAONLY'], required=True, help='Backup code', dest='backup_code') parser.add_argument('--encryption', metavar='[false|true]', default=True, choices=['true', 'false'],required=True, help='Activate encryption', dest='encryption') parser.add_argument('--root-backup-dir', metavar='ROOT-BACKUP-DIR', default=True, required=True, help='Root backup dir', dest='root_backup_dir') parser.add_argument('--extra-backup-parameters', metavar='EXTRA-PARAMETERS', required=False, help='extra pg_dump parameters', dest='extra_backup_parameters') parser.add_argument('--pg-dump-release', metavar='PG-DUMP-RELEASE', required=False, help='pg_dump release', dest='pg_dump_release') - - args = parser.parse_args() - + + args = parser.parse_args() + if args.pgsql_node_fqdn: global_parameters['pgsql_node_fqdn'] = args.pgsql_node_fqdn else: print('PgSQL node fqdn parameter not defined') sys.exit(1) - + if args.pgsql_node_id: global_parameters['pgsql_node_id'] = args.pgsql_node_id else: print('PgSQL node id parameter not defined') sys.exit(1) - + if args.pgsql_node_port: global_parameters['pgsql_node_port'] = args.pgsql_node_port else: @@ -1378,7 +1380,7 @@ if __name__ == '__main__': else: print('PgSQL node admin user parameter not defined') sys.exit(1) - + if args.def_id: global_parameters['def_id'] = args.def_id else: @@ -1399,7 +1401,7 @@ if __name__ == '__main__': else: print('Encryption parameter not defined') sys.exit(1) - + if args.backup_code: global_parameters['backup_code'] = args.backup_code else: @@ -1428,7 +1430,7 @@ if __name__ == '__main__': logs = PgbackmanLogs("pgbackman_dump", "[" + global_parameters['pgsql_node_fqdn'] + "]", "[" + global_parameters['dbname'] + "]") logs.logger.info('**** pgbackman_dump started. ****') - + main() logs.logger.info('**** pgbackman_dump finished. ****') From 86a00c678c49becca9c7b921545877e1fd1c3238 Mon Sep 17 00:00:00 2001 From: Philip McClarence Date: Fri, 26 Jan 2018 11:36:26 +0000 Subject: [PATCH 04/12] adding RDS backup code option --- bin/pgbackman_dump | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/pgbackman_dump b/bin/pgbackman_dump index 9d91b18..428bc73 100755 --- a/bin/pgbackman_dump +++ b/bin/pgbackman_dump @@ -1295,7 +1295,7 @@ def main(): if global_parameters['backup_code'] == 'CLUSTER': pg_dumpall(db) - elif global_parameters['backup_code'] == 'RDSDATAONLY': + elif global_parameters['backup_code'] == 'RDSDATA': pg_dump(db) else: pg_dump(db) @@ -1349,7 +1349,7 @@ if __name__ == '__main__': parser.add_argument('--def-id', metavar='JOBID', required=False, help='Backup job ID', dest='def_id') parser.add_argument('--snapshot-id', metavar='SNAPSHOT-ID', required=False, help='snapshot ID', dest='snapshot_id') parser.add_argument('--dbname', metavar='DBNAME', required=False, help='Database name', dest='dbname') - parser.add_argument('--backup-code', metavar='[FULL|SCHEMA|DATA|CLUSTER|RDSDATAONLY]', choices=['FULL', 'SCHEMA', 'DATA', 'CLUSTER', 'RDSDATAONLY'], required=True, help='Backup code', dest='backup_code') + parser.add_argument('--backup-code', metavar='[FULL|SCHEMA|DATA|CLUSTER|RDSDATAONLY]', choices=['FULL', 'SCHEMA', 'DATA', 'CLUSTER', 'RDSDATA'], required=True, help='Backup code', dest='backup_code') parser.add_argument('--encryption', metavar='[false|true]', default=True, choices=['true', 'false'],required=True, help='Activate encryption', dest='encryption') parser.add_argument('--root-backup-dir', metavar='ROOT-BACKUP-DIR', default=True, required=True, help='Root backup dir', dest='root_backup_dir') parser.add_argument('--extra-backup-parameters', metavar='EXTRA-PARAMETERS', required=False, help='extra pg_dump parameters', dest='extra_backup_parameters') From c506f3cdbb35cc8a06875be2ad4e30ce6a401da3 Mon Sep 17 00:00:00 2001 From: Philip McClarence Date: Fri, 26 Jan 2018 12:34:31 +0000 Subject: [PATCH 05/12] adding RDS backup code option --- bin/pgbackman_dump | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/bin/pgbackman_dump b/bin/pgbackman_dump index 428bc73..5da0e23 100755 --- a/bin/pgbackman_dump +++ b/bin/pgbackman_dump @@ -157,6 +157,20 @@ def pg_dump(db): ' ' + global_parameters['extra_backup_parameters'] + \ ' ' + global_parameters['dbname'] + elif global_parameters['backup_code'] == 'RDSDATA': + + pg_dump_command = global_parameters['backup_server_pgsql_bin_dir'] + '/pg_dump' + \ + ' -h ' + global_parameters['pgsql_node_fqdn'] + \ + ' -p ' + global_parameters['pgsql_node_port'] + \ + ' -U ' + global_parameters['pgsql_node_admin_user'] + \ + ' --data-only' + \ + ' --file=' + global_parameters['database_dump_file'] + \ + ' --format=d' + \ + ' --blobs' + \ + ' --verbose' + \ + ' ' + global_parameters['extra_backup_parameters'] + \ + ' ' + global_parameters['dbname'] + try: with open(global_parameters['database_log_file'],'w') as database_log_file: From cac860eb5a2cad13e471eabc81cd6b26a38793b4 Mon Sep 17 00:00:00 2001 From: Philip McClarence Date: Fri, 26 Jan 2018 13:14:48 +0000 Subject: [PATCH 06/12] adding RDS backup code option --- sql/pgbackman.sql | 858 +++++++++++++++++++++++----------------------- 1 file changed, 429 insertions(+), 429 deletions(-) diff --git a/sql/pgbackman.sql b/sql/pgbackman.sql index 4c8343a..137bb07 100644 --- a/sql/pgbackman.sql +++ b/sql/pgbackman.sql @@ -19,7 +19,7 @@ CREATE USER pgbackman_role_ro; \echo '# [Creating database: pgbackman]\n' CREATE DATABASE pgbackman OWNER pgbackman_role_rw; -\c pgbackman +\c pgbackman BEGIN; @@ -284,7 +284,7 @@ ALTER TABLE at_definition_status OWNER TO pgbackman_role_rw; -- ------------------------------------------------------ -- Table: backup_server_default_config -- --- @Description: Default configuration values for +-- @Description: Default configuration values for -- backup servers. -- -- Attributes: @@ -310,7 +310,7 @@ ALTER TABLE backup_server_default_config OWNER TO pgbackman_role_rw; -- ------------------------------------------------------ -- Table: pgsql_node_default_config -- --- @Description: Default configuration values for +-- @Description: Default configuration values for -- postgresql servers. -- -- Attributes: @@ -335,7 +335,7 @@ ALTER TABLE pgsql_node_default_config OWNER TO pgbackman_role_rw; -- ------------------------------------------------------ -- Table: job_queue -- --- @Description: +-- @Description: -- -- Attributes: -- @@ -362,7 +362,7 @@ ALTER TABLE job_queue OWNER TO pgbackman_role_rw; -- ------------------------------------------------------ -- Table: backup_definition -- --- @Description: Backup jobs defined in Pgbackman +-- @Description: Backup jobs defined in Pgbackman -- -- Attributes: -- @@ -418,7 +418,7 @@ ALTER TABLE backup_definition OWNER TO pgbackman_role_rw; -- ------------------------------------------------------ -- Table: snapshot_definition -- --- @Description: snapshots defined in Pgbackman +-- @Description: snapshots defined in Pgbackman -- -- Attributes: -- @@ -465,7 +465,7 @@ ALTER TABLE snapshot_definition OWNER TO pgbackman_role_rw; -- ------------------------------------------------------ -- Table: restore_definition -- --- @Description: restore definitions +-- @Description: restore definitions -- -- Attributes: -- @@ -583,7 +583,7 @@ CREATE TABLE restore_catalog( target_pgsql_node_id INTEGER NOT NULL, source_dbname TEXT NOT NULL, target_dbname TEXT NOT NULL, - renamed_dbname TEXT, + renamed_dbname TEXT, started TIMESTAMP WITH TIME ZONE, finished TIMESTAMP WITH TIME ZONE, duration INTERVAL, @@ -611,7 +611,7 @@ ALTER TABLE restore_catalog OWNER TO pgbackman_role_rw; -- Table: alerts -- -- @Description: Alerts generated when a backup fails --- +-- -- -- ------------------------------------------------------ @@ -645,7 +645,7 @@ ALTER TABLE alerts OWNER TO pgbackman_role_rw; -- Table: alert_type -- -- @Description: Type of alert --- +-- -- -- ------------------------------------------------------ @@ -777,7 +777,7 @@ ALTER TABLE pgsql_node_config OWNER TO pgbackman_role_rw; CREATE TABLE pgbackman_version( - registered TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), + registered TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), version INTEGER, tag TEXT ); @@ -845,7 +845,7 @@ ALTER TABLE ONLY snapshot_definition ALTER TABLE ONLY snapshot_definition ADD FOREIGN KEY (status) REFERENCES at_definition_status (code) MATCH FULL ON DELETE RESTRICT; - + ALTER TABLE ONLY restore_definition ADD FOREIGN KEY (backup_server_id) REFERENCES backup_server (server_id) MATCH FULL ON DELETE CASCADE; @@ -857,7 +857,7 @@ ALTER TABLE ONLY restore_definition ALTER TABLE ONLY restore_definition ADD FOREIGN KEY (status) REFERENCES at_definition_status (code) MATCH FULL ON DELETE RESTRICT; - + ALTER TABLE ONLY restore_catalog ADD FOREIGN KEY (backup_server_id) REFERENCES backup_server (server_id) MATCH FULL ON DELETE CASCADE; @@ -902,7 +902,7 @@ INSERT INTO backup_code (code,description) VALUES ('SCHEMA','Schema backup of a INSERT INTO backup_code (code,description) VALUES ('DATA','Data backup of the database.'); INSERT INTO backup_code (code,description) VALUES ('CLUSTER','Full backup of the database cluster.'); INSERT INTO backup_code (code,description) VALUES ('CONFIG','Backup of the configuration files'); - +INSERT INTO backup_code (code,description) VALUES ('RDSDATA','Only runs a pg_dump of the data so that it does not try to access globals that postgres cannot access on RDS'); \echo '# [Init: job_definition_status]\n' INSERT INTO job_definition_status (code,description) VALUES ('ACTIVE','Backup job activated and in production'); @@ -987,8 +987,8 @@ INSERT INTO pgbackman_version (version,tag) VALUES ('3','v_1_2_0'); -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION notify_pgsql_node_change() RETURNS TRIGGER - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ BEGIN @@ -996,17 +996,17 @@ CREATE OR REPLACE FUNCTION notify_pgsql_node_change() RETURNS TRIGGER IF NEW.status = 'RUNNING' THEN EXECUTE 'DELETE FROM pgsql_node_stopped WHERE pgsql_node_id = $1' - USING NEW.node_id; + USING NEW.node_id; PERFORM pg_notify('channel_pgsql_node_running','PgSQL node running'); - + ELSEIF NEW.status = 'STOPPED' THEN EXECUTE 'INSERT INTO pgsql_node_stopped (pgsql_node_id) VALUES ($1)' - USING NEW.node_id; + USING NEW.node_id; - PERFORM pg_notify('channel_pgsql_node_stopped','PgSQL node stopped'); - END IF; + PERFORM pg_notify('channel_pgsql_node_stopped','PgSQL node stopped'); + END IF; RETURN NULL; END; @@ -1025,13 +1025,13 @@ CREATE TRIGGER notify_pgsql_node_change AFTER INSERT OR UPDATE -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION notify_pgsql_node_deleted() RETURNS TRIGGER - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ BEGIN PERFORM pg_notify('channel_pgsql_node_deleted','PgSQL node deleted'); - + RETURN NULL; END; $$; @@ -1049,16 +1049,16 @@ CREATE TRIGGER notify_pgsql_node_deleted AFTER DELETE -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION update_pgsql_nodes_to_delete() RETURNS TRIGGER - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ BEGIN - - EXECUTE 'INSERT INTO pgsql_node_to_delete (backup_server_id,pgsql_node_id) + + EXECUTE 'INSERT INTO pgsql_node_to_delete (backup_server_id,pgsql_node_id) SELECT server_id,$1 FROM backup_server ORDER BY server_id' - USING OLD.node_id; + USING OLD.node_id; RETURN NULL; END; @@ -1077,14 +1077,14 @@ CREATE TRIGGER update_pgsql_nodes_to_delete AFTER DELETE -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION delete_pgsql_nodes_stopped() RETURNS TRIGGER - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ BEGIN - + EXECUTE 'DELETE FROM pgsql_node_stopped WHERE pgsql_node_id = $1' - USING OLD.node_id; + USING OLD.node_id; RETURN NULL; END; @@ -1104,13 +1104,13 @@ CREATE TRIGGER delete_pgsql_nodes_stopped AFTER DELETE -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION update_backup_server_configuration() RETURNS TRIGGER - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ BEGIN - EXECUTE 'INSERT INTO backup_server_config (server_id,parameter,value,description) + EXECUTE 'INSERT INTO backup_server_config (server_id,parameter,value,description) SELECT $1,parameter,value,description FROM backup_server_default_config' USING NEW.server_id; @@ -1132,17 +1132,17 @@ CREATE TRIGGER update_backup_server_configuration AFTER INSERT -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION update_pgsql_node_configuration() RETURNS TRIGGER - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ BEGIN - EXECUTE 'INSERT INTO pgsql_node_config (node_id,parameter,value,description) + EXECUTE 'INSERT INTO pgsql_node_config (node_id,parameter,value,description) SELECT $1,parameter,replace(value,''%%pgnode%%'',$2),description FROM pgsql_node_default_config' USING NEW.node_id, NEW.node_id::TEXT; - + RETURN NULL; END; @@ -1161,8 +1161,8 @@ CREATE TRIGGER update_pgsql_node_configuration AFTER INSERT -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION update_backup_definition_registration() RETURNS TRIGGER - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ BEGIN @@ -1186,8 +1186,8 @@ CREATE TRIGGER update_backup_definition_registration BEFORE UPDATE -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION update_job_queue(INTEGER,INTEGER) RETURNS VOID - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -1202,7 +1202,7 @@ CREATE OR REPLACE FUNCTION update_job_queue(INTEGER,INTEGER) RETURNS VOID BEGIN SELECT count(*) FROM job_queue WHERE backup_server_id = backup_server_id_ AND pgsql_node_id = pgsql_node_id_ AND is_assigned IS FALSE INTO srv_cnt; - + IF srv_cnt = 0 THEN EXECUTE 'INSERT INTO job_queue (backup_server_id,pgsql_node_id,is_assigned) VALUES ($1,$2,FALSE)' @@ -1213,7 +1213,7 @@ CREATE OR REPLACE FUNCTION update_job_queue(INTEGER,INTEGER) RETURNS VOID END IF; EXCEPTION WHEN others THEN - GET STACKED DIAGNOSTICS + GET STACKED DIAGNOSTICS v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_context = PG_EXCEPTION_CONTEXT; @@ -1231,8 +1231,8 @@ ALTER FUNCTION update_job_queue(INTEGER,INTEGER) OWNER TO pgbackman_role_rw; -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION update_job_queue() RETURNS TRIGGER - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -1248,8 +1248,8 @@ CREATE OR REPLACE FUNCTION update_job_queue() RETURNS TRIGGER IF (TG_OP = 'INSERT' ) THEN SELECT count(*) FROM job_queue WHERE backup_server_id = NEW.backup_server_id AND pgsql_node_id = NEW.pgsql_node_id AND is_assigned IS FALSE INTO srv_cnt; - SELECT hostname || '.' || domain_name FROM backup_server WHERE server_id = NEW.backup_server_id INTO backup_server_; - SELECT hostname || '.' || domain_name FROM pgsql_node WHERE node_id = NEW.pgsql_node_id INTO pgsql_node_; + SELECT hostname || '.' || domain_name FROM backup_server WHERE server_id = NEW.backup_server_id INTO backup_server_; + SELECT hostname || '.' || domain_name FROM pgsql_node WHERE node_id = NEW.pgsql_node_id INTO pgsql_node_; IF srv_cnt = 0 THEN EXECUTE 'INSERT INTO job_queue (backup_server_id,pgsql_node_id) VALUES ($1,$2)' @@ -1257,7 +1257,7 @@ CREATE OR REPLACE FUNCTION update_job_queue() RETURNS TRIGGER NEW.pgsql_node_id; PERFORM pg_notify('channel_bs' || NEW.backup_server_id || '_pg' || NEW.pgsql_node_id,'Backup jobs for ' || pgsql_node_ || ' updated on ' || backup_server_); - END IF; + END IF; -- -------------------------- -- Updating a backup job @@ -1272,8 +1272,8 @@ CREATE OR REPLACE FUNCTION update_job_queue() RETURNS TRIGGER IF (OLD.backup_server_id = NEW.backup_server_id) THEN SELECT count(*) FROM job_queue WHERE backup_server_id = NEW.backup_server_id AND pgsql_node_id = NEW.pgsql_node_id AND is_assigned IS FALSE INTO srv_cnt; - SELECT hostname || '.' || domain_name FROM backup_server WHERE server_id = NEW.backup_server_id INTO backup_server_; - SELECT hostname || '.' || domain_name FROM pgsql_node WHERE node_id = NEW.pgsql_node_id INTO pgsql_node_; + SELECT hostname || '.' || domain_name FROM backup_server WHERE server_id = NEW.backup_server_id INTO backup_server_; + SELECT hostname || '.' || domain_name FROM pgsql_node WHERE node_id = NEW.pgsql_node_id INTO pgsql_node_; IF srv_cnt = 0 THEN EXECUTE 'INSERT INTO job_queue (backup_server_id,pgsql_node_id) VALUES ($1,$2)' @@ -1281,17 +1281,17 @@ CREATE OR REPLACE FUNCTION update_job_queue() RETURNS TRIGGER NEW.pgsql_node_id; PERFORM pg_notify('channel_bs' || NEW.backup_server_id || '_pg' || NEW.pgsql_node_id,'Backup jobs for ' || pgsql_node_ || ' updated on ' || backup_server_); - END IF; + END IF; -- -- The backup job has been moved to another backup server - -- + -- ELSEIF (OLD.backup_server_id <> NEW.backup_server_id) THEN - + SELECT count(*) FROM job_queue WHERE backup_server_id = NEW.backup_server_id AND pgsql_node_id = NEW.pgsql_node_id AND is_assigned IS FALSE INTO srv_cnt; - SELECT hostname || '.' || domain_name FROM backup_server WHERE server_id = NEW.backup_server_id INTO backup_server_; - SELECT hostname || '.' || domain_name FROM pgsql_node WHERE node_id = NEW.pgsql_node_id INTO pgsql_node_; + SELECT hostname || '.' || domain_name FROM backup_server WHERE server_id = NEW.backup_server_id INTO backup_server_; + SELECT hostname || '.' || domain_name FROM pgsql_node WHERE node_id = NEW.pgsql_node_id INTO pgsql_node_; IF srv_cnt = 0 THEN EXECUTE 'INSERT INTO job_queue (backup_server_id,pgsql_node_id) VALUES ($1,$2)' @@ -1299,11 +1299,11 @@ CREATE OR REPLACE FUNCTION update_job_queue() RETURNS TRIGGER NEW.pgsql_node_id; PERFORM pg_notify('channel_bs' || NEW.backup_server_id || '_pg' || NEW.pgsql_node_id,'Backup jobs for ' || pgsql_node_ || ' updated on ' || backup_server_); - END IF; + END IF; SELECT count(*) FROM job_queue WHERE backup_server_id = OLD.backup_server_id AND pgsql_node_id = NEW.pgsql_node_id AND is_assigned IS FALSE INTO srv_cnt; - SELECT hostname || '.' || domain_name FROM backup_server WHERE server_id = OLD.backup_server_id INTO backup_server_; - SELECT hostname || '.' || domain_name FROM pgsql_node WHERE node_id = NEW.pgsql_node_id INTO pgsql_node_; + SELECT hostname || '.' || domain_name FROM backup_server WHERE server_id = OLD.backup_server_id INTO backup_server_; + SELECT hostname || '.' || domain_name FROM pgsql_node WHERE node_id = NEW.pgsql_node_id INTO pgsql_node_; IF srv_cnt = 0 THEN EXECUTE 'INSERT INTO job_queue (backup_server_id,pgsql_node_id) VALUES ($1,$2)' @@ -1311,7 +1311,7 @@ CREATE OR REPLACE FUNCTION update_job_queue() RETURNS TRIGGER NEW.pgsql_node_id; PERFORM pg_notify('channel_bs' || OLD.backup_server_id || '_pg' || NEW.pgsql_node_id,'Backup jobs for ' || pgsql_node_ || ' updated on ' || backup_server_); - END IF; + END IF; END IF; @@ -1322,8 +1322,8 @@ CREATE OR REPLACE FUNCTION update_job_queue() RETURNS TRIGGER ELSEIF (TG_OP = 'DELETE') THEN SELECT count(*) FROM job_queue WHERE backup_server_id = OLD.backup_server_id AND pgsql_node_id = OLD.pgsql_node_id AND is_assigned IS FALSE INTO srv_cnt; - SELECT hostname || '.' || domain_name FROM backup_server WHERE server_id = OLD.backup_server_id INTO backup_server_; - SELECT hostname || '.' || domain_name FROM pgsql_node WHERE node_id = OLD.pgsql_node_id INTO pgsql_node_; + SELECT hostname || '.' || domain_name FROM backup_server WHERE server_id = OLD.backup_server_id INTO backup_server_; + SELECT hostname || '.' || domain_name FROM pgsql_node WHERE node_id = OLD.pgsql_node_id INTO pgsql_node_; IF srv_cnt = 0 THEN EXECUTE 'INSERT INTO job_queue (backup_server_id,pgsql_node_id) VALUES ($1,$2)' @@ -1331,7 +1331,7 @@ CREATE OR REPLACE FUNCTION update_job_queue() RETURNS TRIGGER OLD.pgsql_node_id; PERFORM pg_notify('channel_bs' || OLD.backup_server_id || '_pg' || OLD.pgsql_node_id,'Backup jobs for ' || pgsql_node_ || ' updated on ' || backup_server_); - END IF; + END IF; END IF; @@ -1342,7 +1342,7 @@ $$; ALTER FUNCTION update_job_queue() OWNER TO pgbackman_role_rw; CREATE TRIGGER update_job_queue AFTER INSERT OR UPDATE OR DELETE - ON backup_definition FOR EACH ROW + ON backup_definition FOR EACH ROW EXECUTE PROCEDURE update_job_queue(); @@ -1351,13 +1351,13 @@ CREATE TRIGGER update_job_queue AFTER INSERT OR UPDATE OR DELETE -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION notify_new_snapshot() RETURNS TRIGGER - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ BEGIN PERFORM pg_notify('channel_snapshot_defined','Snapshot defined'); - + RETURN NULL; END; $$; @@ -1374,13 +1374,13 @@ CREATE TRIGGER notify_new_snapshot AFTER INSERT -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION notify_new_restore() RETURNS TRIGGER - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ BEGIN PERFORM pg_notify('channel_restore_defined','Restore defined'); - + RETURN NULL; END; $$; @@ -1398,8 +1398,8 @@ CREATE TRIGGER notify_new_restore AFTER INSERT -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION update_restore_logs_to_delete() RETURNS TRIGGER - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ BEGIN @@ -1407,7 +1407,7 @@ CREATE OR REPLACE FUNCTION update_restore_logs_to_delete() RETURNS TRIGGER EXECUTE 'INSERT INTO restore_logs_to_delete (backup_server_id,restore_log_file) VALUES ($1,$2)' USING OLD.backup_server_id, OLD.restore_log_file; - + RETURN NULL; END; $$; @@ -1426,8 +1426,8 @@ CREATE TRIGGER update_restore_logs_to_delete AFTER DELETE -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION generate_backup_catalog_alert() RETURNS TRIGGER - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -1448,7 +1448,7 @@ CREATE OR REPLACE FUNCTION generate_backup_catalog_alert() RETURNS TRIGGER NEW.execution_status, NEW.error_message, sendto_, - FALSE; + FALSE; ELSEIF NEW.execution_status = 'ERROR' AND NEW.def_id IS NULL THEN @@ -1462,8 +1462,8 @@ CREATE OR REPLACE FUNCTION generate_backup_catalog_alert() RETURNS TRIGGER NEW.execution_status, NEW.error_message, sendto_, - FALSE; - END IF; + FALSE; + END IF; RETURN NULL; END; @@ -1481,8 +1481,8 @@ CREATE TRIGGER generate_backup_catalog_alert AFTER INSERT OR UPDATE -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION get_next_crontab_id_to_generate(INTEGER) RETURNS INTEGER - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -1492,18 +1492,18 @@ CREATE OR REPLACE FUNCTION get_next_crontab_id_to_generate(INTEGER) RETURNS INTE BEGIN -- --- We got the idea for this function from +-- We got the idea for this function from -- https://github.com/ryandotsmith/queue_classic/ --- +-- -- If we can not get a lock right away for SELECT FOR UPDATE -- we abort the select with NOWAIT, wait random() ms. and try again. --- With this we try to avoid problems in system with a lot of +-- With this we try to avoid problems in system with a lot of -- concurrency processes trying to get a job assigned. -- LOOP BEGIN - EXECUTE 'SELECT id' + EXECUTE 'SELECT id' || ' FROM job_queue' || ' WHERE backup_server_id = $1' || ' AND is_assigned IS FALSE' @@ -1539,16 +1539,16 @@ ALTER FUNCTION get_next_crontab_id_to_generate(INTEGER) OWNER TO pgbackman_role_ -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION register_backup_server(TEXT,TEXT,CHARACTER VARYING,TEXT) RETURNS VOID - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE - + hostname_ ALIAS FOR $1; domain_name_ ALIAS FOR $2; status_ ALIAS FOR $3; - remarks_ ALIAS FOR $4; + remarks_ ALIAS FOR $4; v_msg TEXT; v_detail TEXT; @@ -1571,15 +1571,15 @@ CREATE OR REPLACE FUNCTION register_backup_server(TEXT,TEXT,CHARACTER VARYING,TE USING hostname_, domain_name_, status_, - remarks_; + remarks_; EXCEPTION WHEN others THEN - GET STACKED DIAGNOSTICS + GET STACKED DIAGNOSTICS v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_context = PG_EXCEPTION_CONTEXT; RAISE EXCEPTION E'\n----------------------------------------------\nEXCEPTION:\n----------------------------------------------\nMESSAGE: % \nDETAIL : % \n----------------------------------------------', v_msg, v_detail; - + END; $$; @@ -1591,8 +1591,8 @@ ALTER FUNCTION register_backup_server(TEXT,TEXT,CHARACTER VARYING,TEXT) OWNER TO -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION delete_backup_server(INTEGER) RETURNS VOID - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -1610,16 +1610,16 @@ CREATE OR REPLACE FUNCTION delete_backup_server(INTEGER) RETURNS VOID EXECUTE 'DELETE FROM backup_server_config WHERE server_id = $1' USING backup_server_id_; - + EXECUTE 'DELETE FROM backup_server WHERE server_id = $1' USING backup_server_id_; ELSE - RAISE EXCEPTION 'Backup server % does not exist',backup_server_id_; + RAISE EXCEPTION 'Backup server % does not exist',backup_server_id_; END IF; - + EXCEPTION WHEN others THEN - GET STACKED DIAGNOSTICS + GET STACKED DIAGNOSTICS v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_context = PG_EXCEPTION_CONTEXT; @@ -1634,8 +1634,8 @@ ALTER FUNCTION delete_backup_server(INTEGER) OWNER TO pgbackman_role_rw; -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION update_backup_server(INTEGER,TEXT) RETURNS VOID - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -1655,13 +1655,13 @@ CREATE OR REPLACE FUNCTION update_backup_server(INTEGER,TEXT) RETURNS VOID EXECUTE 'UPDATE backup_server SET remarks = $2 WHERE server_id = $1' USING backup_server_id_, remarks_; - + ELSE - RAISE EXCEPTION 'Backup server % does not exist',backup_server_id_; + RAISE EXCEPTION 'Backup server % does not exist',backup_server_id_; END IF; - + EXCEPTION WHEN others THEN - GET STACKED DIAGNOSTICS + GET STACKED DIAGNOSTICS v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_context = PG_EXCEPTION_CONTEXT; @@ -1676,18 +1676,18 @@ ALTER FUNCTION update_backup_server(INTEGER,TEXT) OWNER TO pgbackman_role_rw; -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION register_pgsql_node(TEXT,TEXT,INTEGER,TEXT,CHARACTER VARYING,TEXT) RETURNS VOID - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE - + hostname_ ALIAS FOR $1; domain_name_ ALIAS FOR $2; - pgport_ ALIAS FOR $3; + pgport_ ALIAS FOR $3; admin_role_ ALIAS FOR $4; status_ ALIAS FOR $5; - remarks_ ALIAS FOR $6; + remarks_ ALIAS FOR $6; v_msg TEXT; v_detail TEXT; @@ -1720,15 +1720,15 @@ CREATE OR REPLACE FUNCTION register_pgsql_node(TEXT,TEXT,INTEGER,TEXT,CHARACTER pgport_, admin_role_, status_, - remarks_; + remarks_; EXCEPTION WHEN others THEN - GET STACKED DIAGNOSTICS + GET STACKED DIAGNOSTICS v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_context = PG_EXCEPTION_CONTEXT; RAISE EXCEPTION E'\n----------------------------------------------\nEXCEPTION:\n----------------------------------------------\nMESSAGE: % \nDETAIL : % \n----------------------------------------------\n', v_msg, v_detail; - + END; $$; @@ -1740,8 +1740,8 @@ ALTER FUNCTION register_pgsql_node(TEXT,TEXT,INTEGER,TEXT,CHARACTER VARYING,TEXT -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION delete_pgsql_node(INTEGER) RETURNS VOID - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -1755,8 +1755,8 @@ CREATE OR REPLACE FUNCTION delete_pgsql_node(INTEGER) RETURNS VOID SELECT count(*) FROM pgsql_node WHERE node_id = pgsql_node_id_ INTO node_cnt; - IF node_cnt !=0 THEN - + IF node_cnt !=0 THEN + EXECUTE 'DELETE FROM pgsql_node_config WHERE node_id = $1' USING pgsql_node_id_; @@ -1768,10 +1768,10 @@ CREATE OR REPLACE FUNCTION delete_pgsql_node(INTEGER) RETURNS VOID ELSE RAISE EXCEPTION 'PgSQL node % does not exist',pgsql_node_id_; - END IF; + END IF; EXCEPTION WHEN others THEN - GET STACKED DIAGNOSTICS + GET STACKED DIAGNOSTICS v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_context = PG_EXCEPTION_CONTEXT; @@ -1788,8 +1788,8 @@ ALTER FUNCTION delete_pgsql_node(INTEGER) OWNER TO pgbackman_role_rw; -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION update_pgsql_node(INTEGER,INTEGER,TEXT,TEXT,TEXT) RETURNS VOID - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -1815,13 +1815,13 @@ CREATE OR REPLACE FUNCTION update_pgsql_node(INTEGER,INTEGER,TEXT,TEXT,TEXT) RET admin_user_, status_, remarks_; - + ELSE - RAISE EXCEPTION 'PgSQL node % does not exist',pgsql_node_id_; + RAISE EXCEPTION 'PgSQL node % does not exist',pgsql_node_id_; END IF; - + EXCEPTION WHEN others THEN - GET STACKED DIAGNOSTICS + GET STACKED DIAGNOSTICS v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_context = PG_EXCEPTION_CONTEXT; @@ -1837,8 +1837,8 @@ ALTER FUNCTION update_pgsql_node(INTEGER,INTEGER,TEXT,TEXT,TEXT) OWNER TO pgback -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION update_pgsql_node_config(INTEGER,TEXT,TEXT,TEXT,TEXT,TEXT,TEXT,INTERVAL,INTEGER,INTERVAL,TEXT,TEXT,TEXT,TEXT,TEXT,TEXT,INTEGER,TEXT,TEXT,TEXT) RETURNS VOID - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -1846,7 +1846,7 @@ CREATE OR REPLACE FUNCTION update_pgsql_node_config(INTEGER,TEXT,TEXT,TEXT,TEXT, backup_minutes_interval_ ALIAS FOR $2; backup_hours_interval_ ALIAS FOR $3; backup_weekday_cron_ ALIAS FOR $4; - backup_month_cron_ ALIAS FOR $5; + backup_month_cron_ ALIAS FOR $5; backup_day_month_cron_ ALIAS FOR $6; backup_code_ ALIAS FOR $7; retention_period_ ALIAS FOR $8; @@ -1876,7 +1876,7 @@ CREATE OR REPLACE FUNCTION update_pgsql_node_config(INTEGER,TEXT,TEXT,TEXT,TEXT, EXECUTE 'UPDATE pgsql_node_config SET value = $2 WHERE node_id = $1 AND parameter = ''backup_minutes_interval''' USING pgsql_node_id_, backup_minutes_interval_; - + EXECUTE 'UPDATE pgsql_node_config SET value = $2 WHERE node_id = $1 AND parameter = ''backup_hours_interval''' USING pgsql_node_id_, backup_hours_interval_; @@ -1907,7 +1907,7 @@ CREATE OR REPLACE FUNCTION update_pgsql_node_config(INTEGER,TEXT,TEXT,TEXT,TEXT, EXECUTE 'UPDATE pgsql_node_config SET value = $2 WHERE node_id = $1 AND parameter = ''automatic_deletion_retention''' USING pgsql_node_id_, - automatic_deletion_retention_; + automatic_deletion_retention_; EXECUTE 'UPDATE pgsql_node_config SET value = $2 WHERE node_id = $1 AND parameter = ''extra_backup_parameters''' USING pgsql_node_id_, @@ -1950,11 +1950,11 @@ CREATE OR REPLACE FUNCTION update_pgsql_node_config(INTEGER,TEXT,TEXT,TEXT,TEXT, pgsql_node_status_; ELSE - RAISE EXCEPTION 'PgSQL node % does not exist',pgsql_node_id_; + RAISE EXCEPTION 'PgSQL node % does not exist',pgsql_node_id_; END IF; - + EXCEPTION WHEN others THEN - GET STACKED DIAGNOSTICS + GET STACKED DIAGNOSTICS v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_context = PG_EXCEPTION_CONTEXT; @@ -1970,8 +1970,8 @@ ALTER FUNCTION update_pgsql_node_config(INTEGER,TEXT,TEXT,TEXT,TEXT,TEXT,TEXT,IN -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION update_backup_server_config(INTEGER,TEXT,TEXT,TEXT,TEXT,TEXT,TEXT,TEXT,TEXT,TEXT) RETURNS VOID - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -2003,7 +2003,7 @@ CREATE OR REPLACE FUNCTION update_backup_server_config(INTEGER,TEXT,TEXT,TEXT,TE EXECUTE 'UPDATE backup_server_config SET value = $2 WHERE server_id = $1 AND parameter = ''pgsql_bin_9_1''' USING backup_server_id_, pgsql_bin_9_1_; - + EXECUTE 'UPDATE backup_server_config SET value = $2 WHERE server_id = $1 AND parameter = ''pgsql_bin_9_2''' USING backup_server_id_, pgsql_bin_9_2_; @@ -2033,11 +2033,11 @@ CREATE OR REPLACE FUNCTION update_backup_server_config(INTEGER,TEXT,TEXT,TEXT,TE root_backup_partition_; ELSE - RAISE EXCEPTION 'Backup server % does not exist',backup_server_id_; + RAISE EXCEPTION 'Backup server % does not exist',backup_server_id_; END IF; - + EXCEPTION WHEN others THEN - GET STACKED DIAGNOSTICS + GET STACKED DIAGNOSTICS v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_context = PG_EXCEPTION_CONTEXT; @@ -2053,20 +2053,20 @@ ALTER FUNCTION update_backup_server_config(INTEGER,TEXT,TEXT,TEXT,TEXT,TEXT,TEXT -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION register_backup_definition(INTEGER,INTEGER,TEXT,CHARACTER VARYING,CHARACTER VARYING,CHARACTER VARYING,CHARACTER VARYING,CHARACTER VARYING,CHARACTER VARYING,BOOLEAN,INTERVAL,INTEGER,TEXT,CHARACTER VARYING,TEXT) RETURNS VOID - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE - + backup_server_id_ ALIAS FOR $1; pgsql_node_id_ ALIAS FOR $2; - dbname_ ALIAS FOR $3; + dbname_ ALIAS FOR $3; minutes_cron_ ALIAS FOR $4; hours_cron_ ALIAS FOR $5; - day_month_cron_ ALIAS FOR $6; + day_month_cron_ ALIAS FOR $6; month_cron_ ALIAS FOR $7; - weekday_cron_ ALIAS FOR $8; + weekday_cron_ ALIAS FOR $8; backup_code_ ALIAS FOR $9; encryption_ ALIAS FOR $10; retention_period_ ALIAS FOR $11; @@ -2076,7 +2076,7 @@ CREATE OR REPLACE FUNCTION register_backup_definition(INTEGER,INTEGER,TEXT,CHARA remarks_ ALIAS FOR $15; server_cnt INTEGER; - node_cnt INTEGER; + node_cnt INTEGER; backup_hours_interval TEXT; backup_minutes_interval TEXT; @@ -2100,7 +2100,7 @@ CREATE OR REPLACE FUNCTION register_backup_definition(INTEGER,INTEGER,TEXT,CHARA IF hours_cron_ = '' OR hours_cron_ IS NULL THEN backup_hours_interval := get_default_pgsql_node_parameter('backup_hours_interval'); hours_cron_ := get_hour_from_interval(backup_hours_interval)::TEXT; - END IF; + END IF; IF minutes_cron_ = '' OR minutes_cron_ IS NULL THEN backup_minutes_interval := get_default_pgsql_node_parameter('backup_minutes_interval'); @@ -2130,7 +2130,7 @@ CREATE OR REPLACE FUNCTION register_backup_definition(INTEGER,INTEGER,TEXT,CHARA IF retention_period_ IS NULL THEN retention_period_ := get_default_pgsql_node_parameter('retention_period')::INTERVAL; END IF; - + IF retention_redundancy_ = 0 OR retention_redundancy_ IS NULL THEN retention_redundancy_ := get_default_pgsql_node_parameter('retention_redundancy')::INTEGER; END IF; @@ -2138,7 +2138,7 @@ CREATE OR REPLACE FUNCTION register_backup_definition(INTEGER,INTEGER,TEXT,CHARA IF extra_backup_parameters_ = '' OR extra_backup_parameters_ IS NULL THEN extra_backup_parameters_ := get_default_pgsql_node_parameter('extra_backup_parameters'); END IF; - + IF job_status_ = '' OR job_status_ IS NULL THEN job_status_ := get_default_pgsql_node_parameter('backup_job_status'); END IF; @@ -2173,10 +2173,10 @@ CREATE OR REPLACE FUNCTION register_backup_definition(INTEGER,INTEGER,TEXT,CHARA retention_redundancy_, extra_backup_parameters_, job_status_, - remarks_; + remarks_; EXCEPTION WHEN others THEN - GET STACKED DIAGNOSTICS + GET STACKED DIAGNOSTICS v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_context = PG_EXCEPTION_CONTEXT; @@ -2193,8 +2193,8 @@ ALTER FUNCTION register_backup_definition(INTEGER,INTEGER,TEXT,CHARACTER VARYING -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION delete_backup_definition_id(INTEGER) RETURNS VOID - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -2212,13 +2212,13 @@ CREATE OR REPLACE FUNCTION delete_backup_definition_id(INTEGER) RETURNS VOID EXECUTE 'DELETE FROM backup_definition WHERE def_id = $1' USING def_id_; - + ELSE - RAISE EXCEPTION 'Backup job definition ID:% does not exist',def_id_; + RAISE EXCEPTION 'Backup job definition ID:% does not exist',def_id_; END IF; - + EXCEPTION WHEN others THEN - GET STACKED DIAGNOSTICS + GET STACKED DIAGNOSTICS v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_context = PG_EXCEPTION_CONTEXT; @@ -2234,8 +2234,8 @@ ALTER FUNCTION delete_backup_definition_id(INTEGER) OWNER TO pgbackman_role_rw; -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION delete_force_backup_definition_id(INTEGER) RETURNS VOID - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -2252,7 +2252,7 @@ CREATE OR REPLACE FUNCTION delete_force_backup_definition_id(INTEGER) RETURNS VO IF def_cnt != 0 THEN EXECUTE 'WITH del_catid AS ( - DELETE FROM backup_catalog + DELETE FROM backup_catalog WHERE def_id = $1 RETURNING def_id, bck_id, @@ -2274,18 +2274,18 @@ CREATE OR REPLACE FUNCTION delete_force_backup_definition_id(INTEGER) RETURNS VO pg_dump_roles_log_file, pg_dump_dbconfig_file, pg_dump_dbconfig_log_file) - SELECT * FROM del_catid + SELECT * FROM del_catid ) DELETE FROM backup_definition WHERE def_id = $1;' USING def_id_; ELSE - RAISE EXCEPTION 'Backup job definition ID:% does not exist',def_id_; + RAISE EXCEPTION 'Backup job definition ID:% does not exist',def_id_; END IF; - + EXCEPTION WHEN others THEN - GET STACKED DIAGNOSTICS + GET STACKED DIAGNOSTICS v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_context = PG_EXCEPTION_CONTEXT; @@ -2301,8 +2301,8 @@ ALTER FUNCTION delete_force_backup_definition_id(INTEGER) OWNER TO pgbackman_rol -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION delete_backup_definition_dbname(INTEGER,TEXT) RETURNS VOID - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -2324,11 +2324,11 @@ CREATE OR REPLACE FUNCTION delete_backup_definition_dbname(INTEGER,TEXT) RETURNS dbname_; ELSE - RAISE EXCEPTION 'No backup job definition for dbname: %s and PgSQL node: %s',dbname_,pgsql_node_id_; + RAISE EXCEPTION 'No backup job definition for dbname: %s and PgSQL node: %s',dbname_,pgsql_node_id_; END IF; - + EXCEPTION WHEN others THEN - GET STACKED DIAGNOSTICS + GET STACKED DIAGNOSTICS v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_context = PG_EXCEPTION_CONTEXT; @@ -2344,8 +2344,8 @@ ALTER FUNCTION delete_backup_definition_dbname(INTEGER,TEXT) OWNER TO pgbackman_ -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION delete_force_backup_definition_dbname(INTEGER,TEXT) RETURNS VOID - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -2363,7 +2363,7 @@ CREATE OR REPLACE FUNCTION delete_force_backup_definition_dbname(INTEGER,TEXT) R IF def_cnt != 0 THEN EXECUTE 'WITH del_catid AS ( - DELETE FROM backup_catalog + DELETE FROM backup_catalog WHERE pgsql_node_id = $1 AND dbname = $2 AND snapshot_id IS NULL @@ -2387,7 +2387,7 @@ CREATE OR REPLACE FUNCTION delete_force_backup_definition_dbname(INTEGER,TEXT) R pg_dump_roles_log_file, pg_dump_dbconfig_file, pg_dump_dbconfig_log_file) - SELECT * FROM del_catid + SELECT * FROM del_catid ) DELETE FROM backup_definition WHERE pgsql_node_id = $1 @@ -2396,11 +2396,11 @@ CREATE OR REPLACE FUNCTION delete_force_backup_definition_dbname(INTEGER,TEXT) R dbname_; ELSE - RAISE EXCEPTION 'No backup job definition for dbname: %s and PgSQL node: %s',dbname_,pgsql_node_id_; + RAISE EXCEPTION 'No backup job definition for dbname: %s and PgSQL node: %s',dbname_,pgsql_node_id_; END IF; - + EXCEPTION WHEN others THEN - GET STACKED DIAGNOSTICS + GET STACKED DIAGNOSTICS v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_context = PG_EXCEPTION_CONTEXT; @@ -2416,8 +2416,8 @@ ALTER FUNCTION delete_force_backup_definition_dbname(INTEGER,TEXT) OWNER TO pgba -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION delete_pgsql_node_to_delete(INTEGER,INTEGER) RETURNS VOID - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -2432,9 +2432,9 @@ CREATE OR REPLACE FUNCTION delete_pgsql_node_to_delete(INTEGER,INTEGER) RETURNS EXECUTE 'DELETE FROM pgsql_node_to_delete WHERE backup_server_id = $1 AND pgsql_node_id = $2' USING backup_server_id_, pgsql_node_id_; - + EXCEPTION WHEN others THEN - GET STACKED DIAGNOSTICS + GET STACKED DIAGNOSTICS v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_context = PG_EXCEPTION_CONTEXT; @@ -2450,8 +2450,8 @@ ALTER FUNCTION delete_pgsql_node_to_delete(INTEGER,INTEGER) OWNER TO pgbackman_r -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION update_backup_definition(INTEGER,TEXT,TEXT,TEXT,TEXT,TEXT,INTERVAL,INTEGER,TEXT,TEXT,TEXT) RETURNS VOID - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -2478,11 +2478,11 @@ CREATE OR REPLACE FUNCTION update_backup_definition(INTEGER,TEXT,TEXT,TEXT,TEXT, IF defid_cnt != 0 THEN - EXECUTE 'UPDATE backup_definition SET minutes_cron = $2, - hours_cron = $3, + EXECUTE 'UPDATE backup_definition SET minutes_cron = $2, + hours_cron = $3, day_month_cron = $4, month_cron = $5, - weekday_cron = $6, + weekday_cron = $6, retention_period = $7, retention_redundancy = $8, extra_backup_parameters = $9, @@ -2501,13 +2501,13 @@ CREATE OR REPLACE FUNCTION update_backup_definition(INTEGER,TEXT,TEXT,TEXT,TEXT, extra_backup_parameters_, job_status_, remarks_; - + ELSE - RAISE EXCEPTION 'Backup definition with DefID: % does not exist',def_id_; + RAISE EXCEPTION 'Backup definition with DefID: % does not exist',def_id_; END IF; - + EXCEPTION WHEN others THEN - GET STACKED DIAGNOSTICS + GET STACKED DIAGNOSTICS v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_context = PG_EXCEPTION_CONTEXT; @@ -2523,24 +2523,24 @@ ALTER FUNCTION update_backup_definition(INTEGER,TEXT,TEXT,TEXT,TEXT,TEXT,INTERVA -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION register_snapshot_definition(INTEGER,INTEGER,TEXT,TIMESTAMP,CHARACTER VARYING,INTERVAL,TEXT,TEXT,TEXT) RETURNS VOID - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE - + backup_server_id_ ALIAS FOR $1; pgsql_node_id_ ALIAS FOR $2; - dbname_ ALIAS FOR $3; + dbname_ ALIAS FOR $3; at_time_ ALIAS FOR $4; backup_code_ ALIAS FOR $5; retention_period_ ALIAS FOR $6; extra_backup_parameters_ ALIAS FOR $7; remarks_ ALIAS FOR $8; - pg_dump_release_ ALIAS FOR $9; + pg_dump_release_ ALIAS FOR $9; server_cnt INTEGER; - node_cnt INTEGER; + node_cnt INTEGER; v_msg TEXT; v_detail TEXT; @@ -2565,11 +2565,11 @@ CREATE OR REPLACE FUNCTION register_snapshot_definition(INTEGER,INTEGER,TEXT,TIM IF retention_period_ IS NULL THEN retention_period_ := get_default_pgsql_node_parameter('retention_period')::INTERVAL; END IF; - + IF extra_backup_parameters_ = '' OR extra_backup_parameters_ IS NULL THEN extra_backup_parameters_ := get_default_pgsql_node_parameter('extra_backup_parameters'); END IF; - + EXECUTE 'INSERT INTO snapshot_definition (backup_server_id, pgsql_node_id, dbname, @@ -2588,10 +2588,10 @@ CREATE OR REPLACE FUNCTION register_snapshot_definition(INTEGER,INTEGER,TEXT,TIM retention_period_, extra_backup_parameters_, remarks_, - pg_dump_release_; + pg_dump_release_; EXCEPTION WHEN others THEN - GET STACKED DIAGNOSTICS + GET STACKED DIAGNOSTICS v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_context = PG_EXCEPTION_CONTEXT; @@ -2608,8 +2608,8 @@ ALTER FUNCTION register_snapshot_definition(INTEGER,INTEGER,TEXT,TIMESTAMP,CHARA -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION update_snapshot_status(INTEGER,TEXT) RETURNS VOID - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -2624,9 +2624,9 @@ CREATE OR REPLACE FUNCTION update_snapshot_status(INTEGER,TEXT) RETURNS VOID EXECUTE 'UPDATE snapshot_definition SET status = $2 WHERE snapshot_id = $1' USING snapshot_id_, upper(status_); - + EXCEPTION WHEN others THEN - GET STACKED DIAGNOSTICS + GET STACKED DIAGNOSTICS v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_context = PG_EXCEPTION_CONTEXT; @@ -2641,8 +2641,8 @@ ALTER FUNCTION update_snapshot_status(INTEGER,TEXT) OWNER TO pgbackman_role_rw; -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION update_restore_status(INTEGER,TEXT) RETURNS VOID - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -2657,9 +2657,9 @@ CREATE OR REPLACE FUNCTION update_restore_status(INTEGER,TEXT) RETURNS VOID EXECUTE 'UPDATE restore_definition SET status = $2 WHERE restore_def = $1' USING restore_def_, upper(status_); - + EXCEPTION WHEN others THEN - GET STACKED DIAGNOSTICS + GET STACKED DIAGNOSTICS v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_context = PG_EXCEPTION_CONTEXT; @@ -2675,8 +2675,8 @@ ALTER FUNCTION update_restore_status(INTEGER,TEXT) OWNER TO pgbackman_role_rw; -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION update_alert_sent(INTEGER,BOOLEAN) RETURNS VOID - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -2698,9 +2698,9 @@ CREATE OR REPLACE FUNCTION update_alert_sent(INTEGER,BOOLEAN) RETURNS VOID EXECUTE 'UPDATE alerts SET alert_sent = $2 WHERE alert_id = $1' USING alert_id_, alert_sent_; - + EXCEPTION WHEN others THEN - GET STACKED DIAGNOSTICS + GET STACKED DIAGNOSTICS v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_context = PG_EXCEPTION_CONTEXT; @@ -2716,14 +2716,14 @@ ALTER FUNCTION update_alert_sent(INTEGER,BOOLEAN) OWNER TO pgbackman_role_rw; -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION delete_alert(INTEGER) RETURNS VOID - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE alert_id_ ALIAS FOR $1; alert_id_cnt INTEGER; - + v_msg TEXT; v_detail TEXT; v_context TEXT; @@ -2738,9 +2738,9 @@ CREATE OR REPLACE FUNCTION delete_alert(INTEGER) RETURNS VOID EXECUTE 'DELETE FROM alerts WHERE alert_id = $1' USING alert_id_; - + EXCEPTION WHEN others THEN - GET STACKED DIAGNOSTICS + GET STACKED DIAGNOSTICS v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_context = PG_EXCEPTION_CONTEXT; @@ -2756,8 +2756,8 @@ ALTER FUNCTION delete_alert(INTEGER) OWNER TO pgbackman_role_rw; -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION update_backup_definition_status_to_delete(INTEGER) RETURNS VOID - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -2770,9 +2770,9 @@ CREATE OR REPLACE FUNCTION update_backup_definition_status_to_delete(INTEGER) RE EXECUTE 'UPDATE backup_definition SET job_status = ''DELETED'' WHERE def_id = $1' USING def_id_; - + EXCEPTION WHEN others THEN - GET STACKED DIAGNOSTICS + GET STACKED DIAGNOSTICS v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_context = PG_EXCEPTION_CONTEXT; @@ -2787,13 +2787,13 @@ ALTER FUNCTION update_backup_definition_status_to_delete(INTEGER) OWNER TO pgbac -- Function: get_default_backup_server_parameter() -- ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION get_default_backup_server_parameter(TEXT) RETURNS TEXT - LANGUAGE plpgsql - SECURITY INVOKER +CREATE OR REPLACE FUNCTION get_default_backup_server_parameter(TEXT) RETURNS TEXT + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE - parameter_ ALIAS FOR $1; + parameter_ ALIAS FOR $1; value_ TEXT := ''; BEGIN @@ -2816,13 +2816,13 @@ ALTER FUNCTION get_default_backup_server_parameter(TEXT) OWNER TO pgbackman_role -- Function: get_default_pgsql_node_parameter() -- ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION get_default_pgsql_node_parameter(TEXT) RETURNS TEXT - LANGUAGE plpgsql - SECURITY INVOKER +CREATE OR REPLACE FUNCTION get_default_pgsql_node_parameter(TEXT) RETURNS TEXT + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE - parameter_ ALIAS FOR $1; + parameter_ ALIAS FOR $1; value_ TEXT := ''; BEGIN @@ -2845,14 +2845,14 @@ ALTER FUNCTION get_default_pgsql_node_parameter(TEXT) OWNER TO pgbackman_role_rw -- Function: get_backup_definition_def_value() -- ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION get_backup_definition_def_value(INTEGER,TEXT) RETURNS TEXT - LANGUAGE plpgsql - SECURITY INVOKER +CREATE OR REPLACE FUNCTION get_backup_definition_def_value(INTEGER,TEXT) RETURNS TEXT + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE def_id_ ALIAS FOR $1; - parameter_ ALIAS FOR $2; + parameter_ ALIAS FOR $2; value_ TEXT := ''; defid_cnt INTEGER; @@ -2876,19 +2876,19 @@ CREATE OR REPLACE FUNCTION get_backup_definition_def_value(INTEGER,TEXT) RETURNS ELSIF parameter_ = 'month_cron' THEN SELECT month_cron FROM backup_definition WHERE def_id = def_id_ INTO value_; - + ELSIF parameter_ = 'weekday_cron' THEN SELECT weekday_cron FROM backup_definition WHERE def_id = def_id_ INTO value_; - + ELSIF parameter_ = 'retention_period' THEN SELECT retention_period FROM backup_definition WHERE def_id = def_id_ INTO value_; - + ELSIF parameter_ = 'retention_redundancy' THEN SELECT retention_redundancy FROM backup_definition WHERE def_id = def_id_ INTO value_; - + ELSIF parameter_ = 'extra_backup_parameters' THEN SELECT extra_backup_parameters FROM backup_definition WHERE def_id = def_id_ INTO value_; - + ELSIF parameter_ = 'job_status' THEN SELECT job_status FROM backup_definition WHERE def_id = def_id_ INTO value_; @@ -2910,14 +2910,14 @@ ALTER FUNCTION get_backup_definition_def_value(INTEGER,TEXT) OWNER TO pgbackman_ -- Function: get_pgsql_node_def_value() -- ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION get_pgsql_node_def_value(INTEGER,TEXT) RETURNS TEXT - LANGUAGE plpgsql - SECURITY INVOKER +CREATE OR REPLACE FUNCTION get_pgsql_node_def_value(INTEGER,TEXT) RETURNS TEXT + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE pgsql_node_id_ ALIAS FOR $1; - parameter_ ALIAS FOR $2; + parameter_ ALIAS FOR $2; value_ TEXT := ''; nodeid_cnt INTEGER; @@ -2956,14 +2956,14 @@ ALTER FUNCTION get_pgsql_node_def_value(INTEGER,TEXT) OWNER TO pgbackman_role_rw -- Function: get_backup_server_def_value() -- ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION get_backup_server_def_value(INTEGER,TEXT) RETURNS TEXT - LANGUAGE plpgsql - SECURITY INVOKER +CREATE OR REPLACE FUNCTION get_backup_server_def_value(INTEGER,TEXT) RETURNS TEXT + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE backup_server_id_ ALIAS FOR $1; - parameter_ ALIAS FOR $2; + parameter_ ALIAS FOR $2; value_ TEXT := ''; serverid_cnt INTEGER; @@ -2994,14 +2994,14 @@ ALTER FUNCTION get_backup_server_def_value(INTEGER,TEXT) OWNER TO pgbackman_role -- Function: get_pgsql_node_config_value() -- ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION get_pgsql_node_config_value(INTEGER,TEXT) RETURNS TEXT - LANGUAGE plpgsql - SECURITY INVOKER +CREATE OR REPLACE FUNCTION get_pgsql_node_config_value(INTEGER,TEXT) RETURNS TEXT + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE pgsql_node_id_ ALIAS FOR $1; - parameter_ ALIAS FOR $2; + parameter_ ALIAS FOR $2; value_ TEXT := ''; pgsql_node_id_cnt INTEGER; @@ -3031,14 +3031,14 @@ ALTER FUNCTION get_pgsql_node_config_value(INTEGER,TEXT) OWNER TO pgbackman_role -- Function: get_backup_server_config_value() -- ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION get_backup_server_config_value(INTEGER,TEXT) RETURNS TEXT - LANGUAGE plpgsql - SECURITY INVOKER +CREATE OR REPLACE FUNCTION get_backup_server_config_value(INTEGER,TEXT) RETURNS TEXT + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE backup_server_id_ ALIAS FOR $1; - parameter_ ALIAS FOR $2; + parameter_ ALIAS FOR $2; value_ TEXT := ''; backup_server_id_cnt INTEGER; @@ -3068,9 +3068,9 @@ ALTER FUNCTION get_backup_server_config_value(INTEGER,TEXT) OWNER TO pgbackman_r -- Function: check_pgsql_node_status() -- ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION check_pgsql_node_status(INTEGER) RETURNS VOID - LANGUAGE plpgsql - SECURITY INVOKER +CREATE OR REPLACE FUNCTION check_pgsql_node_status(INTEGER) RETURNS VOID + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -3100,20 +3100,20 @@ ALTER FUNCTION check_pgsql_node_status(INTEGER) OWNER TO pgbackman_role_rw; -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION get_hour_from_interval(TEXT) RETURNS TEXT - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE - hour_interval_ ALIAS FOR $1; - + hour_interval_ ALIAS FOR $1; + hour_from INTEGER; hour_to INTEGER; value_ INTEGER; v_msg TEXT; v_detail TEXT; - v_context TEXT; + v_context TEXT; BEGIN -- -- This function returns a value from an interval defined as 'Num1-Num2' @@ -3133,7 +3133,7 @@ CREATE OR REPLACE FUNCTION get_hour_from_interval(TEXT) RETURNS TEXT RETURN lpad(value_::TEXT,2,'0'); EXCEPTION WHEN others THEN - GET STACKED DIAGNOSTICS + GET STACKED DIAGNOSTICS v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_context = PG_EXCEPTION_CONTEXT; @@ -3150,13 +3150,13 @@ ALTER FUNCTION get_hour_from_interval(TEXT) OWNER TO pgbackman_role_rw; -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION get_minute_from_interval(TEXT) RETURNS TEXT - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE - minute_interval_ ALIAS FOR $1; - + minute_interval_ ALIAS FOR $1; + minute_from INTEGER; minute_to INTEGER; value_ INTEGER; @@ -3183,7 +3183,7 @@ CREATE OR REPLACE FUNCTION get_minute_from_interval(TEXT) RETURNS TEXT RETURN lpad(value_::TEXT,2,'0'); EXCEPTION WHEN others THEN - GET STACKED DIAGNOSTICS + GET STACKED DIAGNOSTICS v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_context = PG_EXCEPTION_CONTEXT; @@ -3199,13 +3199,13 @@ ALTER FUNCTION get_minute_from_interval(TEXT) OWNER TO pgbackman_role_rw; -- Function: get_backup_server_fqdn() -- ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION get_backup_server_fqdn(INTEGER) RETURNS TEXT - LANGUAGE plpgsql - SECURITY INVOKER +CREATE OR REPLACE FUNCTION get_backup_server_fqdn(INTEGER) RETURNS TEXT + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE - parameter_ ALIAS FOR $1; + parameter_ ALIAS FOR $1; backup_server_ TEXT := ''; BEGIN @@ -3230,13 +3230,13 @@ ALTER FUNCTION get_backup_server_fqdn(INTEGER) OWNER TO pgbackman_role_rw; -- Function: get_backup_server_id() -- ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION get_backup_server_id(TEXT) RETURNS INTEGER - LANGUAGE plpgsql - SECURITY INVOKER +CREATE OR REPLACE FUNCTION get_backup_server_id(TEXT) RETURNS INTEGER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE - backup_server_fqdn ALIAS FOR $1; + backup_server_fqdn ALIAS FOR $1; backup_server_id_ TEXT := ''; BEGIN -- @@ -3260,13 +3260,13 @@ ALTER FUNCTION get_backup_server_id(TEXT) OWNER TO pgbackman_role_rw; -- Function: get_pgsql_node_fqdn() -- ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION get_pgsql_node_fqdn(INTEGER) RETURNS TEXT - LANGUAGE plpgsql - SECURITY INVOKER +CREATE OR REPLACE FUNCTION get_pgsql_node_fqdn(INTEGER) RETURNS TEXT + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE - parameter_ ALIAS FOR $1; + parameter_ ALIAS FOR $1; pgsql_node_ TEXT := ''; BEGIN @@ -3291,13 +3291,13 @@ ALTER FUNCTION get_pgsql_node_fqdn(INTEGER) OWNER TO pgbackman_role_rw; -- Function: get_pgsql_node_id() -- ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION get_pgsql_node_id(TEXT) RETURNS INTEGER - LANGUAGE plpgsql - SECURITY INVOKER +CREATE OR REPLACE FUNCTION get_pgsql_node_id(TEXT) RETURNS INTEGER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE - pgsql_node_fqdn ALIAS FOR $1; + pgsql_node_fqdn ALIAS FOR $1; pgsql_node_id_ TEXT := ''; BEGIN @@ -3322,9 +3322,9 @@ ALTER FUNCTION get_pgsql_node_id(TEXT) OWNER TO pgbackman_role_rw; -- Function: get_listen_channel_names() -- ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION get_listen_channel_names(INTEGER) RETURNS SETOF TEXT +CREATE OR REPLACE FUNCTION get_listen_channel_names(INTEGER) RETURNS SETOF TEXT LANGUAGE sql - SECURITY INVOKER + SECURITY INVOKER SET search_path = public, pg_temp AS $$ SELECT 'channel_pgsql_node_running' AS channel @@ -3347,9 +3347,9 @@ ALTER FUNCTION get_listen_channel_names(INTEGER) OWNER TO pgbackman_role_rw; -- Function: generate_crontab_file() -- ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION generate_crontab_backup_jobs(INTEGER,INTEGER) RETURNS TEXT +CREATE OR REPLACE FUNCTION generate_crontab_backup_jobs(INTEGER,INTEGER) RETURNS TEXT LANGUAGE plpgsql - SECURITY INVOKER + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -3371,7 +3371,7 @@ CREATE OR REPLACE FUNCTION generate_crontab_backup_jobs(INTEGER,INTEGER) RETURNS output TEXT := ''; BEGIN - SELECT count(*) FROM pgsql_node WHERE node_id = pgsql_node_id_ INTO node_cnt; + SELECT count(*) FROM pgsql_node WHERE node_id = pgsql_node_id_ INTO node_cnt; IF node_cnt = 0 THEN RETURN output; @@ -3399,12 +3399,12 @@ BEGIN output := output || 'SHELL=/bin/bash' || E'\n'; output := output || 'PATH=/sbin:/bin:/usr/sbin:/usr/bin' || E'\n'; output := output || 'MAILTO=' || COALESCE(logs_email,'') || E'\n'; - output := output || E'\n'; + output := output || E'\n'; -- -- Generating backup jobs output for jobs -- with job_status = ACTIVE for a backup server - -- and a PgSQL node + -- and a PgSQL node -- FOR job_row IN ( @@ -3421,25 +3421,25 @@ BEGIN output := output || COALESCE(job_row.minutes_cron, '*') || ' ' || COALESCE(job_row.hours_cron, '*') || ' ' || COALESCE(job_row.day_month_cron, '*') || ' ' || COALESCE(job_row.month_cron, '*') || ' ' || COALESCE(job_row.weekday_cron, '*'); output := output || ' pgbackman'; - output := output || ' ' || pgbackman_dump || + output := output || ' ' || pgbackman_dump || ' --node-fqdn ' || pgsql_node_fqdn || ' --node-id ' || pgsql_node_id_ || ' --node-port ' || pgsql_node_port || - ' --node-user ' || admin_user || + ' --node-user ' || admin_user || ' --def-id ' || job_row.def_id; IF job_row.backup_code != 'CLUSTER' THEN output := output || ' --dbname ' || job_row.dbname; END IF; - output := output || ' --encryption ' || job_row.encryption::TEXT || + output := output || ' --encryption ' || job_row.encryption::TEXT || ' --backup-code ' || job_row.backup_code || ' --root-backup-dir ' || root_backup_dir; IF job_row.extra_backup_parameters != '' AND job_row.extra_backup_parameters IS NOT NULL THEN output := output || ' --extra-backup-parameters "''' || job_row.extra_backup_parameters || '''"'; END IF; - + output := output || E'\n'; END LOOP; @@ -3455,9 +3455,9 @@ ALTER FUNCTION generate_crontab_backup_jobs(INTEGER,INTEGER) OWNER TO pgbackman_ -- Function: generate_snapshot_at_file() -- ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION generate_snapshot_at_file(INTEGER) RETURNS TEXT +CREATE OR REPLACE FUNCTION generate_snapshot_at_file(INTEGER) RETURNS TEXT LANGUAGE plpgsql - SECURITY INVOKER + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -3477,7 +3477,7 @@ CREATE OR REPLACE FUNCTION generate_snapshot_at_file(INTEGER) RETURNS TEXT BEGIN SELECT backup_server_id FROM snapshot_definition WHERE snapshot_id = snapshot_id_ INTO backup_server_id_; - SELECT pgsql_node_id FROM snapshot_definition WHERE snapshot_id = snapshot_id_ INTO pgsql_node_id_; + SELECT pgsql_node_id FROM snapshot_definition WHERE snapshot_id = snapshot_id_ INTO pgsql_node_id_; root_backup_dir := get_backup_server_config_value(backup_server_id_,'root_backup_partition'); backup_server_fqdn := get_backup_server_fqdn(backup_server_id_); @@ -3493,11 +3493,11 @@ BEGIN ) LOOP output := output || 'su -l pgbackman -c "'; - output := output || pgbackman_dump || + output := output || pgbackman_dump || ' --node-fqdn ' || pgsql_node_fqdn || ' --node-id ' || pgsql_node_id_ || ' --node-port ' || pgsql_node_port || - ' --node-user ' || admin_user || + ' --node-user ' || admin_user || ' --snapshot-id ' || snapshot_row.snapshot_id::TEXT; IF snapshot_row.pg_dump_release != '' AND snapshot_row.pg_dump_release IS NOT NULL THEN @@ -3508,14 +3508,14 @@ BEGIN output := output || ' --dbname ' || snapshot_row.dbname; END IF; - output := output || ' --encryption ' || snapshot_row.encryption::TEXT || + output := output || ' --encryption ' || snapshot_row.encryption::TEXT || ' --backup-code ' || snapshot_row.backup_code || ' --root-backup-dir ' || root_backup_dir; IF snapshot_row.extra_backup_parameters != '' AND snapshot_row.extra_backup_parameters IS NOT NULL THEN output := output || E' --extra-backup-parameters \\"''' || snapshot_row.extra_backup_parameters || E'''\\"'; END IF; - + output := output || E'"\n'; END LOOP; @@ -3531,9 +3531,9 @@ ALTER FUNCTION generate_snapshot_at_file(INTEGER) OWNER TO pgbackman_role_rw; -- Function: generate_restore_at_file() -- ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION generate_restore_at_file(INTEGER) RETURNS TEXT +CREATE OR REPLACE FUNCTION generate_restore_at_file(INTEGER) RETURNS TEXT LANGUAGE plpgsql - SECURITY INVOKER + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -3552,7 +3552,7 @@ CREATE OR REPLACE FUNCTION generate_restore_at_file(INTEGER) RETURNS TEXT BEGIN SELECT backup_server_id FROM restore_definition WHERE restore_def = restore_def_ INTO backup_server_id_; - SELECT target_pgsql_node_id FROM restore_definition WHERE restore_def = restore_def_ INTO pgsql_node_id_; + SELECT target_pgsql_node_id FROM restore_definition WHERE restore_def = restore_def_ INTO pgsql_node_id_; root_backup_dir := get_backup_server_config_value(backup_server_id_,'root_backup_partition'); pgsql_node_fqdn := get_pgsql_node_fqdn(pgsql_node_id_); @@ -3572,18 +3572,18 @@ BEGIN b.pg_dump_file, b.pg_dump_roles_file, b.pg_dump_dbconfig_file, - b.pg_dump_release - FROM restore_definition a - JOIN backup_catalog b ON a.bck_id = b.bck_id + b.pg_dump_release + FROM restore_definition a + JOIN backup_catalog b ON a.bck_id = b.bck_id WHERE restore_def = restore_def_ ) LOOP output := output || 'su -l pgbackman -c "'; - output := output || pgbackman_restore || + output := output || pgbackman_restore || ' --node-fqdn ' || pgsql_node_fqdn || ' --node-id ' || pgsql_node_id_ || ' --node-port ' || pgsql_node_port || - ' --node-user ' || admin_user || + ' --node-user ' || admin_user || ' --restore-def ' || restore_row.restore_def::TEXT || ' --pgdump-file ' || restore_row.pg_dump_file || ' --pgdump-roles-file ' || restore_row.pg_dump_roles_file || @@ -3605,7 +3605,7 @@ BEGIN output := output || ' --pg-release ' || restore_row.pg_dump_release || ' --root-backup-dir ' || root_backup_dir; - + output := output || E'" \n'; END LOOP; @@ -3621,9 +3621,9 @@ ALTER FUNCTION generate_restore_at_file(INTEGER) OWNER TO pgbackman_role_rw; -- Function: get_pgsql_node_dsn() -- ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION get_pgsql_node_dsn(INTEGER) RETURNS TEXT - LANGUAGE plpgsql - SECURITY INVOKER +CREATE OR REPLACE FUNCTION get_pgsql_node_dsn(INTEGER) RETURNS TEXT + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -3647,9 +3647,9 @@ ALTER FUNCTION get_pgsql_node_dsn(INTEGER) OWNER TO pgbackman_role_rw; -- Function: get_pgsql_node_port() -- ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION get_pgsql_node_port(INTEGER) RETURNS TEXT - LANGUAGE plpgsql - SECURITY INVOKER +CREATE OR REPLACE FUNCTION get_pgsql_node_port(INTEGER) RETURNS TEXT + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -3673,9 +3673,9 @@ ALTER FUNCTION get_pgsql_node_port(INTEGER) OWNER TO pgbackman_role_rw; -- Function: get_pgsql_node_admin_user() -- ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION get_pgsql_node_admin_user(INTEGER) RETURNS TEXT - LANGUAGE plpgsql - SECURITY INVOKER +CREATE OR REPLACE FUNCTION get_pgsql_node_admin_user(INTEGER) RETURNS TEXT + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -3700,8 +3700,8 @@ ALTER FUNCTION get_pgsql_node_admin_user(INTEGER) OWNER TO pgbackman_role_rw; -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION register_backup_catalog(INTEGER,INTEGER,INTEGER,INTEGER,TEXT,TIMESTAMP WITH TIME ZONE,TIMESTAMP WITH TIME ZONE,INTERVAL,TEXT,BIGINT,TEXT,TEXT,BIGINT,TEXT,TEXT,BIGINT,TEXT,TEXT,TEXT,TEXT,TEXT,INTEGER,TEXT[],TEXT,TEXT) RETURNS VOID - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -3734,7 +3734,7 @@ CREATE OR REPLACE FUNCTION register_backup_catalog(INTEGER,INTEGER,INTEGER,INTEG v_msg TEXT; v_detail TEXT; - v_context TEXT; + v_context TEXT; BEGIN EXECUTE 'INSERT INTO backup_catalog (def_id, @@ -3761,7 +3761,7 @@ CREATE OR REPLACE FUNCTION register_backup_catalog(INTEGER,INTEGER,INTEGER,INTEG snapshot_id, role_list, pgsql_node_release, - pg_dump_release) + pg_dump_release) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25)' USING def_id_, procpid_, @@ -3790,12 +3790,12 @@ CREATE OR REPLACE FUNCTION register_backup_catalog(INTEGER,INTEGER,INTEGER,INTEG pg_dump_release_; EXCEPTION WHEN others THEN - GET STACKED DIAGNOSTICS + GET STACKED DIAGNOSTICS v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_context = PG_EXCEPTION_CONTEXT; RAISE EXCEPTION E'\n----------------------------------------------\nEXCEPTION:\n----------------------------------------------\nMESSAGE: % \nDETAIL : % \n----------------------------------------------\n', v_msg, v_detail; - + END; $$; @@ -3807,8 +3807,8 @@ ALTER FUNCTION register_backup_catalog(INTEGER,INTEGER,INTEGER,INTEGER,TEXT,TIME -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION delete_catalog_entries_to_delete(INTEGER) RETURNS VOID - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -3826,13 +3826,13 @@ CREATE OR REPLACE FUNCTION delete_catalog_entries_to_delete(INTEGER) RETURNS VOI EXECUTE 'DELETE FROM catalog_entries_to_delete WHERE del_id = $1' USING del_id_; - + ELSE - RAISE EXCEPTION 'Cataloginfo DelID % does not exist',del_id_; + RAISE EXCEPTION 'Cataloginfo DelID % does not exist',del_id_; END IF; - + EXCEPTION WHEN others THEN - GET STACKED DIAGNOSTICS + GET STACKED DIAGNOSTICS v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_context = PG_EXCEPTION_CONTEXT; @@ -3848,8 +3848,8 @@ ALTER FUNCTION delete_catalog_entries_to_delete(INTEGER) OWNER TO pgbackman_role -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION delete_restore_logs_to_delete(INTEGER) RETURNS VOID - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -3867,13 +3867,13 @@ CREATE OR REPLACE FUNCTION delete_restore_logs_to_delete(INTEGER) RETURNS VOID EXECUTE 'DELETE FROM restore_logs_to_delete WHERE del_id = $1' USING del_id_; - + ELSE - RAISE EXCEPTION 'Restore logs to delete with DelID % does not exist',del_id_; + RAISE EXCEPTION 'Restore logs to delete with DelID % does not exist',del_id_; END IF; - + EXCEPTION WHEN others THEN - GET STACKED DIAGNOSTICS + GET STACKED DIAGNOSTICS v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_context = PG_EXCEPTION_CONTEXT; @@ -3889,8 +3889,8 @@ ALTER FUNCTION delete_restore_logs_to_delete(INTEGER) OWNER TO pgbackman_role_rw -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION delete_backup_catalog(INTEGER) RETURNS VOID - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -3908,13 +3908,13 @@ CREATE OR REPLACE FUNCTION delete_backup_catalog(INTEGER) RETURNS VOID EXECUTE 'DELETE FROM backup_catalog WHERE bck_id = $1' USING bck_id_; - + ELSE - RAISE EXCEPTION 'Catalog entry with BckID % does not exist',bck_id_; + RAISE EXCEPTION 'Catalog entry with BckID % does not exist',bck_id_; END IF; - + EXCEPTION WHEN others THEN - GET STACKED DIAGNOSTICS + GET STACKED DIAGNOSTICS v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_context = PG_EXCEPTION_CONTEXT; @@ -3930,8 +3930,8 @@ ALTER FUNCTION delete_backup_catalog(INTEGER) OWNER TO pgbackman_role_rw; -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION delete_snapshot_definition(INTEGER) RETURNS VOID - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -3954,11 +3954,11 @@ CREATE OR REPLACE FUNCTION delete_snapshot_definition(INTEGER) RETURNS VOID USING snapshot_id_; ELSE - RAISE EXCEPTION 'Snapshot entry with snapshotID % does not exist',snapshot_id_; + RAISE EXCEPTION 'Snapshot entry with snapshotID % does not exist',snapshot_id_; END IF; - + EXCEPTION WHEN others THEN - GET STACKED DIAGNOSTICS + GET STACKED DIAGNOSTICS v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_context = PG_EXCEPTION_CONTEXT; @@ -3974,8 +3974,8 @@ ALTER FUNCTION delete_snapshot_definition(INTEGER) OWNER TO pgbackman_role_rw; -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION register_restore_catalog(BIGINT,INTEGER,INTEGER,INTEGER,TEXT,TEXT,TEXT,TIMESTAMP WITH TIME ZONE,TIMESTAMP WITH TIME ZONE,INTERVAL,TEXT,TEXT,TEXT,TEXT,TEXT[],TEXT,TEXT) RETURNS VOID - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE @@ -4000,7 +4000,7 @@ CREATE OR REPLACE FUNCTION register_restore_catalog(BIGINT,INTEGER,INTEGER,INTEG v_msg TEXT; v_detail TEXT; - v_context TEXT; + v_context TEXT; BEGIN EXECUTE 'INSERT INTO restore_catalog (restore_def, @@ -4019,7 +4019,7 @@ CREATE OR REPLACE FUNCTION register_restore_catalog(BIGINT,INTEGER,INTEGER,INTEG error_message, role_list, target_pgsql_node_release, - backup_pg_release) + backup_pg_release) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17)' USING restore_def_, procpid_, @@ -4040,12 +4040,12 @@ CREATE OR REPLACE FUNCTION register_restore_catalog(BIGINT,INTEGER,INTEGER,INTEG backup_pg_release_; EXCEPTION WHEN others THEN - GET STACKED DIAGNOSTICS + GET STACKED DIAGNOSTICS v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_context = PG_EXCEPTION_CONTEXT; RAISE EXCEPTION E'\n----------------------------------------------\nEXCEPTION:\n----------------------------------------------\nMESSAGE: % \nDETAIL : % \n----------------------------------------------\n', v_msg, v_detail; - + END; $$; @@ -4056,13 +4056,13 @@ ALTER FUNCTION register_restore_catalog(BIGINT,INTEGER,INTEGER,INTEGER,TEXT,TEXT -- Function: get_dbname_from_bckid() -- ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION get_dbname_from_bckid(INTEGER) RETURNS TEXT - LANGUAGE plpgsql - SECURITY INVOKER +CREATE OR REPLACE FUNCTION get_dbname_from_bckid(INTEGER) RETURNS TEXT + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE - bck_id_ ALIAS FOR $1; + bck_id_ ALIAS FOR $1; dbname_ TEXT := ''; BEGIN @@ -4086,13 +4086,13 @@ ALTER FUNCTION get_dbname_from_bckid(INTEGER) OWNER TO pgbackman_role_rw; -- Function: get_backup_server_id_from_bckid() -- ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION get_backup_server_id_from_bckid(INTEGER) RETURNS TEXT - LANGUAGE plpgsql - SECURITY INVOKER +CREATE OR REPLACE FUNCTION get_backup_server_id_from_bckid(INTEGER) RETURNS TEXT + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE - bck_id_ ALIAS FOR $1; + bck_id_ ALIAS FOR $1; backup_server_id_ TEXT := ''; BEGIN @@ -4116,13 +4116,13 @@ ALTER FUNCTION get_backup_server_id_from_bckid(INTEGER) OWNER TO pgbackman_role_ -- Function: get_roles_from_bckid() -- ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION get_role_list_from_bckid(INTEGER) RETURNS TEXT[] - LANGUAGE plpgsql - SECURITY INVOKER +CREATE OR REPLACE FUNCTION get_role_list_from_bckid(INTEGER) RETURNS TEXT[] + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE - bck_id_ ALIAS FOR $1; + bck_id_ ALIAS FOR $1; role_list_ TEXT [] := '{}'; BEGIN @@ -4148,15 +4148,15 @@ ALTER FUNCTION get_role_list_from_bckid(INTEGER) OWNER TO pgbackman_role_rw; -- ------------------------------------------------------------ CREATE OR REPLACE FUNCTION register_restore_definition(TIMESTAMP,INTEGER,INTEGER,INTEGER,TEXT,TEXT,TEXT,TEXT[]) RETURNS VOID - LANGUAGE plpgsql - SECURITY INVOKER + LANGUAGE plpgsql + SECURITY INVOKER SET search_path = public, pg_temp AS $$ DECLARE - + at_time_ ALIAS FOR $1; backup_server_id_ ALIAS FOR $2; - target_pgsql_node_id_ ALIAS FOR $3; + target_pgsql_node_id_ ALIAS FOR $3; bck_id_ ALIAS FOR $4; target_dbname_ ALIAS FOR $5; renamed_dbname_ ALIAS FOR $6; @@ -4164,8 +4164,8 @@ CREATE OR REPLACE FUNCTION register_restore_definition(TIMESTAMP,INTEGER,INTEGER roles_to_restore_ ALIAS FOR $8; server_cnt INTEGER; - node_cnt INTEGER; - bck_cnt INTEGER; + node_cnt INTEGER; + bck_cnt INTEGER; v_msg TEXT; v_detail TEXT; @@ -4183,7 +4183,7 @@ CREATE OR REPLACE FUNCTION register_restore_definition(TIMESTAMP,INTEGER,INTEGER ELSIF bck_cnt = 0 THEN RAISE EXCEPTION 'Backup with BckID: % does not exist',bck_id_; END IF; - + EXECUTE 'INSERT INTO restore_definition (bck_id, roles_to_restore, backup_server_id, @@ -4203,7 +4203,7 @@ CREATE OR REPLACE FUNCTION register_restore_definition(TIMESTAMP,INTEGER,INTEGER at_time_; EXCEPTION WHEN others THEN - GET STACKED DIAGNOSTICS + GET STACKED DIAGNOSTICS v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_context = PG_EXCEPTION_CONTEXT; @@ -4219,7 +4219,7 @@ ALTER FUNCTION register_restore_definition(TIMESTAMP,INTEGER,INTEGER,INTEGER,TEX -- Views -- ------------------------------------------------------------ -CREATE OR REPLACE VIEW show_jobs_queue AS +CREATE OR REPLACE VIEW show_jobs_queue AS SELECT a.id AS "JobID", date_trunc('seconds',a.registered) AS "Registered", a.backup_server_id AS "SrvID", @@ -4235,22 +4235,22 @@ ORDER BY a.registered ASC; ALTER VIEW show_jobs_queue OWNER TO pgbackman_role_rw; CREATE OR REPLACE VIEW show_pgsql_nodes AS -SELECT lpad(node_id::text,6,'0') AS "NodeID", +SELECT lpad(node_id::text,6,'0') AS "NodeID", hostname || '.' || domain_name AS "FQDN", pgport AS "Pgport", admin_user AS "Admin user", status AS "Status", - remarks AS "Remarks" + remarks AS "Remarks" FROM pgsql_node ORDER BY domain_name,hostname,"Pgport","Admin user","Status"; ALTER VIEW show_pgsql_nodes OWNER TO pgbackman_role_rw; CREATE OR REPLACE VIEW show_backup_servers AS -SELECT lpad(server_id::text,5,'0') AS "SrvID", +SELECT lpad(server_id::text,5,'0') AS "SrvID", hostname || '.' || domain_name AS "FQDN", status AS "Status", - remarks AS "Remarks" + remarks AS "Remarks" FROM backup_server ORDER BY domain_name,hostname,"Status"; @@ -4290,9 +4290,9 @@ CREATE OR REPLACE VIEW show_backup_catalog AS pg_size_pretty(a.pg_dump_file_size+a.pg_dump_roles_file_size+a.pg_dump_dbconfig_file_size) AS "Size", b.backup_code AS "Code", a.execution_method AS "Execution", - a.execution_status AS "Status" - FROM backup_catalog a - JOIN backup_definition b ON a.def_id = b.def_id) + a.execution_status AS "Status" + FROM backup_catalog a + JOIN backup_definition b ON a.def_id = b.def_id) UNION (SELECT lpad(a.bck_id::text,9,'0') AS "BckID", '' AS "DefID", @@ -4309,9 +4309,9 @@ CREATE OR REPLACE VIEW show_backup_catalog AS pg_size_pretty(a.pg_dump_file_size+a.pg_dump_roles_file_size+a.pg_dump_dbconfig_file_size) AS "Size", b.backup_code AS "Code", a.execution_method AS "Execution", - a.execution_status AS "Status" - FROM backup_catalog a - JOIN snapshot_definition b ON a.snapshot_id = b.snapshot_id) + a.execution_status AS "Status" + FROM backup_catalog a + JOIN snapshot_definition b ON a.snapshot_id = b.snapshot_id) ORDER BY "Finished" DESC,backup_server_id,pgsql_node_id,"DBname","Code","Status"; ALTER VIEW show_backup_catalog OWNER TO pgbackman_role_rw; @@ -4353,9 +4353,9 @@ CREATE OR REPLACE VIEW show_backup_details AS left(a.error_message,60) AS "Error message", array_to_string(a.role_list,',') AS "Role list", a.pgsql_node_release AS "PgSQL node release", - a.pg_dump_release AS "pg_dump release" - FROM backup_catalog a - JOIN backup_definition b ON a.def_id = b.def_id) + a.pg_dump_release AS "pg_dump release" + FROM backup_catalog a + JOIN backup_definition b ON a.def_id = b.def_id) UNION (SELECT lpad(a.bck_id::text,12,'0') AS "BckID", a.bck_id AS bck_id, @@ -4393,8 +4393,8 @@ CREATE OR REPLACE VIEW show_backup_details AS left(a.error_message,60) AS "Error message", array_to_string(a.role_list,',') AS "Role list", a.pgsql_node_release AS "PgSQL node release", - a.pg_dump_release AS "pg_dump release" - FROM backup_catalog a + a.pg_dump_release AS "pg_dump release" + FROM backup_catalog a JOIN snapshot_definition b ON a.snapshot_id = b.snapshot_id) ORDER BY "Finished" DESC,backup_server_id,pgsql_node_id,"DBname","Code","Status"; @@ -4411,39 +4411,39 @@ CREATE OR REPLACE VIEW get_catalog_entries_to_delete AS pg_dump_roles_file, pg_dump_roles_log_file, pg_dump_dbconfig_file, - pg_dump_dbconfig_log_file + pg_dump_dbconfig_log_file FROM catalog_entries_to_delete ORDER BY del_id; ALTER VIEW get_catalog_entries_to_delete OWNER TO pgbackman_role_rw; CREATE OR REPLACE VIEW get_cron_catalog_entries_to_delete_by_retention AS -WITH +WITH all_backup_jobs_catalog AS ( - SELECT - row_number() OVER (PARTITION BY a.def_id ORDER BY a.def_id,a.finished DESC) AS row_id, - a.bck_id, + SELECT + row_number() OVER (PARTITION BY a.def_id ORDER BY a.def_id,a.finished DESC) AS row_id, + a.bck_id, a.def_id, a.backup_server_id, a.pgsql_node_id, - a.dbname, + a.dbname, a.finished, b.retention_period, b.retention_redundancy, - a.pg_dump_file, + a.pg_dump_file, a.pg_dump_log_file, a.pg_dump_roles_file, a.pg_dump_roles_log_file, a.pg_dump_dbconfig_file, - a.pg_dump_dbconfig_log_file - FROM backup_catalog a + a.pg_dump_dbconfig_log_file + FROM backup_catalog a INNER JOIN backup_definition b ON a.def_id=b.def_id ORDER BY a.def_id,a.finished ASC ) - SELECT * + SELECT * FROM all_backup_jobs_catalog WHERE finished < now() - retention_period - AND row_id > retention_redundancy + AND row_id > retention_redundancy ORDER BY def_id,finished DESC; ALTER VIEW get_cron_catalog_entries_to_delete_by_retention OWNER TO pgbackman_role_rw; @@ -4452,34 +4452,34 @@ ALTER VIEW get_cron_catalog_entries_to_delete_by_retention OWNER TO pgbackman_ro CREATE OR REPLACE VIEW get_restore_logs_to_delete AS SELECT del_id, backup_server_id, - restore_log_file -FROM restore_logs_to_delete; + restore_log_file +FROM restore_logs_to_delete; ALTER VIEW get_restore_logs_to_delete OWNER TO pgbackman_role_rw; CREATE OR REPLACE VIEW get_at_catalog_entries_to_delete_by_retention AS -WITH +WITH all_backup_jobs_catalog AS ( - SELECT - a.bck_id, + SELECT + a.bck_id, a.snapshot_id, a.backup_server_id, a.pgsql_node_id, - a.dbname, + a.dbname, a.finished, b.retention_period, - a.pg_dump_file, + a.pg_dump_file, a.pg_dump_log_file, a.pg_dump_roles_file, a.pg_dump_roles_log_file, a.pg_dump_dbconfig_file, - a.pg_dump_dbconfig_log_file - FROM backup_catalog a + a.pg_dump_dbconfig_log_file + FROM backup_catalog a INNER JOIN snapshot_definition b ON a.snapshot_id=b.snapshot_id ORDER BY a.snapshot_id,a.finished ASC ) - SELECT * + SELECT * FROM all_backup_jobs_catalog WHERE finished < now() - retention_period ORDER BY snapshot_id,finished DESC; @@ -4493,7 +4493,7 @@ SELECT server_id, value AS "Value", description As "Description" FROM backup_server_config -ORDER BY parameter; +ORDER BY parameter; ALTER VIEW show_backup_server_config OWNER TO pgbackman_role_rw; @@ -4503,7 +4503,7 @@ SELECT node_id, value AS "Value", description As "Description" FROM pgsql_node_config -ORDER BY parameter; +ORDER BY parameter; ALTER VIEW show_pgsql_node_config OWNER TO pgbackman_role_rw; @@ -4584,18 +4584,18 @@ CREATE OR REPLACE VIEW show_restore_catalog AS a.target_dbname AS "Target DBname", a.renamed_dbname AS "Renamed DBname", date_trunc('seconds',a.duration) AS "Duration", - a.execution_status AS "Status" - FROM restore_catalog a - JOIN restore_definition b ON a.restore_def = b.restore_def) + a.execution_status AS "Status" + FROM restore_catalog a + JOIN restore_definition b ON a.restore_def = b.restore_def) ORDER BY "Finished" DESC, backup_server_id,target_pgsql_node_id,"Target DBname","Status"; ALTER VIEW show_restore_catalog OWNER TO pgbackman_role_rw; CREATE OR REPLACE VIEW show_restore_details AS -(SELECT a.restore_id, +(SELECT a.restore_id, lpad(a.restore_id::text,10,'0') AS "RestoreID", - a.registered AS "Registered", + a.registered AS "Registered", a.restore_def, lpad(a.restore_def::text,10,'0') AS "RestoreDef", a.procpid AS "ProcPID", @@ -4615,13 +4615,13 @@ CREATE OR REPLACE VIEW show_restore_details AS a.target_pgsql_node_release AS "Target PGnode release", b.at_time AS "AT time", a.restore_log_file AS "Log file", - a.global_log_file AS "Global log file", + a.global_log_file AS "Global log file", array_to_string(a.role_list,',') AS "Roles restored", a.error_message AS "Error message", b.extra_restore_parameters AS "Extra parameters" - FROM restore_catalog a + FROM restore_catalog a JOIN restore_definition b ON a.restore_def = b.restore_def - JOIN backup_catalog c ON b.bck_id = c.bck_id) + JOIN backup_catalog c ON b.bck_id = c.bck_id) ORDER BY "Finished" DESC, backup_server_id,target_pgsql_node_id,"Target DBname","Status"; ALTER VIEW show_restore_details OWNER TO pgbackman_role_rw; @@ -4639,7 +4639,7 @@ CREATE OR REPLACE VIEW show_snapshots_in_progress AS date_trunc('second',now()-a.at_time)::text AS "Elapsed time" FROM snapshot_definition a LEFT JOIN backup_catalog b - ON a.snapshot_id = b.snapshot_id + ON a.snapshot_id = b.snapshot_id WHERE b.snapshot_id IS NULL ORDER BY a.at_time ASC; @@ -4658,19 +4658,19 @@ CREATE OR REPLACE VIEW show_restores_in_progress AS date_trunc('second',now()-a.at_time)::text AS "Elapsed time" FROM restore_definition a LEFT JOIN restore_catalog b - ON a.restore_def = b.restore_def + ON a.restore_def = b.restore_def WHERE b.restore_def IS NULL ORDER BY a.at_time ASC; ALTER VIEW show_restores_in_progress OWNER TO pgbackman_role_rw; CREATE OR REPLACE VIEW get_deleted_backup_definitions_to_delete_by_retention AS - SELECT a.def_id - FROM backup_definition a - INNER JOIN pgsql_node_config b - ON a.pgsql_node_id = b.node_id - WHERE a.job_status = 'DELETED' - AND b.parameter = 'automatic_deletion_retention' + SELECT a.def_id + FROM backup_definition a + INNER JOIN pgsql_node_config b + ON a.pgsql_node_id = b.node_id + WHERE a.job_status = 'DELETED' + AND b.parameter = 'automatic_deletion_retention' AND a.registered < now()-b.value::interval; ALTER VIEW get_deleted_backup_definitions_to_delete_by_retention OWNER TO pgbackman_role_rw; From d01c067a3b7322710f1b783f5c5894266b2109ff Mon Sep 17 00:00:00 2001 From: Philip McClarence Date: Fri, 26 Jan 2018 14:17:28 +0000 Subject: [PATCH 07/12] adding RDS backup code option --- bin/pgbackman_dump | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/pgbackman_dump b/bin/pgbackman_dump index 5da0e23..1ca54e2 100755 --- a/bin/pgbackman_dump +++ b/bin/pgbackman_dump @@ -163,7 +163,7 @@ def pg_dump(db): ' -h ' + global_parameters['pgsql_node_fqdn'] + \ ' -p ' + global_parameters['pgsql_node_port'] + \ ' -U ' + global_parameters['pgsql_node_admin_user'] + \ - ' --data-only' + \ + #' --data-only' + \ ' --file=' + global_parameters['database_dump_file'] + \ ' --format=d' + \ ' --blobs' + \ From 2e0b61efc0b360b641a42cdecb4261f8c579dfcb Mon Sep 17 00:00:00 2001 From: Philip McClarence Date: Fri, 26 Jan 2018 14:25:57 +0000 Subject: [PATCH 08/12] adding RDS backup code option --- bin/pgbackman_dump | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/pgbackman_dump b/bin/pgbackman_dump index 1ca54e2..816e71a 100755 --- a/bin/pgbackman_dump +++ b/bin/pgbackman_dump @@ -163,7 +163,6 @@ def pg_dump(db): ' -h ' + global_parameters['pgsql_node_fqdn'] + \ ' -p ' + global_parameters['pgsql_node_port'] + \ ' -U ' + global_parameters['pgsql_node_admin_user'] + \ - #' --data-only' + \ ' --file=' + global_parameters['database_dump_file'] + \ ' --format=d' + \ ' --blobs' + \ From 822846dfd27a07ec1090207366dbd77861432bac Mon Sep 17 00:00:00 2001 From: Philip McClarence Date: Fri, 26 Jan 2018 14:59:38 +0000 Subject: [PATCH 09/12] adding RDS backup code option --- bin/pgbackman_dump | 4 +- bin/pgbackman_maintenance | 140 +++++++++++++++++++------------------- sql/pgbackman.sql | 2 +- 3 files changed, 73 insertions(+), 73 deletions(-) diff --git a/bin/pgbackman_dump b/bin/pgbackman_dump index 816e71a..77854be 100755 --- a/bin/pgbackman_dump +++ b/bin/pgbackman_dump @@ -157,7 +157,7 @@ def pg_dump(db): ' ' + global_parameters['extra_backup_parameters'] + \ ' ' + global_parameters['dbname'] - elif global_parameters['backup_code'] == 'RDSDATA': + elif global_parameters['backup_code'] == 'RDS': pg_dump_command = global_parameters['backup_server_pgsql_bin_dir'] + '/pg_dump' + \ ' -h ' + global_parameters['pgsql_node_fqdn'] + \ @@ -1362,7 +1362,7 @@ if __name__ == '__main__': parser.add_argument('--def-id', metavar='JOBID', required=False, help='Backup job ID', dest='def_id') parser.add_argument('--snapshot-id', metavar='SNAPSHOT-ID', required=False, help='snapshot ID', dest='snapshot_id') parser.add_argument('--dbname', metavar='DBNAME', required=False, help='Database name', dest='dbname') - parser.add_argument('--backup-code', metavar='[FULL|SCHEMA|DATA|CLUSTER|RDSDATAONLY]', choices=['FULL', 'SCHEMA', 'DATA', 'CLUSTER', 'RDSDATA'], required=True, help='Backup code', dest='backup_code') + parser.add_argument('--backup-code', metavar='[FULL|SCHEMA|DATA|CLUSTER|RDSDATAONLY]', choices=['FULL', 'SCHEMA', 'DATA', 'CLUSTER', 'RDS'], required=True, help='Backup code', dest='backup_code') parser.add_argument('--encryption', metavar='[false|true]', default=True, choices=['true', 'false'],required=True, help='Activate encryption', dest='encryption') parser.add_argument('--root-backup-dir', metavar='ROOT-BACKUP-DIR', default=True, required=True, help='Root backup dir', dest='root_backup_dir') parser.add_argument('--extra-backup-parameters', metavar='EXTRA-PARAMETERS', required=False, help='extra pg_dump parameters', dest='extra_backup_parameters') diff --git a/bin/pgbackman_maintenance b/bin/pgbackman_maintenance index b5b6437..b5ad424 100755 --- a/bin/pgbackman_maintenance +++ b/bin/pgbackman_maintenance @@ -43,7 +43,7 @@ This program is used by PgBackMan to run some maintenance tasks in this backup s These are the task implemented: * Delete restore logs files when restore definitions/catalogs are deleted. -* Delete backup and log files from catalog entries associated to a backup definition after +* Delete backup and log files from catalog entries associated to a backup definition after this definition has been deleted with the force parameter. * Enforce retentions for backup definitions. * Enforce retentions of snapshot backups @@ -60,10 +60,10 @@ def delete_restore_logs(db,backup_server_id): '''Delete restore log files after restore definitions/catalogs are deleted''' logs.logger.debug('## Deleting restore logs after restore definitions/catalogs are deleted ##') - + try: for record in db.get_restore_logs_to_delete(backup_server_id): - + error_cnt = 0 # @@ -73,7 +73,7 @@ def delete_restore_logs(db,backup_server_id): try: os.unlink(record[2]) logs.logger.debug('File: %s deleted',record[2]) - + except OSError as e: if e.errno != errno.ENOENT: logs.logger.error('Problems deleting restore log files with DelID: %s - %s',record[0],e) @@ -82,7 +82,7 @@ def delete_restore_logs(db,backup_server_id): pass # - # We can delete the delID entry from the database only if we can + # We can delete the delID entry from the database only if we can # delete all the files in the delID entry without errors # @@ -90,18 +90,18 @@ def delete_restore_logs(db,backup_server_id): try: db.delete_restore_logs_to_delete(record[0]) logs.logger.info('Restore Log file for DelID: %s deleted',record[0]) - + except psycopg2.OperationalError as e: raise e except Exception as e: logs.logger.error('Problems deleting restore logs to delete entry DelID: %s - %s',record[0],e) - + except psycopg2.OperationalError as e: raise e except Exception as e: logs.logger.error('Could not get the restore log to delete information - %s',e) - + # ############################################ # Function delete_files_from_force_deletes() # ############################################ @@ -110,10 +110,10 @@ def delete_files_from_force_deletes(db,backup_server_id): '''Delete dump and log files from force deletions of backup definitions''' logs.logger.debug('## Deleting files from forced DefID deletions ##') - + try: for record in db.get_catalog_entries_to_delete(backup_server_id): - + error_cnt = 0 # @@ -131,7 +131,7 @@ def delete_files_from_force_deletes(db,backup_server_id): elif os.path.isdir(record[index]): shutil.rmtree(record[index]) logs.logger.debug('Directory: %s deleted',record[index]) - + except OSError as e: if e.errno != errno.ENOENT: logs.logger.error('Problems deleting files from force deletions of DefIDs: %s',e) @@ -141,7 +141,7 @@ def delete_files_from_force_deletes(db,backup_server_id): # - # We can delete the delID entry from the database only if we can + # We can delete the delID entry from the database only if we can # delete all the files in the delID entry without errors # @@ -149,18 +149,18 @@ def delete_files_from_force_deletes(db,backup_server_id): try: db.delete_catalog_entries_to_delete(record[0]) logs.logger.info('Files for catalog ID: %s / DefID: %s deleted',record[3],record[2]) - + except psycopg2.OperationalError as e: raise e except Exception as e: logs.logger.error('Problems deleting cataloginfo from force defid deletions - %s',e) - + except psycopg2.OperationalError as e: raise e except Exception as e: logs.logger.error('Could not get the catalog information for defid force deletions - %s',e) - + # ############################################ # Function enforce_backup_retentions() # ############################################ @@ -174,7 +174,7 @@ def enforce_backup_retentions(db,backup_server_id): for record in db.get_cron_catalog_entries_to_delete_by_retention(backup_server_id): error_cnt = 0 - + # # record[9...14] are the files to delete # @@ -190,16 +190,16 @@ def enforce_backup_retentions(db,backup_server_id): elif os.path.isdir(record[index]): shutil.rmtree(record[index]) logs.logger.debug('Directory: %s deleted',record[index]) - + except OSError as e: if e.errno != errno.ENOENT: logs.logger.error('Problems deleting files from backup enforce retentions: %s',e) error_cnt = error_cnt + 1 else: pass - + # - # We can delete the BckID entry from the catalog table only if we can + # We can delete the BckID entry from the catalog table only if we can # delete all the files for the BckID entry without errors # @@ -217,7 +217,7 @@ def enforce_backup_retentions(db,backup_server_id): raise e except Exception as e: logs.logger.error('Could not get information to enforce backup file retentions - %s',e) - + # ############################################ # Function enforce_snapshot_retentions() @@ -232,7 +232,7 @@ def enforce_snapshot_retentions(db,backup_server_id): for record in db.get_at_catalog_entries_to_delete_by_retention(backup_server_id): error_cnt = 0 - + # # record[7...12] are the files to delete # @@ -240,7 +240,7 @@ def enforce_snapshot_retentions(db,backup_server_id): for index in range(7,13): try: - + if os.path.isfile(record[index]): os.unlink(record[index]) logs.logger.debug('File: %s deleted',record[index]) @@ -248,16 +248,16 @@ def enforce_snapshot_retentions(db,backup_server_id): elif os.path.isdir(record[index]): shutil.rmtree(record[index]) logs.logger.debug('Directory: %s deleted',record[index]) - + except OSError as e: if e.errno != errno.ENOENT: logs.logger.error('Problems deleting files from snapshot enforce retentions: %s',e) error_cnt = error_cnt + 1 else: pass - + # - # We can delete the snapshotID entry from the catalog and definition table only if we can + # We can delete the snapshotID entry from the catalog and definition table only if we can # delete all the files for the SnapshotID entry without errors # @@ -275,7 +275,7 @@ def enforce_snapshot_retentions(db,backup_server_id): raise e except Exception as e: logs.logger.error('Could not get information to enforce snapshot file retentions - %s',e) - + # ################################################## # Function process_pending_backup_catalog_log_file() @@ -287,19 +287,19 @@ def process_pending_backup_catalog_log_file(db,backup_server_id): role_list = [] logs.logger.debug('## Processing pending backup catalog log files ##') - + try: db.pg_connect() - + root_backup_partition = db.get_backup_server_config_value(backup_server_id,'root_backup_partition') pending_catalog = root_backup_partition + '/pending_updates' - + for pending_log_file in os.listdir(pending_catalog): if pending_log_file.find('backup_jobs_pending_log_updates_nodeid') != -1: with open(pending_catalog + '/' + pending_log_file,'r') as pending_file: for line in pending_file: parameters = line.split('::') - + if len(parameters) == 25: # @@ -309,13 +309,13 @@ def process_pending_backup_catalog_log_file(db,backup_server_id): def_id = parameters[0] snapshot_id = parameters[21] - + if def_id == '': def_id = None elif snapshot_id == '': snapshot_id = None - - # Generate role list + + # Generate role list role_list = parameters[22].split(' ') @@ -350,15 +350,15 @@ def process_pending_backup_catalog_log_file(db,backup_server_id): parameters[24].replace('\n','')) logs.logger.info('Backup job catalog for DefID: %s or snapshotID: %s in pending file %s updated in the database',def_id,snapshot_id,pending_log_file) - + # # Deleting the pending file if we can update the database with # the information in the file # - + os.unlink(pending_catalog + '/' + pending_log_file) logs.logger.info('Pending backup file: %s deleted',pending_log_file) - + else: logs.logger.error('Wrong format in pending backup file: %s',pending_log_file) @@ -378,19 +378,19 @@ def process_pending_restore_catalog_log_file(db,backup_server_id): role_list = [] logs.logger.debug('## Processing pending restore catalog log files ##') - + try: db.pg_connect() - + root_backup_partition = db.get_backup_server_config_value(backup_server_id,'root_backup_partition') pending_catalog = root_backup_partition + '/pending_updates' - + for pending_log_file in os.listdir(pending_catalog): if pending_log_file.find('restore_jobs_pending_log_updates_nodeid') != -1: with open(pending_catalog + '/' + pending_log_file,'r') as pending_file: for line in pending_file: parameters = line.split('::') - + if len(parameters) == 17: # @@ -416,15 +416,15 @@ def process_pending_restore_catalog_log_file(db,backup_server_id): parameters[16].replace('\n','')) logs.logger.info('Restore job catalog for restoreDef: %s in pending file %s updated in the database',parameters[0],pending_log_file) - + # # Deleting the pending file if we can update the database with # the information in the file # - + os.unlink(pending_catalog + '/' + pending_log_file) logs.logger.info('Pending restore file: %s deleted',pending_log_file) - + else: logs.logger.error('Wrong format in pending restore file: %s',pending_log_file) @@ -438,15 +438,15 @@ def process_pending_restore_catalog_log_file(db,backup_server_id): # ############################################################ # Function process_backup_definitions_from_deleted_databases() # ############################################################ - + def process_backup_definitions_from_deleted_databases(db,backup_server_id): ''' This function stops backup definitions for databases that have been deleted - in the PgSQL nodes running them. + in the PgSQL nodes running them. ''' logs.logger.debug('## Processing backup definitions for deleted databases ##') - + # # Processing data for all PgSQL nodes with status "RUNNING" # @@ -467,7 +467,7 @@ def process_backup_definitions_from_deleted_databases(db,backup_server_id): dsn_value = db.get_pgsql_node_dsn(pgsql_node_id) db_node = PgbackmanDB(dsn_value, 'pgbackman_maintenance') - + # # Get all backup definitions for this Backup server - PgSQL node # @@ -477,7 +477,7 @@ def process_backup_definitions_from_deleted_databases(db,backup_server_id): # # Get backup definitions for databases that exist in the PgSQL_node # - + for database in db_node.get_pgsql_node_database_list(): for backup_def_id in db.get_database_backup_definitions(backup_server_id,pgsql_node_id,database[0]): @@ -486,9 +486,9 @@ def process_backup_definitions_from_deleted_databases(db,backup_server_id): # # List of backup definitions to process # - + backup_def_list_to_process = set(backup_def_full_list) - set(backup_def_database_list) - + # # Update the backup def status to "DELETED" # @@ -508,7 +508,7 @@ def process_backup_definitions_from_deleted_databases(db,backup_server_id): # automatic_deletion_retention for the PgSQL node that was # running the deleted database. # - + for backup_def_id in db.get_deleted_backup_definitions_to_delete_by_retention(): try: db.delete_force_backup_definition_id(backup_def_id[0]) @@ -525,7 +525,7 @@ def process_backup_definitions_from_deleted_databases(db,backup_server_id): # ############################################ # Function signal_handler() # ############################################ - + def signal_handler(signum, frame): logs.logger.info('**** pgbackman_maintenance stopped. ****') sys.exit(0) @@ -534,14 +534,14 @@ def signal_handler(signum, frame): # ############################################ # Function check_database_connection() # ############################################ - + def check_database_connection(db): '''Check if we can connect to the database server and the pgbackman database''' try: db.pg_connect() return True - except Exception as e: + except Exception as e: return False @@ -562,18 +562,18 @@ def main(): db = PgbackmanDB(dsn, 'pgbackman_maintenance') # - # We check before starting if the database is available. - # If it is not available we will wait conf.pg_connect_retry_interval - # and try again + # We check before starting if the database is available. + # If it is not available we will wait conf.pg_connect_retry_interval + # and try again check_db = check_database_connection(db) while not check_db: logs.logger.critical('The pgbackman database is not available. Waiting %s seconds before trying again',conf.pg_connect_retry_interval) - + time.sleep(conf.pg_connect_retry_interval) check_db = check_database_connection(db) - + logs.logger.debug('Database server is up and running and pgbackman database is available') # @@ -588,12 +588,12 @@ def main(): try: backup_server_id = db.get_backup_server_id(backup_server_fqdn) logs.logger.info('Backup server: %s is registered in pgbackman',backup_server_fqdn) - + except psycopg2.Error as e: logs.logger.critical('Cannot find backup server %s in pgbackman. Stopping pgbackman2cron.',backup_server_fqdn) logs.logger.info('**** pgbackman_maintenance stopped. ****') - sys.exit(1) - + sys.exit(1) + loop = 0 while loop == 0: @@ -605,21 +605,21 @@ def main(): process_pending_backup_catalog_log_file(db,backup_server_id) process_pending_restore_catalog_log_file(db,backup_server_id) process_backup_definitions_from_deleted_databases(db,backup_server_id) - + except psycopg2.OperationalError as e: # # If we lose the connection to the database, we will wait conf.pg_connect_retry_interval - # before trying to connect again. + # before trying to connect again. # logs.logger.critical('Operational error: %s',e) check_db = check_database_connection(db) - + while not check_db: logs.logger.critical('We have lost the connection to the database. Waiting %s seconds before trying again',conf.pg_connect_retry_interval) - + time.sleep(conf.pg_connect_retry_interval) check_db = check_database_connection(db) @@ -628,12 +628,12 @@ def main(): else: # Wait for next maintenance run if in loop mode time.sleep(conf.maintenance_interval) - + db.pg_close() # ############################################ -# +# # ############################################ if __name__ == '__main__': @@ -646,10 +646,10 @@ if __name__ == '__main__': parser = argparse.ArgumentParser(prog=sys.argv[0]) parser.add_argument('--cron', required=False, help='Single run to use via cron', action="store_true") - args = parser.parse_args() + args = parser.parse_args() logs.logger.info('**** pgbackman_maintenance started. ****') - + if args.cron: cron = True logs.logger.info('Running in cron mode') diff --git a/sql/pgbackman.sql b/sql/pgbackman.sql index 137bb07..02b17e5 100644 --- a/sql/pgbackman.sql +++ b/sql/pgbackman.sql @@ -902,7 +902,7 @@ INSERT INTO backup_code (code,description) VALUES ('SCHEMA','Schema backup of a INSERT INTO backup_code (code,description) VALUES ('DATA','Data backup of the database.'); INSERT INTO backup_code (code,description) VALUES ('CLUSTER','Full backup of the database cluster.'); INSERT INTO backup_code (code,description) VALUES ('CONFIG','Backup of the configuration files'); -INSERT INTO backup_code (code,description) VALUES ('RDSDATA','Only runs a pg_dump of the data so that it does not try to access globals that postgres cannot access on RDS'); +INSERT INTO backup_code (code,description) VALUES ('RDS','Only runs a pg_dump of the data so that it does not try to access globals that postgres cannot access on RDS'); \echo '# [Init: job_definition_status]\n' INSERT INTO job_definition_status (code,description) VALUES ('ACTIVE','Backup job activated and in production'); From e40c62354480750c7a29fb351ee6e57f03fce5b2 Mon Sep 17 00:00:00 2001 From: Philip McClarence Date: Mon, 14 Jan 2019 12:32:39 +0000 Subject: [PATCH 10/12] updating the pg_recovery_x functions to use the renamed functions on pg 10 --- pgbackman/database.py | 1361 +++++++++++++++++++++-------------------- 1 file changed, 686 insertions(+), 675 deletions(-) diff --git a/pgbackman/database.py b/pgbackman/database.py index 8fef535..c48a5ba 100644 --- a/pgbackman/database.py +++ b/pgbackman/database.py @@ -55,7 +55,7 @@ def __init__(self, dsn,application): self.output_format = 'table' - + # ############################################ # Method pg_connect() # ############################################ @@ -65,7 +65,7 @@ def pg_connect(self): try: self.conn = psycopg2.connect(self.dsn) - + if self.conn: self.conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) psycopg2.extras.wait_select(self.conn) @@ -75,13 +75,13 @@ def pg_connect(self): self.server_version = self.conn.server_version if (self.server_version >= 90000 and 'application_name=' not in self.dsn): - + try: self.cur.execute('SET application_name TO %s',(self.application,)) self.conn.commit() except psycopg2.Error as e: raise e - + except psycopg2.Error as e: raise e @@ -97,17 +97,17 @@ def pg_close(self): try: self.cur.close() except psycopg2.Error as e: - raise e + raise e if self.conn: try: - self.conn.close() + self.conn.close() except psycopg2.Error as e: raise e - - + + # ############################################ - # Method + # Method # ############################################ def get_server_version(self): @@ -115,13 +115,13 @@ def get_server_version(self): try: return self.server_version - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ def show_backup_servers(self): @@ -139,15 +139,15 @@ def show_backup_servers(self): except psycopg2.Error as e: raise e - + self.pg_close() except psycopg2.Error as e: raise e - - + + # ############################################ - # Method + # Method # ############################################ def register_backup_server(self,hostname,domain,status,remarks): @@ -159,21 +159,21 @@ def register_backup_server(self,hostname,domain,status,remarks): if self.cur: try: self.cur.execute('SELECT register_backup_server(%s,%s,%s,%s)',(hostname,domain,status,remarks)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def delete_backup_server(self,server_id): """A function to delete a backup server""" @@ -183,19 +183,19 @@ def delete_backup_server(self,server_id): if self.cur: try: self.cur.execute('SELECT delete_backup_server(%s)',(server_id,)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def show_pgsql_nodes(self): @@ -209,19 +209,19 @@ def show_pgsql_nodes(self): self.cur.execute('SELECT * FROM show_pgsql_nodes') self.conn.commit() - return self.cur + return self.cur except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e - - + + # ############################################ - # Method + # Method # ############################################ def register_pgsql_node(self,hostname,domain,port,admin_user,status,remarks): @@ -233,19 +233,19 @@ def register_pgsql_node(self,hostname,domain,port,admin_user,status,remarks): if self.cur: try: self.cur.execute('SELECT register_pgsql_node(%s,%s,%s,%s,%s,%s)',(hostname,domain,port,admin_user,status,remarks)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - - + + # ############################################ - # Method + # Method # ############################################ def delete_pgsql_node(self,node_id): @@ -257,19 +257,19 @@ def delete_pgsql_node(self,node_id): if self.cur: try: self.cur.execute('SELECT delete_pgsql_node(%s)',(node_id,)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - - + + # ############################################ - # Method + # Method # ############################################ def register_backup_definition(self,backup_server,pgsql_node,dbname,minutes_cron,hours_cron,day_month_cron, \ @@ -285,19 +285,19 @@ def register_backup_definition(self,backup_server,pgsql_node,dbname,minutes_cron self.cur.execute('SELECT register_backup_definition(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)',(backup_server,pgsql_node,dbname,minutes_cron,hours_cron,day_month_cron, \ month_cron,weekday_cron,backup_code,encryption, \ retention_period,retention_redundancy,extra_backup_parameters,job_status,remarks)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def delete_backup_definition_id(self,def_id): @@ -309,19 +309,19 @@ def delete_backup_definition_id(self,def_id): if self.cur: try: self.cur.execute('SELECT delete_backup_definition_id(%s)',(def_id,)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def delete_force_backup_definition_id(self,def_id): @@ -333,19 +333,19 @@ def delete_force_backup_definition_id(self,def_id): if self.cur: try: self.cur.execute('SELECT delete_force_backup_definition_id(%s)',(def_id,)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def delete_backup_definition_dbname(self,pgsql_node_id,dbname): @@ -357,19 +357,19 @@ def delete_backup_definition_dbname(self,pgsql_node_id,dbname): if self.cur: try: self.cur.execute('SELECT delete_backup_definition_dbname(%s,%s)',(pgsql_node_id,dbname)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def delete_force_backup_definition_dbname(self,pgsql_node_id,dbname): @@ -381,19 +381,19 @@ def delete_force_backup_definition_dbname(self,pgsql_node_id,dbname): if self.cur: try: self.cur.execute('SELECT delete_force_backup_definition_dbname(%s,%s)',(pgsql_node_id,dbname)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def show_backup_definitions(self,backup_server_list,pgsql_node_list,dbname_list): @@ -407,59 +407,59 @@ def show_backup_definitions(self,backup_server_list,pgsql_node_list,dbname_list) if backup_server_list != None: server_sql = 'AND (FALSE ' - + for server in backup_server_list: if server.isdigit(): server_sql = server_sql + 'OR backup_server_id = ' + str(server) + ' ' else: server_sql = server_sql + 'OR backup_server_id = ' + str(self.get_backup_server_id(server.lower())) + ' ' - + server_sql = server_sql + ') ' - + else: server_sql = '' - + if pgsql_node_list != None: node_sql = 'AND (FALSE ' - + for node in pgsql_node_list: if node.isdigit(): node_sql = node_sql + 'OR pgsql_node_id = ' + str(node) + ' ' else: node_sql = node_sql + 'OR pgsql_node_id = ' + str(self.get_pgsql_node_id(node.lower())) + ' ' - + node_sql = node_sql + ') ' - + else: - node_sql = '' + node_sql = '' if dbname_list != None: dbname_sql = 'AND (FALSE ' - + for dbname in dbname_list: dbname_sql = dbname_sql + 'OR "DBname" = \'' + dbname + '\' ' - + dbname_sql = dbname_sql + ') ' - + else: dbname_sql = '' - + self.cur.execute('SELECT \"DefID\",backup_server_id AS \"ID.\",\"Backup server\",pgsql_node_id AS \"ID\",\"PgSQL node\",\"DBname\",\"Schedule\",\"Code\",\"Retention\",\"Status\",\"Parameters\" FROM show_backup_definitions WHERE TRUE ' + server_sql + node_sql + dbname_sql) - + return self.cur - + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def register_snapshot_definition(self,backup_server,pgsql_node,dbname,at_time,backup_code,retention_period,extra_backup_parameters,remarks,pg_dump_release): @@ -472,19 +472,19 @@ def register_snapshot_definition(self,backup_server,pgsql_node,dbname,at_time,ba try: self.cur.execute('SELECT register_snapshot_definition(%s,%s,%s,%s,%s,%s,%s,%s,%s)',(backup_server,pgsql_node,dbname,at_time,backup_code, \ retention_period,extra_backup_parameters,remarks,pg_dump_release)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def show_snapshot_definitions(self,backup_server_list,pgsql_node_list,dbname_list): @@ -498,58 +498,58 @@ def show_snapshot_definitions(self,backup_server_list,pgsql_node_list,dbname_lis if backup_server_list != None: server_sql = 'AND (FALSE ' - + for server in backup_server_list: if server.isdigit(): server_sql = server_sql + 'OR backup_server_id = ' + str(server) + ' ' else: server_sql = server_sql + 'OR backup_server_id = ' + str(self.get_backup_server_id(server.lower())) + ' ' - + server_sql = server_sql + ') ' - + else: server_sql = '' - + if pgsql_node_list != None: node_sql = 'AND (FALSE ' - + for node in pgsql_node_list: if node.isdigit(): node_sql = node_sql + 'OR pgsql_node_id = ' + str(node) + ' ' else: node_sql = node_sql + 'OR pgsql_node_id = ' + str(self.get_pgsql_node_id(node.lower())) + ' ' - + node_sql = node_sql + ') ' - + else: - node_sql = '' + node_sql = '' if dbname_list != None: dbname_sql = 'AND (FALSE ' - + for dbname in dbname_list: dbname_sql = dbname_sql + 'OR "DBname" = \'' + dbname + '\' ' - + dbname_sql = dbname_sql + ') ' - + else: dbname_sql = '' - + self.cur.execute('SELECT \"SnapshotID\",\"Registered\",backup_server_id AS \"ID.\",\"Backup server\",pgsql_node_id AS \"ID\",\"PgSQL node\",\"DBname\",\"AT time\",\"Code\",\"Retention\",\"Parameters\",\"Status\" FROM show_snapshot_definitions WHERE TRUE ' + server_sql + node_sql + dbname_sql) return self.cur - + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def show_restore_definitions(self,backup_server_list,pgsql_node_list,dbname_list): @@ -563,59 +563,59 @@ def show_restore_definitions(self,backup_server_list,pgsql_node_list,dbname_list if backup_server_list != None: server_sql = 'AND (FALSE ' - + for server in backup_server_list: if server.isdigit(): server_sql = server_sql + 'OR backup_server_id = ' + str(server) + ' ' else: server_sql = server_sql + 'OR backup_server_id = ' + str(self.get_backup_server_id(server.lower())) + ' ' - + server_sql = server_sql + ') ' - + else: server_sql = '' - + if pgsql_node_list != None: node_sql = 'AND (FALSE ' - + for node in pgsql_node_list: if node.isdigit(): node_sql = node_sql + 'OR target_pgsql_node_id = ' + str(node) + ' ' else: node_sql = node_sql + 'OR target_pgsql_node_id = ' + str(self.get_pgsql_node_id(node.lower())) + ' ' - + node_sql = node_sql + ') ' - + else: - node_sql = '' + node_sql = '' if dbname_list != None: dbname_sql = 'AND (FALSE ' - + for dbname in dbname_list: dbname_sql = dbname_sql + 'OR "Target DBname" = \'' + dbname + '\' ' - + dbname_sql = dbname_sql + ') ' - + else: dbname_sql = '' - + self.cur.execute('SELECT \"RestoreDef\",\"Registered\",\"BckID\",target_pgsql_node_id AS \"ID\",\"Target PgSQL node\",\"Target DBname\",\"Renamed database\",\"AT time\",\"Extra parameters\",\"Status\" FROM show_restore_definitions WHERE TRUE ' + server_sql + node_sql + dbname_sql) return self.cur except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def show_backup_catalog(self,backup_server_list,pgsql_node_list,dbname_list,def_id_list,status_list): @@ -629,80 +629,80 @@ def show_backup_catalog(self,backup_server_list,pgsql_node_list,dbname_list,def_ if backup_server_list != None: server_sql = 'AND (FALSE ' - + for server in backup_server_list: if server.isdigit(): server_sql = server_sql + 'OR backup_server_id = ' + str(server) + ' ' else: server_sql = server_sql + 'OR backup_server_id = ' + str(self.get_backup_server_id(server.lower())) + ' ' - + server_sql = server_sql + ') ' - + else: server_sql = '' - + if pgsql_node_list != None: node_sql = 'AND (FALSE ' - + for node in pgsql_node_list: if node.isdigit(): node_sql = node_sql + 'OR pgsql_node_id = ' + str(node) + ' ' else: node_sql = node_sql + 'OR pgsql_node_id = ' + str(self.get_pgsql_node_id(node.lower())) + ' ' - + node_sql = node_sql + ') ' - + else: - node_sql = '' + node_sql = '' if dbname_list != None: dbname_sql = 'AND (FALSE ' - + for dbname in dbname_list: dbname_sql = dbname_sql + 'OR "DBname" = \'' + dbname + '\' ' - + dbname_sql = dbname_sql + ') ' - + else: dbname_sql = '' - + if def_id_list != None: def_id_sql = 'AND (FALSE ' - + for def_id in def_id_list: def_id_sql = def_id_sql + 'OR def_id = \'' + def_id + '\' ' - + def_id_sql = def_id_sql + ') ' - + else: def_id_sql = '' if status_list != None: status_sql = 'AND (FALSE ' - + for status in status_list: status_sql = status_sql + 'OR "Status" = \'' + status + '\' ' - + status_sql = status_sql + ') ' - + else: status_sql = '' - - + + self.cur.execute('SELECT \"BckID\",\"DefID\",\"SnapshotID\",\"Finished\",backup_server_id AS \"ID.\",\"Backup server\",pgsql_node_id AS \"ID\",\"PgSQL node\",\"DBname\",\"Duration\",\"Size\",\"Code\",\"Execution\",\"Status\" FROM show_backup_catalog WHERE TRUE ' + server_sql + node_sql + dbname_sql + def_id_sql + status_sql) - + return self.cur except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def show_restore_catalog(self,backup_server_list,pgsql_node_list,dbname_list): @@ -716,58 +716,58 @@ def show_restore_catalog(self,backup_server_list,pgsql_node_list,dbname_list): if backup_server_list != None: server_sql = 'AND (FALSE ' - + for server in backup_server_list: if server.isdigit(): server_sql = server_sql + 'OR backup_server_id = ' + str(server) + ' ' else: server_sql = server_sql + 'OR backup_server_id = ' + str(self.get_backup_server_id(server.lower())) + ' ' - + server_sql = server_sql + ') ' - + else: server_sql = '' - + if pgsql_node_list != None: node_sql = 'AND (FALSE ' - + for node in pgsql_node_list: if node.isdigit(): node_sql = node_sql + 'OR target_pgsql_node_id = ' + str(node) + ' ' else: node_sql = node_sql + 'OR target_pgsql_node_id = ' + str(self.get_pgsql_node_id(node.lower())) + ' ' - + node_sql = node_sql + ') ' - + else: - node_sql = '' + node_sql = '' if dbname_list != None: dbname_sql = 'AND (FALSE ' - + for dbname in dbname_list: dbname_sql = dbname_sql + 'OR "Target DBname" = \'' + dbname + '\' ' - + dbname_sql = dbname_sql + ') ' - + else: dbname_sql = '' - + self.cur.execute('SELECT \"RestoreID\",\"RestoreDef\",\"BckID\",\"Finished\",backup_server_id AS \"ID.\",\"Backup server\",target_pgsql_node_id AS \"ID\",\"Target PgSQL node\",\"Target DBname\",\"Duration\",\"Status\" FROM show_restore_catalog WHERE TRUE ' + server_sql + node_sql + dbname_sql) return self.cur except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def show_backup_details(self,bck_id): @@ -779,7 +779,7 @@ def show_backup_details(self,bck_id): if self.cur: try: self.cur.execute("SELECT * FROM show_backup_details WHERE bck_id= %s",(bck_id,)) - + result = OrderedDict() for record in self.cur: @@ -827,14 +827,14 @@ def show_backup_details(self,bck_id): except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: - raise e - + raise e + # ############################################ - # Method + # Method # ############################################ def show_restore_details(self,restore_id): @@ -846,11 +846,11 @@ def show_restore_details(self,restore_id): if self.cur: try: self.cur.execute("SELECT * FROM show_restore_details WHERE restore_id= %s",(restore_id,)) - + result = OrderedDict() for record in self.cur: - + result['RestoreID'] = str(record[0]) result['ProcPID'] = str(record[5]) result['Registered'] = str(record[2]) @@ -883,15 +883,15 @@ def show_restore_details(self,restore_id): except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: - raise e + raise e # ############################################ - # Method + # Method # ############################################ def show_databases_without_backup_definitions(self,pgsql_node): @@ -900,11 +900,11 @@ def show_databases_without_backup_definitions(self,pgsql_node): try: result = [] - + pgsql_node_list = [] if pgsql_node.lower() in ['all','*']: - + for id,fqdn in self.get_pgsql_nodes_list(): pgsql_node_list.append(str(id)) @@ -922,7 +922,7 @@ def show_databases_without_backup_definitions(self,pgsql_node): dsn_value = self.get_pgsql_node_dsn(pgsql_node_id) db_node = self.__class__(dsn_value,'pgbackman_cli') - + database_list = [] all_databases = [] databases_with_bckdef = [] @@ -934,11 +934,11 @@ def show_databases_without_backup_definitions(self,pgsql_node): # If it is down, we continue with the next one in # pgsql_node_list # - + try: for database in db_node.get_pgsql_node_database_list(): all_databases.append(database[0]) - + except Exception as e: continue @@ -951,13 +951,13 @@ def show_databases_without_backup_definitions(self,pgsql_node): result.append((pgsql_node_fqdn,database)) return result - + except Exception as e: raise e - + # ############################################ - # Method + # Method # ############################################ def show_snapshots_in_progress(self): @@ -968,22 +968,22 @@ def show_snapshots_in_progress(self): if self.cur: try: - + self.cur.execute('SELECT \"SnapshotID\",\"Registered\",backup_server_id AS \"ID.\",\"Backup server\",pgsql_node_id AS \"ID\",\"PgSQL node\",\"DBname\",\"AT time\",\"Code\",\"Elapsed time\" FROM show_snapshots_in_progress') return self.cur - + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def show_restores_in_progress(self): @@ -994,22 +994,22 @@ def show_restores_in_progress(self): if self.cur: try: - + self.cur.execute('SELECT \"RestoreDef\",\"Registered\",\"BckID\",backup_server_id AS \"ID.\",\"Backup server\",target_pgsql_node_id AS \"ID\",\"Target PgSQL node\",\"Target DBname\",\"AT time\",\"Elapsed time\" FROM show_restores_in_progress') - + return self.cur except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def get_default_backup_server_parameter(self,param): @@ -1021,7 +1021,7 @@ def get_default_backup_server_parameter(self,param): if self.cur: try: self.cur.execute('SELECT get_default_backup_server_parameter(%s)',(param,)) - + data = self.cur.fetchone()[0] return data @@ -1029,15 +1029,15 @@ def get_default_backup_server_parameter(self,param): raise e self.pg_close() - + except psycopg2.Error as e: - raise e + raise e + - # ############################################ - # Method + # Method # ############################################ - + def get_default_pgsql_node_parameter(self,param): """A function to get the default value of a configuration parameter""" @@ -1047,23 +1047,23 @@ def get_default_pgsql_node_parameter(self,param): if self.cur: try: self.cur.execute('SELECT get_default_pgsql_node_parameter(%s)',(param,)) - + data = self.cur.fetchone()[0] return data except psycopg2.Error as e: raise e - + self.pg_close() except psycopg2.Error as e: - raise e - + raise e + # ############################################ - # Method + # Method # ############################################ - + def get_minute_from_interval(self,param): """A function to get a random minute from an interval""" @@ -1073,13 +1073,13 @@ def get_minute_from_interval(self,param): if self.cur: try: self.cur.execute('SELECT get_minute_from_interval(%s)',(param,)) - + data = self.cur.fetchone()[0] return data - + except psycopg2.Error as e: raise e - + self.pg_close() except psycopg2.Error as e: @@ -1087,35 +1087,35 @@ def get_minute_from_interval(self,param): # ############################################ - # Method + # Method # ############################################ - + def get_hour_from_interval(self,param): """A function to get a random hour from an interval""" - + try: self.pg_connect() if self.cur: try: self.cur.execute('SELECT get_hour_from_interval(%s)',(param,)) - + data = self.cur.fetchone()[0] return data except psycopg2.Error as e: raise e - + self.pg_close() except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def get_backup_server_fqdn(self,param): """A function to get the FQDN of a backup server""" @@ -1125,7 +1125,7 @@ def get_backup_server_fqdn(self,param): if self.cur: try: self.cur.execute('SELECT get_backup_server_fqdn(%s)',(param,)) - + data = self.cur.fetchone()[0] return data @@ -1133,15 +1133,15 @@ def get_backup_server_fqdn(self,param): raise e self.pg_close() - + except Exception as e: raise e # ############################################ - # Method + # Method # ############################################ - + def get_backup_server_id(self,param): """A function to get the ID of a backup server""" @@ -1151,23 +1151,23 @@ def get_backup_server_id(self,param): if self.cur: try: self.cur.execute('SELECT get_backup_server_id(%s)',(param,)) - + data = self.cur.fetchone()[0] return data except psycopg2.Error as e: raise e - + self.pg_close() - + except Exception as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def get_pgsql_node_fqdn(self,param): """A function to get the FQDN of a PgSQL node""" @@ -1177,13 +1177,13 @@ def get_pgsql_node_fqdn(self,param): if self.cur: try: self.cur.execute('SELECT get_pgsql_node_fqdn(%s)',(param,)) - + data = self.cur.fetchone()[0] return data except psycopg2.Error as e: raise e - + self.pg_close() except Exception as e: @@ -1191,9 +1191,9 @@ def get_pgsql_node_fqdn(self,param): # ############################################ - # Method + # Method # ############################################ - + def get_pgsql_node_id(self,param): """A function to get the ID of a PgSQL node""" @@ -1203,34 +1203,34 @@ def get_pgsql_node_id(self,param): if self.cur: try: self.cur.execute('SELECT get_pgsql_node_id(%s)',(param,)) - + data = self.cur.fetchone()[0] return data except psycopg2.Error as e: raise e - + self.pg_close() except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def get_next_crontab_id_to_generate(self,param): """A function to get the next PgSQL node ID to generate a crontab file for""" try: self.pg_connect() - + if self.cur: try: self.cur.execute('SELECT get_next_crontab_id_to_generate(%s)',(param,)) self.conn.commit() - + data = self.cur.fetchone()[0] return data @@ -1238,28 +1238,28 @@ def get_next_crontab_id_to_generate(self,param): raise e self.pg_close() - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ - + def generate_crontab_backup_jobs(self,backup_server_id,pgsql_node_id): """A function to get the crontab file for a PgSQL node in a backup server""" try: self.pg_connect() - + if self.cur: try: self.cur.execute('SELECT generate_crontab_backup_jobs(%s,%s)',(backup_server_id,pgsql_node_id)) - + data = self.cur.fetchone()[0] return data - + except psycopg2.Error as e: raise e @@ -1267,36 +1267,36 @@ def generate_crontab_backup_jobs(self,backup_server_id,pgsql_node_id): except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def update_job_queue(self,backup_server_id,pgsql_node_id): """A function to update the backup job queue if the crontab generation fails""" try: self.pg_connect() - + if self.cur: try: self.cur.execute('SELECT update_job_queue(%s,%s)',(backup_server_id,pgsql_node_id)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - self.pg_close() + self.pg_close() except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def get_pgsql_node_dsn(self,pgsql_node_id): """A function to DSN value for a PgSQL node in a backup server""" @@ -1306,34 +1306,34 @@ def get_pgsql_node_dsn(self,pgsql_node_id): if self.cur: try: self.cur.execute('SELECT get_pgsql_node_dsn(%s)',(pgsql_node_id,)) - + data = self.cur.fetchone()[0] return data - + except psycopg2.Error as e: return None self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def register_backup_catalog_1_0_0(self,def_id,procpid,backup_server_id,pgsql_node_id,dbname,started,finished,duration,pg_dump_file, pg_dump_file_size,pg_dump_log_file,pg_dump_roles_file,pg_dump_roles_file_size,pg_dump_roles_log_file, pg_dump_dbconfig_file,pg_dump_dbconfig_file_size,pg_dump_dbconfig_log_file,global_log_file,execution_status, execution_method,error_message,snapshot_id,role_list,pgsql_node_release): - + """A function to update the backup job catalog in a 1.0.0 version""" try: self.pg_connect() - + if self.cur: try: @@ -1361,32 +1361,32 @@ def register_backup_catalog_1_0_0(self,def_id,procpid,backup_server_id,pgsql_nod snapshot_id, role_list, pgsql_node_release)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - self.pg_close() + self.pg_close() except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ - + def register_backup_catalog(self,def_id,procpid,backup_server_id,pgsql_node_id,dbname,started,finished,duration,pg_dump_file, pg_dump_file_size,pg_dump_log_file,pg_dump_roles_file,pg_dump_roles_file_size,pg_dump_roles_log_file, pg_dump_dbconfig_file,pg_dump_dbconfig_file_size,pg_dump_dbconfig_log_file,global_log_file,execution_status, execution_method,error_message,snapshot_id,role_list,pgsql_node_release,pg_dump_release): - + """A function to update the backup job catalog""" try: self.pg_connect() - + if self.cur: try: @@ -1415,30 +1415,30 @@ def register_backup_catalog(self,def_id,procpid,backup_server_id,pgsql_node_id,d role_list, pgsql_node_release, pg_dump_release)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - self.pg_close() + self.pg_close() except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ - + def register_restore_catalog(self,restore_def,procpid,backup_server_id,target_pgsql_node_id,source_dbname,target_dbname,renamed_dbname,started,finished,duration,restore_log_file, global_log_file,execution_status,error_message,role_list,target_pgsql_node_release,backup_pg_release): - + """A function to update the restore job catalog""" try: self.pg_connect() - + if self.cur: try: @@ -1460,32 +1460,32 @@ def register_restore_catalog(self,restore_def,procpid,backup_server_id,target_pg target_pgsql_node_release, backup_pg_release)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - self.pg_close() + self.pg_close() except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ - + def print_results_table(self,cur,colnames,left_columns): '''A function to print a table with sql results''' - + if self.output_format == 'table': - + x = PrettyTable(colnames) x.padding_width = 1 - + for column in left_columns: x.align[column] = "l" - + for records in cur: columns = [] @@ -1493,28 +1493,28 @@ def print_results_table(self,cur,colnames,left_columns): columns.append(records[index]) x.add_row(columns) - + print x.get_string() print elif self.output_format == 'csv': - + for records in cur: columns = [] - + for index in range(len(colnames)): columns.append(str(records[index])) - + print ','.join(columns) # ############################################ - # Method + # Method # ############################################ - + def get_catalog_entries_to_delete(self,backup_server_id): """A function to get catalog information about force deletion of backup job definitions""" - + try: self.pg_connect() @@ -1522,9 +1522,9 @@ def get_catalog_entries_to_delete(self,backup_server_id): try: self.cur.execute('SELECT * FROM get_catalog_entries_to_delete WHERE backup_server_id = %s',(backup_server_id,)) self.conn.commit() - + return self.cur - + except psycopg2.Error as e: raise e @@ -1535,9 +1535,9 @@ def get_catalog_entries_to_delete(self,backup_server_id): # ############################################ - # Method + # Method # ############################################ - + def delete_catalog_entries_to_delete(self,del_id): """A function to delete catalog info from defid force deletions""" @@ -1547,21 +1547,21 @@ def delete_catalog_entries_to_delete(self,del_id): if self.cur: try: self.cur.execute('SELECT delete_catalog_entries_to_delete(%s)',(del_id,)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def get_cron_catalog_entries_to_delete_by_retention(self,backup_server_id): """A function to get backup catalog entries to delete""" @@ -1571,23 +1571,23 @@ def get_cron_catalog_entries_to_delete_by_retention(self,backup_server_id): if self.cur: try: self.cur.execute('SELECT * FROM get_cron_catalog_entries_to_delete_by_retention WHERE backup_server_id = %s',(backup_server_id,)) - self.conn.commit() - + self.conn.commit() + return self.cur except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def get_at_catalog_entries_to_delete_by_retention(self,backup_server_id): """A function to get snapshot catalog entries to delete""" @@ -1597,26 +1597,26 @@ def get_at_catalog_entries_to_delete_by_retention(self,backup_server_id): if self.cur: try: self.cur.execute('SELECT * FROM get_at_catalog_entries_to_delete_by_retention WHERE backup_server_id = %s',(backup_server_id,)) - self.conn.commit() - + self.conn.commit() + return self.cur except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def get_restore_logs_to_delete(self,backup_server_id): """A function to get restore log files to delete""" - + try: self.pg_connect() @@ -1624,9 +1624,9 @@ def get_restore_logs_to_delete(self,backup_server_id): try: self.cur.execute('SELECT * FROM get_restore_logs_to_delete WHERE backup_server_id = %s',(backup_server_id,)) self.conn.commit() - + return self.cur - + except psycopg2.Error as e: raise e @@ -1636,9 +1636,9 @@ def get_restore_logs_to_delete(self,backup_server_id): raise e # ############################################ - # Method + # Method # ############################################ - + def delete_restore_logs_to_delete(self,del_id): """A function to delete restore logs to delete information""" @@ -1648,21 +1648,21 @@ def delete_restore_logs_to_delete(self,del_id): if self.cur: try: self.cur.execute('SELECT delete_restore_logs_to_delete(%s)',(del_id,)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def delete_backup_catalog(self,bck_id): """A function to delete entries from backup job catalog""" @@ -1672,20 +1672,20 @@ def delete_backup_catalog(self,bck_id): if self.cur: try: self.cur.execute('SELECT delete_backup_catalog(%s)',(bck_id,)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def delete_snapshot_definition(self,snapshot_id): """A function to delete entries from snapshot_definition""" @@ -1695,41 +1695,41 @@ def delete_snapshot_definition(self,snapshot_id): if self.cur: try: self.cur.execute('SELECT delete_snapshot_definition(%s)',(snapshot_id,)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def add_listen(self,channel): '''Subscribe to a PostgreSQL NOTIFY''' replace_list = ['.','-'] - + for i,j in enumerate(replace_list): channel = channel.replace(j, '_') - + sql = "LISTEN %s" % channel - + try: self.cur.execute(sql) self.conn.commit() - + except psycopg2.Error as e: raise e - - + + # ############################################ - # Method + # Method # ############################################ def delete_listen(self,channel): @@ -1739,27 +1739,27 @@ def delete_listen(self,channel): for i,j in enumerate(replace_list): channel = channel.replace(j, '_') - + sql = "UNLISTEN %s" % channel try: self.cur.execute(sql) self.conn.commit() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def get_listen_channel_names(self,param): """A function to get a list of channels to LISTEN for a backup_server""" try: list = [] - + self.cur.execute('SELECT get_listen_channel_names(%s)',(param,)) self.conn.commit() @@ -1767,13 +1767,13 @@ def get_listen_channel_names(self,param): list.append(row[0]) return list - + except psycopg2.Error as e: return e # ############################################ - # Method + # Method # ############################################ def show_jobs_queue(self): @@ -1788,18 +1788,18 @@ def show_jobs_queue(self): self.conn.commit() return self.cur - + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def show_backup_server_config(self,backup_server_id): @@ -1819,12 +1819,12 @@ def show_backup_server_config(self,backup_server_id): raise e self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def show_pgsql_node_config(self,pgsql_node_id): @@ -1839,23 +1839,23 @@ def show_pgsql_node_config(self,pgsql_node_id): self.conn.commit() return self.cur - + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def show_pgbackman_stats(self): """A function to get pgbackman global stats""" - + try: self.pg_connect() @@ -1872,16 +1872,16 @@ def show_pgbackman_stats(self): self.cur.execute("SELECT count(*) FROM pgsql_node WHERE status = 'STOPPED'") pgsql_node_stopped_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM (SELECT DISTINCT ON (pgsql_node_id,dbname) def_id FROM backup_definition) AS cnt") dbname_cnt = self.cur.fetchone()[0] self.cur.execute("SELECT count(*) FROM backup_definition WHERE job_status = 'ACTIVE'") backup_jobs_active_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM backup_definition WHERE job_status = 'STOPPED'") backup_jobs_stopped_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM backup_definition WHERE backup_code = 'CLUSTER'") backup_jobs_cluster_cnt = self.cur.fetchone()[0] @@ -1893,28 +1893,28 @@ def show_pgbackman_stats(self): self.cur.execute("SELECT count(*) FROM backup_definition WHERE backup_code = 'SCHEMA'") backup_jobs_schema_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM backup_catalog WHERE execution_status = 'SUCCEEDED'") backup_catalog_succeeded_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM backup_catalog WHERE execution_status = 'ERROR'") backup_catalog_error_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT pg_size_pretty(sum(pg_dump_file_size+pg_dump_roles_file_size+pg_dump_dbconfig_file_size)) FROM backup_catalog") backup_space = self.cur.fetchone()[0] - + self.cur.execute("SELECT sum(duration) FROM backup_catalog") backup_duration = self.cur.fetchone()[0] - + self.cur.execute("select date_trunc('seconds',min(finished)) from backup_catalog;") oldest_backup_job = self.cur.fetchone()[0] - + self.cur.execute("select date_trunc('seconds',max(finished)) from backup_catalog;") newest_backup_job = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM job_queue") job_queue_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM catalog_entries_to_delete") defid_force_deletion_cnt = self.cur.fetchone()[0] @@ -1943,25 +1943,25 @@ def show_pgbackman_stats(self): result['####'] = '' result['Jobs waiting to be processed by pgbackman_control'] = str(job_queue_cnt) result['Forced deletion of backups waiting to be processed'] = str(defid_force_deletion_cnt) - + return result except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: - raise e - + raise e + # ############################################ - # Method + # Method # ############################################ def show_backup_server_stats(self,backup_server_id): """A function to get global stats for a backup server""" - + try: self.pg_connect() @@ -1972,16 +1972,16 @@ def show_backup_server_stats(self,backup_server_id): self.cur.execute("SELECT count(*) FROM (SELECT DISTINCT ON (pgsql_node_id) pgsql_node_id FROM backup_definition WHERE backup_server_id = %s) AS cnt",(backup_server_id,)) pgsql_node_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM (SELECT DISTINCT ON (pgsql_node_id,dbname) def_id FROM backup_definition WHERE backup_server_id = %s) AS cnt",(backup_server_id,)) dbname_cnt = self.cur.fetchone()[0] self.cur.execute("SELECT count(*) FROM backup_definition WHERE job_status = 'ACTIVE' AND backup_server_id = %s",(backup_server_id,)) backup_jobs_active_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM backup_definition WHERE job_status = 'STOPPED' AND backup_server_id = %s",(backup_server_id,)) backup_jobs_stopped_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM backup_definition WHERE backup_code = 'CLUSTER' AND backup_server_id = %s",(backup_server_id,)) backup_jobs_cluster_cnt = self.cur.fetchone()[0] @@ -1993,33 +1993,33 @@ def show_backup_server_stats(self,backup_server_id): self.cur.execute("SELECT count(*) FROM backup_definition WHERE backup_code = 'SCHEMA' AND backup_server_id = %s",(backup_server_id,)) backup_jobs_schema_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM backup_catalog WHERE execution_status = 'SUCCEEDED' AND backup_server_id = %s",(backup_server_id,)) backup_catalog_succeeded_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM backup_catalog WHERE execution_status = 'ERROR' AND backup_server_id = %s",(backup_server_id,)) backup_catalog_error_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT pg_size_pretty(sum(pg_dump_file_size+pg_dump_roles_file_size+pg_dump_dbconfig_file_size)) FROM backup_catalog WHERE backup_server_id = %s",(backup_server_id,)) backup_space = self.cur.fetchone()[0] - + self.cur.execute("SELECT sum(duration) FROM backup_catalog WHERE backup_server_id = %s",(backup_server_id,)) backup_duration = self.cur.fetchone()[0] - + self.cur.execute("select date_trunc('seconds',min(finished)) from backup_catalog WHERE backup_server_id = %s",(backup_server_id,)) oldest_backup_job = self.cur.fetchone()[0] - + self.cur.execute("select date_trunc('seconds',max(finished)) from backup_catalog WHERE backup_server_id = %s",(backup_server_id,)) newest_backup_job = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM job_queue WHERE backup_server_id = %s",(backup_server_id,)) job_queue_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM catalog_entries_to_delete WHERE backup_server_id = %s",(backup_server_id,)) defid_force_deletion_cnt = self.cur.fetchone()[0] - + result = OrderedDict() - + result['Backup server'] = backup_server_fqdn result['#'] = '' result['PgSQL nodes using this backup server'] = str(pgsql_node_cnt) @@ -2043,23 +2043,23 @@ def show_backup_server_stats(self,backup_server_id): result['Forced deletion of backups waiting to be processed'] = str(defid_force_deletion_cnt) return result - + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: - raise e + raise e + - # ############################################ - # Method + # Method # ############################################ def show_pgsql_node_stats(self,pgsql_node_id): """A function to get global stats for a backup server""" - + try: self.pg_connect() @@ -2070,16 +2070,16 @@ def show_pgsql_node_stats(self,pgsql_node_id): self.cur.execute("SELECT count(*) FROM (SELECT DISTINCT ON (backup_server_id) backup_server_id FROM backup_definition WHERE pgsql_node_id = %s) AS cnt",(pgsql_node_id,)) backup_server_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM (SELECT DISTINCT ON (pgsql_node_id,dbname) def_id FROM backup_definition WHERE pgsql_node_id = %s) AS cnt",(pgsql_node_id,)) dbname_cnt = self.cur.fetchone()[0] self.cur.execute("SELECT count(*) FROM backup_definition WHERE job_status = 'ACTIVE' AND pgsql_node_id = %s",(pgsql_node_id,)) backup_jobs_active_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM backup_definition WHERE job_status = 'STOPPED' AND pgsql_node_id = %s",(pgsql_node_id,)) backup_jobs_stopped_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM backup_definition WHERE backup_code = 'CLUSTER' AND pgsql_node_id = %s",(pgsql_node_id,)) backup_jobs_cluster_cnt = self.cur.fetchone()[0] @@ -2091,30 +2091,30 @@ def show_pgsql_node_stats(self,pgsql_node_id): self.cur.execute("SELECT count(*) FROM backup_definition WHERE backup_code = 'SCHEMA' AND pgsql_node_id = %s",(pgsql_node_id,)) backup_jobs_schema_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM backup_catalog WHERE execution_status = 'SUCCEEDED' AND pgsql_node_id = %s",(pgsql_node_id,)) backup_catalog_succeeded_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM backup_catalog WHERE execution_status = 'ERROR' AND pgsql_node_id = %s",(pgsql_node_id,)) backup_catalog_error_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT pg_size_pretty(sum(pg_dump_file_size+pg_dump_roles_file_size+pg_dump_dbconfig_file_size)) FROM backup_catalog WHERE pgsql_node_id = %s",(pgsql_node_id,)) backup_space = self.cur.fetchone()[0] - + self.cur.execute("SELECT sum(duration) FROM backup_catalog WHERE pgsql_node_id = %s",(pgsql_node_id,)) backup_duration = self.cur.fetchone()[0] - + self.cur.execute("select date_trunc('seconds',min(finished)) from backup_catalog WHERE pgsql_node_id = %s",(pgsql_node_id,)) oldest_backup_job = self.cur.fetchone()[0] - + self.cur.execute("select date_trunc('seconds',max(finished)) from backup_catalog WHERE pgsql_node_id = %s",(pgsql_node_id,)) newest_backup_job = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM job_queue WHERE pgsql_node_id = %s",(pgsql_node_id,)) job_queue_cnt = self.cur.fetchone()[0] - + result = OrderedDict() - + result['PgSQL node'] = "[" + str(pgsql_node_id) + "] " + pgsql_node_fqdn result['#'] = '' result['Backup servers running backups for this Node'] = str(backup_server_cnt) @@ -2135,25 +2135,25 @@ def show_pgsql_node_stats(self,pgsql_node_id): result['Newest backup in catalog'] = str(newest_backup_job) result['####'] = '' result['Jobs waiting to be processed by pgbackman_control'] = str(job_queue_cnt) - + return result except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: - raise e - + raise e + # ############################################ - # Method + # Method # ############################################ - + def get_pgsql_node_to_delete(self,backup_server_id): """A function to get the PgSQL node data from nodes that has been deleted""" - + try: self.pg_connect() @@ -2161,9 +2161,9 @@ def get_pgsql_node_to_delete(self,backup_server_id): try: self.cur.execute('SELECT backup_server_id,pgsql_node_id FROM pgsql_node_to_delete WHERE backup_server_id = %s',(backup_server_id,)) self.conn.commit() - + return self.cur - + except psycopg2.Error as e: raise e @@ -2173,12 +2173,12 @@ def get_pgsql_node_to_delete(self,backup_server_id): raise e # ############################################ - # Method + # Method # ############################################ - + def delete_pgsql_node_to_delete(self,backup_server_id,pgsql_node_id): """A function to delete the PgSQL node data from a node that has been deleted""" - + try: self.pg_connect() @@ -2186,7 +2186,7 @@ def delete_pgsql_node_to_delete(self,backup_server_id,pgsql_node_id): try: self.cur.execute('SELECT delete_pgsql_node_to_delete(%s,%s)',(backup_server_id,pgsql_node_id)) self.conn.commit() - + except psycopg2.Error as e: raise e @@ -2197,12 +2197,12 @@ def delete_pgsql_node_to_delete(self,backup_server_id,pgsql_node_id): # ############################################ - # Method + # Method # ############################################ - + def get_pgsql_node_stopped(self): """A function to get data for PgSQL nodes stopped when pgbackman_control was down""" - + try: self.pg_connect() @@ -2210,9 +2210,9 @@ def get_pgsql_node_stopped(self): try: self.cur.execute('SELECT pgsql_node_id FROM pgsql_node_stopped') self.conn.commit() - + return self.cur - + except psycopg2.Error as e: raise e @@ -2223,7 +2223,7 @@ def get_pgsql_node_stopped(self): # ############################################ - # Method + # Method # ############################################ def show_empty_backup_catalogs(self): @@ -2238,18 +2238,18 @@ def show_empty_backup_catalogs(self): self.conn.commit() return self.cur - + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ def database_exists(self,dbname): @@ -2267,20 +2267,20 @@ def database_exists(self,dbname): if dbname_cnt > 0: return True - + elif dbname_cnt == 0: return False - + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ def role_exists(self,role): @@ -2298,21 +2298,21 @@ def role_exists(self,role): if role_cnt > 0: return True - + elif role_cnt == 0: return False - + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ def get_pgsql_node_database_list(self): @@ -2327,20 +2327,20 @@ def get_pgsql_node_database_list(self): self.conn.commit() return self.cur - + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ - + def update_backup_server(self,backup_server_id,remarks): """A function to update a backup server""" @@ -2350,21 +2350,21 @@ def update_backup_server(self,backup_server_id,remarks): if self.cur: try: self.cur.execute('SELECT update_backup_server(%s,%s)',(backup_server_id,remarks)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def update_pgsql_node(self,pgsql_node_id,port,admin_user,status,remarks): """A function to update a PgSQL node""" @@ -2374,21 +2374,21 @@ def update_pgsql_node(self,pgsql_node_id,port,admin_user,status,remarks): if self.cur: try: self.cur.execute('SELECT update_pgsql_node(%s,%s,%s,%s,%s)',(pgsql_node_id,port,admin_user,status,remarks)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def update_pgsql_node_config(self,pgsql_node_id,backup_minutes_interval,backup_hours_interval,backup_weekday_cron, backup_month_cron,backup_day_month_cron,backup_code,retention_period,retention_redundancy,automatic_deletion_retention, extra_backup_parameters,extra_restore_parameters,backup_job_status,domain,logs_email,admin_user,pgport,pgnode_backup_partition, @@ -2420,22 +2420,22 @@ def update_pgsql_node_config(self,pgsql_node_id,backup_minutes_interval,backup_h pgnode_backup_partition, pgnode_crontab_file, pgsql_node_status)) - - self.conn.commit() - + + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def update_backup_server_config(self,backup_server_id,pgsql_bin_9_0,pgsql_bin_9_1,pgsql_bin_9_2,pgsql_bin_9_3,pgsql_bin_9_4,pgsql_bin_9_5,pgsql_bin_9_6,pgsql_bin_10,root_backup_partition): """A function to update the configuration of a backup server""" @@ -2454,21 +2454,21 @@ def update_backup_server_config(self,backup_server_id,pgsql_bin_9_0,pgsql_bin_9_ pgsql_bin_9_6, pgsql_bin_10, root_backup_partition)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def check_pgsql_node_status(self,pgsql_node_id): """A function to check if a PgSQL node is stopped""" @@ -2478,24 +2478,24 @@ def check_pgsql_node_status(self,pgsql_node_id): if self.cur: try: self.cur.execute('SELECT check_pgsql_node_status(%s)',(pgsql_node_id,)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ - + def get_new_snapshots(self,backup_server_id): """A function to get new snapshots to be run in a backup server""" - + try: self.pg_connect() @@ -2503,9 +2503,9 @@ def get_new_snapshots(self,backup_server_id): try: self.cur.execute('SELECT "SnapshotID","AT time" FROM show_snapshot_definitions WHERE backup_server_id = %s AND "Status" = %s',(backup_server_id,'WAITING')) self.conn.commit() - + return self.cur - + except psycopg2.Error as e: raise e @@ -2516,12 +2516,12 @@ def get_new_snapshots(self,backup_server_id): # ############################################ - # Method + # Method # ############################################ - + def generate_snapshot_at_file(self,snapshot_id): """A function to generate a at file for a snapshot""" - + try: self.pg_connect() @@ -2529,9 +2529,9 @@ def generate_snapshot_at_file(self,snapshot_id): try: self.cur.execute('SELECT generate_snapshot_at_file(%s)',(snapshot_id,)) self.conn.commit() - + return self.cur.fetchone()[0] - + except psycopg2.Error as e: raise e @@ -2542,12 +2542,12 @@ def generate_snapshot_at_file(self,snapshot_id): # ############################################ - # Method + # Method # ############################################ - + def update_snapshot_status(self,snapshot_id,status): """A function to update the status for a snapshot""" - + try: self.pg_connect() @@ -2555,7 +2555,7 @@ def update_snapshot_status(self,snapshot_id,status): try: self.cur.execute('SELECT update_snapshot_status(%s,%s)',(snapshot_id,status)) self.conn.commit() - + except psycopg2.Error as e: raise e @@ -2566,9 +2566,9 @@ def update_snapshot_status(self,snapshot_id,status): # ############################################ - # Method + # Method # ############################################ - + def get_dbname_from_bckid(self,bck_id): """A function to get the dbname from a bckID""" @@ -2578,13 +2578,13 @@ def get_dbname_from_bckid(self,bck_id): if self.cur: try: self.cur.execute('SELECT get_dbname_from_bckid(%s)',(bck_id,)) - + data = self.cur.fetchone()[0] return data except psycopg2.Error as e: raise e - + self.pg_close() except Exception as e: @@ -2592,9 +2592,9 @@ def get_dbname_from_bckid(self,bck_id): # ############################################ - # Method + # Method # ############################################ - + def get_backup_server_id_from_bckid(self,bck_id): """A function to get the backup server ID from a bckID""" @@ -2604,13 +2604,13 @@ def get_backup_server_id_from_bckid(self,bck_id): if self.cur: try: self.cur.execute('SELECT get_backup_server_id_from_bckid(%s)',(bck_id,)) - + data = self.cur.fetchone()[0] return data except psycopg2.Error as e: raise e - + self.pg_close() except Exception as e: @@ -2618,9 +2618,9 @@ def get_backup_server_id_from_bckid(self,bck_id): # ############################################ - # Method + # Method # ############################################ - + def get_role_list_from_bckid(self,bck_id): """A function to get the role list from a bckID""" @@ -2630,21 +2630,21 @@ def get_role_list_from_bckid(self,bck_id): if self.cur: try: self.cur.execute('SELECT get_role_list_from_bckid(%s)',(bck_id,)) - + data = self.cur.fetchone()[0] return data except psycopg2.Error as e: raise e - + self.pg_close() except Exception as e: raise e - + # ############################################ - # Method + # Method # ############################################ def register_restore_definition(self,at_time,backup_server_id,pgsql_node_id,bck_id,target_dbname,renamed_dbname,extra_restore_parameters,roles_to_restore): @@ -2663,24 +2663,24 @@ def register_restore_definition(self,at_time,backup_server_id,pgsql_node_id,bck_ renamed_dbname, extra_restore_parameters, roles_to_restore)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def get_new_restore(self,backup_server_id): """A function to get new restore jobs to be run in a backup server""" - + try: self.pg_connect() @@ -2688,9 +2688,9 @@ def get_new_restore(self,backup_server_id): try: self.cur.execute('SELECT "RestoreDef","AT time" FROM show_restore_definitions WHERE backup_server_id = %s AND "Status" = %s',(backup_server_id,'WAITING')) self.conn.commit() - + return self.cur - + except psycopg2.Error as e: raise e @@ -2701,12 +2701,12 @@ def get_new_restore(self,backup_server_id): # ############################################ - # Method + # Method # ############################################ - + def generate_restore_at_file(self,restore_def): """A function to generate a at file for a restore""" - + try: self.pg_connect() @@ -2714,9 +2714,9 @@ def generate_restore_at_file(self,restore_def): try: self.cur.execute('SELECT generate_restore_at_file(%s)',(restore_def,)) self.conn.commit() - + return self.cur.fetchone()[0] - + except psycopg2.Error as e: raise e @@ -2727,12 +2727,12 @@ def generate_restore_at_file(self,restore_def): # ############################################ - # Method + # Method # ############################################ - + def update_restore_status(self,restore_id,status): """A function to update the status for a restore""" - + try: self.pg_connect() @@ -2740,7 +2740,7 @@ def update_restore_status(self,restore_id,status): try: self.cur.execute('SELECT update_restore_status(%s,%s)',(restore_id,status)) self.conn.commit() - + except psycopg2.Error as e: raise e @@ -2751,22 +2751,22 @@ def update_restore_status(self,restore_id,status): # ############################################ - # Method + # Method # ############################################ - + def rename_existing_database(self,from_name,to_name): """A function to rename an existing database before restoring a backup""" - + try: self.pg_connect() if self.cur: try: sql = 'ALTER DATABASE ' + from_name + ' RENAME TO ' + to_name - + self.cur.execute(sql) self.conn.commit() - + except psycopg2.Error as e: raise e @@ -2776,9 +2776,9 @@ def rename_existing_database(self,from_name,to_name): raise e # ############################################ - # Method + # Method # ############################################ - + def update_backup_definition(self,def_id,minutes_cron,hours_cron,weekday_cron,month_cron,day_month_cron,retention_period, retention_redundancy,extra_backup_parameters,job_status,remarks): """A function to update a backup definition""" @@ -2799,19 +2799,19 @@ def update_backup_definition(self,def_id,minutes_cron,hours_cron,weekday_cron,mo extra_backup_parameters, job_status, remarks)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ def move_backup_definition(self,from_backup_server, to_backup_server,pgsql_node_list,dbname_list,def_id_list): @@ -2835,61 +2835,61 @@ def move_backup_definition(self,from_backup_server, to_backup_server,pgsql_node_ if pgsql_node_list != None: node_sql = 'AND (FALSE ' - + for node in pgsql_node_list: if node.isdigit(): node_sql = node_sql + 'OR pgsql_node_id = ' + str(node) + ' ' else: node_sql = node_sql + 'OR pgsql_node_id = ' + str(self.get_pgsql_node_id(node.lower())) + ' ' - + node_sql = node_sql + ') ' - + else: - node_sql = '' + node_sql = '' if dbname_list != None: dbname_sql = 'AND (FALSE ' - + for dbname in dbname_list: dbname_sql = dbname_sql + 'OR "dbname" = \'' + dbname + '\' ' - + dbname_sql = dbname_sql + ') ' - + else: dbname_sql = '' - + if def_id_list != None: def_id_sql = 'AND (FALSE ' - + for def_id in def_id_list: def_id_sql = def_id_sql + 'OR def_id = \'' + def_id + '\' ' - + def_id_sql = def_id_sql + ') ' - + else: def_id_sql = '' - + self.cur.execute('UPDATE backup_definition SET backup_server_id = ' + to_server_id + ' WHERE backup_server_id = ' + from_server_id + ' ' + node_sql + dbname_sql + def_id_sql) - - self.conn.commit() + + self.conn.commit() except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ - + def get_backup_definition_def_value(self,def_id,parameter): """A function to get the value of an attribute from a backup_definition""" - + try: self.pg_connect() @@ -2900,7 +2900,7 @@ def get_backup_definition_def_value(self,def_id,parameter): data = self.cur.fetchone()[0] return data - + except psycopg2.Error as e: raise e @@ -2911,12 +2911,12 @@ def get_backup_definition_def_value(self,def_id,parameter): # ############################################ - # Method + # Method # ############################################ - + def get_pgsql_node_config_value(self,pgsql_node_id,parameter): """A function to get the value of a default configuration parameter for a PgSQL node""" - + try: self.pg_connect() @@ -2927,7 +2927,7 @@ def get_pgsql_node_config_value(self,pgsql_node_id,parameter): data = self.cur.fetchone()[0] return data - + except psycopg2.Error as e: raise e @@ -2937,12 +2937,12 @@ def get_pgsql_node_config_value(self,pgsql_node_id,parameter): raise e # ############################################ - # Method + # Method # ############################################ - + def get_backup_server_config_value(self,backup_server_id,parameter): """A function to get the value of a default configuration parameter for a backup server""" - + try: self.pg_connect() @@ -2953,7 +2953,7 @@ def get_backup_server_config_value(self,backup_server_id,parameter): data = self.cur.fetchone()[0] return data - + except psycopg2.Error as e: raise e @@ -2964,12 +2964,12 @@ def get_backup_server_config_value(self,backup_server_id,parameter): # ############################################ - # Method + # Method # ############################################ - + def get_pgsql_node_def_value(self,pgsql_node_id,parameter): """A function to get the value of an attribute from pgsql_node""" - + try: self.pg_connect() @@ -2980,7 +2980,7 @@ def get_pgsql_node_def_value(self,pgsql_node_id,parameter): data = self.cur.fetchone()[0] return data - + except psycopg2.Error as e: raise e @@ -2991,12 +2991,12 @@ def get_pgsql_node_def_value(self,pgsql_node_id,parameter): # ############################################ - # Method + # Method # ############################################ - + def get_backup_server_def_value(self,backup_server_id,parameter): """A function to get the value of an attribute from backup_server""" - + try: self.pg_connect() @@ -3007,7 +3007,7 @@ def get_backup_server_def_value(self,backup_server_id,parameter): data = self.cur.fetchone()[0] return data - + except psycopg2.Error as e: raise e @@ -3018,7 +3018,7 @@ def get_backup_server_def_value(self,backup_server_id,parameter): # ############################################ - # Method + # Method # ############################################ def get_pgbackman_database_version(self): @@ -3033,18 +3033,18 @@ def get_pgbackman_database_version(self): self.conn.commit() return self.cur - + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ def run_sql_file(self,sqlfile): @@ -3057,19 +3057,19 @@ def run_sql_file(self,sqlfile): try: self.cur.execute(open(sqlfile,'r').read()) - return - + return + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e - - + + # ############################################ - # Method + # Method # ############################################ def get_pgsql_nodes_list(self): @@ -3084,19 +3084,19 @@ def get_pgsql_nodes_list(self): self.conn.commit() return self.cur - + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def get_deleted_backup_definitions_to_delete_by_retention(self): @@ -3115,19 +3115,19 @@ def get_deleted_backup_definitions_to_delete_by_retention(self): self.conn.commit() return self.cur - + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e - - + + # ############################################ - # Method + # Method # ############################################ def get_all_backup_definitions(self,backup_server_id,pgsql_node_id): @@ -3147,17 +3147,17 @@ def get_all_backup_definitions(self,backup_server_id,pgsql_node_id): self.conn.commit() return self.cur - + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ def get_database_backup_definitions(self,backup_server_id,pgsql_node_id,dbname): @@ -3178,12 +3178,12 @@ def get_database_backup_definitions(self,backup_server_id,pgsql_node_id,dbname): self.conn.commit() return self.cur - + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e @@ -3191,12 +3191,12 @@ def get_database_backup_definitions(self,backup_server_id,pgsql_node_id,dbname): # ############################################ - # Method + # Method # ############################################ - + def update_backup_definition_status_to_delete(self,def_id): """A function to update the status for a backup definition to DELETE""" - + try: self.pg_connect() @@ -3204,7 +3204,7 @@ def update_backup_definition_status_to_delete(self,def_id): try: self.cur.execute('SELECT update_backup_definition_status_to_delete(%s)',(def_id,)) self.conn.commit() - + except psycopg2.Error as e: raise e @@ -3215,7 +3215,7 @@ def update_backup_definition_status_to_delete(self,def_id): # ############################################ - # Method + # Method # ############################################ def get_pgsql_node_database_with_bckdef_list(self,pgsql_node_id): @@ -3234,17 +3234,17 @@ def get_pgsql_node_database_with_bckdef_list(self,pgsql_node_id): self.conn.commit() return self.cur - + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ def get_backup_server_bckdef_list(self,backup_server_id): @@ -3262,129 +3262,140 @@ def get_backup_server_bckdef_list(self,backup_server_id): self.conn.commit() return self.cur - + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ - + def get_pgbackman_dump_count(self): """A function to get the number of pgbackman_dump processes running in a PgSQL node""" - + try: if self.cur: - + self.cur.execute('SELECT count(*) FROM pg_stat_activity WHERE application_name = \'pgbackman_dump\'') self.conn.commit() data = self.cur.fetchone()[0] return data - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ - + def pg_recovery_in_progress(self): """A function to find out if a PgSQL node is in recovery modus. This means that the PgSQL node is a slave/standby node in a replication system. """ - + try: if self.cur: - + self.cur.execute('SELECT pg_is_in_recovery()') self.conn.commit() data = self.cur.fetchone()[0] - + if data == True: return True elif data == False: return False else: return False - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ - + def pg_recovery_paused(self): """A function to find out if a PgSQL node in recovery modus has the recovery process paused""" try: if self.cur: - - self.cur.execute('SELECT pg_is_xlog_replay_paused()') - self.conn.commit() + if self.conn.version() >= 100000: + self.cur.execute('SELECT pg_is_wal_replay_paused()') + self.conn.commit() + + data = self.cur.fetchone()[0] + else: + self.cur.execute('SELECT pg_is_xlog_replay_paused()') + self.conn.commit() + + data = self.cur.fetchone()[0] - data = self.cur.fetchone()[0] - if data == True: return True elif data == False: return False else: return False - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ - + def pause_pg_recovery(self): """A function to pause a postgres recovery process""" try: if self.cur: - - self.cur.execute('SELECT pg_xlog_replay_pause()') - self.conn.commit() - + if self.conn.version() >= 100000: + self.cur.execute('SELECT pg_wal_replay_pause()') + self.conn.commit() + else: + self.cur.execute('SELECT pg_xlog_replay_pause()') + self.conn.commit() + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ - + def resume_pg_recovery(self): """A function to resume a postgres recovery process""" try: if self.cur: - - self.cur.execute('SELECT pg_xlog_replay_resume()') - self.conn.commit() - + if self.conn.version() >= 100000: + self.cur.execute('SELECT pg_wal_replay_resume()') + self.conn.commit() + else: + self.cur.execute('SELECT pg_xlog_replay_resume()') + self.conn.commit() + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ def get_status_info(self,parameter_status,backup_server,def_id): @@ -3429,7 +3440,7 @@ def get_status_info(self,parameter_status,backup_server,def_id): # # This value can be: succeeded, error, warning. # - + elif parameter_status in ['backup_last_status']: self.cur.execute('SELECT lower("Status") FROM show_backup_catalog WHERE def_id = %s ORDER BY "Finished" DESC LIMIT 1',(def_id,)) @@ -3444,20 +3455,20 @@ def get_status_info(self,parameter_status,backup_server,def_id): else: raise Exception("Invalid parameter status [%s]" % parameter_status) - + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ - + def delete_alert(self,alert_id): """A function to delete an alert""" @@ -3467,21 +3478,21 @@ def delete_alert(self,alert_id): if self.cur: try: self.cur.execute('SELECT delete_alert(%s)',(alert_id,)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ - + def update_alert_sent(self,alert_id,status): """A function to update an alert sent status""" @@ -3491,19 +3502,19 @@ def update_alert_sent(self,alert_id,status): if self.cur: try: self.cur.execute('SELECT update_alert_sent(%s,%s)',(alert_id,status)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ def get_alerts(self,backup_server_id): @@ -3521,11 +3532,11 @@ def get_alerts(self,backup_server_id): self.conn.commit() return self.cur - + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e From 4609ef13cee292a7e8345e43cca35ff87ddf5deb Mon Sep 17 00:00:00 2001 From: Philip McClarence Date: Tue, 15 Jan 2019 10:33:32 +0000 Subject: [PATCH 11/12] updating the pg_recovery_x functions to use the renamed functions on pg 10 --- pgbackman/database.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pgbackman/database.py b/pgbackman/database.py index c48a5ba..33cce06 100644 --- a/pgbackman/database.py +++ b/pgbackman/database.py @@ -3332,7 +3332,7 @@ def pg_recovery_paused(self): try: if self.cur: - if self.conn.version() >= 100000: + if self.conn.server_version >= 100000: self.cur.execute('SELECT pg_is_wal_replay_paused()') self.conn.commit() @@ -3363,7 +3363,7 @@ def pause_pg_recovery(self): try: if self.cur: - if self.conn.version() >= 100000: + if self.conn.server_version >= 100000: self.cur.execute('SELECT pg_wal_replay_pause()') self.conn.commit() else: @@ -3383,7 +3383,7 @@ def resume_pg_recovery(self): try: if self.cur: - if self.conn.version() >= 100000: + if self.server_version >= 100000: self.cur.execute('SELECT pg_wal_replay_resume()') self.conn.commit() else: From 2d3bf30830ea7cff0ee29791878a883f82f95f86 Mon Sep 17 00:00:00 2001 From: Philip McClarence Date: Mon, 13 May 2019 14:55:06 +0100 Subject: [PATCH 12/12] Adding postgres 10 as a node version into the pgbackman_restore file --- bin/pgbackman_restore | 179 +++++++++++++++++++++--------------------- 1 file changed, 90 insertions(+), 89 deletions(-) diff --git a/bin/pgbackman_restore b/bin/pgbackman_restore index c8bf522..e37e319 100755 --- a/bin/pgbackman_restore +++ b/bin/pgbackman_restore @@ -31,7 +31,7 @@ import signal import argparse from pgbackman.logs import * -from pgbackman.database import * +from pgbackman.database import * from pgbackman.config import * ''' @@ -55,15 +55,15 @@ def restore_roles(db): try: pg_restore_roles_temp_file = tempfile.NamedTemporaryFile(delete=True,dir=global_parameters['tmp_dir']) logs.logger.debug('Restore roles temp file created %s',pg_restore_roles_temp_file.name) - + with open(global_parameters['pgdump_roles_file'], 'r') as sqldump_in: with open(pg_restore_roles_temp_file.name, 'w') as sqldump_out: sqldump_out.write('BEGIN;\n') - + for line in sqldump_in: - + for role in global_parameters['role_list']: - + # # CREATE ROLE statements # @@ -82,7 +82,7 @@ def restore_roles(db): if 'GRANT ' in line: if ' TO ' + role + ' GRANTED BY ' in line: sqldump_out.write(line) - + sqldump_out.write('COMMIT;\n') logs.logger.debug('Role restore file generated.') sqldump_out.flush() @@ -96,12 +96,12 @@ def restore_roles(db): ' -f ' + pg_restore_roles_temp_file.name with open(global_parameters['restore_log_file'],'a') as restore_log_file: - + restore_log_file.write('------------------------------------\n') restore_log_file.write('Timestamp:' + str(datetime.datetime.now()) + '\n') restore_log_file.write('Command: ' + role_restore_command + '\n') restore_log_file.write('------------------------------------\n\n') - + restore_log_file.flush() proc = subprocess.Popen([role_restore_command],stdout=restore_log_file,stderr=subprocess.STDOUT,shell=True) @@ -109,23 +109,23 @@ def restore_roles(db): if proc.returncode != 0: logs.logger.critical('The command used to restore the roles needed by the target database has a return value != 0') - + global_parameters['execution_status'] = 'ERROR' global_parameters['error_message'] = 'Roles returncode: ' + str(proc.returncode) register_restore_catalog(db) sys.exit(1) - + else: logs.logger.info('Roles for database restored.') except Exception as e: logs.logger.critical('Could not restore the roles needed by the database - %s',e) - + global_parameters['execution_status'] = 'ERROR' global_parameters['error_message'] = 'Problems restoring roles - ' + str(e) register_restore_catalog(db) sys.exit(1) - + # ############################################ @@ -140,10 +140,10 @@ def restore_dbconfig(db): try: pg_restore_dbconfig_temp_file = tempfile.NamedTemporaryFile(delete=True,dir=global_parameters['tmp_dir']) logs.logger.debug('Temp dbconfig restore file created %s',pg_restore_dbconfig_temp_file.name) - + with open(global_parameters['pgdump_dbconfig_file'], 'r') as sqldump_in: with open(pg_restore_dbconfig_temp_file.name, 'w') as sqldump_out: - + for line in sqldump_in: # @@ -153,7 +153,7 @@ def restore_dbconfig(db): if global_parameters['source_dbname'] == global_parameters['target_dbname']: sqldump_out.write(line) - + elif global_parameters['source_dbname'] != global_parameters['target_dbname']: sqldump_out.write(line.replace('CREATE DATABASE ' + global_parameters['source_dbname'],'CREATE DATABASE ' + global_parameters['target_dbname'])) @@ -166,10 +166,10 @@ def restore_dbconfig(db): if global_parameters['source_dbname'] == global_parameters['target_dbname']: sqldump_out.write(line) - + elif global_parameters['source_dbname'] != global_parameters['target_dbname']: sqldump_out.write(line.replace(' ON DATABASE ' + global_parameters['source_dbname'],' ON DATABASE ' + global_parameters['target_dbname'])) - + # # ALTER DATABASE statements # @@ -177,12 +177,12 @@ def restore_dbconfig(db): if global_parameters['source_dbname'] == global_parameters['target_dbname']: sqldump_out.write(line) - + elif global_parameters['source_dbname'] != global_parameters['target_dbname']: sqldump_out.write(line.replace('ALTER DATABASE ' + global_parameters['source_dbname'],'ALTER DATABASE ' + global_parameters['target_dbname'])) - - sqldump_out.write('COMMIT;\n') + + sqldump_out.write('COMMIT;\n') logs.logger.debug('Temp dbconfig restore file generated.') sqldump_out.flush() @@ -196,36 +196,36 @@ def restore_dbconfig(db): ' -f ' + pg_restore_dbconfig_temp_file.name with open(global_parameters['restore_log_file'],'a') as restore_log_file: - + restore_log_file.write('------------------------------------\n') restore_log_file.write('Timestamp:' + str(datetime.datetime.now()) + '\n') restore_log_file.write('Command: ' + dbconfig_restore_command + '\n') restore_log_file.write('------------------------------------\n\n') restore_log_file.flush() - + proc = subprocess.Popen([dbconfig_restore_command],stdout=restore_log_file,stderr=subprocess.STDOUT,shell=True) proc.wait() if proc.returncode != 0: logs.logger.critical('The command used to restore the global database statements needed by the target database has a return value != 0') - + global_parameters['execution_status'] = 'ERROR' global_parameters['error_message'] = 'DBconfig for database returncode: ' + str(proc.returncode) register_restore_catalog(db) sys.exit(1) - + else: logs.logger.info('DBconfig for database restored.') except Exception as e: logs.logger.critical('Could not restore the global database statements needed by the database - %s',e) - + global_parameters['execution_status'] = 'ERROR' global_parameters['error_message'] = 'Problems restoring DBconfig - ' + str(e) register_restore_catalog(db) sys.exit(1) - + # ############################################ # Function restore_database() @@ -233,7 +233,7 @@ def restore_dbconfig(db): def restore_database(db): '''Restore database''' - + global global_parameters try: @@ -247,31 +247,31 @@ def restore_database(db): ' ' + global_parameters['pgdump_file'] with open(global_parameters['restore_log_file'],'a') as restore_log_file: - + restore_log_file.write('------------------------------------\n') restore_log_file.write('Timestamp:' + str(datetime.datetime.now()) + '\n') restore_log_file.write('Command: ' + database_restore_command + '\n') restore_log_file.write('------------------------------------\n\n') - + restore_log_file.flush() proc = subprocess.Popen([database_restore_command],stdout=restore_log_file,stderr=subprocess.STDOUT,shell=True) proc.wait() - + if proc.returncode != 0: logs.logger.critical('The command used to restore the database has a return value != 0') - + global_parameters['execution_status'] = 'ERROR' global_parameters['error_message'] = 'Database restore returncode: ' + str(proc.returncode) register_restore_catalog(db) sys.exit(1) - + else: logs.logger.info('Database restored.') except Exception as e: logs.logger.critical('Could not restore the database - %s',e) - + global_parameters['execution_status'] = 'ERROR' global_parameters['error_message'] = 'Problems restoring Database - ' + str(e) register_restore_catalog(db) @@ -281,7 +281,7 @@ def restore_database(db): # ############################################ # Function get_pgsql_node_dsn() # ############################################ - + def get_pgsql_node_dsn(): '''Get the DSN values needed to connect to a PgSQL node''' @@ -296,10 +296,10 @@ def get_pgsql_node_dsn(): # ############################################ # Function get_pgsql_node_release() # ############################################ - + def get_pgsql_node_release(db,db_pgnode): '''Get the postgreSQL release version a PgSQL node is running''' - + pgsql_node_version = '' try: @@ -308,10 +308,12 @@ def get_pgsql_node_release(db,db_pgnode): db_pgnode.pg_close() except Exception as e: logs.logger.critical('Problems getting the postgreSQL version running on Pgsql node - %s',e) - + logs.logger.debug('PgSQL node version: %s',pgsql_node_version) - if pgsql_node_version == '905': + if pgsql_node_version == '100': + pgsql_node_release = '10' + elif pgsql_node_version == '905': pgsql_node_release = '9_5' elif pgsql_node_version == '904': pgsql_node_release = '9_4' @@ -335,12 +337,12 @@ def get_pgsql_node_release(db,db_pgnode): global_parameters['error_message'] = 'Problems getting pgsql release' register_restore_catalog(db) sys.exit(1) - + # ############################################ # Function get_backup_server_pgsql_bin_dir() # ############################################ - + def get_backup_server_pgsql_bin_dir(db): '''Get the directory with postgreSQL binaries to use''' @@ -349,22 +351,22 @@ def get_backup_server_pgsql_bin_dir(db): try: pgsql_bin_dir = db.get_backup_server_config_value(global_parameters['backup_server_id'],'pgsql_bin_' + global_parameters['pg_release']) logs.logger.debug('pgsql bin directory to use: %s',pgsql_bin_dir) - + return pgsql_bin_dir except Exception as e: pgsql_bin_dir = backup_server_cache_data['pgsql_bin_' + global_parameters['pg_release']] logs.logger.debug('pgsql bin directory to use: %s',pgsql_bin_dir) - + return pgsql_bin_dir - + # ################################################## # Function get_backup_server_parameters_from_cache() # ################################################## -def get_backup_server_parameters_from_cache(db,backup_server_fqdn): +def get_backup_server_parameters_from_cache(db,backup_server_fqdn): '''Get backup server parameters from cache file''' global backup_server_cache_data @@ -372,12 +374,12 @@ def get_backup_server_parameters_from_cache(db,backup_server_fqdn): try: backup_server_cache_file = global_parameters['root_backup_dir'] + '/cache_dir/backup_server_' + backup_server_fqdn + '.cache' - + with open(backup_server_cache_file,'r') as backup_server_cache: for line in backup_server_cache: (key, val) = line.split('::') backup_server_cache_data[key] = val.replace('\n','') - + except Exception as e: logs.logger.error('Could not read the cache file for the backup server: %s - %s',backup_server_fqdn,e) global_parameters['execution_status'] = 'ERROR' @@ -391,7 +393,7 @@ def get_backup_server_parameters_from_cache(db,backup_server_fqdn): # Function get_pgsql_node_parameters_from_cache() # ############################################### -def get_pgsql_node_parameters_from_cache(db): +def get_pgsql_node_parameters_from_cache(db): '''Get pgsql_node parameters from cache file''' global pgsql_node_cache_data @@ -404,7 +406,7 @@ def get_pgsql_node_parameters_from_cache(db): for line in pgsql_node_cache: (key, val) = line.split('::') pgsql_node_cache_data[key] = val.replace('\n','') - + except Exception as e: logs.logger.critical('Could not read the cache file for the PgSQL node - %s',e) global_parameters['execution_status'] = 'ERROR' @@ -427,14 +429,14 @@ def rename_existing_database(db,db_pgnode): global_parameters['execution_status'] = 'ERROR' global_parameters['error_message'] = 'Problems renaming an existing database' register_restore_catalog(db) - + sys.exit(1) - - + + # ############################################ # Function register_restore_catalog() # ############################################ - + def register_restore_catalog(db): '''Update the restore catalog information in the database''' @@ -453,7 +455,7 @@ def register_restore_catalog(db): procpid = os.getpid() db.register_restore_catalog(global_parameters['restore_def'], procpid, - global_parameters['backup_server_id'], + global_parameters['backup_server_id'], global_parameters['pgsql_node_id'], global_parameters['source_dbname'], global_parameters['target_dbname'], @@ -469,30 +471,30 @@ def register_restore_catalog(db): global_parameters['target_pgsql_node_release'].replace('_','.'), global_parameters['pg_release'].replace('_','.') ) - + logs.logger.info('Restore job catalog for RestoreDef: %s updated in the database',str(global_parameters['restore_def'])) - + except Exception as e: # # We create a pending log file if we can not update the # database. This file will be processed by # pgbackman_maintenance later. - # + # logs.logger.warning('Problems updating the restore job catalog for RestoreDef: %s in the database - %s',str(global_parameters['restore_def']),e) pending_log_file = '' - + try: procpid = os.getpid() pending_log_file = global_parameters['backup_server_pending_registration_dir'] + '/restore_jobs_pending_log_updates_nodeid_' + str(global_parameters['pgsql_node_id']) + '_' + str(procpid) + '.log' - + with open(pending_log_file,'w+') as catalog_pending: catalog_pending.write(str(global_parameters['restore_def']) + '::' + str(procpid) + '::' + - str(global_parameters['backup_server_id']) + '::' + + str(global_parameters['backup_server_id']) + '::' + str(global_parameters['pgsql_node_id']) + '::' + global_parameters['source_dbname'] + '::' + global_parameters['target_dbname'] + '::' + @@ -507,19 +509,19 @@ def register_restore_catalog(db): " ".join(global_parameters['role_list']) + '::' + global_parameters['target_pgsql_node_release'].replace('_','.') + '::' + global_parameters['pg_release'].replace('_','.') + '\n') - + logs.logger.info('Restore catalog pending log file: %s created',pending_log_file) - + except Exception as e: logs.logger.error('Could not generate the restore catalog pending log file: %s - %s',pending_log_file,e) # ############################################ # Function get_logfile_id() # ############################################ - + def get_logfile_id(): '''Generate the filename used for the log files of a restore job''' - + global global_parameters timestamp = datetime.datetime.now().strftime('%Y%m%dT%H%M%S') @@ -531,7 +533,7 @@ def get_logfile_id(): # ############################################## # Function check_pgbackman_database_connection() # ############################################## - + def check_pgbackman_database_connection(db): '''Check if we can connect to the pgbackman database''' @@ -540,7 +542,7 @@ def check_pgbackman_database_connection(db): db.pg_close() return True - except Exception as e: + except Exception as e: logs.logger.error('The pgbackman database is not available - %s',e) return False @@ -548,7 +550,7 @@ def check_pgbackman_database_connection(db): # ############################################### # Function check_pgsql_node_database_connection() # ############################################### - + def check_pgsql_node_database_connection(db_pgnode): '''Check if we can connect to the pgsql node database''' @@ -557,18 +559,18 @@ def check_pgsql_node_database_connection(db_pgnode): db_pgnode.pg_close() return True - except Exception as e: + except Exception as e: logs.logger.critical('The PgSQL node is not available - %s',e) - return False + return False # ############################################ # Function Main() # ############################################ - + def main(): '''Main function''' - + global global_parameters conf = PgbackmanConfiguration() @@ -583,11 +585,11 @@ def main(): global_parameters['target_pgsql_node_release'] = '' db = PgbackmanDB(pgbackman_dsn, 'pgbackman_restore') - + pgsql_node_dsn = get_pgsql_node_dsn() db_pgnode = PgbackmanDB(pgsql_node_dsn, 'pgbackman_restore') - # + # # The backup server FQDN to be used can be defined in the # pgbackman configuration file. If the configuration parameter # 'backup_server' is not defined, the return value of @@ -628,18 +630,18 @@ def main(): if not check_pgnode_db: logs.logger.critical('The PgSQL node is not available. Shutting down the restore job with RestoreID: %s',str(global_parameters['restore_def'])) - + global_parameters['execution_status'] = 'ERROR' global_parameters['error_message'] = 'Target PgSQL node not available' register_restore_catalog(db) - sys.exit(1) - + sys.exit(1) + try: global_parameters['pgsql_node_backup_dir'] = db.get_pgsql_node_config_value(global_parameters['pgsql_node_id'],'pgnode_backup_partition') except psycopg2.Error as e: global_parameters['pgsql_node_backup_dir'] = pgsql_node_cache_data['pgnode_backup_partition'] - + global_parameters['backup_server_pgsql_bin_dir'] = get_backup_server_pgsql_bin_dir(db) global_parameters['target_pgsql_node_release'] = get_pgsql_node_release(db,db_pgnode) @@ -663,24 +665,24 @@ def main(): global_parameters['execution_status'] = 'ERROR' global_parameters['error_message'] = 'Backup pgsql release > Target pgsql node release' register_restore_catalog(db) - sys.exit(1) + sys.exit(1) if global_parameters['renamed_dbname'] != '': rename_existing_database(db,db_pgnode) logs.logger.info('Renaming an existing database to %s before restoring the backup in the target database',global_parameters['renamed_dbname']) - + # # Check if the target database already exists in the target PgSQL # node. Stop the backup with an error it it exists. # - + if db_pgnode.database_exists(global_parameters['target_dbname']): logs.logger.critical('The target database already exists in the target PgSQL node') global_parameters['execution_status'] = 'ERROR' global_parameters['error_message'] = 'The target database already exists in the target PgSQL node' register_restore_catalog(db) - sys.exit(1) + sys.exit(1) restore_roles(db) restore_dbconfig(db) @@ -693,14 +695,14 @@ def main(): # ############################################ # Function signal_handler() # ############################################ - + def signal_handler(signum, frame): logs.logger.info('**** pgbackman_dump stopped. ****') sys.exit(0) # ############################################ -# +# # ############################################ if __name__ == '__main__': @@ -725,7 +727,7 @@ if __name__ == '__main__': parser.add_argument('--pg-release', metavar='PG-RELEASE', required=True, help='PG release from backup', dest='pg_release') parser.add_argument('--root-backup-dir', metavar='ROOT-BACKUP-DIR', default=True, required=True, help='Root backup dir', dest='root_backup_dir') - args = parser.parse_args() + args = parser.parse_args() if args.pgsql_node_fqdn: global_parameters['pgsql_node_fqdn'] = args.pgsql_node_fqdn @@ -744,19 +746,19 @@ if __name__ == '__main__': else: print('PgSQL node port parameter not defined') sys.exit(1) - + if args.pgsql_node_admin_user: global_parameters['pgsql_node_admin_user'] = args.pgsql_node_admin_user else: print('PgSQL node admin user parameter not defined') sys.exit(1) - + if args.restore_def: global_parameters['restore_def'] = args.restore_def else: print('RestoreID parameter not defined') sys.exit(1) - + if args.pgdump_file: global_parameters['pgdump_file'] = args.pgdump_file else: @@ -779,7 +781,7 @@ if __name__ == '__main__': global_parameters['source_dbname'] = args.source_dbname else: print('Source dbname parameter not defined') - sys.exit(1) + sys.exit(1) if args.target_dbname: global_parameters['target_dbname'] = args.target_dbname @@ -820,8 +822,7 @@ if __name__ == '__main__': logs = PgbackmanLogs("pgbackman_restore", "[" + global_parameters['pgsql_node_fqdn'] + "]", "[" + global_parameters['target_dbname'] + "]") logs.logger.info('**** pgbackman_restore started. ****') - + main() logs.logger.info('**** pgbackman_restore finished. ****') -