From 0381cb86882348270eba2950c14d1578d0ab18a2 Mon Sep 17 00:00:00 2001 From: Gravelord Date: Tue, 19 May 2026 23:49:21 +0300 Subject: [PATCH 1/3] feat: new test --- tests/robot/Lib/pgsLibrary.py | 59 ++++++++++ .../check_pgbackrest_restore.robot | 102 ++++++++++++++++++ 2 files changed, 161 insertions(+) create mode 100644 tests/robot/check_pgbackrest_restore/check_pgbackrest_restore.robot diff --git a/tests/robot/Lib/pgsLibrary.py b/tests/robot/Lib/pgsLibrary.py index 7f82f7d1..b576936c 100644 --- a/tests/robot/Lib/pgsLibrary.py +++ b/tests/robot/Lib/pgsLibrary.py @@ -781,6 +781,65 @@ def check_last_backup_id(self): last_backup_id = health_json["storage"]["lastSuccessful"]["id"] return last_backup_id + def pgbackrest_backup_exists(self, backup_id): + response = requests.get(f"{self._scheme}://postgres-backup-daemon:8081/list", verify=False, timeout=10) + response.raise_for_status() + backups = response.json() + logging.info("Backup daemon backup list: {}".format(backups)) + return backup_id in backups + + def restore_pgbackrest_backup(self, backup_id): + pod = self.get_pod(label='app:postgres-backup-daemon', status='Running') + command = "cd /maintenance/recovery && SET={} python3 pg_back_rest_recovery.py".format(backup_id) + logging.info("Start PgBackRest restore from backup daemon pod {} with backup id {}".format( + pod.metadata.name, backup_id)) + output, errors = self.execute_in_pod(pod.metadata.name, command) + logging.info("PgBackRest restore output: {}".format(output)) + if errors: + logging.info("PgBackRest restore stderr: {}".format(errors)) + return output + + def get_backup_daemon_restart_count(self): + pod = self.get_pod(label='app:postgres-backup-daemon', status='Running') + restart_count = 0 + for container_status in pod.status.container_statuses or []: + restart_count += container_status.restart_count + logging.info("Backup daemon pod {} restart count: {}".format(pod.metadata.name, restart_count)) + return restart_count + + def get_pgbackrest_prerequisite_status(self): + status = { + "storage_type": None, + "backup_daemon_pod": None, + "pgbackrest_configmap_exists": False, + "pgbackrest_sidecar_pods": [], + "missing": [] + } + + backup_daemon = self.get_pod(label='app:postgres-backup-daemon', status='Running') + status["backup_daemon_pod"] = backup_daemon.metadata.name + status["storage_type"] = self.get_env_for_pod(backup_daemon, "STORAGE_TYPE") + if status["storage_type"] != "pgbackrest": + status["missing"].append("backup daemon STORAGE_TYPE is not pgbackrest") + + try: + self.pl_lib.get_config_map("pgbackrest-conf", self._namespace) + status["pgbackrest_configmap_exists"] = True + except Exception: + status["missing"].append("pgbackrest-conf config map is absent") + + pg_cluster_name = os.getenv("PG_CLUSTER_NAME", "patroni") + for pod in self.get_pods(label="pgcluster:{}".format(pg_cluster_name), status="Running"): + for container in pod.spec.containers: + if container.name == "pgbackrest-sidecar": + status["pgbackrest_sidecar_pods"].append(pod.metadata.name) + break + if not status["pgbackrest_sidecar_pods"]: + status["missing"].append("pgbackrest-sidecar container is absent in running patroni pods") + + logging.info("PgBackRest prerequisite status: {}".format(status)) + return status + def schedule_evict(self, last_backup_id): health_json = requests.delete(f"{self._scheme}://postgres-backup-daemon:8081/evict?id={last_backup_id}", verify=False).json() return health_json diff --git a/tests/robot/check_pgbackrest_restore/check_pgbackrest_restore.robot b/tests/robot/check_pgbackrest_restore/check_pgbackrest_restore.robot new file mode 100644 index 00000000..6249e743 --- /dev/null +++ b/tests/robot/check_pgbackrest_restore/check_pgbackrest_restore.robot @@ -0,0 +1,102 @@ +*** Settings *** +Documentation Check positive full restore cycle with PgBackRest storage +Library Collections +Library OperatingSystem +Library String +Resource ../Lib/lib.robot + +*** Variables *** +${OPERATION_RETRY_COUNT} 60 +${OPERATION_RETRY_INTERVAL} 5s + +*** Test Cases *** +Check PgBackRest Full Backup Restore + [Tags] pgbackrest pgbackrest_restore + [Documentation] + ... Positive PgBackRest cycle: + ... 1. Verify backup daemon uses PgBackRest storage. + ... 2. Create database and seed data. + ... 3. Create full backup through backup daemon. + ... 4. Add data after backup. + ... 5. Restore Patroni cluster from the created PgBackRest backup. + ... 6. Verify cluster is healthy and data state matches the backup. + ${pg_cluster_name}= Get Environment Variable PG_CLUSTER_NAME default=patroni + ${postfix}= Generate Random String 5 [LOWER] + ${db_name}= Set Variable pgbackrest_restore_${postfix} + Set Test Variable \${db_name} ${db_name} + Log To Console \n[pgbackrest-debug] cluster=${pg_cluster_name}, database=${db_name} + Skip Test If PgBackRest Is Not Configured + Log To Console [pgbackrest-debug] creating database ${db_name} + Create Database ${db_name} + Wait Until Keyword Succeeds ${OPERATION_RETRY_COUNT} ${OPERATION_RETRY_INTERVAL} + ... Check Database Exists ${pg_cluster_name} ${db_name} + Log To Console [pgbackrest-debug] inserting data before backup into ${db_name} + ${rid_before} ${expected_before}= Insert Test Record database=${db_name} + Log To Console [pgbackrest-debug] record before backup: id=${rid_before}, expected=${expected_before} + ${restart_count_before}= Get Backup Daemon Restart Count + Log To Console [pgbackrest-debug] backup daemon restart count before restore: ${restart_count_before} + Log To Console [pgbackrest-debug] requesting full pgBackRest backup through backup daemon + ${backup_id}= Create PgBackRest Full Backup + Log To Console [pgbackrest-debug] created pgBackRest backup_id=${backup_id} + Log To Console [pgbackrest-debug] inserting data after backup into ${db_name} + ${rid_after} ${expected_after}= Insert Test Record database=${db_name} + Log To Console [pgbackrest-debug] record after backup: id=${rid_after}, expected=${expected_after} + Log To Console [pgbackrest-debug] starting restore for backup_id=${backup_id} + ${restore_output}= Restore Pgbackrest Backup ${backup_id} + Log ${restore_output} + Log To Console [pgbackrest-debug] restore command output: ${restore_output} + Log To Console [pgbackrest-debug] waiting for Patroni cluster to become ready after restore + Wait Until Keyword Succeeds 20 min 10 sec Patroni Ready + Log To Console [pgbackrest-debug] checking that pre-backup record exists after restore + Check Test Record pg-${pg_cluster_name} ${rid_before} ${expected_before} ${db_name} + Log To Console [pgbackrest-debug] checking that post-backup record is absent after restore + Check Test Record Is Absent pg-${pg_cluster_name} ${rid_after} ${expected_after} ${db_name} + ${restart_count_after}= Get Backup Daemon Restart Count + Log To Console [pgbackrest-debug] backup daemon restart count after restore: ${restart_count_after} + Should Be Equal As Integers ${restart_count_after} ${restart_count_before} + [Teardown] Delete Database ${db_name} + +*** Keywords *** +Skip Test If PgBackRest Is Not Configured + ${status}= Get Pgbackrest Prerequisite Status + Log To Console [pgbackrest-debug] prerequisites: ${status} + Log PgBackRest prerequisites: ${status} + ${missing}= Get From Dictionary ${status} missing + ${missing_count}= Get Length ${missing} + Run Keyword If ${missing_count} > 0 Pass Execution PgBackRest is not configured for this environment: ${missing} + +Check Database Exists + [Arguments] ${pg_cluster_name} ${db_name} + Log To Console [pgbackrest-debug] checking database existence: cluster=${pg_cluster_name}, database=${db_name} + ${databases}= Execute Query pg-${pg_cluster_name} SELECT datname FROM pg_database + Log To Console [pgbackrest-debug] databases query result: ${databases} + Should Contain str(${databases}) ${db_name} + +Create PgBackRest Full Backup + ${pod}= Get Pod label=app:postgres-backup-daemon status=Running + Log To Console [pgbackrest-debug] backup daemon pod=${pod.metadata.name} + ${dump_count}= Get Backup Count + Log To Console [pgbackrest-debug] dump count before backup=${dump_count} + ${schedule_response}= Schedule Backup + Log To Console [pgbackrest-debug] schedule response=${schedule_response} + Dictionary Should Contain Key ${schedule_response} backup_id + ${backup_id}= Get From Dictionary ${schedule_response} backup_id + Log To Console [pgbackrest-debug] waiting backup completion: backup_id=${backup_id}, previous_dump_count=${dump_count} + Wait For Backup To Complete ${pod.metadata.name} ${dump_count} + Log To Console [pgbackrest-debug] backup daemon reports backup completed: backup_id=${backup_id} + Wait Until Keyword Succeeds 10 min 10 sec Check PgBackRest Backup Exists ${backup_id} + [Return] ${backup_id} + +Check PgBackRest Backup Exists + [Arguments] ${backup_id} + Log To Console [pgbackrest-debug] checking backup in daemon list: backup_id=${backup_id} + ${exists}= Pgbackrest Backup Exists ${backup_id} + Log To Console [pgbackrest-debug] backup exists=${exists} + Should Be True ${exists} msg=PgBackRest backup ${backup_id} was not found in backrest list + +Check Test Record Is Absent + [Arguments] ${pod_name} ${rid} ${expected} ${database} + Log To Console [pgbackrest-debug] checking absent record: host=${pod_name}, database=${database}, id=${rid} + ${res}= Execute Query ${pod_name} select * from test_insert_robot where id=${rid} dbname=${database} + Log To Console [pgbackrest-debug] absent record query result: ${res} + Should Not Be True """${expected}""" in """${res}""" msg=Record added after backup is still present after restore: ${res} From dfe12758c1ff587f71bd22ef850c51cd73fff349 Mon Sep 17 00:00:00 2001 From: Gravelord Date: Thu, 21 May 2026 02:09:33 +0300 Subject: [PATCH 2/3] feat: fix errors --- tests/robot/Lib/pgsLibrary.py | 6 +++++- .../check_pgbackrest_restore.robot | 13 ++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/robot/Lib/pgsLibrary.py b/tests/robot/Lib/pgsLibrary.py index b576936c..eeea374b 100644 --- a/tests/robot/Lib/pgsLibrary.py +++ b/tests/robot/Lib/pgsLibrary.py @@ -782,11 +782,15 @@ def check_last_backup_id(self): return last_backup_id def pgbackrest_backup_exists(self, backup_id): + backups = self.get_pgbackrest_backup_list() + return backup_id in backups + + def get_pgbackrest_backup_list(self): response = requests.get(f"{self._scheme}://postgres-backup-daemon:8081/list", verify=False, timeout=10) response.raise_for_status() backups = response.json() logging.info("Backup daemon backup list: {}".format(backups)) - return backup_id in backups + return backups def restore_pgbackrest_backup(self, backup_id): pod = self.get_pod(label='app:postgres-backup-daemon', status='Running') diff --git a/tests/robot/check_pgbackrest_restore/check_pgbackrest_restore.robot b/tests/robot/check_pgbackrest_restore/check_pgbackrest_restore.robot index 6249e743..5e052a63 100644 --- a/tests/robot/check_pgbackrest_restore/check_pgbackrest_restore.robot +++ b/tests/robot/check_pgbackrest_restore/check_pgbackrest_restore.robot @@ -81,15 +81,18 @@ Create PgBackRest Full Backup Log To Console [pgbackrest-debug] schedule response=${schedule_response} Dictionary Should Contain Key ${schedule_response} backup_id ${backup_id}= Get From Dictionary ${schedule_response} backup_id - Log To Console [pgbackrest-debug] waiting backup completion: backup_id=${backup_id}, previous_dump_count=${dump_count} - Wait For Backup To Complete ${pod.metadata.name} ${dump_count} - Log To Console [pgbackrest-debug] backup daemon reports backup completed: backup_id=${backup_id} - Wait Until Keyword Succeeds 10 min 10 sec Check PgBackRest Backup Exists ${backup_id} - [Return] ${backup_id} + Log To Console [pgbackrest-debug] waiting pgBackRest backup in daemon list: backup_id=${backup_id}, previous_dump_count=${dump_count} + Wait Until Keyword Succeeds 30 min 15 sec Check PgBackRest Backup Exists ${backup_id} + ${dump_count_after}= Get Backup Count + Log To Console [pgbackrest-debug] pgBackRest backup is listed: backup_id=${backup_id}, dump_count_after=${dump_count_after} + RETURN ${backup_id} Check PgBackRest Backup Exists [Arguments] ${backup_id} Log To Console [pgbackrest-debug] checking backup in daemon list: backup_id=${backup_id} + ${backups}= Get Pgbackrest Backup List + @{backup_keys}= Get Dictionary Keys ${backups} + Log To Console [pgbackrest-debug] current daemon backup list keys: ${backup_keys} ${exists}= Pgbackrest Backup Exists ${backup_id} Log To Console [pgbackrest-debug] backup exists=${exists} Should Be True ${exists} msg=PgBackRest backup ${backup_id} was not found in backrest list From c490c83d7daa5088b905d3c9c007be936738a1f7 Mon Sep 17 00:00:00 2001 From: Gravelord Date: Fri, 22 May 2026 12:36:44 +0300 Subject: [PATCH 3/3] feat: fix tracking --- tests/robot/Lib/pgsLibrary.py | 70 ++++++++++++++++++- .../check_pgbackrest_restore.robot | 37 ++-------- 2 files changed, 75 insertions(+), 32 deletions(-) diff --git a/tests/robot/Lib/pgsLibrary.py b/tests/robot/Lib/pgsLibrary.py index eeea374b..dcb12966 100644 --- a/tests/robot/Lib/pgsLibrary.py +++ b/tests/robot/Lib/pgsLibrary.py @@ -782,8 +782,25 @@ def check_last_backup_id(self): return last_backup_id def pgbackrest_backup_exists(self, backup_id): - backups = self.get_pgbackrest_backup_list() - return backup_id in backups + try: + backups = self.get_pgbackrest_backup_list() + if backup_id in backups: + logging.info("PgBackRest backup %s found through backup daemon /list", backup_id) + return True + except Exception as e: + logging.info("Cannot check backup daemon list for PgBackRest backup %s: %s", backup_id, e) + + status = self.get_pgbackrest_sidecar_backup_status(backup_id) + if self._pgbackrest_backup_matches(backup_id, status): + logging.info("PgBackRest backup %s found through backrest /status", backup_id) + return True + + backups = self.get_pgbackrest_sidecar_backup_list() + if self._pgbackrest_backup_found_in_list(backup_id, backups): + logging.info("PgBackRest backup %s found through backrest /list", backup_id) + return True + + return False def get_pgbackrest_backup_list(self): response = requests.get(f"{self._scheme}://postgres-backup-daemon:8081/list", verify=False, timeout=10) @@ -792,6 +809,55 @@ def get_pgbackrest_backup_list(self): logging.info("Backup daemon backup list: {}".format(backups)) return backups + def get_pgbackrest_sidecar_backup_status(self, backup_id): + for service in ["backrest", "backrest-headless"]: + try: + response = requests.get( + "http://{}:3000/status?timestamp={}".format(service, backup_id), + timeout=10 + ) + response.raise_for_status() + status = response.json() + logging.info("PgBackRest status from %s for %s: %s", service, backup_id, status) + if self._pgbackrest_backup_matches(backup_id, status): + return status + except Exception as e: + logging.info("Cannot get PgBackRest status from %s for %s: %s", service, backup_id, e) + return {} + + def get_pgbackrest_sidecar_backup_list(self): + errors = {} + for service in ["backrest", "backrest-headless"]: + try: + response = requests.get("http://{}:3000/list".format(service), timeout=10) + response.raise_for_status() + backups = response.json() + logging.info("PgBackRest list from %s: %s", service, backups) + return backups + except Exception as e: + errors[service] = str(e) + logging.info("Cannot get PgBackRest list from %s: %s", service, e) + logging.info("Cannot get PgBackRest list from sidecar services: %s", errors) + return [] + + def _pgbackrest_backup_matches(self, backup_id, backup): + if not isinstance(backup, dict): + return False + annotation = backup.get("annotation") or {} + return annotation.get("timestamp") == backup_id and not backup.get("error", False) + + def _pgbackrest_backup_found_in_list(self, backup_id, backups): + if isinstance(backups, dict): + if backup_id in backups: + return True + backups = backups.values() + if not isinstance(backups, list): + return False + for backup in backups: + if self._pgbackrest_backup_matches(backup_id, backup): + return True + return False + def restore_pgbackrest_backup(self, backup_id): pod = self.get_pod(label='app:postgres-backup-daemon', status='Running') command = "cd /maintenance/recovery && SET={} python3 pg_back_rest_recovery.py".format(backup_id) diff --git a/tests/robot/check_pgbackrest_restore/check_pgbackrest_restore.robot b/tests/robot/check_pgbackrest_restore/check_pgbackrest_restore.robot index 5e052a63..79a969ef 100644 --- a/tests/robot/check_pgbackrest_restore/check_pgbackrest_restore.robot +++ b/tests/robot/check_pgbackrest_restore/check_pgbackrest_restore.robot @@ -24,42 +24,30 @@ Check PgBackRest Full Backup Restore ${postfix}= Generate Random String 5 [LOWER] ${db_name}= Set Variable pgbackrest_restore_${postfix} Set Test Variable \${db_name} ${db_name} - Log To Console \n[pgbackrest-debug] cluster=${pg_cluster_name}, database=${db_name} + Log To Console \n[pgbackrest] cluster=${pg_cluster_name}, database=${db_name} Skip Test If PgBackRest Is Not Configured - Log To Console [pgbackrest-debug] creating database ${db_name} Create Database ${db_name} Wait Until Keyword Succeeds ${OPERATION_RETRY_COUNT} ${OPERATION_RETRY_INTERVAL} ... Check Database Exists ${pg_cluster_name} ${db_name} - Log To Console [pgbackrest-debug] inserting data before backup into ${db_name} ${rid_before} ${expected_before}= Insert Test Record database=${db_name} - Log To Console [pgbackrest-debug] record before backup: id=${rid_before}, expected=${expected_before} ${restart_count_before}= Get Backup Daemon Restart Count - Log To Console [pgbackrest-debug] backup daemon restart count before restore: ${restart_count_before} - Log To Console [pgbackrest-debug] requesting full pgBackRest backup through backup daemon ${backup_id}= Create PgBackRest Full Backup - Log To Console [pgbackrest-debug] created pgBackRest backup_id=${backup_id} - Log To Console [pgbackrest-debug] inserting data after backup into ${db_name} + Log To Console [pgbackrest] backup_id=${backup_id}, restart_count_before=${restart_count_before} ${rid_after} ${expected_after}= Insert Test Record database=${db_name} - Log To Console [pgbackrest-debug] record after backup: id=${rid_after}, expected=${expected_after} - Log To Console [pgbackrest-debug] starting restore for backup_id=${backup_id} ${restore_output}= Restore Pgbackrest Backup ${backup_id} Log ${restore_output} - Log To Console [pgbackrest-debug] restore command output: ${restore_output} - Log To Console [pgbackrest-debug] waiting for Patroni cluster to become ready after restore + Log To Console [pgbackrest] restore started for backup_id=${backup_id} Wait Until Keyword Succeeds 20 min 10 sec Patroni Ready - Log To Console [pgbackrest-debug] checking that pre-backup record exists after restore Check Test Record pg-${pg_cluster_name} ${rid_before} ${expected_before} ${db_name} - Log To Console [pgbackrest-debug] checking that post-backup record is absent after restore Check Test Record Is Absent pg-${pg_cluster_name} ${rid_after} ${expected_after} ${db_name} ${restart_count_after}= Get Backup Daemon Restart Count - Log To Console [pgbackrest-debug] backup daemon restart count after restore: ${restart_count_after} + Log To Console [pgbackrest] restart_count_after=${restart_count_after} Should Be Equal As Integers ${restart_count_after} ${restart_count_before} [Teardown] Delete Database ${db_name} *** Keywords *** Skip Test If PgBackRest Is Not Configured ${status}= Get Pgbackrest Prerequisite Status - Log To Console [pgbackrest-debug] prerequisites: ${status} Log PgBackRest prerequisites: ${status} ${missing}= Get From Dictionary ${status} missing ${missing_count}= Get Length ${missing} @@ -67,39 +55,28 @@ Skip Test If PgBackRest Is Not Configured Check Database Exists [Arguments] ${pg_cluster_name} ${db_name} - Log To Console [pgbackrest-debug] checking database existence: cluster=${pg_cluster_name}, database=${db_name} ${databases}= Execute Query pg-${pg_cluster_name} SELECT datname FROM pg_database - Log To Console [pgbackrest-debug] databases query result: ${databases} Should Contain str(${databases}) ${db_name} Create PgBackRest Full Backup ${pod}= Get Pod label=app:postgres-backup-daemon status=Running - Log To Console [pgbackrest-debug] backup daemon pod=${pod.metadata.name} ${dump_count}= Get Backup Count - Log To Console [pgbackrest-debug] dump count before backup=${dump_count} ${schedule_response}= Schedule Backup - Log To Console [pgbackrest-debug] schedule response=${schedule_response} + Log PgBackRest backup schedule response: ${schedule_response} Dictionary Should Contain Key ${schedule_response} backup_id ${backup_id}= Get From Dictionary ${schedule_response} backup_id - Log To Console [pgbackrest-debug] waiting pgBackRest backup in daemon list: backup_id=${backup_id}, previous_dump_count=${dump_count} + Log To Console [pgbackrest] waiting backup_id=${backup_id} Wait Until Keyword Succeeds 30 min 15 sec Check PgBackRest Backup Exists ${backup_id} ${dump_count_after}= Get Backup Count - Log To Console [pgbackrest-debug] pgBackRest backup is listed: backup_id=${backup_id}, dump_count_after=${dump_count_after} + Log PgBackRest backup ${backup_id} is listed, dump_count_before=${dump_count}, dump_count_after=${dump_count_after} RETURN ${backup_id} Check PgBackRest Backup Exists [Arguments] ${backup_id} - Log To Console [pgbackrest-debug] checking backup in daemon list: backup_id=${backup_id} - ${backups}= Get Pgbackrest Backup List - @{backup_keys}= Get Dictionary Keys ${backups} - Log To Console [pgbackrest-debug] current daemon backup list keys: ${backup_keys} ${exists}= Pgbackrest Backup Exists ${backup_id} - Log To Console [pgbackrest-debug] backup exists=${exists} Should Be True ${exists} msg=PgBackRest backup ${backup_id} was not found in backrest list Check Test Record Is Absent [Arguments] ${pod_name} ${rid} ${expected} ${database} - Log To Console [pgbackrest-debug] checking absent record: host=${pod_name}, database=${database}, id=${rid} ${res}= Execute Query ${pod_name} select * from test_insert_robot where id=${rid} dbname=${database} - Log To Console [pgbackrest-debug] absent record query result: ${res} Should Not Be True """${expected}""" in """${res}""" msg=Record added after backup is still present after restore: ${res}