diff --git a/tests/robot/Lib/pgsLibrary.py b/tests/robot/Lib/pgsLibrary.py index 7f82f7d1..dcb12966 100644 --- a/tests/robot/Lib/pgsLibrary.py +++ b/tests/robot/Lib/pgsLibrary.py @@ -781,6 +781,135 @@ def check_last_backup_id(self): last_backup_id = health_json["storage"]["lastSuccessful"]["id"] return last_backup_id + def pgbackrest_backup_exists(self, backup_id): + try: + backups = self.get_pgbackrest_backup_list() + if backup_id in backups: + logging.info("PgBackRest backup %s found through backup daemon /list", backup_id) + return True + except Exception as e: + logging.info("Cannot check backup daemon list for PgBackRest backup %s: %s", backup_id, e) + + status = self.get_pgbackrest_sidecar_backup_status(backup_id) + if self._pgbackrest_backup_matches(backup_id, status): + logging.info("PgBackRest backup %s found through backrest /status", backup_id) + return True + + backups = self.get_pgbackrest_sidecar_backup_list() + if self._pgbackrest_backup_found_in_list(backup_id, backups): + logging.info("PgBackRest backup %s found through backrest /list", backup_id) + return True + + return False + + def get_pgbackrest_backup_list(self): + response = requests.get(f"{self._scheme}://postgres-backup-daemon:8081/list", verify=False, timeout=10) + response.raise_for_status() + backups = response.json() + logging.info("Backup daemon backup list: {}".format(backups)) + return backups + + def get_pgbackrest_sidecar_backup_status(self, backup_id): + for service in ["backrest", "backrest-headless"]: + try: + response = requests.get( + "http://{}:3000/status?timestamp={}".format(service, backup_id), + timeout=10 + ) + response.raise_for_status() + status = response.json() + logging.info("PgBackRest status from %s for %s: %s", service, backup_id, status) + if self._pgbackrest_backup_matches(backup_id, status): + return status + except Exception as e: + logging.info("Cannot get PgBackRest status from %s for %s: %s", service, backup_id, e) + return {} + + def get_pgbackrest_sidecar_backup_list(self): + errors = {} + for service in ["backrest", "backrest-headless"]: + try: + response = requests.get("http://{}:3000/list".format(service), timeout=10) + response.raise_for_status() + backups = response.json() + logging.info("PgBackRest list from %s: %s", service, backups) + return backups + except Exception as e: + errors[service] = str(e) + logging.info("Cannot get PgBackRest list from %s: %s", service, e) + logging.info("Cannot get PgBackRest list from sidecar services: %s", errors) + return [] + + def _pgbackrest_backup_matches(self, backup_id, backup): + if not isinstance(backup, dict): + return False + annotation = backup.get("annotation") or {} + return annotation.get("timestamp") == backup_id and not backup.get("error", False) + + def _pgbackrest_backup_found_in_list(self, backup_id, backups): + if isinstance(backups, dict): + if backup_id in backups: + return True + backups = backups.values() + if not isinstance(backups, list): + return False + for backup in backups: + if self._pgbackrest_backup_matches(backup_id, backup): + return True + return False + + def restore_pgbackrest_backup(self, backup_id): + pod = self.get_pod(label='app:postgres-backup-daemon', status='Running') + command = "cd /maintenance/recovery && SET={} python3 pg_back_rest_recovery.py".format(backup_id) + logging.info("Start PgBackRest restore from backup daemon pod {} with backup id {}".format( + pod.metadata.name, backup_id)) + output, errors = self.execute_in_pod(pod.metadata.name, command) + logging.info("PgBackRest restore output: {}".format(output)) + if errors: + logging.info("PgBackRest restore stderr: {}".format(errors)) + return output + + def get_backup_daemon_restart_count(self): + pod = self.get_pod(label='app:postgres-backup-daemon', status='Running') + restart_count = 0 + for container_status in pod.status.container_statuses or []: + restart_count += container_status.restart_count + logging.info("Backup daemon pod {} restart count: {}".format(pod.metadata.name, restart_count)) + return restart_count + + def get_pgbackrest_prerequisite_status(self): + status = { + "storage_type": None, + "backup_daemon_pod": None, + "pgbackrest_configmap_exists": False, + "pgbackrest_sidecar_pods": [], + "missing": [] + } + + backup_daemon = self.get_pod(label='app:postgres-backup-daemon', status='Running') + status["backup_daemon_pod"] = backup_daemon.metadata.name + status["storage_type"] = self.get_env_for_pod(backup_daemon, "STORAGE_TYPE") + if status["storage_type"] != "pgbackrest": + status["missing"].append("backup daemon STORAGE_TYPE is not pgbackrest") + + try: + self.pl_lib.get_config_map("pgbackrest-conf", self._namespace) + status["pgbackrest_configmap_exists"] = True + except Exception: + status["missing"].append("pgbackrest-conf config map is absent") + + pg_cluster_name = os.getenv("PG_CLUSTER_NAME", "patroni") + for pod in self.get_pods(label="pgcluster:{}".format(pg_cluster_name), status="Running"): + for container in pod.spec.containers: + if container.name == "pgbackrest-sidecar": + status["pgbackrest_sidecar_pods"].append(pod.metadata.name) + break + if not status["pgbackrest_sidecar_pods"]: + status["missing"].append("pgbackrest-sidecar container is absent in running patroni pods") + + logging.info("PgBackRest prerequisite status: {}".format(status)) + return status + def schedule_evict(self, last_backup_id): health_json = requests.delete(f"{self._scheme}://postgres-backup-daemon:8081/evict?id={last_backup_id}", verify=False).json() return health_json diff --git a/tests/robot/check_pgbackrest_restore/check_pgbackrest_restore.robot b/tests/robot/check_pgbackrest_restore/check_pgbackrest_restore.robot new file mode 100644 index 00000000..79a969ef --- /dev/null +++ b/tests/robot/check_pgbackrest_restore/check_pgbackrest_restore.robot @@ -0,0 +1,82 @@ +*** Settings *** +Documentation Check positive full restore cycle with PgBackRest storage +Library Collections +Library OperatingSystem +Library String +Resource ../Lib/lib.robot + +*** Variables *** +${OPERATION_RETRY_COUNT} 60 +${OPERATION_RETRY_INTERVAL} 5s + +*** Test Cases *** +Check PgBackRest Full Backup Restore + [Tags] pgbackrest pgbackrest_restore + [Documentation] + ... Positive PgBackRest cycle: + ... 1. Verify backup daemon uses PgBackRest storage. + ... 2. Create database and seed data. + ... 3. Create full backup through backup daemon. + ... 4. Add data after backup. + ... 5. Restore Patroni cluster from the created PgBackRest backup. + ... 6. Verify cluster is healthy and data state matches the backup. + ${pg_cluster_name}= Get Environment Variable PG_CLUSTER_NAME default=patroni + ${postfix}= Generate Random String 5 [LOWER] + ${db_name}= Set Variable pgbackrest_restore_${postfix} + Set Test Variable \${db_name} ${db_name} + Log To Console \n[pgbackrest] cluster=${pg_cluster_name}, database=${db_name} + Skip Test If PgBackRest Is Not Configured + Create Database ${db_name} + Wait Until Keyword Succeeds ${OPERATION_RETRY_COUNT} ${OPERATION_RETRY_INTERVAL} + ... Check Database Exists ${pg_cluster_name} ${db_name} + ${rid_before} ${expected_before}= Insert Test Record database=${db_name} + ${restart_count_before}= Get Backup Daemon Restart Count + ${backup_id}= Create PgBackRest Full Backup + Log To Console [pgbackrest] backup_id=${backup_id}, restart_count_before=${restart_count_before} + ${rid_after} ${expected_after}= Insert Test Record database=${db_name} + ${restore_output}= Restore Pgbackrest Backup ${backup_id} + Log ${restore_output} + Log To Console [pgbackrest] restore started for backup_id=${backup_id} + Wait Until Keyword Succeeds 20 min 10 sec Patroni Ready + Check Test Record pg-${pg_cluster_name} ${rid_before} ${expected_before} ${db_name} + Check Test Record Is Absent pg-${pg_cluster_name} ${rid_after} ${expected_after} ${db_name} + ${restart_count_after}= Get Backup Daemon Restart Count + Log To Console [pgbackrest] restart_count_after=${restart_count_after} + Should Be Equal As Integers ${restart_count_after} ${restart_count_before} + [Teardown] Delete Database ${db_name} + +*** Keywords *** +Skip Test If PgBackRest Is Not Configured + ${status}= Get Pgbackrest Prerequisite Status + Log PgBackRest prerequisites: ${status} + ${missing}= Get From Dictionary ${status} missing + ${missing_count}= Get Length ${missing} + Run Keyword If ${missing_count} > 0 Pass Execution PgBackRest is not configured for this environment: ${missing} + +Check Database Exists + [Arguments] ${pg_cluster_name} ${db_name} + ${databases}= Execute Query pg-${pg_cluster_name} SELECT datname FROM pg_database + Should Contain str(${databases}) ${db_name} + +Create PgBackRest Full Backup + ${pod}= Get Pod label=app:postgres-backup-daemon status=Running + ${dump_count}= Get Backup Count + ${schedule_response}= Schedule Backup + Log PgBackRest backup schedule response: ${schedule_response} + Dictionary Should Contain Key ${schedule_response} backup_id + ${backup_id}= Get From Dictionary ${schedule_response} backup_id + Log To Console [pgbackrest] waiting backup_id=${backup_id} + Wait Until Keyword Succeeds 30 min 15 sec Check PgBackRest Backup Exists ${backup_id} + ${dump_count_after}= Get Backup Count + Log PgBackRest backup ${backup_id} is listed, dump_count_before=${dump_count}, dump_count_after=${dump_count_after} + RETURN ${backup_id} + +Check PgBackRest Backup Exists + [Arguments] ${backup_id} + ${exists}= Pgbackrest Backup Exists ${backup_id} + Should Be True ${exists} msg=PgBackRest backup ${backup_id} was not found in backrest list + +Check Test Record Is Absent + [Arguments] ${pod_name} ${rid} ${expected} ${database} + ${res}= Execute Query ${pod_name} select * from test_insert_robot where id=${rid} dbname=${database} + Should Not Be True """${expected}""" in """${res}""" msg=Record added after backup is still present after restore: ${res}