[shiftstack] Use GitHub for shiftstack-qa clone and fix false-positive test results

shiftstack-qa moved code review to GitHub PRs (shiftstack/
shiftstack-qa#2). The role default still clones from
GerritHub, which is no longer maintained.

- Change cifmw_shiftstack_qa_repo default to GitHub
- Rename cifmw_shiftstack_qa_gerrithub_change to
  cifmw_shiftstack_qa_change_ref

Also fix a false-positive bug in exec_command_in_pod.yml
where oc rsh could return rc=0 before the inner command
finished, causing the test to report PASS when the playbook
actually failed. Observed in tp!2297 build d078ccbd - the
shiftstack test ran for only 145s (vs typical 1.5-2h), OCP
was never installed, yet the job reported SUCCESS.

The fix runs the inner command detached via nohup and polls
a marker file for the exit code. This survives connection
drops. Uses kubernetes.core.k8s_exec and k8s_cp modules
for setup tasks; the poll step uses oc exec for resilience
to transient network errors. Process-crash detection via
kill -0 inside the pod catches OOM without polling for 8h.
Diagnostic output fetched in rescue/always blocks restores
stderr visibility lost by the async approach.

Configurable via cifmw_shiftstack_exec_retries (default 5760)
and cifmw_shiftstack_exec_delay (default 5s) for an 8h timeout.

Related-Issue: #OSPRH-30385

Assisted-By: Claude Code
Signed-off-by: Itay Matza <imatza@redhat.com>

Author: imatza-rh

roles/shiftstack/README.md

@@ -14,13 +14,15 @@ Role for triggering Openshift on Openstack QA automation (installation and tests
 * `cifmw_shiftstack_client_pvc_manifest`: (*string*) The file name for the shiftstackclient pvc manifest. Defaults to `"{{ cifmw_shiftstack_client_pod_name }}_pvc.yml"`.
 * `cifmw_shiftstack_cluster_name`: (*string*) The Openshift cluster name. Defaults to `ostest`.
 * `cifmw_shiftstack_hypervisor`: (*string*) The hypervisor where RHOSO is deployed. Defaults to `"{{ hostvars[hostvars['controller-0']['cifmw_hypervisor_host'] | default ('')]['ansible_host'] | default('') }}"`.
+* `cifmw_shiftstack_exec_delay`: (*integer*) Seconds between polling attempts when waiting for a command to complete in the pod. Defaults to `5`.
+* `cifmw_shiftstack_exec_retries`: (*integer*) Maximum polling attempts when waiting for a command to complete in the pod. With the default delay of 5s, 5760 retries gives an 8-hour timeout. Defaults to `5760`.
 * `cifmw_shiftstack_exclude_artifacts_regex`: (*string*) Regex that will be passed on `oc rsync` command as `--exclude` param, so the role does not gather the artifacts matching it.
 * `cifmw_shiftstack_installation_dir`: (*string*) Directory to place installation files. Defaults to `"{{ cifmw_shiftstack_shiftstackclient_artifacts_dir }}/installation"`.
 * `cifmw_shiftstack_manifests_dir`: (*string*) Directory name for the role generated Openshift manifests. Defaults to `"{{ cifmw_shiftstack_basedir }}/manifests"`.
 * `cifmw_shiftstack_project_name`: (*string*) The Openstack project name. Defaults to `shiftstack`.
 * `cifmw_shiftstack_proxy`: (*string*) The proxy url that should be used to reach the underlying OCP. Defaults to omit.
-* `cifmw_shiftstack_qa_gerrithub_change`: (*string*) The gerrithub change to fetch from the `cifmw_shiftstack_qa_repo` repository (i.e. 'refs/changes/29/1188429/50)'. Defaults to ''.
-* `cifmw_shiftstack_qa_repo`: (*string*) The repository containing the Openshift on Openstack QA automation. Defaults to `https://review.gerrithub.io/shiftstack/shiftstack-qa`.
+* `cifmw_shiftstack_qa_change_ref`: (*string*) A git ref to fetch and checkout from the `cifmw_shiftstack_qa_repo` repository. Supports GitHub PR refs (e.g. `refs/pull/5/head`) and GerritHub change refs (e.g. `refs/changes/29/1188429/50`). Defaults to `''`.
+* `cifmw_shiftstack_qa_repo`: (*string*) The repository containing the Openshift on Openstack QA automation. Defaults to `https://github.com/shiftstack/shiftstack-qa`.
 * `cifmw_shiftstack_run_playbook`: (*string*) The playbook to be run from the `cifmw_shiftstack_qa_repo` repository. Defaults to `ocp_testing.yaml`.
 * `cifmw_shiftstack_sc`: (*string*) The storage class to be used for PVC for the shiftstackclient pod. Defaults to `local-storage`.
 * `cifmw_shiftstack_shiftstackclient_artifacts_dir`: (*string*) The artifacts directory path for the shiftstackclient pod. Defaults to `/home/cloud-admin/artifacts`.
@@ -40,7 +42,7 @@ cifmw_run_test_role: shiftstack
 cifmw_run_test_shiftstack_testconfig:
   - 4.17_ovnkubernetes_ipi.yaml
   - 4.17_ovnkubernetes_upi.yaml
-cifmw_shiftstack_qa_gerrithub_change: refs/changes/29/1188429/50 #optional
+cifmw_shiftstack_qa_change_ref: refs/pull/5/head #optional
 
-$ ansible-playbook deploy-edpm.yml --extra-vars "cifmw_run_tests=true cifmw_run_test_role=shiftstack cifmw_test_operator_stages=[] cifmw_openshift_kubeconfig={{ ansible_user_dir }}/.kube/config cifmw_run_test_shiftstack_testconfig=4.15_ovnkubernetes_ipi_va1.yaml cifmw_shiftstack_qa_gerrithub_change=refs/changes/29/1188429/50"
+$ ansible-playbook deploy-edpm.yml --extra-vars "cifmw_run_tests=true cifmw_run_test_role=shiftstack cifmw_test_operator_stages=[] cifmw_openshift_kubeconfig={{ ansible_user_dir }}/.kube/config cifmw_run_test_shiftstack_testconfig=4.15_ovnkubernetes_ipi_va1.yaml cifmw_shiftstack_qa_change_ref=refs/pull/5/head"
 ```

roles/shiftstack/defaults/main.yml

@@ -28,13 +28,15 @@ cifmw_shiftstack_client_pod_name: "shiftstackclient-{{ cifmw_shiftstack_project_
 cifmw_shiftstack_client_pod_namespace: "openstack"
 cifmw_shiftstack_client_pvc_manifest: "{{ cifmw_shiftstack_client_pod_name }}_pvc.yml"
 cifmw_shiftstack_cluster_name: "ostest"
+cifmw_shiftstack_exec_delay: 5
+cifmw_shiftstack_exec_retries: 5760
 cifmw_shiftstack_exclude_artifacts_regex: "openshift-install"
 cifmw_shiftstack_hypervisor: "{{ hostvars[hostvars['controller-0']['cifmw_hypervisor_host'] | default ('')]['ansible_host'] | default('') }}"
 cifmw_shiftstack_installation_dir: "{{ cifmw_shiftstack_basedir }}/installation"
 cifmw_shiftstack_manifests_dir: "{{ cifmw_shiftstack_basedir }}/manifests"
 cifmw_shiftstack_project_name: "shiftstack"
-cifmw_shiftstack_qa_gerrithub_change: ""
-cifmw_shiftstack_qa_repo: "https://review.gerrithub.io/shiftstack/shiftstack-qa"
+cifmw_shiftstack_qa_change_ref: ""
+cifmw_shiftstack_qa_repo: "https://github.com/shiftstack/shiftstack-qa"
 cifmw_shiftstack_run_playbook: "ocp_testing.yaml"
 cifmw_shiftstack_sc: "local-storage"
 cifmw_shiftstack_shiftstackclient_artifacts_dir: "/home/cloud-admin/artifacts"

roles/shiftstack/tasks/exec_command_in_pod.yml

@@ -16,23 +16,130 @@
 
 - name: Command execution block
   block:
-    - name: Execute in the pod the command '{{ command }}'
+    - name: Kill any previous background command and clear markers
+      kubernetes.core.k8s_exec:
+        kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
+        namespace: "{{ namespace }}"
+        pod: "{{ pod_name }}"
+        command: >-
+          bash -c 'if [ -f /tmp/cifmw_cmd_pid ]; then
+          kill $(cat /tmp/cifmw_cmd_pid) 2>/dev/null;
+          wait $(cat /tmp/cifmw_cmd_pid) 2>/dev/null; fi;
+          rm -f /tmp/cifmw_cmd_rc /tmp/cifmw_cmd_pid
+          /tmp/cifmw_cmd_script.sh /tmp/cifmw_cmd_output.log'
+      changed_when: false
+      failed_when: false
+
+    - name: Copy the command script into the pod
+      kubernetes.core.k8s_cp:
+        kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
+        namespace: "{{ namespace }}"
+        pod: "{{ pod_name }}"
+        remote_path: /tmp/cifmw_cmd_script.sh
+        content: |
+          #!/bin/bash
+          {{ command }}
+        state: to_pod
+
+    - name: Execute '{{ command }}' detached in the pod
+      kubernetes.core.k8s_exec:
+        kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
+        namespace: "{{ namespace }}"
+        pod: "{{ pod_name }}"
+        command: >-
+          bash -c 'nohup bash -c "bash /tmp/cifmw_cmd_script.sh;
+          echo \$? > /tmp/cifmw_cmd_rc"
+          > /tmp/cifmw_cmd_output.log 2>&1 &
+          echo $! > /tmp/cifmw_cmd_pid && cat /tmp/cifmw_cmd_pid'
+      register: pod_command_result
+
+    - name: Wait for the command to complete in the pod
       environment:
         PATH: "{{ cifmw_path }}"
         KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
       ansible.builtin.command:
-        cmd: "oc rsh -n {{ namespace }} {{ pod_name }} bash -c '{{ command }}'"
-      register: pod_command_result
+        cmd: >-
+          oc exec -n {{ namespace }} {{ pod_name }} --
+          bash -c 'if cat /tmp/cifmw_cmd_rc 2>/dev/null; then exit 0;
+          elif [ -f /tmp/cifmw_cmd_pid ] && kill -0 $(cat /tmp/cifmw_cmd_pid) 2>/dev/null; then exit 1;
+          else exit 2; fi'
+      register: pod_exit_code
+      changed_when: false
+      failed_when: false
+      until: pod_exit_code.rc == 0 or pod_exit_code.rc == 2
+      retries: "{{ cifmw_shiftstack_exec_retries }}"
+      delay: "{{ cifmw_shiftstack_exec_delay }}"
+
+    - name: Fail if the background process died without writing the marker
+      ansible.builtin.fail:
+        msg: >-
+          The background process in pod {{ pod_name }} exited without writing a result.
+          The command may have been killed by OOM or a signal.
+      when: pod_exit_code.rc == 2
+
+    - name: Fail if polling timed out
+      ansible.builtin.fail:
+        msg: >-
+          Timed out waiting for "{{ command }}" to complete
+          in pod {{ pod_name }} after {{ cifmw_shiftstack_exec_retries }} attempts.
+      when: pod_exit_code.rc != 0 and pod_exit_code.rc != 2
+
+    - name: Fail if the command failed inside the pod
+      ansible.builtin.fail:
+        msg: >-
+          "{{ command }}" failed inside the pod with
+          exit code {{ pod_exit_code.stdout | trim }}.
+      when:
+        - pod_exit_code.rc == 0
+        - (pod_exit_code.stdout | default('') | trim) != '0'
 
   rescue:
-    - name: Fail when the command module fails
+    - name: Kill background process on failure
+      kubernetes.core.k8s_exec:
+        kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
+        namespace: "{{ namespace }}"
+        pod: "{{ pod_name }}"
+        command: >-
+          bash -c 'if [ -f /tmp/cifmw_cmd_pid ]; then
+          kill $(cat /tmp/cifmw_cmd_pid) 2>/dev/null; fi'
+      failed_when: false
+      changed_when: false
+
+    - name: Retrieve command output from pod
+      kubernetes.core.k8s_exec:
+        kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
+        namespace: "{{ namespace }}"
+        pod: "{{ pod_name }}"
+        command: >-
+          bash -c 'tail -c 8192 /tmp/cifmw_cmd_output.log 2>/dev/null
+          || echo "(output not available)"'
+      register: pod_output_log
+      failed_when: false
+      changed_when: false
+
+    - name: Fail with diagnostic information
       ansible.builtin.fail:
-        msg: '"{{ command }}" command execution failed with exit code "{{ pod_command_result.rc }}" and error: "{{ pod_command_result.stderr }}".'
+        msg: >-
+          "{{ command }}" failed.
+          pod exit code: {{ pod_exit_code.stdout | default('N/A') | trim }},
+          output (last 8KB): {{ pod_output_log.stdout | default('(not retrieved)') }}
 
   always:
     - name: Command execution log saving block
       when: log_file_name | default(false)
       block:
+        - name: Retrieve command output for log
+          kubernetes.core.k8s_exec:
+            kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
+            namespace: "{{ namespace }}"
+            pod: "{{ pod_name }}"
+            command: >-
+              bash -c 'tail -c 16384 /tmp/cifmw_cmd_output.log 2>/dev/null
+              || echo "(output not available)"'
+          register: pod_output_for_log
+          failed_when: false
+          changed_when: false
+
         - name: Get current date and time
           ansible.builtin.command:
             cmd: date +"%Y_%m_%d-%H_%M_%S"
@@ -44,7 +151,9 @@
               namespace: "{{ namespace }}"
               pod_name: "{{ pod_name }}"
               command: "{{ command }}"
-              pod_command_result: "{{ pod_command_result }}"
+              pod_command_result: "{{ pod_command_result | default('N/A') }}"
+              pod_exit_code: "{{ pod_exit_code | default('N/A') }}"
+              pod_output: "{{ pod_output_for_log.stdout | default('N/A') }}"
           ansible.builtin.copy:
             content: "{{ command_info | to_nice_yaml }}"
             dest: "{{ cifmw_shiftstack_ansible_command_logs_dir }}/{{ current_date.stdout }}.{{ log_file_name }}"

roles/shiftstack/tasks/test_shiftstack.yml

@@ -23,18 +23,18 @@
     log_file_name: "clone_shiftstack_qa_repo.log"
   ansible.builtin.include_tasks: exec_command_in_pod.yml
 
-- name: Fetch the gerrithub change '{{ cifmw_shiftstack_qa_gerrithub_change }}'
+- name: Fetch the change ref '{{ cifmw_shiftstack_qa_change_ref }}'
   vars:
     namespace: "{{ cifmw_shiftstack_client_pod_namespace }}"
     pod_name: "{{ cifmw_shiftstack_client_pod_name }}"
     command: >-
       cd shiftstack-qa &&
-      git fetch {{ cifmw_shiftstack_qa_repo }} {{ cifmw_shiftstack_qa_gerrithub_change }} &&
+      git fetch {{ cifmw_shiftstack_qa_repo }} {{ cifmw_shiftstack_qa_change_ref }} &&
       git checkout FETCH_HEAD
-    log_file_name: "fetch_shiftstack_qa_gerrithub_change.log"
+    log_file_name: "fetch_shiftstack_qa_change_ref.log"
   when:
-    - cifmw_shiftstack_qa_gerrithub_change is defined
-    - cifmw_shiftstack_qa_gerrithub_change != ''
+    - cifmw_shiftstack_qa_change_ref is defined
+    - cifmw_shiftstack_qa_change_ref != ''
   ansible.builtin.include_tasks: exec_command_in_pod.yml
 
 - name: Install the ansible collections