Skip to content

Commit 378fd15

Browse files
ekuris-redhatekuris-redhat
authored
Merge pull request #9 from shiftstack/support_cpms_test_role_tj
Add CPMS e2e-presubmit test stage role for shiftstack-qa
2 parents 736ee2b + ef55d76 commit 378fd15

10 files changed

Lines changed: 359 additions & 4 deletions

File tree

‎collection/stages/roles/cpms_test/defaults/main.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
---
2+
# defaults file for cpms_test
3+
cpms_test_name: cluster-control-plane-machine-set-operator
4+
cpms_test_dir: "{{ artifacts_dir }}/{{ cpms_test_name }}"
5+
cpms_test_results_dir: "{{ artifacts_dir }}/cpms_test-results"
6+
cpms_test_testsuite_name: cpms
7+
cpms_tests_go_version: "{{ tests.default_go_version_target }}"
8+
cpms_test_junit_filename: junit_control_plane_machine_set_operator.xml
9+
cpms_test_suites:
10+
- e2e-presubmit

‎collection/stages/roles/cpms_test/meta/main.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
---
2+
collections:
3+
- shiftstack.tools

‎collection/stages/roles/cpms_test/tasks/main.yml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
---
2+
# tasks file for cpms_test
3+
- name: Prepare CPMS test
4+
ansible.builtin.include_role:
5+
name: tools_openshift_tests
6+
tasks_from: prepare_openshift_tests.yml
7+
vars:
8+
repo_name: "{{ cpms_test_name }}"
9+
results_dir: "{{ cpms_test_results_dir }}"
10+
go_version_target: "{{ cpms_tests_go_version }}"
11+
12+
- name: Run CPMS {{ item }} tests
13+
ansible.builtin.include_tasks: run_cpms_test.yml
14+
vars:
15+
cpms_tests_type: "{{ item }}"
16+
loop: "{{ cpms_test_suites }}"
17+
18+
- name: Remove the source directory after tests complete
19+
ansible.builtin.file:
20+
path: "{{ cpms_test_dir }}"
21+
state: absent

‎collection/stages/roles/cpms_test/tasks/run_cpms_test.yml‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
---
2+
- name: Set result paths for {{ cpms_tests_type }}
3+
ansible.builtin.set_fact:
4+
cpms_results_subdir: "{{ cpms_test_results_dir }}/{{ cpms_tests_type }}"
5+
6+
- name: Create results directory for {{ cpms_tests_type }}
7+
ansible.builtin.file:
8+
path: "{{ cpms_results_subdir }}"
9+
state: directory
10+
mode: u=rwx,g=rw,o=r
11+
12+
- name: Run {{ cpms_test_name }} {{ cpms_tests_type }} tests
13+
block:
14+
- name: Run make {{ cpms_tests_type }}
15+
ansible.builtin.shell: |
16+
source {{ home_dir }}/.bashrc
17+
make {{ cpms_tests_type }}
18+
args:
19+
chdir: "{{ cpms_test_dir }}"
20+
environment:
21+
KUBECONFIG: "{{ kubeconfig }}"
22+
OS_CLOUD: "{{ user_cloud }}"
23+
OPENSHIFT_CI: "true"
24+
ARTIFACT_DIR: "{{ cpms_results_subdir }}"
25+
changed_when: true
26+
27+
rescue:
28+
- name: Mark the CPMS {{ cpms_tests_type }} tests as UNSTABLE
29+
ansible.builtin.include_role:
30+
name: tools_stage_results
31+
tasks_from: mark_stage_unstable.yml
32+
vars:
33+
unstable_msg: >-
34+
The {{ cpms_test_name }} {{ cpms_tests_type }} test suite failed.
35+
36+
- name: Run must-gather
37+
ansible.builtin.include_role:
38+
name: tools_must-gather
39+
vars:
40+
must_gather_suffix: "cpms-{{ cpms_tests_type }}"
41+
42+
- name: Check if JUnit file exists
43+
ansible.builtin.stat:
44+
path: "{{ cpms_results_subdir }}/{{ cpms_test_junit_filename }}"
45+
register: _cpms_junit_file
46+
47+
- name: Rename JUnit XML to match expected prefix
48+
ansible.builtin.copy:
49+
src: "{{ cpms_results_subdir }}/{{ cpms_test_junit_filename }}"
50+
dest: "{{ cpms_results_subdir }}/junit_e2e_{{ cpms_tests_type }}.xml"
51+
remote_src: yes
52+
mode: u=rw,g=rw,o=r
53+
when: _cpms_junit_file.stat.exists
54+
55+
- name: Post openshift-test
56+
ansible.builtin.include_role:
57+
name: tools_openshift_tests
58+
tasks_from: post_openshift_tests.yml
59+
vars:
60+
testsuite_name: "{{ cpms_test_testsuite_name }}_{{ cpms_tests_type }}"
61+
key_for_filtering_results: "cpms"
62+
test_name: "{{ cpms_test_name }}-{{ cpms_tests_type }}"
63+
results_dir: "{{ cpms_results_subdir }}"
64+
when: _cpms_junit_file.stat.exists

‎collection/stages/roles/day2ops/tasks/procedures/cpms_replace_attrs.yml‎

Lines changed: 217 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,217 @@
1+
---
2+
# Procedure that updates the CPMS object by adding a fake extra network/subnet and SG to the
3+
# masters and replacing failureDomain params on master-0 to have the same attributes as master-1.
4+
# After reconciliation, master-0 and master-1 will have same nova AZ, cinderAZ, and volumeType.
5+
- name: Set log directory for cpms_replace_attrs procedure
6+
ansible.builtin.set_fact:
7+
cpms_log_directory: "{{ artifacts_dir }}/cpms_replace_attrs"
8+
9+
- name: Run cpms_replace_attrs procedure
10+
block:
11+
- name: Create {{ cpms_log_directory }} directory
12+
ansible.builtin.file:
13+
path: "{{ cpms_log_directory }}"
14+
state: directory
15+
mode: u=rwx,g=rw,o=r
16+
17+
- name: Set artifact paths
18+
ansible.builtin.set_fact:
19+
cpms_orig_path: "{{ cpms_log_directory }}/orig_cpms.yaml"
20+
cpms_patch_to_apply_path: "{{ cpms_log_directory }}/cpms_patch_to_apply.yaml"
21+
cpms_applied_path: "{{ cpms_log_directory }}/applied_cpms.yaml"
22+
cpms_patch_to_restore_path: "{{ cpms_log_directory }}/cpms_patch_to_restore_NOTAPPLIED.yaml"
23+
24+
- name: Get original CPMS definition
25+
kubernetes.core.k8s_info:
26+
kubeconfig: "{{ kubeconfig }}"
27+
api_version: machine.openshift.io/v1
28+
kind: ControlPlaneMachineSet
29+
namespace: openshift-machine-api
30+
register: original_cpms
31+
32+
- name: Save the original CPMS definition
33+
ansible.builtin.copy:
34+
content: "{{ original_cpms.resources[0] | to_nice_yaml(indent=2) }}"
35+
dest: "{{ cpms_orig_path }}"
36+
mode: u=rw,g=rw,o=r
37+
38+
- name: Create network {{ cpms_replacements.network_name }}
39+
openstack.cloud.network:
40+
cloud: "{{ user_cloud }}"
41+
name: "{{ cpms_replacements.network_name }}"
42+
state: present
43+
register: network_to_add
44+
45+
- name: Create subnet {{ cpms_replacements.subnet_name }}
46+
openstack.cloud.subnet:
47+
cloud: "{{ user_cloud }}"
48+
name: "{{ cpms_replacements.subnet_name }}"
49+
cidr: "{{ cpms_replacements.cidr }}"
50+
network_name: "{{ cpms_replacements.network_name }}"
51+
enable_dhcp: false
52+
state: present
53+
54+
- name: Create security group {{ cpms_replacements.sg_name }}
55+
openstack.cloud.security_group:
56+
cloud: "{{ user_cloud }}"
57+
name: "{{ cpms_replacements.sg_name }}"
58+
state: present
59+
register: sg_to_add
60+
61+
- name: Store original CPMS attributes that will be updated
62+
ansible.builtin.set_fact:
63+
orig_failuredomains: >-
64+
{{ original_cpms.resources[0].spec.template.machines_v1beta1_machine_openshift_io.failureDomains.openstack | list }}
65+
orig_networks: >-
66+
{{ original_cpms.resources[0].spec.template.machines_v1beta1_machine_openshift_io.spec.providerSpec.value.networks | list }}
67+
orig_sgs: >-
68+
{{ original_cpms.resources[0].spec.template.machines_v1beta1_machine_openshift_io.spec.providerSpec.value.securityGroups | list }}
69+
70+
- name: Create patch for CPMS
71+
ansible.builtin.template:
72+
src: control-plane-machine-set-patch.yaml.j2
73+
dest: "{{ cpms_patch_to_apply_path }}"
74+
mode: u=rw,g=rw,o=r
75+
vars:
76+
cpms_failuredomains: >-
77+
{{ orig_failuredomains |
78+
replace(orig_failuredomains[0].availabilityZone, orig_failuredomains[1].availabilityZone) |
79+
replace(orig_failuredomains[0].rootVolume.availabilityZone, orig_failuredomains[1].rootVolume.availabilityZone) |
80+
replace(orig_failuredomains[0].rootVolume.volumeType, orig_failuredomains[1].rootVolume.volumeType) }}
81+
cpms_networks: "{{ orig_networks + [{'filter': {}, 'uuid': network_to_add.id}] }}"
82+
cpms_sgs: "{{ orig_sgs + [{'filter': {}, 'name': cpms_replacements.sg_name}] }}"
83+
84+
- name: Apply patch for CPMS
85+
kubernetes.core.k8s:
86+
kubeconfig: "{{ kubeconfig }}"
87+
state: present
88+
apply: true
89+
src: "{{ cpms_patch_to_apply_path }}"
90+
91+
- name: Wait for CPMS reconciliation to complete
92+
ansible.builtin.include_role:
93+
name: tools_cluster_checks
94+
tasks_from: wait_cpms_updated.yml
95+
vars:
96+
wait_retries: 180
97+
wait_delay: 60
98+
99+
- name: Check cluster health after CPMS patch application
100+
block:
101+
- name: Check the Control Plane MachineSet is healthy
102+
ansible.builtin.include_role:
103+
name: tools_cluster_checks
104+
tasks_from: check_controlplane_machinesets.yml
105+
106+
- name: Wait until OCP cluster is healthy
107+
ansible.builtin.include_role:
108+
name: tools_cluster_checks
109+
tasks_from: wait_until_cluster_is_healthy.yml
110+
111+
- name: Get resulting CPMS definition after application
112+
kubernetes.core.k8s_info:
113+
kubeconfig: "{{ kubeconfig }}"
114+
api_version: machine.openshift.io/v1
115+
kind: ControlPlaneMachineSet
116+
namespace: openshift-machine-api
117+
register: applied_cpms
118+
119+
- name: Save the resulting CPMS after application
120+
ansible.builtin.copy:
121+
content: "{{ applied_cpms.resources[0] | to_nice_yaml(indent=2) }}"
122+
dest: "{{ cpms_applied_path }}"
123+
mode: u=rw,g=rw,o=r
124+
125+
- name: Get all the master VMs after procedure
126+
openstack.cloud.server_info:
127+
cloud: "{{ user_cloud }}"
128+
server: "*master*"
129+
register: result
130+
131+
- name: Store the master VMs after procedure
132+
ansible.builtin.set_fact:
133+
master_after: "{{ result.servers | sort(attribute='name') | list }}"
134+
135+
- name: Get root volumes info
136+
ansible.builtin.shell: >
137+
openstack volume show {{ item }} -c type -c availability_zone -f shell
138+
environment:
139+
OS_CLOUD: "{{ user_cloud }}"
140+
register: root_volumes_after
141+
changed_when: false
142+
with_items:
143+
- "{{ master_after[0].volumes[0].id }}"
144+
- "{{ master_after[1].volumes[0].id }}"
145+
- "{{ master_after[2].volumes[0].id }}"
146+
147+
- name: Confirm that master VMs reflect FailureDomain changes after procedure
148+
ansible.builtin.assert:
149+
that:
150+
- master_after | length == 3
151+
- master_after | selectattr('location.zone', 'eq', expected_fd_az) | list | length == 2
152+
- root_volumes_after.results | selectattr('stdout', 'eq',
153+
'availability_zone="' + expected_fd_cinderaz + '"\ntype="' + expected_fd_volume_type + '"') | list | length == 2
154+
vars:
155+
expected_fd_az: "{{ orig_failuredomains[1].availabilityZone }}"
156+
expected_fd_cinderaz: "{{ orig_failuredomains[1].rootVolume.availabilityZone }}"
157+
expected_fd_volume_type: "{{ orig_failuredomains[1].rootVolume.volumeType }}"
158+
159+
- name: Confirm that the VMs include the added network and SG
160+
ansible.builtin.assert:
161+
that:
162+
- cpms_replacements.network_name in item.addresses.keys()
163+
- cpms_replacements.sg_name in item.security_groups | json_query('[*].name')
164+
with_items: "{{ master_after }}"
165+
166+
always:
167+
- name: Create restore patch for CPMS
168+
ansible.builtin.template:
169+
src: control-plane-machine-set-patch.yaml.j2
170+
dest: "{{ cpms_patch_to_restore_path }}"
171+
mode: u=rw,g=rw,o=r
172+
vars:
173+
cpms_failuredomains: "{{ orig_failuredomains }}"
174+
cpms_networks: "{{ orig_networks }}"
175+
cpms_sgs: "{{ orig_sgs }}"
176+
177+
- name: Apply restore patch for CPMS
178+
kubernetes.core.k8s:
179+
kubeconfig: "{{ kubeconfig }}"
180+
state: present
181+
apply: true
182+
src: "{{ cpms_patch_to_restore_path }}"
183+
184+
- name: Wait for CPMS restore reconciliation to complete
185+
ansible.builtin.include_role:
186+
name: tools_cluster_checks
187+
tasks_from: wait_cpms_updated.yml
188+
vars:
189+
wait_retries: 180
190+
wait_delay: 60
191+
192+
- name: Check cluster health after restore
193+
block:
194+
- name: Check the Control Plane MachineSet is healthy
195+
ansible.builtin.include_role:
196+
name: tools_cluster_checks
197+
tasks_from: check_controlplane_machinesets.yml
198+
199+
- name: Wait until OCP cluster is healthy
200+
ansible.builtin.include_role:
201+
name: tools_cluster_checks
202+
tasks_from: wait_until_cluster_is_healthy.yml
203+
204+
- name: Clean up test OpenStack resources
205+
failed_when: false
206+
block:
207+
- name: Delete test network
208+
openstack.cloud.network:
209+
cloud: "{{ user_cloud }}"
210+
name: "{{ cpms_replacements.network_name }}"
211+
state: absent
212+
213+
- name: Delete test security group
214+
openstack.cloud.security_group:
215+
cloud: "{{ user_cloud }}"
216+
name: "{{ cpms_replacements.sg_name }}"
217+
state: absent

‎collection/stages/roles/day2ops/templates/control-plane-machine-set-patch.yaml.j2‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
apiVersion: machine.openshift.io/v1
2+
metadata:
3+
name: cluster
4+
namespace: openshift-machine-api
5+
kind: ControlPlaneMachineSet
6+
spec:
7+
template:
8+
machines_v1beta1_machine_openshift_io:
9+
failureDomains:
10+
openstack:
11+
{{cpms_failuredomains| to_nice_yaml| indent(8, false)}}
12+
spec:
13+
providerSpec:
14+
value:
15+
networks:
16+
{{cpms_networks| to_nice_yaml| indent(12, false)}}
17+
securityGroups:
18+
{{cpms_sgs| to_nice_yaml| indent(12, false)}}
19+
status: {}

‎collection/stages/roles/verification/tasks/main.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,8 +69,8 @@
6969
name: tools_cluster_checks
7070
tasks_from: check_pods.yml
7171
vars:
72-
wait_retries: 20
73-
wait_delay: 10
72+
wait_retries: 30
73+
wait_delay: 30
7474

7575
- name: Check cluster alerts
7676
ansible.builtin.include_role:

‎jobs_definitions/osp_verification.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,5 +55,5 @@ ocp_deployment_topology:
5555
master: 3
5656
worker: 2
5757
servergroups:
58-
master: "anti-affinity"
59-
worker: "anti-affinity"
58+
master: "soft-anti-affinity"
59+
worker: "soft-anti-affinity"

‎playbooks/ocp_testing.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,10 @@
100100
ansible.builtin.import_playbook: plays/conformance_test.yaml
101101
when: "'conformance_test' in stages"
102102

103+
- name: Run CPMS e2e tests on OpenShift
104+
ansible.builtin.import_playbook: plays/cpms_test.yaml
105+
when: "'cpms_test' in stages"
106+
103107
- name: Run Openshift Cinder CSI Tests on OpenShift
104108
ansible.builtin.import_playbook: plays/cinder_csi.yaml
105109
when: "'cinder_csi_tests' in stages"

‎playbooks/plays/cpms_test.yaml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
---
2+
- name: Run CPMS e2e tests on OpenShift
3+
hosts: installer
4+
gather_facts: no
5+
vars_files:
6+
- "../../configs/global.yml"
7+
tasks:
8+
- name: Main block
9+
block:
10+
- name: Run CPMS e2e tests on OCP
11+
ansible.builtin.include_role:
12+
name: shiftstack.stages.cpms_test
13+
always:
14+
- name: Synchronize artifacts from the Ansible Managed Node to Ansible Controller
15+
ansible.builtin.include_role:
16+
name: shiftstack.tools.tools_ansible_inventory
17+
tasks_from: sync_artifacts.yml

0 commit comments

Comments
 (0)