From 375a97aecd4251252c96e06b044c2f74811bfc48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Harald=20Jens=C3=A5s?= Date: Thu, 15 Jan 2026 18:23:39 +0100 Subject: [PATCH] WIP: Add networking-lab with devstack-nxsw-vxlan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces a new networking-lab directory for network-focused scenarios that don't require full OpenShift deployments. Includes first scenario: devstack-nxsw-vxlan with spine-and-leaf topology featuring: - 4 Cisco NX-OS switches (2 spine + 2 leaf) with OSPF underlay - Devstack node with trunk port for VLAN-tagged traffic - 2 Ironic baremetal nodes attached to leaf switches - Individual POAP scripts and configs per switch - VXLAN/EVPN ready configuration - SVG topology diagram - Automated POAP md5sum management script and pre-commit hook Point-to-point links consolidated in 10.1.1.0/24 to preserve 10.1.2.0/24+ address space for future use. Assisted-By: Claude (claude-4.5-sonnet) Signed-off-by: Harald Jensås --- .../devstack-nxsw-vxlan/README.md | 347 +++++ .../devstack-nxsw-vxlan/TROUBLESHOOTING.md | 523 +++++++ .../devstack-nxsw-vxlan/automation-vars.yml | 130 ++ .../devstack-nxsw-vxlan/bootstrap_vars.yml | 63 + .../devstack-nxsw-vxlan/heat_template.yaml | 1242 +++++++++++++++++ .../l2vni_network_nodes.yaml | 16 + .../devstack-nxsw-vxlan/leaf01-poap.cfg | 114 ++ .../devstack-nxsw-vxlan/leaf01-poap.py | 437 ++++++ .../devstack-nxsw-vxlan/leaf02-poap.cfg | 107 ++ .../devstack-nxsw-vxlan/leaf02-poap.py | 437 ++++++ .../devstack-nxsw-vxlan/local.conf.j2 | 158 +++ .../devstack-nxsw-vxlan/spine01-poap.cfg | 86 ++ .../devstack-nxsw-vxlan/spine01-poap.py | 437 ++++++ .../devstack-nxsw-vxlan/spine02-poap.cfg | 86 ++ .../devstack-nxsw-vxlan/spine02-poap.py | 437 ++++++ .../devstack-nxsw-vxlan/topology-diagram.svg | 132 ++ 16 files changed, 4752 insertions(+) create mode 100644 scenarios/networking-lab/devstack-nxsw-vxlan/README.md create mode 100644 scenarios/networking-lab/devstack-nxsw-vxlan/TROUBLESHOOTING.md create mode 100644 scenarios/networking-lab/devstack-nxsw-vxlan/automation-vars.yml create mode 100644 scenarios/networking-lab/devstack-nxsw-vxlan/bootstrap_vars.yml create mode 100644 scenarios/networking-lab/devstack-nxsw-vxlan/heat_template.yaml create mode 100644 scenarios/networking-lab/devstack-nxsw-vxlan/l2vni_network_nodes.yaml create mode 100644 scenarios/networking-lab/devstack-nxsw-vxlan/leaf01-poap.cfg create mode 100644 scenarios/networking-lab/devstack-nxsw-vxlan/leaf01-poap.py create mode 100644 scenarios/networking-lab/devstack-nxsw-vxlan/leaf02-poap.cfg create mode 100644 scenarios/networking-lab/devstack-nxsw-vxlan/leaf02-poap.py create mode 100644 scenarios/networking-lab/devstack-nxsw-vxlan/local.conf.j2 create mode 100644 scenarios/networking-lab/devstack-nxsw-vxlan/spine01-poap.cfg create mode 100644 scenarios/networking-lab/devstack-nxsw-vxlan/spine01-poap.py create mode 100644 scenarios/networking-lab/devstack-nxsw-vxlan/spine02-poap.cfg create mode 100644 scenarios/networking-lab/devstack-nxsw-vxlan/spine02-poap.py create mode 100644 scenarios/networking-lab/devstack-nxsw-vxlan/topology-diagram.svg diff --git a/scenarios/networking-lab/devstack-nxsw-vxlan/README.md b/scenarios/networking-lab/devstack-nxsw-vxlan/README.md new file mode 100644 index 00000000..b12bb5e5 --- /dev/null +++ b/scenarios/networking-lab/devstack-nxsw-vxlan/README.md @@ -0,0 +1,347 @@ +# Networking Lab: Devstack with Spine-and-Leaf VXLAN Topology + +## Overview + +This scenario provides a networking laboratory environment featuring a spine-and-leaf Cisco NX-OS switch topology with VXLAN overlay capabilities. The setup includes: + +- **4 Cisco NX-OS switches** in a spine-and-leaf topology + - 2 Spine switches: `spine01` and `spine02` + - 2 Leaf switches: `leaf01` and `leaf02` +- **1 Devstack node** for running OpenStack development environment +- **2 Ironic nodes** for bare metal provisioning testing +- **1 Controller node** providing DNS, DHCP, and TFTP services for POAP + +## Network Topology + +![Topology Diagram](topology-diagram.svg) + +The topology features a full-mesh spine-and-leaf design where: +- Both leaf switches connect to both spine switches (4 uplinks total per leaf) +- Spine switches are interconnected for redundancy +- Server nodes attach to leaf switches only +- All switches participate in OSPF for underlay routing +- Loopback interfaces (10.255.255.x/32) serve as VTEPs for VXLAN overlay + +## Network Details + +### Management Network +- **machine-net**: `192.168.32.0/24` + - Controller: `192.168.32.254` + - Spine01: `192.168.32.11` + - Spine02: `192.168.32.12` + - Leaf01: `192.168.32.13` + - Leaf02: `192.168.32.14` + - Devstack: `192.168.32.20` + +### Inter-Switch Links (Underlay) +All point-to-point links are allocated from `10.1.1.0/24` to preserve `10.1.2.0/24`, `10.1.3.0/24`, etc. for other uses: +- **spine-link-net**: `10.1.1.0/30` - Link between Spine01 and Spine02 +- **leaf01-spine01-net**: `10.1.1.4/30` - Link between Leaf01 and Spine01 +- **leaf01-spine02-net**: `10.1.1.8/30` - Link between Leaf01 and Spine02 +- **leaf02-spine01-net**: `10.1.1.12/30` - Link between Leaf02 and Spine01 +- **leaf02-spine02-net**: `10.1.1.16/30` - Link between Leaf02 and Spine02 + +### Loopback Addresses (VTEP) +- Spine01: `10.255.255.1/32` +- Spine02: `10.255.255.2/32` +- Leaf01: `10.255.255.3/32` +- Leaf02: `10.255.255.4/32` + +### BGP EVPN Topology + +The fabric uses BGP EVPN for VXLAN overlay control plane with the following iBGP configuration: + +``` + spine01 (RR) spine02 (RR) + 10.255.255.1 10.255.255.2 + | | | | + | +-------+--------+ | + | | | + iBGP iBGP iBGP + | | | + +----+ +----+--------+ + | | + leaf01 (RRC) leaf02 (RRC) + 10.255.255.3 10.255.255.4 + NVE1 (VTEP) NVE1 (VTEP) +``` + +**Configuration Details:** +- **AS Number**: 65001 (iBGP) +- **Route Reflectors**: spine01 and spine02 serve as BGP route reflectors +- **Route Reflector Clients**: leaf01 and leaf02 are route reflector clients +- **Address Family**: L2VPN EVPN +- **VTEP Interfaces**: NVE1 on each leaf switch (source-interface loopback0) +- **Host Reachability**: BGP-based VXLAN (ingress-replication via BGP) + +This configuration allows the ML2 networking-generic-switch plugin to dynamically create VNIs and map VLANs to VXLANs across the fabric. + +### Networks + +**Bridge Networks** (Simple L2 connectivity for servers): +- `devstack-br-net` (`172.20.10.0/29`): Devstack ↔ Leaf01 (Ethernet1/5) +- `ironic0-br-net` (`172.20.11.0/29`): Ironic0 ↔ Leaf01 (Ethernet1/4) +- `ironic1-br-net` (`172.20.12.0/29`): Ironic1 ↔ Leaf02 (Ethernet1/4) + +**Leaf01 Tenant VLANs** (on trunk port Ethernet1/3): +- `leaf01-trunk-net` (`172.20.20.0/24`): Parent/native VLAN +- `leaf01-public-vlan100` (`172.20.0.0/24`): VLAN 100 - Public network +- `leaf01-tenant-vlan103` (`172.20.3.0/24`): VLAN 103 - Tenant network +- `leaf01-tenant-vlan104` (`172.20.4.0/24`): VLAN 104 - Tenant network +- `leaf01-tenant-vlan105` (`172.20.5.0/24`): VLAN 105 - Tenant network + +**Leaf02 Tenant VLANs** (on trunk port Ethernet1/3): +- `leaf02-trunk-net` (`172.20.21.0/24`): Parent/native VLAN +- `leaf02-public-vlan100` (`172.20.1.0/24`): VLAN 100 - Public network +- `leaf02-tenant-vlan103` (`172.20.6.0/24`): VLAN 103 - Tenant network +- `leaf02-tenant-vlan104` (`172.20.7.0/24`): VLAN 104 - Tenant network +- `leaf02-tenant-vlan105` (`172.20.8.0/24`): VLAN 105 - Tenant network + +> **Note**: Leaf switches receive these as Neutron provider networks via trunk ports. ML2 plugins can dynamically manage VLAN configurations on edge ports. + +## Switch Configuration + +### Spine Switches (Spine01 & Spine02) +The spine switches are configured with: +- OSPF for underlay routing (Area 0.0.0.0) +- BGP AS 65001 configured as route reflectors for L2VPN EVPN +- NV overlay features enabled +- Point-to-point links to all leaf switches +- iBGP neighbors to both leaf switches + +### Leaf Switches (Leaf01 & Leaf02) +The leaf switches are configured with: +- OSPF for underlay routing (Area 0.0.0.0) +- BGP AS 65001 as route reflector clients for L2VPN EVPN +- NVE1 interface configured for VXLAN (source-interface loopback0) +- VN-segment-vlan-based feature enabled +- **Minimal base configuration** - VLANs, VNIs, and port configurations managed dynamically via ML2 plugins +- Edge ports (Ethernet1/3, 1/4, 1/5) shutdown by default - to be configured by networking-generic-switch ML2 plugin +- Port assignments: + - **Leaf01**: Ethernet1/3 (trunk for VXLAN VNIs), Ethernet1/4 (Ironic0), Ethernet1/5 (Devstack) + - **Leaf02**: Ethernet1/3 (trunk for VXLAN VNIs), Ethernet1/4 (Ironic1) + +## POAP Configuration + +Each switch receives a unique configuration via POAP (Power-On Auto Provisioning): + +| Switch | MAC Address | IP Address | POAP Script | POAP Config File | Role | +|---------|-------------------|----------------|--------------------|--------------------|-------| +| spine01 | 22:57:f8:dd:01:01 | 192.168.32.11 | spine01-poap.py | spine01-poap.cfg | Spine | +| spine02 | 22:57:f8:dd:02:01 | 192.168.32.12 | spine02-poap.py | spine02-poap.cfg | Spine | +| leaf01 | 22:57:f8:dd:03:01 | 192.168.32.13 | leaf01-poap.py | leaf01-poap.cfg | Leaf | +| leaf02 | 22:57:f8:dd:04:01 | 192.168.32.14 | leaf02-poap.py | leaf02-poap.cfg | Leaf | + +The POAP process uses DHCP options: +- Option 66 (TFTP Server): Points to controller IP (192.168.32.254) +- Option 67 (Boot File): Points to switch-specific Python script (e.g., `core01-poap.py`) + +Each switch has its own POAP script that points to its specific configuration file. The POAP script downloads and applies the configuration, then installs the NX-OS image. + +## Components + +### Controller Node +The controller provides: +- **DNS** (dnsmasq): Name resolution for all nodes +- **DHCP** (dnsmasq): IP address assignment with POAP options +- **TFTP** (dnsmasq): POAP script and configuration file distribution +- **HTTP** (httpd on port 8081): Additional file serving + +### Devstack Node +OpenStack development environment with two network interfaces: +- **eth0**: Management network (192.168.32.20) +- **eth1**: Trunk port with VLANs 100, 103, 104, 105 (to Leaf01 Ethernet1/5) + - Native VLAN: devstack-br-net + - VLAN 100: leaf01-public-vlan100 + - VLAN 103: leaf01-tenant-vlan103 + - VLAN 104: leaf01-tenant-vlan104 + - VLAN 105: leaf01-tenant-vlan105 + +### Ironic Nodes +Bare metal nodes for provisioning testing: +- **ironic0**: Attached to Leaf01 (Ethernet1/4) via ironic0-br-net +- **ironic1**: Attached to Leaf02 (Ethernet1/4) via ironic1-br-net +- Managed via Redfish virtual BMC (sushy-tools) +- Configured for UEFI boot mode + +## Deployment + +### Prerequisites +1. OpenStack cloud with Heat support +2. Required images uploaded: + - `hotstack-controller` + - `nexus9300v.9.3.15` (or your NX-OS version) + - `CentOS-Stream-GenericCloud-9` + - `sushy-tools-blank-image` +3. SSH key pair configured +4. Sufficient quota for: + - 7 instances (1 controller + 4 switches + 1 devstack + 1 ironic) + - Multiple networks and subnets + - Floating IP + +### Deploy the Stack + +1. Update variables in `bootstrap_vars.yml`: +```yaml +os_cloud: your-cloud-name +controller_ssh_pub_key: "{{ lookup('file', '~/.ssh/id_rsa.pub') }}" +dataplane_ssh_pub_key: "{{ lookup('file', '~/.ssh/id_rsa.pub') }}" +``` + +2. Deploy the infrastructure: +```bash +ansible-playbook -e @scenarios/networking-lab/devstack-nxsw-vxlan/bootstrap_vars.yml 01-infra.yml +``` + +3. Bootstrap the controller: +```bash +ansible-playbook -e @scenarios/networking-lab/devstack-nxsw-vxlan/bootstrap_vars.yml 02-bootstrap_controller.yml +``` + +4. Wait for switches to complete POAP (~10-15 minutes) + +5. Verify switch connectivity: +```bash +# SSH to controller +ssh zuul@ + +# From controller, SSH to each switch +ssh admin@192.168.32.11 # spine01 +ssh admin@192.168.32.12 # spine02 +ssh admin@192.168.32.13 # leaf01 +ssh admin@192.168.32.14 # leaf02 +``` + +6. Install DevStack: +```bash +ansible-playbook -e @scenarios/networking-lab/devstack-nxsw-vxlan/bootstrap_vars.yml 04-install_devstack.yml +``` + +This will: +- Configure the DevStack node with proper networking +- Run stack.sh to install OpenStack with OVN, Ironic, and networking-generic-switch +- Set up neutron networks and bridge mappings +- Configure networking-generic-switch with leaf switch credentials (via local.conf) +- Enable L2VNI trunk reconciliation for automatic VLAN management (via local.conf) +- Create clouds.yaml on the controller for remote DevStack access + +7. Enroll Ironic nodes (optional, for testing baremetal provisioning): +```bash +ansible-playbook -e @scenarios/networking-lab/devstack-nxsw-vxlan/bootstrap_vars.yml 05-hotloop-stages.yml +``` + +This will: +- Enroll the Ironic nodes in DevStack's Ironic service using definitions from Heat stack outputs +- Wait for nodes to reach 'enroll' state (validates BMC connectivity) +- Transition nodes to 'manageable' state (prepares for provisioning operations) + +The enrollment creates: +- Ironic nodes with Redfish BMC configuration pointing to sushy-tools instances +- Ironic ports with local_link_information for networking-generic-switch integration +- Port bindings that trigger VLAN configuration on physical switches when nodes are deployed + + +## Validation + +### Verify Switch Underlay +```bash +# On any switch +show ip route +show ip ospf neighbors +``` + +### Verify BGP EVPN Overlay +```bash +# On spine switches (route reflectors) +show bgp l2vpn evpn summary + +# On leaf switches +show bgp l2vpn evpn summary +show nve peers +show interface nve1 +show nve vni +``` + +Expected output: +- **Spines**: BGP neighbors (leaf switches) should be in "Established" state +- **Leafs**: BGP neighbors (spine switches) should be in "Established" state +- **Leafs**: NVE peers will appear once VNIs are created dynamically by ML2 plugin +- **Leafs**: VNI list will populate as ML2 creates VLAN-to-VNI mappings + +### Verify Switch Connectivity +```bash +# From controller +ssh admin@192.168.32.11 # spine01 +ssh admin@192.168.32.12 # spine02 +ssh admin@192.168.32.13 # leaf01 +ssh admin@192.168.32.14 # leaf02 +``` + +## L2VNI Trunk Reconciliation + +This scenario includes the L2VNI trunk reconciliation feature from networking-baremetal, which automatically manages trunk subports on the devstack network node. This enables: + +- Automatic VLAN management on the trunk port connecting to the physical switch +- Dynamic addition/removal of VLANs as overlay networks are created/deleted +- Synchronization with OVN ha_chassis_group membership + +## Use Cases + +This scenario is ideal for: +- **ML2 Plugin Development**: Test networking-generic-switch and other ML2 mechanism drivers +- **Dynamic Switch Configuration**: Validate automatic VLAN creation and port configuration +- **VXLAN/EVPN Testing**: Validate VXLAN overlay configurations +- **Ironic Bare Metal Testing**: Validate bare metal provisioning with network integration +- **Network Automation**: Develop and test OpenStack-driven network automation +- **Spine-Leaf Architecture**: Learn and validate spine-leaf design patterns +- **L2VNI Trunk Reconciliation**: Test automatic trunk port management for network nodes + +## Customization + +### Adding More VLANs/VNIs +Edit the leaf switch POAP configs to add more VLANs: +``` +vlan 200 + name Another_VLAN + vn-segment 10200 +``` + +### Changing Topology +Modify `heat_template.yaml` to: +- Add more spine or leaf switches +- Add more server ports +- Adjust IP addressing schemes + +### BGP Configuration +BGP EVPN is pre-configured in the POAP files with: +- AS 65001 (iBGP) +- Spines as route reflectors +- Leafs as route reflector clients +- L2VPN EVPN address-family enabled + +If you need to modify BGP parameters (e.g., add more neighbors or change policies), edit the POAP configuration files (`*-poap.cfg`) and redeploy the stack. + +## Troubleshooting + +### Switch POAP Failed +- Check DHCP options: `tail -f /var/log/messages` on controller +- Verify TFTP connectivity: `curl tftp://192.168.32.254/poap.py` +- Check switch console logs via OpenStack console + +### Switches Not Reachable +- Verify management interface configuration on switches +- Check DNS resolution from controller +- Ensure routes are configured in management VRF + +### Devstack Deployment Issues +- Check network connectivity on eth1 +- Verify VLAN configuration on leaf switch +- Review devstack logs: `/opt/stack/logs/stack.sh.log` + +## References + +- [Cisco NX-OS POAP Documentation](https://www.cisco.com/c/en/us/td/docs/switches/datacenter/nexus9000/sw/poap/guide/b_poap.html) +- [OpenStack Networking-generic-switch](https://opendev.org/openstack/networking-generic-switch) +- [DevStack Documentation](https://docs.openstack.org/devstack/latest/) +- [OpenStack Ironic Documentation](https://docs.openstack.org/ironic/latest/) +- [VXLAN BGP EVPN Design Guide](https://www.cisco.com/c/en/us/products/collateral/switches/nexus-9000-series-switches/guide-c07-734107.html) diff --git a/scenarios/networking-lab/devstack-nxsw-vxlan/TROUBLESHOOTING.md b/scenarios/networking-lab/devstack-nxsw-vxlan/TROUBLESHOOTING.md new file mode 100644 index 00000000..cd9023d6 --- /dev/null +++ b/scenarios/networking-lab/devstack-nxsw-vxlan/TROUBLESHOOTING.md @@ -0,0 +1,523 @@ +# Troubleshooting Guide: Cisco NX-OS VXLAN EVPN + +This guide provides useful Cisco NX-OS commands for troubleshooting the spine-and-leaf VXLAN EVPN topology. + +## Table of Contents +- [BGP EVPN Control Plane](#bgp-evpn-control-plane) +- [VXLAN Overlay](#vxlan-overlay) +- [VLAN Configuration](#vlan-configuration) +- [Underlay Routing](#underlay-routing) +- [Interface Status](#interface-status) +- [MAC Address Tables](#mac-address-tables) +- [Port Configuration](#port-configuration) +- [General System Information](#general-system-information) + +## BGP EVPN Control Plane + +### Check BGP EVPN Session Status +```bash +show bgp l2vpn evpn summary +``` +Shows BGP neighbor status for L2VPN EVPN address family. All neighbors should be in "Established" state. + +**Expected Output:** +- **Spine switches**: Should show both leaf switches as established neighbors +- **Leaf switches**: Should show both spine switches as established neighbors + +### View BGP EVPN Routes +```bash +show bgp l2vpn evpn +``` +Displays all EVPN routes (Type 2 MAC/IP, Type 3 IMET, etc.) learned via BGP. + +### View Specific EVPN Route Types +```bash +# Type 2 routes (MAC/IP Advertisement) +show bgp l2vpn evpn route-type 2 + +# Type 3 routes (Inclusive Multicast Ethernet Tag) +show bgp l2vpn evpn route-type 3 + +# Type 5 routes (IP Prefix) +show bgp l2vpn evpn route-type 5 +``` + +### Check BGP Neighbors +```bash +show bgp l2vpn evpn neighbors +``` +Detailed information about BGP EVPN neighbors including capabilities and statistics. + +### View BGP Configuration +```bash +show running-config bgp +``` +Shows the complete BGP configuration including AS number, neighbors, and address families. + +## VXLAN Overlay + +### Check NVE Interface Status +```bash +show interface nve1 +``` +Shows the status of the Network Virtualization Endpoint (VTEP) interface. + +**Key fields to check:** +- State: Should be "Up" +- Source-Interface: Should be loopback0 +- Primary IP: Should show the loopback IP (10.255.255.x) + +### View NVE Peers +```bash +show nve peers +``` +Lists discovered VXLAN tunnel endpoints (VTEPs). Peers are discovered via BGP EVPN. + +**Expected Output:** +- Leaf switches should show each other as peers once VNIs are configured +- Peer state should be "Up" + +### Check VNI Status +```bash +show nve vni +``` +Shows all configured VNIs and their associated VLANs. + +**Key information:** +- VNI number +- Associated VLAN +- Mode (L2 or L3) +- State + +### View Detailed VNI Information +```bash +show nve vni +``` +Detailed information for a specific VNI including peer information and statistics. + +### Check L2 Route Table +```bash +show l2route evpn mac all +``` +Shows MAC addresses learned via EVPN across all VNIs. + +**Key information:** +- MAC address +- Next-hop (remote VTEP) +- VNI +- Topology ID + +### View L2 Routes for Specific VNI +```bash +show l2route evpn mac vni +``` +Shows MAC addresses for a specific VNI. + +### Check EVPN IMET Routes +```bash +show l2route evpn imet all +``` +Shows Inclusive Multicast Ethernet Tag routes (Type 3) used for BUM traffic. + +## VLAN Configuration + +### View All VLANs +```bash +show vlan +``` +Lists all configured VLANs with their status and associated ports. + +### View VLAN Configuration +```bash +show running-config vlan +``` +Shows VLAN configuration including VN-segment mappings. + +**Key information:** +- VLAN ID +- VLAN name +- VN-segment (VNI) mapping + +### Check Specific VLAN +```bash +show vlan id +``` +Detailed information for a specific VLAN including member ports. + +### View VLAN Brief +```bash +show vlan brief +``` +Compact view of all VLANs with their status and ports. + +## Underlay Routing + +### Check OSPF Neighbors +```bash +show ip ospf neighbors +``` +Shows OSPF adjacencies. All inter-switch links should have OSPF neighbors in "FULL" state. + +**Expected Output:** +- **Spine switches**: Should show both leaf switches and the other spine +- **Leaf switches**: Should show both spine switches + +### View OSPF Routes +```bash +show ip route ospf +``` +Shows routes learned via OSPF, including loopback addresses used as VTEPs. + +### Check Full IP Routing Table +```bash +show ip route +``` +Complete routing table including connected, static, and dynamic routes. + +**Key routes to verify:** +- Loopback addresses (10.255.255.x/32) for all switches +- Inter-switch link subnets (10.1.1.x/30) + +### View OSPF Interface Status +```bash +show ip ospf interface brief +``` +Shows which interfaces are participating in OSPF. + +## Interface Status + +### Check All Interfaces +```bash +show interface status +``` +Shows status of all interfaces including speed, duplex, and VLAN. + +### View Specific Interface +```bash +show interface ethernet1/3 +``` +Detailed information for a specific interface including counters and errors. + +### Check Interface Brief +```bash +show interface brief +``` +Compact view of all interfaces with status and IP addresses. + +### View Trunk Interfaces +```bash +show interface trunk +``` +Shows trunk port configuration and allowed VLANs. + +### Check Interface Counters +```bash +show interface counters +``` +Shows packet and byte counters for all interfaces. + +### Check Interface Errors +```bash +show interface counters errors +``` +Shows error counters that can indicate physical layer issues. + +## MAC Address Tables + +### View MAC Address Table +```bash +show mac address-table +``` +Shows locally learned MAC addresses on all VLANs. + +### Check MAC Table for Specific VLAN +```bash +show mac address-table vlan +``` +Shows MAC addresses learned on a specific VLAN. + +### View Dynamic MAC Entries +```bash +show mac address-table dynamic +``` +Shows only dynamically learned MAC addresses (excludes static entries). + +## Port Configuration + +### Check Port Channel Status +```bash +show port-channel summary +``` +Shows status of port channels (if configured). + +### View Switchport Configuration +```bash +show running-config interface ethernet1/3 +``` +Shows configuration for a specific interface. + +### Check Switchport Mode +```bash +show interface switchport +``` +Shows switchport mode (access/trunk) and VLAN assignments for all interfaces. + +## General System Information + +### Check System Version +```bash +show version +``` +Shows NX-OS version, uptime, and hardware information. + +### View Running Configuration +```bash +show running-config +``` +Complete running configuration. Use with filters for specific sections: +```bash +show running-config | section bgp +show running-config | section interface +show running-config | section vlan +``` + +### Check Feature Status +```bash +show feature +``` +Shows which features are enabled (ospf, bgp, nv overlay, etc.). + +### View System Resources +```bash +show system resources +``` +Shows CPU, memory, and process information. + +### Check Logging +```bash +show logging last 50 +``` +Shows the last 50 log messages. Useful for diagnosing recent issues. + +## Common Troubleshooting Scenarios + +### Scenario 1: BGP EVPN Sessions Not Establishing + +**Symptoms:** `show bgp l2vpn evpn summary` shows neighbors in "Idle" or "Active" state + +**Diagnosis:** +```bash +# Check BGP configuration +show running-config bgp + +# Verify loopback reachability (BGP uses loopback IPs) +ping 10.255.255.1 # From leaf to spine +ping 10.255.255.3 # From spine to leaf + +# Check OSPF is advertising loopbacks +show ip route 10.255.255.0/24 + +# Check for BGP errors +show logging | include BGP +``` + +**Common Causes:** +- OSPF not running or not advertising loopbacks +- BGP neighbor IP misconfigured +- Route reflector configuration missing on spines +- Firewall blocking TCP port 179 + +### Scenario 2: No NVE Peers Appearing + +**Symptoms:** `show nve peers` returns empty on leaf switches + +**Diagnosis:** +```bash +# Verify NVE interface is up +show interface nve1 + +# Check if any VNIs are configured +show nve vni + +# Verify BGP EVPN is receiving Type 3 (IMET) routes +show bgp l2vpn evpn route-type 3 + +# Check NVE configuration +show running-config interface nve1 +``` + +**Common Causes:** +- No VNIs configured yet (expected before ML2 creates networks) +- NVE interface not up +- BGP EVPN not exchanging routes +- Source interface (loopback0) not reachable + +### Scenario 3: VMs Cannot Communicate Across Leaf Switches + +**Symptoms:** VMs on different leaf switches cannot ping each other + +**Diagnosis:** +```bash +# On both leaf switches, check if VNI exists +show nve vni + +# Verify VLAN-to-VNI mapping +show running-config vlan + +# Check if MAC addresses are learned via EVPN +show l2route evpn mac vni + +# Verify NVE peers are up +show nve peers + +# Check for local MAC addresses +show mac address-table vlan + +# Look for VXLAN encapsulation counters +show interface nve1 counters +``` + +**Common Causes:** +- VNI not configured on one of the leaf switches +- VLAN-to-VNI mapping mismatch +- BGP EVPN not advertising MAC addresses (Type 2 routes) +- NVE peer down + +### Scenario 4: ML2 Plugin Not Configuring Switch Ports + +**Symptoms:** Ironic nodes cannot get network connectivity; switch ports remain shutdown + +**Diagnosis:** +```bash +# Check if port is administratively down +show interface ethernet1/4 + +# Check port configuration +show running-config interface ethernet1/4 + +# Check if VLAN exists +show vlan id + +# Check switch logs for NETCONF/API access +show logging | include NETCONF +show logging | include httpd +``` + +**Common Causes:** +- networking-generic-switch credentials incorrect in DevStack config +- NETCONF/NX-API not enabled on switch +- Port configuration locked or in wrong mode +- VLAN not created before port assignment + +### Scenario 5: VLAN Not Extending Across Fabric + +**Symptoms:** VLAN works locally but not across spine switches + +**Diagnosis:** +```bash +# Verify VLAN has VN-segment configured +show running-config vlan + +# Check if VNI exists +show nve vni + +# Verify BGP is advertising the VNI (Type 3 IMET routes) +show bgp l2vpn evpn | include + +# Check remote leaf switch has matching configuration +# (Run on other leaf switch) +show running-config vlan +show nve vni +``` + +**Common Causes:** +- VN-segment (VNI) not configured on VLAN +- VNI mismatch between leaf switches +- BGP EVPN not advertising IMET routes +- NVE interface down + +## TCAM Region Configuration Error for ARP Suppression + +**Symptom:** +``` +ERROR: Please configure TCAM region for Ingress ARP-Ether ACL before configuring ARP suppression. +``` + +When trying to configure `suppress-arp` under a VNI member, even though the POAP config includes the TCAM configuration. + +**Root Cause:** + +The TCAM (Ternary Content Addressable Memory) regions are allocated at boot time. If the switch was already running when the TCAM configuration was added, or if it wasn't properly saved to startup-config before reload, the hardware allocation won't match the config. + +Additionally, the default TCAM allocation may not have enough free slices for the `arp-ether` region. + +**Diagnosis:** + +```bash +# Check if TCAM config is in startup +show startup-config | include "hardware access-list tcam" + +# Check current TCAM allocation +show hardware access-list tcam region | include arp-ether + +# Check all TCAM regions to see what's consuming space +show hardware access-list tcam region +``` + +**Solution:** + +If `arp-ether` size is 0 and you get an error about exceeding available TCAM slices when trying to configure it, you need to reduce other TCAM regions first: + +```bash +# Reduce RACL region (default 1536 is often oversized) +hardware access-list tcam region racl 1024 + +# Allocate arp-ether region (256 double-wide = 512 slices) +hardware access-list tcam region arp-ether 256 double-wide + +# Save and reload for hardware changes to take effect +copy running-config startup-config +reload +``` + +After reload, verify: + +```bash +show hardware access-list tcam region | include arp-ether +# Should show: Ingress ARP-Ether ACL [arp-ether] size = 256 +``` + +Now you can configure `suppress-arp` under VNI members without errors. + +## Quick Health Check Script + +Run these commands in sequence for a quick health check: + +```bash +# 1. Check BGP EVPN +show bgp l2vpn evpn summary + +# 2. Check NVE status +show interface nve1 + +# 3. Check NVE peers +show nve peers + +# 4. Check VNIs +show nve vni + +# 5. Check OSPF neighbors +show ip ospf neighbors + +# 6. Check loopback reachability +show ip route 10.255.255.0/24 + +# 7. Check for recent errors +show logging last 20 +``` + +## Additional Resources + +- [Cisco NX-OS VXLAN Configuration Guide](https://www.cisco.com/c/en/us/td/docs/switches/datacenter/nexus9000/sw/vxlan/guide/b-nxos-vxlan-config-guide.html) +- [Cisco NX-OS BGP Configuration Guide](https://www.cisco.com/c/en/us/td/docs/switches/datacenter/nexus9000/sw/routing/config-guide/b-nxos-routing-config-guide.html) +- [OpenStack networking-generic-switch Documentation](https://docs.openstack.org/networking-generic-switch/latest/) diff --git a/scenarios/networking-lab/devstack-nxsw-vxlan/automation-vars.yml b/scenarios/networking-lab/devstack-nxsw-vxlan/automation-vars.yml new file mode 100644 index 00000000..6e00064c --- /dev/null +++ b/scenarios/networking-lab/devstack-nxsw-vxlan/automation-vars.yml @@ -0,0 +1,130 @@ +--- +# Networking lab automation stages + +stages: + - name: Configure provisioning network route + documentation: >- + Configures a static route on the devstack node to enable communication + between the ironic-conductor and the provisioning network (where + ironic-python-agent runs during node deployment/cleaning). This retrieves + the router gateway IP from Neutron and adds a route via that gateway. + The provisioning network subnet is configured in local.conf.j2 via + IRONIC_PROVISION_SUBNET_PREFIX (default: 10.0.5.0/24). + shell: | + set -xe -o pipefail + + IRONIC_PROVISION_SUBNET_PREFIX="10.0.5.0/24" + + EXTERNAL_GW_INFO=$(openstack --os-cloud devstack-admin router show router1 -c external_gateway_info -f json) + + ROUTER_GW_IP=$(echo "$EXTERNAL_GW_INFO" | python3 -c ' + import sys, json, ipaddress + data = json.load(sys.stdin) + for ip_info in data["external_gateway_info"]["external_fixed_ips"]: + addr = ipaddress.ip_address(ip_info["ip_address"]) + if addr.version == 4: + print(ip_info["ip_address"]) + break + ') + + if [ -z "$ROUTER_GW_IP" ]; then + echo "ERROR: Could not determine router gateway IP" + exit 1 + fi + + ssh -o StrictHostKeyChecking=no stack@devstack.netlab.example.com " + ROUTES=\$(ip -j r) + ROUTE_EXISTS=\$(echo \"\$ROUTES\" | python3 -c ' + import sys, json + routes = json.load(sys.stdin) + target_dst = \"$IRONIC_PROVISION_SUBNET_PREFIX\" + target_gw = \"$ROUTER_GW_IP\" + for route in routes: + if route.get(\"dst\") == target_dst and route.get(\"gateway\") == target_gw: + print(\"exists\") + break + ') + if [ -z \"\$ROUTE_EXISTS\" ]; then + echo \"Adding route: $IRONIC_PROVISION_SUBNET_PREFIX via $ROUTER_GW_IP\" + sudo ip route add $IRONIC_PROVISION_SUBNET_PREFIX via $ROUTER_GW_IP + else + echo \"Route already exists: $IRONIC_PROVISION_SUBNET_PREFIX via $ROUTER_GW_IP\" + fi + " + + - name: Enroll nodes in devstack ironic + documentation: >- + Registers physical baremetal nodes with the Ironic service in the DevStack + deployment using the node definitions from ironic_nodes.yaml. This creates + Ironic node records with BMC access credentials, hardware profiles, and port + configurations for networking-generic-switch integration. + shell: | + set -xe -o pipefail + + NODES_FILE=/home/zuul/data/ironic_nodes.yaml + + # Enroll the nodes + openstack --os-cloud devstack-admin baremetal create "$NODES_FILE" + + echo "Nodes enrolled successfully" + openstack --os-cloud devstack-admin baremetal node list + + - name: Wait for ironic nodes to reach enroll state + documentation: >- + Monitors node state transition to 'enroll' status, indicating that Ironic + has successfully registered the nodes and validated basic BMC connectivity. + This is the first state in the baremetal provisioning lifecycle. + shell: | + set -xe -o pipefail + + counter=0 + max_retries=60 + sleep_interval=5 + + echo "Waiting for all nodes to reach 'enroll' state..." + + until ! openstack --os-cloud devstack-admin baremetal node list -f value -c "Provisioning State" | grep -v "enroll"; do + ((counter++)) + if (( counter > max_retries )); then + echo "ERROR: Timeout waiting for nodes to reach enroll state" + openstack --os-cloud devstack-admin baremetal node list + exit 1 + fi + echo "Attempt $counter/$max_retries - waiting ${sleep_interval}s..." + sleep ${sleep_interval} + done + + echo "All nodes successfully reached enroll state" + openstack --os-cloud devstack-admin baremetal node list + + - name: Manage nodes + documentation: >- + Transitions nodes from 'enroll' to 'manageable' state. This validates + basic hardware connectivity and prepares nodes for further operations. + shell: | + set -xe -o pipefail + + # Get list of node UUIDs + node_uuids=$(openstack --os-cloud devstack-admin baremetal node list -f value -c UUID) + + # Manage each node + for uuid in $node_uuids; do + echo "Managing node: $uuid" + openstack --os-cloud devstack-admin baremetal node manage $uuid + done + + # Wait for manageable state + counter=0 + max_retries=60 + until ! openstack --os-cloud devstack-admin baremetal node list -f value -c "Provisioning State" | grep -v "manageable"; do + ((counter++)) + if (( counter > max_retries )); then + echo "ERROR: Timeout waiting for nodes to reach manageable state" + openstack --os-cloud devstack-admin baremetal node list + exit 1 + fi + sleep 5 + done + + echo "All nodes successfully reached manageable state" + openstack --os-cloud devstack-admin baremetal node list diff --git a/scenarios/networking-lab/devstack-nxsw-vxlan/bootstrap_vars.yml b/scenarios/networking-lab/devstack-nxsw-vxlan/bootstrap_vars.yml new file mode 100644 index 00000000..e843aeb1 --- /dev/null +++ b/scenarios/networking-lab/devstack-nxsw-vxlan/bootstrap_vars.yml @@ -0,0 +1,63 @@ +--- +# Bootstrap configuration for networking lab scenario + +# OpenStack cloud configuration +os_cloud: default +os_floating_network: public +os_router_external_network: public + +# Scenario configuration +scenario: devstack-nxsw-vxlan +scenario_dir: scenarios/networking-lab +stack_template_path: "{{ scenario_dir }}/{{ scenario }}/heat_template.yaml" +automation_vars_file: "{{ scenario_dir }}/{{ scenario }}/automation-vars.yml" + +# Files to compress for user data (relative to heat template directory) +compress_heat_files: + - archive: tftpboot-poap + files: + - spine01-poap.py + - spine01-poap.cfg + - spine02-poap.py + - spine02-poap.cfg + - leaf01-poap.py + - leaf01-poap.cfg + - leaf02-poap.py + - leaf02-poap.cfg + +# DNS and NTP +ntp_servers: [] +dns_servers: + - 8.8.8.8 + - 8.8.4.4 + +# Pull secret for container images (if needed) +# pull_secret_file: ~/pull-secret.txt + +# Stack naming +stack_name: "hs-{{ scenario | replace('/', '-') }}-{{ zuul.build[:8] | default('no-zuul') }}" + +# Stack parameters +stack_parameters: + dns_servers: "{{ dns_servers }}" + ntp_servers: "{{ ntp_servers }}" + controller_ssh_pub_key: "{{ controller_ssh_pub_key | default('') }}" + dataplane_ssh_pub_key: "{{ dataplane_ssh_pub_key | default('') }}" + router_external_network: "{{ os_router_external_network | default('public') }}" + floating_ip_network: "{{ os_floating_network | default('public') }}" + controller_params: + image: hotstack-controller + flavor: hotstack.small + devstack_params: + image: ubuntu-noble-server + flavor: hotstack.xxlarge + switch_params: + image: nexus9300v.9.3.15 + flavor: hotstack.large + ironic_params: + image: CentOS-Stream-GenericCloud-9 + cd_image: sushy-tools-blank-image + flavor: hotstack.medium + +# Controller role configuration +controller_install_openstack_client: true diff --git a/scenarios/networking-lab/devstack-nxsw-vxlan/heat_template.yaml b/scenarios/networking-lab/devstack-nxsw-vxlan/heat_template.yaml new file mode 100644 index 00000000..ec9cdbfb --- /dev/null +++ b/scenarios/networking-lab/devstack-nxsw-vxlan/heat_template.yaml @@ -0,0 +1,1242 @@ +--- +heat_template_version: rocky + +description: > + Heat template for networking lab with spine-and-leaf Cisco NXOS setup. + Includes 4 switches (core01, core02, leaf01, leaf02), 1 devstack node, and 2 ironic nodes. + +parameters: + dns_servers: + type: comma_delimited_list + default: + - 8.8.8.8 + - 8.8.4.4 + ntp_servers: + type: comma_delimited_list + default: [] + controller_ssh_pub_key: + type: string + dataplane_ssh_pub_key: + type: string + router_external_network: + type: string + default: public + floating_ip_network: + type: string + default: public + net_value_specs: + type: json + default: {} + + controller_params: + type: json + default: + image: hotstack-controller + flavor: hotstack.small + devstack_params: + type: json + default: + image: ubuntu-noble-server + flavor: hotstack.xxlarge + ironic_params: + type: json + default: + image: CentOS-Stream-GenericCloud-9 + cd_image: sushy-tools-blank-image + flavor: hotstack.medium + switch_params: + type: json + default: + image: nexus9300v.9.3.15 + flavor: hotstack.large + cdrom_disk_bus: + type: string + description: > + Disk bus type for CDROM device. 'sata' may be required for older versions + of OpenStack. Heat patch https://review.opendev.org/c/openstack/heat/+/966688 + is needed for 'sata' support. + default: scsi + constraints: + - allowed_values: + - sata + - scsi + +resources: + # + # Networks + # + machine-net: + type: OS::Neutron::Net + properties: + port_security_enabled: false + value_specs: {get_param: net_value_specs} + + # Spine switch interconnect + spine-link-net: + type: OS::Neutron::Net + properties: + port_security_enabled: false + value_specs: {get_param: net_value_specs} + + # Leaf to spine links + leaf01-spine01-net: + type: OS::Neutron::Net + properties: + port_security_enabled: false + value_specs: {get_param: net_value_specs} + + leaf01-spine02-net: + type: OS::Neutron::Net + properties: + port_security_enabled: false + value_specs: {get_param: net_value_specs} + + leaf02-spine01-net: + type: OS::Neutron::Net + properties: + port_security_enabled: false + value_specs: {get_param: net_value_specs} + + leaf02-spine02-net: + type: OS::Neutron::Net + properties: + port_security_enabled: false + value_specs: {get_param: net_value_specs} + + # Simple bridge networks for server attachments + # These are just L2 connectivity - VLANs and configuration managed by ML2 + devstack-br-net: + type: OS::Neutron::Net + properties: + port_security_enabled: false + value_specs: {get_param: net_value_specs} + + ironic0-br-net: + type: OS::Neutron::Net + properties: + port_security_enabled: false + value_specs: {get_param: net_value_specs} + + ironic1-br-net: + type: OS::Neutron::Net + properties: + port_security_enabled: false + value_specs: {get_param: net_value_specs} + + # Leaf01 trunk networks for tenant VLANs + leaf01-trunk-net: + type: OS::Neutron::Net + properties: + port_security_enabled: false + value_specs: {get_param: net_value_specs} + + leaf01-public-vlan100: + type: OS::Neutron::Net + properties: + port_security_enabled: false + value_specs: {get_param: net_value_specs} + + leaf01-tenant-vlan103: + type: OS::Neutron::Net + properties: + port_security_enabled: false + value_specs: {get_param: net_value_specs} + + leaf01-tenant-vlan104: + type: OS::Neutron::Net + properties: + port_security_enabled: false + value_specs: {get_param: net_value_specs} + + leaf01-tenant-vlan105: + type: OS::Neutron::Net + properties: + port_security_enabled: false + value_specs: {get_param: net_value_specs} + + # Leaf02 trunk networks for tenant VLANs + leaf02-trunk-net: + type: OS::Neutron::Net + properties: + port_security_enabled: false + value_specs: {get_param: net_value_specs} + + leaf02-public-vlan100: + type: OS::Neutron::Net + properties: + port_security_enabled: false + value_specs: {get_param: net_value_specs} + + leaf02-tenant-vlan103: + type: OS::Neutron::Net + properties: + port_security_enabled: false + value_specs: {get_param: net_value_specs} + + leaf02-tenant-vlan104: + type: OS::Neutron::Net + properties: + port_security_enabled: false + value_specs: {get_param: net_value_specs} + + leaf02-tenant-vlan105: + type: OS::Neutron::Net + properties: + port_security_enabled: false + value_specs: {get_param: net_value_specs} + + # + # Subnets + # + machine-subnet: + type: OS::Neutron::Subnet + properties: + network: {get_resource: machine-net} + ip_version: 4 + cidr: 192.168.32.0/24 + enable_dhcp: true + dns_nameservers: + - 192.168.32.254 + + spine-link-subnet: + type: OS::Neutron::Subnet + properties: + network: {get_resource: spine-link-net} + ip_version: 4 + cidr: 10.1.1.0/30 + enable_dhcp: false + gateway_ip: null + + leaf01-spine01-subnet: + type: OS::Neutron::Subnet + properties: + network: {get_resource: leaf01-spine01-net} + ip_version: 4 + cidr: 10.1.1.4/30 + enable_dhcp: false + gateway_ip: null + + leaf01-spine02-subnet: + type: OS::Neutron::Subnet + properties: + network: {get_resource: leaf01-spine02-net} + ip_version: 4 + cidr: 10.1.1.8/30 + enable_dhcp: false + gateway_ip: null + + leaf02-spine01-subnet: + type: OS::Neutron::Subnet + properties: + network: {get_resource: leaf02-spine01-net} + ip_version: 4 + cidr: 10.1.1.12/30 + enable_dhcp: false + gateway_ip: null + + leaf02-spine02-subnet: + type: OS::Neutron::Subnet + properties: + network: {get_resource: leaf02-spine02-net} + ip_version: 4 + cidr: 10.1.1.16/30 + enable_dhcp: false + gateway_ip: null + + devstack-br-subnet: + type: OS::Neutron::Subnet + properties: + network: {get_resource: devstack-br-net} + ip_version: 4 + cidr: 172.20.10.0/29 + enable_dhcp: false + gateway_ip: null + + ironic0-br-subnet: + type: OS::Neutron::Subnet + properties: + network: {get_resource: ironic0-br-net} + ip_version: 4 + cidr: 172.20.11.0/29 + enable_dhcp: false + gateway_ip: null + + ironic1-br-subnet: + type: OS::Neutron::Subnet + properties: + network: {get_resource: ironic1-br-net} + ip_version: 4 + cidr: 172.20.12.0/29 + enable_dhcp: false + gateway_ip: null + + # Leaf01 trunk subnets + leaf01-trunk-subnet: + type: OS::Neutron::Subnet + properties: + network: {get_resource: leaf01-trunk-net} + ip_version: 4 + cidr: 172.20.20.0/24 + enable_dhcp: false + allocation_pools: + - start: 172.20.20.100 + end: 172.20.20.150 + + leaf01-public-vlan100-subnet: + type: OS::Neutron::Subnet + properties: + network: {get_resource: leaf01-public-vlan100} + ip_version: 4 + cidr: 172.20.0.0/24 + gateway_ip: 172.20.0.1 + enable_dhcp: false + + leaf01-tenant-vlan103-subnet: + type: OS::Neutron::Subnet + properties: + network: {get_resource: leaf01-tenant-vlan103} + ip_version: 4 + cidr: 172.20.3.0/24 + gateway_ip: null + enable_dhcp: false + + leaf01-tenant-vlan104-subnet: + type: OS::Neutron::Subnet + properties: + network: {get_resource: leaf01-tenant-vlan104} + ip_version: 4 + cidr: 172.20.4.0/24 + gateway_ip: null + enable_dhcp: false + + leaf01-tenant-vlan105-subnet: + type: OS::Neutron::Subnet + properties: + network: {get_resource: leaf01-tenant-vlan105} + ip_version: 4 + cidr: 172.20.5.0/24 + gateway_ip: null + enable_dhcp: false + + # Leaf02 trunk subnets + leaf02-trunk-subnet: + type: OS::Neutron::Subnet + properties: + network: {get_resource: leaf02-trunk-net} + ip_version: 4 + cidr: 172.20.21.0/24 + enable_dhcp: false + allocation_pools: + - start: 172.20.21.100 + end: 172.20.21.150 + + leaf02-public-vlan100-subnet: + type: OS::Neutron::Subnet + properties: + network: {get_resource: leaf02-public-vlan100} + ip_version: 4 + cidr: 172.20.1.0/24 + gateway_ip: 172.20.1.1 + enable_dhcp: false + + leaf02-tenant-vlan103-subnet: + type: OS::Neutron::Subnet + properties: + network: {get_resource: leaf02-tenant-vlan103} + ip_version: 4 + cidr: 172.20.6.0/24 + gateway_ip: null + enable_dhcp: false + + leaf02-tenant-vlan104-subnet: + type: OS::Neutron::Subnet + properties: + network: {get_resource: leaf02-tenant-vlan104} + ip_version: 4 + cidr: 172.20.7.0/24 + gateway_ip: null + enable_dhcp: false + + leaf02-tenant-vlan105-subnet: + type: OS::Neutron::Subnet + properties: + network: {get_resource: leaf02-tenant-vlan105} + ip_version: 4 + cidr: 172.20.8.0/24 + gateway_ip: null + enable_dhcp: false + + # + # Routers + # + router: + type: OS::Neutron::Router + properties: + admin_state_up: true + external_gateway_info: + network: {get_param: router_external_network} + + machine-net-router-interface: + type: OS::Neutron::RouterInterface + properties: + router: {get_resource: router} + subnet: {get_resource: machine-subnet} + + # + # Controller Instance + # + controller_users: + type: OS::Heat::CloudConfig + properties: + cloud_config: + users: + - default + - name: zuul + gecos: "Zuul user" + sudo: ALL=(ALL) NOPASSWD:ALL + ssh_authorized_keys: + - {get_param: controller_ssh_pub_key} + + controller-write-files: + type: OS::Heat::CloudConfig + properties: + cloud_config: + write_files: + - path: /etc/dnsmasq.conf + content: | + # dnsmasq service config + # Include all files in /etc/dnsmasq.d except RPM backup files + conf-dir=/etc/dnsmasq.d,.rpmnew,.rpmsave,.rpmorig + no-resolv + owner: root:dnsmasq + - path: /etc/dnsmasq.d/forwarders.conf + content: + str_replace: + template: | + # DNS forwarders records + server=$dns1 + server=$dns2 + params: + $dns1: {get_param: [dns_servers, 0]} + $dns2: {get_param: [dns_servers, 1]} + owner: root:dnsmasq + - path: /etc/dnsmasq.d/host_records.conf + content: + str_replace: + template: | + # Host records + host-record=controller-0.netlab.example.com,$controller0 + host-record=spine01.netlab.example.com,$spine01 + host-record=spine02.netlab.example.com,$spine02 + host-record=leaf01.netlab.example.com,$leaf01 + host-record=leaf02.netlab.example.com,$leaf02 + host-record=devstack.netlab.example.com,$devstack + params: + $controller0: {get_attr: [controller-machine-port, fixed_ips, 0, ip_address]} + $spine01: {get_attr: [spine01-machine-port, fixed_ips, 0, ip_address]} + $spine02: {get_attr: [spine02-machine-port, fixed_ips, 0, ip_address]} + $leaf01: {get_attr: [leaf01-machine-port, fixed_ips, 0, ip_address]} + $leaf02: {get_attr: [leaf02-machine-port, fixed_ips, 0, ip_address]} + $devstack: {get_attr: [devstack-machine-port, fixed_ips, 0, ip_address]} + owner: root:dnsmasq + - path: /etc/resolv.conf + content: | + nameserver: 127.0.0.1 + owner: root:root + - path: /etc/NetworkManager/conf.d/98-rc-manager.conf + content: | + [main] + rc-manager=unmanaged + owner: root:root + - path: /etc/dnsmasq.d/tftpboot.conf + content: | + enable-tftp + tftp-root=/var/lib/tftpboot + tftp-mtu=1442 + owner: root:root + - path: /var/lib/tftpboot/tftpboot-poap.tar.gz + encoding: b64 + content: {get_file: tftpboot-poap.tar.gz.b64} + owner: root:root + permissions: '0644' + + controller-runcmd: + type: OS::Heat::CloudConfig + properties: + cloud_config: + runcmd: + - ['setenforce', 'permissive'] + - ['sed', '-i', 's/Listen 80/Listen 8081/g', '/etc/httpd/conf/httpd.conf'] + - ['systemctl', 'enable', 'httpd.service'] + - ['systemctl', 'start', 'httpd.service'] + - ['systemctl', 'enable', 'dnsmasq.service'] + - ['systemctl', 'start', 'dnsmasq.service'] + # Extract POAP files from tar archive + - ['tar', '-xzf', '/var/lib/tftpboot/tftpboot-poap.tar.gz', '-C', '/var/lib/tftpboot/'] + + controller-init: + type: OS::Heat::MultipartMime + properties: + parts: + - config: {get_resource: controller_users} + - config: {get_resource: controller-write-files} + - config: {get_resource: controller-runcmd} + + controller-machine-port: + type: OS::Neutron::Port + properties: + network: {get_resource: machine-net} + mac_address: "fa:16:9e:81:f6:05" + fixed_ips: + - ip_address: 192.168.32.254 + + controller-floating-ip: + depends_on: machine-net-router-interface + type: OS::Neutron::FloatingIP + properties: + floating_network: {get_param: floating_ip_network} + port_id: {get_resource: controller-machine-port} + + controller: + type: OS::Nova::Server + properties: + image: {get_param: [controller_params, image]} + flavor: {get_param: [controller_params, flavor]} + networks: + - port: {get_resource: controller-machine-port} + user_data_format: RAW + user_data: {get_resource: controller-init} + + # + # Spine Switches + # + + # Spine01 Switch + spine01-extra-dhcp-opts-value: + type: OS::Heat::Value + properties: + type: json + value: + extra_dhcp_opts: + - opt_name: "66" + opt_value: + str_replace: + template: "$server_address" + params: + $server_address: {get_attr: [controller-machine-port, fixed_ips, 0, ip_address]} + ip_version: 4 + - opt_name: "67" + opt_value: "spine01-poap.py" + ip_version: 4 + + spine01-machine-port: + type: OS::Neutron::Port + properties: + network: {get_resource: machine-net} + port_security_enabled: false + mac_address: "22:57:f8:dd:01:01" + fixed_ips: + - ip_address: 192.168.32.11 + value_specs: {get_attr: [spine01-extra-dhcp-opts-value, value]} + + spine01-spine-link-port: + type: OS::Neutron::Port + properties: + network: {get_resource: spine-link-net} + port_security_enabled: false + mac_address: "22:57:f8:dd:01:02" + + spine01-leaf01-port: + type: OS::Neutron::Port + properties: + network: {get_resource: leaf01-spine01-net} + port_security_enabled: false + mac_address: "22:57:f8:dd:01:03" + + spine01-leaf02-port: + type: OS::Neutron::Port + properties: + network: {get_resource: leaf02-spine01-net} + port_security_enabled: false + mac_address: "22:57:f8:dd:01:04" + + spine01: + type: OS::Nova::Server + properties: + image: {get_param: [switch_params, image]} + flavor: {get_param: [switch_params, flavor]} + config_drive: false + diskConfig: MANUAL + networks: + - port: {get_resource: spine01-machine-port} + - port: {get_resource: spine01-spine-link-port} + - port: {get_resource: spine01-leaf01-port} + - port: {get_resource: spine01-leaf02-port} + + # Spine02 Switch + spine02-extra-dhcp-opts-value: + type: OS::Heat::Value + properties: + type: json + value: + extra_dhcp_opts: + - opt_name: "66" + opt_value: + str_replace: + template: "$server_address" + params: + $server_address: {get_attr: [controller-machine-port, fixed_ips, 0, ip_address]} + ip_version: 4 + - opt_name: "67" + opt_value: "spine02-poap.py" + ip_version: 4 + + spine02-machine-port: + type: OS::Neutron::Port + properties: + network: {get_resource: machine-net} + port_security_enabled: false + mac_address: "22:57:f8:dd:02:01" + fixed_ips: + - ip_address: 192.168.32.12 + value_specs: {get_attr: [spine02-extra-dhcp-opts-value, value]} + + spine02-spine-link-port: + type: OS::Neutron::Port + properties: + network: {get_resource: spine-link-net} + port_security_enabled: false + mac_address: "22:57:f8:dd:02:02" + + spine02-leaf01-port: + type: OS::Neutron::Port + properties: + network: {get_resource: leaf01-spine02-net} + port_security_enabled: false + mac_address: "22:57:f8:dd:02:03" + + spine02-leaf02-port: + type: OS::Neutron::Port + properties: + network: {get_resource: leaf02-spine02-net} + port_security_enabled: false + mac_address: "22:57:f8:dd:02:04" + + spine02: + type: OS::Nova::Server + properties: + image: {get_param: [switch_params, image]} + flavor: {get_param: [switch_params, flavor]} + config_drive: false + diskConfig: MANUAL + networks: + - port: {get_resource: spine02-machine-port} + - port: {get_resource: spine02-spine-link-port} + - port: {get_resource: spine02-leaf01-port} + - port: {get_resource: spine02-leaf02-port} + + # + # Leaf Switches + # + + # Leaf01 Switch + leaf01-extra-dhcp-opts-value: + type: OS::Heat::Value + properties: + type: json + value: + extra_dhcp_opts: + - opt_name: "66" + opt_value: + str_replace: + template: "$server_address" + params: + $server_address: {get_attr: [controller-machine-port, fixed_ips, 0, ip_address]} + ip_version: 4 + - opt_name: "67" + opt_value: "leaf01-poap.py" + ip_version: 4 + + leaf01-machine-port: + type: OS::Neutron::Port + properties: + network: {get_resource: machine-net} + port_security_enabled: false + mac_address: "22:57:f8:dd:03:01" + fixed_ips: + - ip_address: 192.168.32.13 + value_specs: {get_attr: [leaf01-extra-dhcp-opts-value, value]} + + leaf01-spine01-port: + type: OS::Neutron::Port + properties: + network: {get_resource: leaf01-spine01-net} + port_security_enabled: false + mac_address: "22:57:f8:dd:03:02" + + leaf01-spine02-port: + type: OS::Neutron::Port + properties: + network: {get_resource: leaf01-spine02-net} + port_security_enabled: false + mac_address: "22:57:f8:dd:03:03" + + leaf01-devstack-br-port: + type: OS::Neutron::Port + properties: + network: {get_resource: devstack-br-net} + port_security_enabled: false + mac_address: "22:57:f8:dd:03:04" + + leaf01-trunk-parent-port: + type: OS::Neutron::Port + properties: + network: {get_resource: leaf01-trunk-net} + port_security_enabled: false + mac_address: "22:57:f8:dd:03:05" + + leaf01-trunk-public-vlan100-port: + type: OS::Neutron::Port + properties: + network: {get_resource: leaf01-public-vlan100} + port_security_enabled: false + mac_address: "22:57:f8:dd:03:06" + + leaf01-trunk-tenant-vlan103-port: + type: OS::Neutron::Port + properties: + network: {get_resource: leaf01-tenant-vlan103} + port_security_enabled: false + mac_address: "22:57:f8:dd:03:07" + + leaf01-trunk-tenant-vlan104-port: + type: OS::Neutron::Port + properties: + network: {get_resource: leaf01-tenant-vlan104} + port_security_enabled: false + mac_address: "22:57:f8:dd:03:08" + + leaf01-trunk-tenant-vlan105-port: + type: OS::Neutron::Port + properties: + network: {get_resource: leaf01-tenant-vlan105} + port_security_enabled: false + mac_address: "22:57:f8:dd:03:09" + + leaf01-trunk: + type: OS::Neutron::Trunk + properties: + port: {get_resource: leaf01-trunk-parent-port} + sub_ports: + - port: {get_resource: leaf01-trunk-public-vlan100-port} + segmentation_id: 100 + segmentation_type: vlan + - port: {get_resource: leaf01-trunk-tenant-vlan103-port} + segmentation_id: 103 + segmentation_type: vlan + - port: {get_resource: leaf01-trunk-tenant-vlan104-port} + segmentation_id: 104 + segmentation_type: vlan + - port: {get_resource: leaf01-trunk-tenant-vlan105-port} + segmentation_id: 105 + segmentation_type: vlan + + leaf01-ironic0-br-port: + type: OS::Neutron::Port + properties: + network: {get_resource: ironic0-br-net} + port_security_enabled: false + mac_address: "22:57:f8:dd:03:0a" + + leaf01: + type: OS::Nova::Server + properties: + image: {get_param: [switch_params, image]} + flavor: {get_param: [switch_params, flavor]} + config_drive: false + diskConfig: MANUAL + networks: + - port: {get_resource: leaf01-machine-port} + - port: {get_resource: leaf01-spine01-port} + - port: {get_resource: leaf01-spine02-port} + - port: {get_attr: [leaf01-trunk, port_id]} + - port: {get_resource: leaf01-ironic0-br-port} + - port: {get_resource: leaf01-devstack-br-port} + + # Leaf02 Switch + leaf02-extra-dhcp-opts-value: + type: OS::Heat::Value + properties: + type: json + value: + extra_dhcp_opts: + - opt_name: "66" + opt_value: + str_replace: + template: "$server_address" + params: + $server_address: {get_attr: [controller-machine-port, fixed_ips, 0, ip_address]} + ip_version: 4 + - opt_name: "67" + opt_value: "leaf02-poap.py" + ip_version: 4 + + leaf02-machine-port: + type: OS::Neutron::Port + properties: + network: {get_resource: machine-net} + port_security_enabled: false + mac_address: "22:57:f8:dd:04:01" + fixed_ips: + - ip_address: 192.168.32.14 + value_specs: {get_attr: [leaf02-extra-dhcp-opts-value, value]} + + leaf02-spine01-port: + type: OS::Neutron::Port + properties: + network: {get_resource: leaf02-spine01-net} + port_security_enabled: false + mac_address: "22:57:f8:dd:04:02" + + leaf02-spine02-port: + type: OS::Neutron::Port + properties: + network: {get_resource: leaf02-spine02-net} + port_security_enabled: false + mac_address: "22:57:f8:dd:04:03" + + leaf02-trunk-parent-port: + type: OS::Neutron::Port + properties: + network: {get_resource: leaf02-trunk-net} + port_security_enabled: false + mac_address: "22:57:f8:dd:04:04" + + leaf02-trunk-public-vlan100-port: + type: OS::Neutron::Port + properties: + network: {get_resource: leaf02-public-vlan100} + port_security_enabled: false + mac_address: "22:57:f8:dd:04:05" + + leaf02-trunk-tenant-vlan103-port: + type: OS::Neutron::Port + properties: + network: {get_resource: leaf02-tenant-vlan103} + port_security_enabled: false + mac_address: "22:57:f8:dd:04:06" + + leaf02-trunk-tenant-vlan104-port: + type: OS::Neutron::Port + properties: + network: {get_resource: leaf02-tenant-vlan104} + port_security_enabled: false + mac_address: "22:57:f8:dd:04:07" + + leaf02-trunk-tenant-vlan105-port: + type: OS::Neutron::Port + properties: + network: {get_resource: leaf02-tenant-vlan105} + port_security_enabled: false + mac_address: "22:57:f8:dd:04:08" + + leaf02-trunk: + type: OS::Neutron::Trunk + properties: + port: {get_resource: leaf02-trunk-parent-port} + sub_ports: + - port: {get_resource: leaf02-trunk-public-vlan100-port} + segmentation_id: 100 + segmentation_type: vlan + - port: {get_resource: leaf02-trunk-tenant-vlan103-port} + segmentation_id: 103 + segmentation_type: vlan + - port: {get_resource: leaf02-trunk-tenant-vlan104-port} + segmentation_id: 104 + segmentation_type: vlan + - port: {get_resource: leaf02-trunk-tenant-vlan105-port} + segmentation_id: 105 + segmentation_type: vlan + + leaf02-ironic1-br-port: + type: OS::Neutron::Port + properties: + network: {get_resource: ironic1-br-net} + port_security_enabled: false + mac_address: "22:57:f8:dd:04:09" + + leaf02: + type: OS::Nova::Server + properties: + image: {get_param: [switch_params, image]} + flavor: {get_param: [switch_params, flavor]} + config_drive: false + diskConfig: MANUAL + networks: + - port: {get_resource: leaf02-machine-port} + - port: {get_resource: leaf02-spine01-port} + - port: {get_resource: leaf02-spine02-port} + - port: {get_attr: [leaf02-trunk, port_id]} + - port: {get_resource: leaf02-ironic1-br-port} + + # + # Devstack Instance + # + devstack_users: + type: OS::Heat::CloudConfig + properties: + cloud_config: + users: + - default + - name: stack + gecos: "Stack user" + sudo: ALL=(ALL) NOPASSWD:ALL + homedir: /opt/stack + shell: /bin/bash + ssh_authorized_keys: + - {get_param: controller_ssh_pub_key} + - {get_param: dataplane_ssh_pub_key} + + devstack-network-config: + type: OS::Heat::CloudConfig + properties: + cloud_config: + hostname: devstack + fqdn: devstack.netlab.example.com + + + devstack-write-files: + type: OS::Heat::CloudConfig + properties: + cloud_config: + write_files: + - path: /etc/hotstack/local.conf.j2 + content: + get_file: local.conf.j2 + owner: root:root + permissions: '0644' + - path: /etc/neutron/l2vni_network_nodes.yaml + content: + get_file: l2vni_network_nodes.yaml + owner: root:root + permissions: '0644' + + devstack-init: + type: OS::Heat::MultipartMime + properties: + parts: + - config: {get_resource: devstack_users} + - config: {get_resource: devstack-network-config} + - config: {get_resource: devstack-write-files} + + devstack-machine-port: + type: OS::Neutron::Port + properties: + network: {get_resource: machine-net} + port_security_enabled: false + mac_address: "fa:16:9e:81:f6:20" + fixed_ips: + - ip_address: 192.168.32.20 + + devstack-trunk-parent-port: + type: OS::Neutron::Port + properties: + network: {get_resource: devstack-br-net} + port_security_enabled: false + mac_address: "fa:16:9e:81:f6:21" + + devstack-public-vlan100-port: + type: OS::Neutron::Port + properties: + network: {get_resource: leaf01-public-vlan100} + port_security_enabled: false + + devstack-tenant-vlan103-port: + type: OS::Neutron::Port + properties: + network: {get_resource: leaf01-tenant-vlan103} + port_security_enabled: false + + devstack-tenant-vlan104-port: + type: OS::Neutron::Port + properties: + network: {get_resource: leaf01-tenant-vlan104} + port_security_enabled: false + + devstack-tenant-vlan105-port: + type: OS::Neutron::Port + properties: + network: {get_resource: leaf01-tenant-vlan105} + port_security_enabled: false + + devstack-trunk: + type: OS::Neutron::Trunk + properties: + port: {get_resource: devstack-trunk-parent-port} + sub_ports: + - port: {get_resource: devstack-public-vlan100-port} + segmentation_id: 100 + segmentation_type: vlan + - port: {get_resource: devstack-tenant-vlan103-port} + segmentation_id: 103 + segmentation_type: vlan + - port: {get_resource: devstack-tenant-vlan104-port} + segmentation_id: 104 + segmentation_type: vlan + - port: {get_resource: devstack-tenant-vlan105-port} + segmentation_id: 105 + segmentation_type: vlan + + devstack: + type: OS::Nova::Server + properties: + image: {get_param: [devstack_params, image]} + flavor: {get_param: [devstack_params, flavor]} + networks: + - port: {get_resource: devstack-machine-port} + - port: {get_attr: [devstack-trunk, port_id]} + user_data_format: RAW + user_data: {get_resource: devstack-init} + + # + # Ironic Nodes + # + ironic0-port: + type: OS::Neutron::Port + properties: + network: {get_resource: ironic0-br-net} + port_security_enabled: false + + ironic0: + type: OS::Nova::Server + properties: + flavor: {get_param: [ironic_params, flavor]} + block_device_mapping_v2: + - device_type: disk + boot_index: 1 + image_id: {get_param: [ironic_params, image]} + volume_size: 40 + delete_on_termination: true + - device_type: cdrom + disk_bus: {get_param: cdrom_disk_bus} + boot_index: 0 + image_id: {get_param: [ironic_params, cd_image]} + volume_size: 5 + delete_on_termination: true + networks: + - port: {get_resource: ironic0-port} + + ironic1-port: + type: OS::Neutron::Port + properties: + network: {get_resource: ironic1-br-net} + port_security_enabled: false + + ironic1: + type: OS::Nova::Server + properties: + flavor: {get_param: [ironic_params, flavor]} + block_device_mapping_v2: + - device_type: disk + boot_index: 1 + image_id: {get_param: [ironic_params, image]} + volume_size: 40 + delete_on_termination: true + - device_type: cdrom + disk_bus: {get_param: cdrom_disk_bus} + boot_index: 0 + image_id: {get_param: [ironic_params, cd_image]} + volume_size: 5 + delete_on_termination: true + networks: + - port: {get_resource: ironic1-port} + +outputs: + controller_floating_ip: + description: Controller Floating IP + value: {get_attr: [controller-floating-ip, floating_ip_address]} + + controller_ansible_host: + description: > + Controller ansible host, this struct can be passed to the ansible.builtin.add_host module + value: + name: controller-0 + ansible_ssh_user: zuul + ansible_host: {get_attr: [controller-floating-ip, floating_ip_address]} + ansible_port: 22 + ansible_ssh_common_args: '-o StrictHostKeyChecking=no' + groups: controllers + + devstack_ansible_host: + description: > + Devstack ansible host, this struct can be passed to the ansible.builtin.add_host module. + Uses ProxyJump through the controller for SSH access. + value: + name: devstack + ansible_user: stack + ansible_host: {get_attr: [devstack-machine-port, fixed_ips, 0, ip_address]} + ansible_port: 22 + ansible_ssh_common_args: + str_replace: + template: '-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyJump=zuul@$controller_ip' + params: + $controller_ip: {get_attr: [controller-floating-ip, floating_ip_address]} + ansible_ssh_private_key_file: '~/.ssh/id_rsa' + groups: devstack_nodes + + devstack_netplan_config: + description: > + Complete netplan configuration for devstack node to be written by Ansible + value: + network: + version: 2 + ethernets: + enp3s0: + match: + macaddress: "fa:16:9e:81:f6:20" + dhcp4: true + set-name: "enp3s0" + mtu: 1442 + trunk0: + match: + macaddress: "fa:16:9e:81:f6:21" + dhcp4: false + dhcp6: false + set-name: trunk0 + mtu: 1442 + vlans: + trunk0.100: + id: 100 + link: trunk0 + mtu: 1442 + dhcp4: false + addresses: + - list_join: + - '' + - - {get_attr: [devstack-public-vlan100-port, fixed_ips, 0, ip_address]} + - '/' + - str_split: + - '/' + - {get_attr: [devstack-public-vlan100-port, subnets, 0, cidr]} + - 1 + + sushy_emulator_uuids: + description: UUIDs of instances to manage with sushy-tools - RedFish virtual BMC + value: + ironic0: {get_resource: ironic0} + ironic1: {get_resource: ironic1} + + sushy_tools_vmedia_type: + description: Virtual media implementation type for sushy-tools (rescue or volumeRebuild) + value: rescue + + ironic_nodes: + description: Ironic nodes YAML, used with openstack baremetal create to enroll nodes in Openstack Ironic + value: + nodes: + - name: ironic0 + driver: redfish + bios_interface: no-bios + boot_interface: redfish-virtual-media + network_interface: neutron + driver_info: + redfish_address: http://controller-0.netlab.example.com:8000 + redfish_system_id: + str_replace: + template: "/redfish/v1/Systems/$SYS_ID" + params: + $SYS_ID: {get_resource: ironic0} + redfish_username: admin + redfish_password: password + properties: + cpu_arch: x86_64 + cpus: 1 + memory_mb: 1024 + local_gb: 15 + capabilities: boot_mode:uefi + ports: + - address: {get_attr: [ironic0-port, mac_address]} + physical_network: public + local_link_connection: + switch_info: leaf01.netlab.example.com + switch_id: "22:57:f8:dd:03:01" + port_id: "ethernet1/4" + - name: ironic1 + driver: redfish + bios_interface: no-bios + boot_interface: redfish-virtual-media + network_interface: neutron + driver_info: + redfish_address: http://controller-0.netlab.example.com:8000 + redfish_system_id: + str_replace: + template: "/redfish/v1/Systems/$SYS_ID" + params: + $SYS_ID: {get_resource: ironic1} + redfish_username: admin + redfish_password: password + properties: + cpu_arch: x86_64 + cpus: 1 + memory_mb: 1024 + local_gb: 15 + capabilities: boot_mode:uefi + ports: + - address: {get_attr: [ironic1-port, mac_address]} + physical_network: public + local_link_connection: + switch_info: leaf02.netlab.example.com + switch_id: "22:57:f8:dd:04:01" + port_id: "ethernet1/4" + + ansible_inventory: + description: Ansible inventory + value: + all: + children: + controllers: + vars: + switches: + vars: + devstack_nodes: + vars: + localhosts: + hosts: + localhost: + ansible_connection: local + controllers: + hosts: + controller0: + ansible_host: {get_attr: [controller-machine-port, fixed_ips, 0, ip_address]} + ansible_user: zuul + ansible_ssh_common_args: '-o StrictHostKeyChecking=no' + ansible_ssh_private_key_file: '~/.ssh/id_rsa' + switches: + hosts: + spine01: + ansible_host: {get_attr: [spine01-machine-port, fixed_ips, 0, ip_address]} + ansible_user: admin + ansible_ssh_common_args: '-o StrictHostKeyChecking=no' + ansible_ssh_private_key_file: '~/.ssh/id_rsa' + spine02: + ansible_host: {get_attr: [spine02-machine-port, fixed_ips, 0, ip_address]} + ansible_user: admin + ansible_ssh_common_args: '-o StrictHostKeyChecking=no' + ansible_ssh_private_key_file: '~/.ssh/id_rsa' + leaf01: + ansible_host: {get_attr: [leaf01-machine-port, fixed_ips, 0, ip_address]} + ansible_user: admin + ansible_ssh_common_args: '-o StrictHostKeyChecking=no' + ansible_ssh_private_key_file: '~/.ssh/id_rsa' + leaf02: + ansible_host: {get_attr: [leaf02-machine-port, fixed_ips, 0, ip_address]} + ansible_user: admin + ansible_ssh_common_args: '-o StrictHostKeyChecking=no' + ansible_ssh_private_key_file: '~/.ssh/id_rsa' + devstack_nodes: + hosts: + devstack: + ansible_host: {get_attr: [devstack-machine-port, fixed_ips, 0, ip_address]} + ansible_user: stack + ansible_ssh_common_args: '-o StrictHostKeyChecking=no' + ansible_ssh_private_key_file: '~/.ssh/id_rsa' diff --git a/scenarios/networking-lab/devstack-nxsw-vxlan/l2vni_network_nodes.yaml b/scenarios/networking-lab/devstack-nxsw-vxlan/l2vni_network_nodes.yaml new file mode 100644 index 00000000..3b2a7fb6 --- /dev/null +++ b/scenarios/networking-lab/devstack-nxsw-vxlan/l2vni_network_nodes.yaml @@ -0,0 +1,16 @@ +--- +# L2VNI Network Nodes Configuration +# This file provides fallback local_link_connection information for network nodes +# when LLDP data is not available from OVN and Ironic. +# +# The system_id must match the OVN chassis system-id (from external-ids:system-id in OVS). +# For this devstack deployment, the system-id is the hostname: devstack + +network_nodes: + - hostname: "devstack" + trunks: + - physical_network: public + local_link_connection: + switch_id: "22:57:f8:dd:03:01" + port_id: "Ethernet1/5" + switch_info: "leaf01.netlab.example.com" diff --git a/scenarios/networking-lab/devstack-nxsw-vxlan/leaf01-poap.cfg b/scenarios/networking-lab/devstack-nxsw-vxlan/leaf01-poap.cfg new file mode 100644 index 00000000..6b02a5e7 --- /dev/null +++ b/scenarios/networking-lab/devstack-nxsw-vxlan/leaf01-poap.cfg @@ -0,0 +1,114 @@ +hostname leaf01.netlab.example.com +vdc leaf01 id 1 + limit-resource vlan minimum 16 maximum 4094 + limit-resource vrf minimum 2 maximum 4096 + limit-resource port-channel minimum 0 maximum 511 + limit-resource m4route-mem minimum 58 maximum 58 + limit-resource m6route-mem minimum 8 maximum 8 + +feature netconf +feature lacp +feature ospf +feature bgp +feature nv overlay +feature vn-segment-vlan-based +feature vlan-interface + +! TCAM region configuration for ARP suppression on VXLANs +! Note: Spine/core switches don't need this - only required on leaf switches with VTEPs +! Reduce multiple regions to free TCAM slices for arp-ether region (needs 512 slices for 256 double-wide) +! TCAM sizes: <=256 must be multiples of 256, >256 must be multiples of 512 +hardware access-list tcam region racl 512 +hardware access-list tcam region vpc-convergence 0 +hardware access-list tcam region e-racl 512 +hardware access-list tcam region arp-ether 256 double-wide + +nv overlay evpn + +no password strength-check +username admin password 5 $5$LFGHKE$ND3U7npkwgMUzxLjjgDQxCx8JFJ5.ZlFyQTTt1vZgA5 role network-admin +ssh key rsa 2048 force + +system default switchport + +vrf context management + ip name-server 192.168.32.254 + ip route 0.0.0.0/0 192.168.32.1 + +interface mgmt0 + description "Management interface" + ip address dhcp + vrf member management + +interface Ethernet1/1 + description "Link to Spine01" + no switchport + mtu 1442 + ip address 10.1.1.6/30 + ip ospf network point-to-point + ip router ospf 1 area 0.0.0.0 + no shutdown + +interface Ethernet1/2 + description "Link to Spine02" + no switchport + mtu 1442 + ip address 10.1.1.10/30 + ip ospf network point-to-point + ip router ospf 1 area 0.0.0.0 + no shutdown + +interface Ethernet1/3 + description "Trunk port for VLANs 100,103-105 - managed by ML2" + switchport mode trunk + switchport trunk allowed vlan 100,103-105 + mtu 1442 + no shutdown + +interface Ethernet1/4 + description "Ironic0 access port - managed by ML2" + mtu 1442 + spanning-tree port type edge + shutdown + +interface Ethernet1/5 + description "Devstack connection to br-ex - trunk for dynamic VLANs" + switchport mode trunk + switchport trunk allowed vlan 103-105 + mtu 1442 + spanning-tree port type edge trunk + no shutdown + +interface loopback0 + description "Router ID and VTEP Source" + ip address 10.255.255.3/32 + ip router ospf 1 area 0.0.0.0 + +interface nve1 + description "VXLAN Tunnel Endpoint" + no shutdown + host-reachability protocol bgp + source-interface loopback0 + +router ospf 1 + router-id 10.255.255.3 + +router bgp 65001 + router-id 10.255.255.3 + address-family l2vpn evpn + advertise-pip + neighbor 10.255.255.1 remote-as 65001 + description spine01 + update-source loopback0 + address-family l2vpn evpn + send-community + send-community extended + neighbor 10.255.255.2 remote-as 65001 + description spine02 + update-source loopback0 + address-family l2vpn evpn + send-community + send-community extended + +line console +line vty diff --git a/scenarios/networking-lab/devstack-nxsw-vxlan/leaf01-poap.py b/scenarios/networking-lab/devstack-nxsw-vxlan/leaf01-poap.py new file mode 100644 index 00000000..cfa26c1c --- /dev/null +++ b/scenarios/networking-lab/devstack-nxsw-vxlan/leaf01-poap.py @@ -0,0 +1,437 @@ +#!/bin/env python3 +# md5sum="a0650e27861590d6c430479b179ac936" + +# If any changes are made to this script, please run the below command +# in bash shell to update the above md5sum. This is used for integrity check. +# f=poap.py ; sed '/^# *md5sum/d' "$f" > "$f.md5" ; sed -i \ +# "s/^# *md5sum=.*/#md5sum=\"$(md5sum $f.md5 | sed 's/ .*//')\"/" $f + +# Protocol Authentication Support: +# - SCP: Always requires username and password +# - HTTP/HTTPS: Authentication optional (anonymous or username/password) +# - TFTP: No authentication support (anonymous only) +# +# Authentication parameters (username/password) are only required for SCP. +# For HTTP/HTTPS, if you provide username, you must also provide password. +# +# Example usage: +# - Anonymous HTTP: protocol="http", hostname="server.com" (no auth needed) +# - Authenticated HTTP: protocol="http", hostname="server.com", port=8080, username="user", password="pass" +# - Anonymous TFTP: protocol="tftp", hostname="server.com" (no auth supported) +# - Authenticated SCP: protocol="scp", hostname="server.com", username="user", password="pass" (auth required) + +import os +import re +import signal +import sys +import syslog +import traceback +import time + +try: + from cisco import cli +except ImportError: + from cli import * + +# Default configuration options +DEFAULT_OPTS = { + "hostname": "192.168.32.254", + "protocol": "tftp", + "cfg_path": "/leaf01-poap.cfg", + "dest_path": "bootflash:poap.cfg", + "ignore_cert": True, + # username, password and port are optional - only needed for SCP or authenticated HTTP/HTTPS + # port is optional for custom ports on any protocol (e.g., SCP:2222, HTTP:8080, HTTPS:8088, TFTP:6969) +} + +# Valid configuration options +VALID_OPTS = { + "username", + "password", + "hostname", + "protocol", + "port", + "cfg_path", + "dest_path", + "ignore_cert", + "vrf", +} + +# Required configuration parameters (always required) +REQUIRED_OPTS = {"hostname"} + +# Logging prefix for syslog messages +SYSLOG_PREFIX = "POAPHandler" + + +def get_log_file_path(): + """Generate the log file path with timestamp and PID""" + return "/bootflash/%s_poap_%s_script.log" % ( + time.strftime("%Y%m%d%H%M%S", time.gmtime()), + os.environ["POAP_PID"], + ) + + +class POAPHandler: + """ + POAP (Power-On Auto Provisioning) handler for Cisco NXOS switches. + Handles configuration download and application for switch bootstrap. + """ + + def __init__(self): + """Initialize the POAP handler with default settings.""" + + # Set up signal handler + signal.signal(signal.SIGTERM, self.sigterm_handler) + + # Initialize logging + self.syslog_prefix = SYSLOG_PREFIX + self.log_file_handler = None # Will be set when using context manager + + self.opts = DEFAULT_OPTS.copy() + + # Validate required parameters + self._validate_required_opts() + + # Check that options are valid + self.validate_opts() + + @property + def dest_path(self): + """Normalized destination path without trailing slashes""" + return self.opts["dest_path"].rstrip("/") + + @property + def cfg_path(self): + """Normalized config path without trailing slashes""" + return self.opts["cfg_path"].rstrip("/") + + def _validate_required_opts(self): + """Validates that required options are provided""" + missing_params = REQUIRED_OPTS.difference(self.opts.keys()) + + if missing_params: + self._log("Required parameters are missing:") + self.abort("Missing %s" % ", ".join(missing_params)) + + # Protocol-specific validation for authentication parameters + protocol = self.opts.get("protocol", "http") + + # SCP always requires authentication + if protocol == "scp": + username = self.opts.get("username") + password = self.opts.get("password") + if not username or not password: + self.abort("SCP protocol requires both username and password") + + # HTTP/HTTPS with authentication requires both username and password + if protocol in ["http", "https"]: + username = self.opts.get("username") + password = self.opts.get("password") + if (username and not password) or (password and not username): + self.abort( + "HTTP/HTTPS authentication requires both username and password" + ) + + # Validate port if provided + port = self.opts.get("port") + if port is not None: + try: + port_int = int(port) + if not (1 <= port_int <= 65535): + self.abort("Port must be between 1 and 65535") + except (ValueError, TypeError): + self.abort("Port must be a valid integer") + + def validate_opts(self): + """ + Validates that the options provided by the user are valid. + Aborts the script if they are not. + """ + # Find any invalid options (ones not in VALID_OPTS) + invalid_opts = set(self.opts.keys()) - VALID_OPTS + if invalid_opts: + self._log( + "Invalid options detected: %s (check spelling, capitalization, and underscores)" + % ", ".join(invalid_opts) + ) + self.abort() + + def abort(self, msg=None): + """Aborts the POAP script + + :param msg: The message to log before aborting + """ + if msg: + self._log(msg) + + # Destination config + self.cleanup_file_from_option("dest_cfg") + + # Log file will be closed by context manager + exit(1) + + def _redact_passwords(self, message): + """Redacts passwords from log messages for security + + :param message: The log message to redact passwords from + :return: The message with passwords replaced with '' + """ + parts = re.split("\s+", message.strip()) + for index, part in enumerate(parts): + # blank out the password after the password keyword (terminal password *****, etc.) + if part == "password" and len(parts) >= index + 2: + parts[index + 1] = "" + + return " ".join(parts) + + def _log(self, info): + """ + Log the trace into console and poap_script log file in bootflash + + :param info: The information that needs to be logged. + """ + # Redact sensitive information before logging + info = self._redact_passwords(info) + + # Add syslog prefix + info = "%s - %s" % (self.syslog_prefix, info) + + syslog.syslog(9, info) + if self.log_file_handler is not None: + print(info, file=self.log_file_handler, flush=True) + + def remove_file(self, filename): + """Removes a file if it exists and it's not a directory. + + :param filename: The file to remove + """ + if os.path.isfile(filename): + try: + os.remove(filename) + except (IOError, OSError) as e: + self._log("Failed to remove %s: %s" % (filename, str(e))) + + def cleanup_file_from_option(self, option, bootflash_root=False): + """Removes a file indicated by the option in the POAP opts and removes it if it exists. + + Handle the cases where the variable is unused or not set yet. + + :param option: The option to remove + :param bootflash_root: Whether to remove the file from the bootflash root + """ + try: + filename = self.opts[option] + if filename is None: + return # Nothing to clean up + + if bootflash_root: + path = "/bootflash" + else: + path = self.dest_path + + self.remove_file(os.path.join(path, filename)) + self.remove_file(os.path.join(path, "%s.tmp" % filename)) + except KeyError: + # Option doesn't exist, nothing to clean up + pass + + def sigterm_handler(self, signum, stack): + """ + A signal handler for the SIGTERM signal. Cleans up and exits + + :param signum: The signal number + :param stack: The stack trace + """ + self.abort("SIGTERM signal received") + + def process_cfg_file(self): + """ + Processes the downloaded switch configuration file. + Copies the config to the scheduled config file for bootstrap replay. + """ + self._log("Processing Config file") + + # Copy config directly to scheduled-config + self._log("Command: copy %s scheduled-config" % self.opts["dest_path"]) + cli("copy %s scheduled-config" % self.opts["dest_path"]) + + self._log("Config processed and prepared for scheduled application") + + def _build_copy_cmd(self, source, dest): + """Build the copy command with all necessary options + + :param source: Source file path on remote server + :param dest: Destination path on local switch + :return: Complete copy command string + """ + # Extract parameters from opts + protocol = self.opts["protocol"] + host = self.opts["hostname"] + user = self.opts.get("username") + password = self.opts.get("password") + vrf = self.opts["vrf"] + ignore_ssl = self.opts["ignore_cert"] + port = self.opts.get("port") + # Build copy command with Cisco NX-OS terminal automation features + parts = [] + + # terminal dont-ask: Auto-answer "yes" to all confirmation prompts + parts.append("terminal dont-ask") + + # Determine if authentication is needed based on protocol and credentials + auth_needed = protocol in ["scp"] or ( + protocol in ["http", "https"] and user and password + ) + + if auth_needed: + if password: + # terminal password: Pre-store password for automatic authentication + # The password will be used automatically for any auth prompts during copy + parts.append("terminal password %s" % password) + + # Build the copy URL - only include user@ if authentication is needed + url = "%s://" % protocol + if auth_needed and user: + url += "%s@" % user + + # Add hostname with optional port + url += host + if port: + url += ":%s" % port + + # Ensure source path starts with / for proper URL construction + if not source.startswith("/"): + source = "/" + source + url += source + + # Build the copy command with URL and destination + cmd = "copy %s %s" % (url, dest) + + # Add ignore-certificate if needed + if protocol == "https" and ignore_ssl: + cmd += " ignore-certificate" + + # Add VRF + cmd += " vrf %s" % vrf + + # Add the complete copy command to the parts + parts.append(cmd) + + # Join all command parts with semicolon separator + return " ; ".join(parts) + + def copy(self, source, dest): + """Copies the files + + Copy the provided file from source to destination via network transfer. + + :param source: The source file to copy + :param dest: The destination file to copy + """ + self._log("Copying %s to %s" % (source, dest)) + + # Build the copy command using the dedicated method + cmd = self._build_copy_cmd(source, dest) + self._log("Copy command: %s" % cmd) + + try: + cli(cmd) + except Exception as e: + exc_type, exc_value, exc_tb = sys.exc_info() + traceback_str = "".join( + traceback.format_exception(exc_type, exc_value, exc_tb) + ) + self._log("Copy failed, Traceback: %s" % traceback_str) + self.abort("Copy failed: %s" % str(e)) + + self._log("Copy completed successfully") + + def get_currently_booted_image_filename(self): + match = None + try: + output = cli("show version") + except Exception as e: + self.abort("Show version failed: %s" % str(e)) + + match = re.search("NXOS image file is:\s+(.+)", output) + + if match: + directory, image = os.path.split(match.group(1)) + return image.strip() + + def install_nxos(self): + """Install the NXOS image on the switch + + Assume the current image is the one we want to install. + """ + boot_image = self.get_currently_booted_image_filename() + if not boot_image: + self.abort("No image found to install") + + # Build the install command + parts = [] + parts.append("terminal dont-ask ") + parts.append("install all nxos %s no-reload non-interruptive" % boot_image) + parts.append("terminal dont-ask") + parts.append("write erase") + cmd = " ; ".join(parts) + self._log("Install command: %s" % cmd) + + try: + cli(cmd) + time.sleep(5) + except Exception as e: + self._log("Failed to ISSU to image %s" % boot_image) + self.abort(str(e)) + + def run_with_logging(self): + """Execute the POAP provisioning process with proper log file management""" + log_file = get_log_file_path() + + with open(log_file, "w+") as log_file_handler: + self.log_file_handler = log_file_handler + self._log("Logfile name: %s" % log_file) + + try: + self.run() + finally: + # Ensure log_file_handler is reset when context exits + self.log_file_handler = None + + def run(self): + """Execute the POAP provisioning process""" + + # Set dynamic VRF value from environment + self.opts.setdefault("vrf", os.environ.get("POAP_VRF", "management")) + + # # create the directory structure needed, if any + # self.create_dest_dirs() + + # Copy config + self.copy(self.opts["cfg_path"], self.opts["dest_path"]) + self.process_cfg_file() + + # Install the NXOS image + self.install_nxos() + + +def main(): + """Main entry point for the POAP script""" + poap_handler = POAPHandler() + poap_handler.run_with_logging() + + +if __name__ == "__main__": + try: + main() + except Exception: + exc_type, exc_value, exc_tb = sys.exc_info() + # Create a temporary handler just for error logging + handler = POAPHandler() + + # Log the full traceback as a single formatted string + traceback_str = "".join(traceback.format_exception(exc_type, exc_value, exc_tb)) + handler._log("Exception occurred:\n%s" % traceback_str) + + handler.abort() diff --git a/scenarios/networking-lab/devstack-nxsw-vxlan/leaf02-poap.cfg b/scenarios/networking-lab/devstack-nxsw-vxlan/leaf02-poap.cfg new file mode 100644 index 00000000..d1b61b8f --- /dev/null +++ b/scenarios/networking-lab/devstack-nxsw-vxlan/leaf02-poap.cfg @@ -0,0 +1,107 @@ +hostname leaf02.netlab.example.com +vdc leaf02 id 1 + limit-resource vlan minimum 16 maximum 4094 + limit-resource vrf minimum 2 maximum 4096 + limit-resource port-channel minimum 0 maximum 511 + limit-resource m4route-mem minimum 58 maximum 58 + limit-resource m6route-mem minimum 8 maximum 8 + +feature netconf +feature lacp +feature ospf +feature bgp +feature nv overlay +feature vn-segment-vlan-based +feature vlan-interface + +! TCAM region configuration for ARP suppression on VXLANs +! Note: Spine/core switches don't need this - only required on leaf switches with VTEPs +! Reduce multiple regions to free TCAM slices for arp-ether region (needs 512 slices for 256 double-wide) +! TCAM sizes: <=256 must be multiples of 256, >256 must be multiples of 512 +hardware access-list tcam region racl 512 +hardware access-list tcam region vpc-convergence 0 +hardware access-list tcam region e-racl 512 +hardware access-list tcam region arp-ether 256 double-wide + +nv overlay evpn + +no password strength-check +username admin password 5 $5$LFGHKE$ND3U7npkwgMUzxLjjgDQxCx8JFJ5.ZlFyQTTt1vZgA5 role network-admin +ssh key rsa 2048 force + +system default switchport + +vrf context management + ip name-server 192.168.32.254 + ip route 0.0.0.0/0 192.168.32.1 + +interface mgmt0 + description "Management interface" + ip address dhcp + vrf member management + +interface Ethernet1/1 + description "Link to Spine01" + no switchport + mtu 1442 + ip address 10.1.1.14/30 + ip ospf network point-to-point + ip router ospf 1 area 0.0.0.0 + no shutdown + +interface Ethernet1/2 + description "Link to Spine02" + no switchport + mtu 1442 + ip address 10.1.1.18/30 + ip ospf network point-to-point + ip router ospf 1 area 0.0.0.0 + no shutdown + +interface Ethernet1/3 + description "Trunk port for VLANs 100,103-105 - managed by ML2" + switchport mode trunk + switchport trunk allowed vlan 100,103-105 + mtu 1442 + spanning-tree port type edge trunk + no shutdown + +interface Ethernet1/4 + description "Ironic1 access port - managed by ML2" + mtu 1442 + spanning-tree port type edge + shutdown + +interface loopback0 + description "Router ID and VTEP Source" + ip address 10.255.255.4/32 + ip router ospf 1 area 0.0.0.0 + +interface nve1 + description "VXLAN Tunnel Endpoint" + no shutdown + host-reachability protocol bgp + source-interface loopback0 + +router ospf 1 + router-id 10.255.255.4 + +router bgp 65001 + router-id 10.255.255.4 + address-family l2vpn evpn + advertise-pip + neighbor 10.255.255.1 remote-as 65001 + description spine01 + update-source loopback0 + address-family l2vpn evpn + send-community + send-community extended + neighbor 10.255.255.2 remote-as 65001 + description spine02 + update-source loopback0 + address-family l2vpn evpn + send-community + send-community extended + +line console +line vty diff --git a/scenarios/networking-lab/devstack-nxsw-vxlan/leaf02-poap.py b/scenarios/networking-lab/devstack-nxsw-vxlan/leaf02-poap.py new file mode 100644 index 00000000..4b678443 --- /dev/null +++ b/scenarios/networking-lab/devstack-nxsw-vxlan/leaf02-poap.py @@ -0,0 +1,437 @@ +#!/bin/env python3 +# md5sum="f5a88b84c3edafdb51efd8c74246676a" + +# If any changes are made to this script, please run the below command +# in bash shell to update the above md5sum. This is used for integrity check. +# f=poap.py ; sed '/^# *md5sum/d' "$f" > "$f.md5" ; sed -i \ +# "s/^# *md5sum=.*/#md5sum=\"$(md5sum $f.md5 | sed 's/ .*//')\"/" $f + +# Protocol Authentication Support: +# - SCP: Always requires username and password +# - HTTP/HTTPS: Authentication optional (anonymous or username/password) +# - TFTP: No authentication support (anonymous only) +# +# Authentication parameters (username/password) are only required for SCP. +# For HTTP/HTTPS, if you provide username, you must also provide password. +# +# Example usage: +# - Anonymous HTTP: protocol="http", hostname="server.com" (no auth needed) +# - Authenticated HTTP: protocol="http", hostname="server.com", port=8080, username="user", password="pass" +# - Anonymous TFTP: protocol="tftp", hostname="server.com" (no auth supported) +# - Authenticated SCP: protocol="scp", hostname="server.com", username="user", password="pass" (auth required) + +import os +import re +import signal +import sys +import syslog +import traceback +import time + +try: + from cisco import cli +except ImportError: + from cli import * + +# Default configuration options +DEFAULT_OPTS = { + "hostname": "192.168.32.254", + "protocol": "tftp", + "cfg_path": "/leaf02-poap.cfg", + "dest_path": "bootflash:poap.cfg", + "ignore_cert": True, + # username, password and port are optional - only needed for SCP or authenticated HTTP/HTTPS + # port is optional for custom ports on any protocol (e.g., SCP:2222, HTTP:8080, HTTPS:8088, TFTP:6969) +} + +# Valid configuration options +VALID_OPTS = { + "username", + "password", + "hostname", + "protocol", + "port", + "cfg_path", + "dest_path", + "ignore_cert", + "vrf", +} + +# Required configuration parameters (always required) +REQUIRED_OPTS = {"hostname"} + +# Logging prefix for syslog messages +SYSLOG_PREFIX = "POAPHandler" + + +def get_log_file_path(): + """Generate the log file path with timestamp and PID""" + return "/bootflash/%s_poap_%s_script.log" % ( + time.strftime("%Y%m%d%H%M%S", time.gmtime()), + os.environ["POAP_PID"], + ) + + +class POAPHandler: + """ + POAP (Power-On Auto Provisioning) handler for Cisco NXOS switches. + Handles configuration download and application for switch bootstrap. + """ + + def __init__(self): + """Initialize the POAP handler with default settings.""" + + # Set up signal handler + signal.signal(signal.SIGTERM, self.sigterm_handler) + + # Initialize logging + self.syslog_prefix = SYSLOG_PREFIX + self.log_file_handler = None # Will be set when using context manager + + self.opts = DEFAULT_OPTS.copy() + + # Validate required parameters + self._validate_required_opts() + + # Check that options are valid + self.validate_opts() + + @property + def dest_path(self): + """Normalized destination path without trailing slashes""" + return self.opts["dest_path"].rstrip("/") + + @property + def cfg_path(self): + """Normalized config path without trailing slashes""" + return self.opts["cfg_path"].rstrip("/") + + def _validate_required_opts(self): + """Validates that required options are provided""" + missing_params = REQUIRED_OPTS.difference(self.opts.keys()) + + if missing_params: + self._log("Required parameters are missing:") + self.abort("Missing %s" % ", ".join(missing_params)) + + # Protocol-specific validation for authentication parameters + protocol = self.opts.get("protocol", "http") + + # SCP always requires authentication + if protocol == "scp": + username = self.opts.get("username") + password = self.opts.get("password") + if not username or not password: + self.abort("SCP protocol requires both username and password") + + # HTTP/HTTPS with authentication requires both username and password + if protocol in ["http", "https"]: + username = self.opts.get("username") + password = self.opts.get("password") + if (username and not password) or (password and not username): + self.abort( + "HTTP/HTTPS authentication requires both username and password" + ) + + # Validate port if provided + port = self.opts.get("port") + if port is not None: + try: + port_int = int(port) + if not (1 <= port_int <= 65535): + self.abort("Port must be between 1 and 65535") + except (ValueError, TypeError): + self.abort("Port must be a valid integer") + + def validate_opts(self): + """ + Validates that the options provided by the user are valid. + Aborts the script if they are not. + """ + # Find any invalid options (ones not in VALID_OPTS) + invalid_opts = set(self.opts.keys()) - VALID_OPTS + if invalid_opts: + self._log( + "Invalid options detected: %s (check spelling, capitalization, and underscores)" + % ", ".join(invalid_opts) + ) + self.abort() + + def abort(self, msg=None): + """Aborts the POAP script + + :param msg: The message to log before aborting + """ + if msg: + self._log(msg) + + # Destination config + self.cleanup_file_from_option("dest_cfg") + + # Log file will be closed by context manager + exit(1) + + def _redact_passwords(self, message): + """Redacts passwords from log messages for security + + :param message: The log message to redact passwords from + :return: The message with passwords replaced with '' + """ + parts = re.split("\s+", message.strip()) + for index, part in enumerate(parts): + # blank out the password after the password keyword (terminal password *****, etc.) + if part == "password" and len(parts) >= index + 2: + parts[index + 1] = "" + + return " ".join(parts) + + def _log(self, info): + """ + Log the trace into console and poap_script log file in bootflash + + :param info: The information that needs to be logged. + """ + # Redact sensitive information before logging + info = self._redact_passwords(info) + + # Add syslog prefix + info = "%s - %s" % (self.syslog_prefix, info) + + syslog.syslog(9, info) + if self.log_file_handler is not None: + print(info, file=self.log_file_handler, flush=True) + + def remove_file(self, filename): + """Removes a file if it exists and it's not a directory. + + :param filename: The file to remove + """ + if os.path.isfile(filename): + try: + os.remove(filename) + except (IOError, OSError) as e: + self._log("Failed to remove %s: %s" % (filename, str(e))) + + def cleanup_file_from_option(self, option, bootflash_root=False): + """Removes a file indicated by the option in the POAP opts and removes it if it exists. + + Handle the cases where the variable is unused or not set yet. + + :param option: The option to remove + :param bootflash_root: Whether to remove the file from the bootflash root + """ + try: + filename = self.opts[option] + if filename is None: + return # Nothing to clean up + + if bootflash_root: + path = "/bootflash" + else: + path = self.dest_path + + self.remove_file(os.path.join(path, filename)) + self.remove_file(os.path.join(path, "%s.tmp" % filename)) + except KeyError: + # Option doesn't exist, nothing to clean up + pass + + def sigterm_handler(self, signum, stack): + """ + A signal handler for the SIGTERM signal. Cleans up and exits + + :param signum: The signal number + :param stack: The stack trace + """ + self.abort("SIGTERM signal received") + + def process_cfg_file(self): + """ + Processes the downloaded switch configuration file. + Copies the config to the scheduled config file for bootstrap replay. + """ + self._log("Processing Config file") + + # Copy config directly to scheduled-config + self._log("Command: copy %s scheduled-config" % self.opts["dest_path"]) + cli("copy %s scheduled-config" % self.opts["dest_path"]) + + self._log("Config processed and prepared for scheduled application") + + def _build_copy_cmd(self, source, dest): + """Build the copy command with all necessary options + + :param source: Source file path on remote server + :param dest: Destination path on local switch + :return: Complete copy command string + """ + # Extract parameters from opts + protocol = self.opts["protocol"] + host = self.opts["hostname"] + user = self.opts.get("username") + password = self.opts.get("password") + vrf = self.opts["vrf"] + ignore_ssl = self.opts["ignore_cert"] + port = self.opts.get("port") + # Build copy command with Cisco NX-OS terminal automation features + parts = [] + + # terminal dont-ask: Auto-answer "yes" to all confirmation prompts + parts.append("terminal dont-ask") + + # Determine if authentication is needed based on protocol and credentials + auth_needed = protocol in ["scp"] or ( + protocol in ["http", "https"] and user and password + ) + + if auth_needed: + if password: + # terminal password: Pre-store password for automatic authentication + # The password will be used automatically for any auth prompts during copy + parts.append("terminal password %s" % password) + + # Build the copy URL - only include user@ if authentication is needed + url = "%s://" % protocol + if auth_needed and user: + url += "%s@" % user + + # Add hostname with optional port + url += host + if port: + url += ":%s" % port + + # Ensure source path starts with / for proper URL construction + if not source.startswith("/"): + source = "/" + source + url += source + + # Build the copy command with URL and destination + cmd = "copy %s %s" % (url, dest) + + # Add ignore-certificate if needed + if protocol == "https" and ignore_ssl: + cmd += " ignore-certificate" + + # Add VRF + cmd += " vrf %s" % vrf + + # Add the complete copy command to the parts + parts.append(cmd) + + # Join all command parts with semicolon separator + return " ; ".join(parts) + + def copy(self, source, dest): + """Copies the files + + Copy the provided file from source to destination via network transfer. + + :param source: The source file to copy + :param dest: The destination file to copy + """ + self._log("Copying %s to %s" % (source, dest)) + + # Build the copy command using the dedicated method + cmd = self._build_copy_cmd(source, dest) + self._log("Copy command: %s" % cmd) + + try: + cli(cmd) + except Exception as e: + exc_type, exc_value, exc_tb = sys.exc_info() + traceback_str = "".join( + traceback.format_exception(exc_type, exc_value, exc_tb) + ) + self._log("Copy failed, Traceback: %s" % traceback_str) + self.abort("Copy failed: %s" % str(e)) + + self._log("Copy completed successfully") + + def get_currently_booted_image_filename(self): + match = None + try: + output = cli("show version") + except Exception as e: + self.abort("Show version failed: %s" % str(e)) + + match = re.search("NXOS image file is:\s+(.+)", output) + + if match: + directory, image = os.path.split(match.group(1)) + return image.strip() + + def install_nxos(self): + """Install the NXOS image on the switch + + Assume the current image is the one we want to install. + """ + boot_image = self.get_currently_booted_image_filename() + if not boot_image: + self.abort("No image found to install") + + # Build the install command + parts = [] + parts.append("terminal dont-ask ") + parts.append("install all nxos %s no-reload non-interruptive" % boot_image) + parts.append("terminal dont-ask") + parts.append("write erase") + cmd = " ; ".join(parts) + self._log("Install command: %s" % cmd) + + try: + cli(cmd) + time.sleep(5) + except Exception as e: + self._log("Failed to ISSU to image %s" % boot_image) + self.abort(str(e)) + + def run_with_logging(self): + """Execute the POAP provisioning process with proper log file management""" + log_file = get_log_file_path() + + with open(log_file, "w+") as log_file_handler: + self.log_file_handler = log_file_handler + self._log("Logfile name: %s" % log_file) + + try: + self.run() + finally: + # Ensure log_file_handler is reset when context exits + self.log_file_handler = None + + def run(self): + """Execute the POAP provisioning process""" + + # Set dynamic VRF value from environment + self.opts.setdefault("vrf", os.environ.get("POAP_VRF", "management")) + + # # create the directory structure needed, if any + # self.create_dest_dirs() + + # Copy config + self.copy(self.opts["cfg_path"], self.opts["dest_path"]) + self.process_cfg_file() + + # Install the NXOS image + self.install_nxos() + + +def main(): + """Main entry point for the POAP script""" + poap_handler = POAPHandler() + poap_handler.run_with_logging() + + +if __name__ == "__main__": + try: + main() + except Exception: + exc_type, exc_value, exc_tb = sys.exc_info() + # Create a temporary handler just for error logging + handler = POAPHandler() + + # Log the full traceback as a single formatted string + traceback_str = "".join(traceback.format_exception(exc_type, exc_value, exc_tb)) + handler._log("Exception occurred:\n%s" % traceback_str) + + handler.abort() diff --git a/scenarios/networking-lab/devstack-nxsw-vxlan/local.conf.j2 b/scenarios/networking-lab/devstack-nxsw-vxlan/local.conf.j2 new file mode 100644 index 00000000..1f753795 --- /dev/null +++ b/scenarios/networking-lab/devstack-nxsw-vxlan/local.conf.j2 @@ -0,0 +1,158 @@ +[[local|localrc]] +# Credentials +ADMIN_PASSWORD=secret +DATABASE_PASSWORD=$ADMIN_PASSWORD +RABBIT_PASSWORD=$ADMIN_PASSWORD +SERVICE_PASSWORD=$ADMIN_PASSWORD + +# Service timeouts +SERVICE_TIMEOUT=120 + +# MTU - running inside an encapsulated environment, restrict to 1442 on the +# physical network so VXLAN tenant networks get 1442 - 50 = 1392 effective MTU. +PUBLIC_BRIDGE_MTU=1442 + +# Networking +HOST_IP=192.168.32.20 +SERVICE_HOST=$HOST_IP +MYSQL_HOST=$HOST_IP +RABBIT_HOST=$HOST_IP +GLANCE_HOSTPORT=$HOST_IP:9292 + +# Network ranges (avoiding Heat template allocations) +FIXED_RANGE=172.20.100.0/24 +IPV4_ADDRS_SAFE_TO_USE=172.20.100.0/24 +FLOATING_RANGE=172.20.200.0/24 +PUBLIC_NETWORK_GATEWAY=172.20.200.1 + +# Disable IPv6 - use IPv4 only +IP_VERSION=4 + +# Enable Neutron with OVN +enable_plugin neutron https://opendev.org/openstack/neutron +disable_service n-net +enable_service q-svc +# Disable traditional neutron agents +disable_service q-agt +disable_service q-dhcp +disable_service q-l3 +disable_service q-meta +# Enable OVN services +enable_service ovn-northd +enable_service ovn-controller +enable_service q-ovn-metadata-agent +# Enable Neutron trunk service +enable_service neutron-trunk +# Enable Neutron segments service +enable_service neutron-segments + +enable_service ir-api +enable_service ir-cond +enable_service ir-neutronagt + +# Ironic configuration +VIRT_DRIVER=ironic +DEFAULT_INSTANCE_TYPE=baremetal +IRONIC_BAREMETAL_BASIC_OPS=True +IRONIC_IS_HARDWARE=True +IRONIC_VM_COUNT=0 +IRONIC_NETWORK_SIMULATOR=none +IRONIC_BUILD_DEPLOY_RAMDISK=False +IRONIC_DEPLOY_DRIVER=redfish +IRONIC_ENABLED_HARDWARE_TYPES=redfish +IRONIC_ENABLED_BOOT_INTERFACES=ipxe,redfish-virtual-media,http-ipxe +IRONIC_ENABLED_POWER_INTERFACES=redfish +IRONIC_ENABLED_MANAGEMENT_INTERFACES=redfish +IRONIC_ENABLED_DEPLOY_INTERFACES=direct,ramdisk +IRONIC_NETWORK_INTERFACE=neutron +IRONIC_ENABLED_NETWORK_INTERFACES=neutron +IRONIC_AUTOMATED_CLEAN_ENABLED=True +FORCE_CONFIG_DRIVE=True + +# Ironic network configuration - use provisioning vxlan network for all operations +IRONIC_PROVISION_NETWORK_NAME=provisioning +IRONIC_PROVISION_PROVIDER_NETWORK_TYPE=vxlan +IRONIC_PROVISION_SUBNET_PREFIX=10.0.5.0/24 +IRONIC_PROVISION_SUBNET_GATEWAY=10.0.5.1 +IRONIC_CLEAN_NET_NAME=provisioning +IRONIC_RESCUE_NET_NAME=provisioning +IRONIC_INSPECTION_NET_NAME=provisioning + +# Networking configuration for ML2 with OVN and Generic Switch +Q_PLUGIN=ml2 +Q_ML2_TENANT_NETWORK_TYPE=vxlan +Q_ML2_PLUGIN_MECHANISM_DRIVERS=ovn,baremetal-l2vni,genericswitch,baremetal +Q_ML2_PLUGIN_TYPE_DRIVERS=vxlan,geneve,vlan,flat +ENABLE_TENANT_VLANS=True +TENANT_VLAN_RANGE=103:105 +PHYSICAL_NETWORK=public + +# Physical interface mapping +# The second interface (trunk port) will be added to br-ex +# trunk0 is matched by MAC address fa:16:9e:81:f6:21 and renamed by netplan +PUBLIC_INTERFACE=trunk0 +OVS_PHYSICAL_BRIDGE=br-ex +PUBLIC_BRIDGE=br-ex + +# OVN Configuration +Q_USE_PROVIDERNET_FOR_PUBLIC=True +OVN_L3_CREATE_PUBLIC_NETWORK=True +OVN_BRIDGE_MAPPINGS=public:br-ex + +# Enable Ironic +enable_plugin ironic https://opendev.org/openstack/ironic + +# Enable networking-generic-switch plugin +enable_plugin networking-generic-switch https://opendev.org/openstack/networking-generic-switch + +# Enable networking-baremetal plugin +enable_plugin networking-baremetal https://opendev.org/openstack/networking-baremetal refs/changes/75/980375/2 + + + +# Disable Swift (optional, not needed for this setup) +disable_service s-proxy s-object s-container s-account + +# Disable Horizon dashboard +disable_service horizon + +[[post-config|$NEUTRON_CONF]] +[DEFAULT] +global_physnet_mtu = 1442 + +[baremetal_l2vni] +# Set to False for pure EVPN deployments where switches handle VXLAN via BGP +# Set to True if you need OVN localnet ports for overlay-to-physical bridging +create_localnet_ports = True +default_physical_network = public + +[l2vni] +# Enable L2VNI trunk reconciliation for network nodes +enable_l2vni_trunk_reconciliation = True +l2vni_reconciliation_interval = 300 +l2vni_auto_create_networks = True +l2vni_subport_anchor_network = l2vni-subport-anchor +l2vni_subport_anchor_network_type = vxlan +l2vni_startup_jitter_max = 60 +l2vni_network_nodes_config = /etc/neutron/l2vni_network_nodes.yaml + +[[post-config|/etc/neutron/plugins/ml2/ml2_conf_genericswitch.ini]] +[genericswitch:leaf01] +device_type = netmiko_cisco_nxos +ip = 192.168.32.13 +username = admin +password = password +ngs_mac_address = 22:57:f8:dd:03:01 +ngs_disable_inactive_ports = true +ngs_physical_networks = public +ngs_nve_interface = nve1 + +[genericswitch:leaf02] +device_type = netmiko_cisco_nxos +ip = 192.168.32.14 +username = admin +password = password +ngs_mac_address = 22:57:f8:dd:04:01 +ngs_disable_inactive_ports = true +ngs_physical_networks = public +ngs_nve_interface = nve1 diff --git a/scenarios/networking-lab/devstack-nxsw-vxlan/spine01-poap.cfg b/scenarios/networking-lab/devstack-nxsw-vxlan/spine01-poap.cfg new file mode 100644 index 00000000..4e2b5854 --- /dev/null +++ b/scenarios/networking-lab/devstack-nxsw-vxlan/spine01-poap.cfg @@ -0,0 +1,86 @@ +hostname spine01.netlab.example.com +vdc spine01 id 1 + limit-resource vlan minimum 16 maximum 4094 + limit-resource vrf minimum 2 maximum 4096 + limit-resource port-channel minimum 0 maximum 511 + limit-resource m4route-mem minimum 58 maximum 58 + limit-resource m6route-mem minimum 8 maximum 8 + +feature netconf +feature lacp +feature ospf +feature bgp +feature nv overlay + +nv overlay evpn + +no password strength-check +username admin password 5 $5$LFGHKE$ND3U7npkwgMUzxLjjgDQxCx8JFJ5.ZlFyQTTt1vZgA5 role network-admin +ssh key rsa 2048 force + +system default switchport + +vrf context management + ip name-server 192.168.32.254 + ip route 0.0.0.0/0 192.168.32.1 + +interface mgmt0 + description "Management interface" + ip address dhcp + vrf member management + +interface Ethernet1/1 + description "Link to Spine02" + no switchport + mtu 1442 + ip address 10.1.1.1/30 + ip ospf network point-to-point + ip router ospf 1 area 0.0.0.0 + no shutdown + +interface Ethernet1/2 + description "Link to Leaf01" + no switchport + mtu 1442 + ip address 10.1.1.5/30 + ip ospf network point-to-point + ip router ospf 1 area 0.0.0.0 + no shutdown + +interface Ethernet1/3 + description "Link to Leaf02" + no switchport + mtu 1442 + ip address 10.1.1.13/30 + ip ospf network point-to-point + ip router ospf 1 area 0.0.0.0 + no shutdown + +interface loopback0 + description "Router ID and VTEP Source" + ip address 10.255.255.1/32 + ip router ospf 1 area 0.0.0.0 + +router ospf 1 + router-id 10.255.255.1 + +router bgp 65001 + router-id 10.255.255.1 + address-family l2vpn evpn + neighbor 10.255.255.3 remote-as 65001 + description leaf01 + update-source loopback0 + address-family l2vpn evpn + send-community + send-community extended + route-reflector-client + neighbor 10.255.255.4 remote-as 65001 + description leaf02 + update-source loopback0 + address-family l2vpn evpn + send-community + send-community extended + route-reflector-client + +line console +line vty diff --git a/scenarios/networking-lab/devstack-nxsw-vxlan/spine01-poap.py b/scenarios/networking-lab/devstack-nxsw-vxlan/spine01-poap.py new file mode 100644 index 00000000..c6287fbb --- /dev/null +++ b/scenarios/networking-lab/devstack-nxsw-vxlan/spine01-poap.py @@ -0,0 +1,437 @@ +#!/bin/env python3 +# md5sum="b8d7ee75fee55dc65e32d30e50fcd332" + +# If any changes are made to this script, please run the below command +# in bash shell to update the above md5sum. This is used for integrity check. +# f=poap.py ; sed '/^# *md5sum/d' "$f" > "$f.md5" ; sed -i \ +# "s/^# *md5sum=.*/#md5sum=\"$(md5sum $f.md5 | sed 's/ .*//')\"/" $f + +# Protocol Authentication Support: +# - SCP: Always requires username and password +# - HTTP/HTTPS: Authentication optional (anonymous or username/password) +# - TFTP: No authentication support (anonymous only) +# +# Authentication parameters (username/password) are only required for SCP. +# For HTTP/HTTPS, if you provide username, you must also provide password. +# +# Example usage: +# - Anonymous HTTP: protocol="http", hostname="server.com" (no auth needed) +# - Authenticated HTTP: protocol="http", hostname="server.com", port=8080, username="user", password="pass" +# - Anonymous TFTP: protocol="tftp", hostname="server.com" (no auth supported) +# - Authenticated SCP: protocol="scp", hostname="server.com", username="user", password="pass" (auth required) + +import os +import re +import signal +import sys +import syslog +import traceback +import time + +try: + from cisco import cli +except ImportError: + from cli import * + +# Default configuration options +DEFAULT_OPTS = { + "hostname": "192.168.32.254", + "protocol": "tftp", + "cfg_path": "/spine01-poap.cfg", + "dest_path": "bootflash:poap.cfg", + "ignore_cert": True, + # username, password and port are optional - only needed for SCP or authenticated HTTP/HTTPS + # port is optional for custom ports on any protocol (e.g., SCP:2222, HTTP:8080, HTTPS:8088, TFTP:6969) +} + +# Valid configuration options +VALID_OPTS = { + "username", + "password", + "hostname", + "protocol", + "port", + "cfg_path", + "dest_path", + "ignore_cert", + "vrf", +} + +# Required configuration parameters (always required) +REQUIRED_OPTS = {"hostname"} + +# Logging prefix for syslog messages +SYSLOG_PREFIX = "POAPHandler" + + +def get_log_file_path(): + """Generate the log file path with timestamp and PID""" + return "/bootflash/%s_poap_%s_script.log" % ( + time.strftime("%Y%m%d%H%M%S", time.gmtime()), + os.environ["POAP_PID"], + ) + + +class POAPHandler: + """ + POAP (Power-On Auto Provisioning) handler for Cisco NXOS switches. + Handles configuration download and application for switch bootstrap. + """ + + def __init__(self): + """Initialize the POAP handler with default settings.""" + + # Set up signal handler + signal.signal(signal.SIGTERM, self.sigterm_handler) + + # Initialize logging + self.syslog_prefix = SYSLOG_PREFIX + self.log_file_handler = None # Will be set when using context manager + + self.opts = DEFAULT_OPTS.copy() + + # Validate required parameters + self._validate_required_opts() + + # Check that options are valid + self.validate_opts() + + @property + def dest_path(self): + """Normalized destination path without trailing slashes""" + return self.opts["dest_path"].rstrip("/") + + @property + def cfg_path(self): + """Normalized config path without trailing slashes""" + return self.opts["cfg_path"].rstrip("/") + + def _validate_required_opts(self): + """Validates that required options are provided""" + missing_params = REQUIRED_OPTS.difference(self.opts.keys()) + + if missing_params: + self._log("Required parameters are missing:") + self.abort("Missing %s" % ", ".join(missing_params)) + + # Protocol-specific validation for authentication parameters + protocol = self.opts.get("protocol", "http") + + # SCP always requires authentication + if protocol == "scp": + username = self.opts.get("username") + password = self.opts.get("password") + if not username or not password: + self.abort("SCP protocol requires both username and password") + + # HTTP/HTTPS with authentication requires both username and password + if protocol in ["http", "https"]: + username = self.opts.get("username") + password = self.opts.get("password") + if (username and not password) or (password and not username): + self.abort( + "HTTP/HTTPS authentication requires both username and password" + ) + + # Validate port if provided + port = self.opts.get("port") + if port is not None: + try: + port_int = int(port) + if not (1 <= port_int <= 65535): + self.abort("Port must be between 1 and 65535") + except (ValueError, TypeError): + self.abort("Port must be a valid integer") + + def validate_opts(self): + """ + Validates that the options provided by the user are valid. + Aborts the script if they are not. + """ + # Find any invalid options (ones not in VALID_OPTS) + invalid_opts = set(self.opts.keys()) - VALID_OPTS + if invalid_opts: + self._log( + "Invalid options detected: %s (check spelling, capitalization, and underscores)" + % ", ".join(invalid_opts) + ) + self.abort() + + def abort(self, msg=None): + """Aborts the POAP script + + :param msg: The message to log before aborting + """ + if msg: + self._log(msg) + + # Destination config + self.cleanup_file_from_option("dest_cfg") + + # Log file will be closed by context manager + exit(1) + + def _redact_passwords(self, message): + """Redacts passwords from log messages for security + + :param message: The log message to redact passwords from + :return: The message with passwords replaced with '' + """ + parts = re.split("\s+", message.strip()) + for index, part in enumerate(parts): + # blank out the password after the password keyword (terminal password *****, etc.) + if part == "password" and len(parts) >= index + 2: + parts[index + 1] = "" + + return " ".join(parts) + + def _log(self, info): + """ + Log the trace into console and poap_script log file in bootflash + + :param info: The information that needs to be logged. + """ + # Redact sensitive information before logging + info = self._redact_passwords(info) + + # Add syslog prefix + info = "%s - %s" % (self.syslog_prefix, info) + + syslog.syslog(9, info) + if self.log_file_handler is not None: + print(info, file=self.log_file_handler, flush=True) + + def remove_file(self, filename): + """Removes a file if it exists and it's not a directory. + + :param filename: The file to remove + """ + if os.path.isfile(filename): + try: + os.remove(filename) + except (IOError, OSError) as e: + self._log("Failed to remove %s: %s" % (filename, str(e))) + + def cleanup_file_from_option(self, option, bootflash_root=False): + """Removes a file indicated by the option in the POAP opts and removes it if it exists. + + Handle the cases where the variable is unused or not set yet. + + :param option: The option to remove + :param bootflash_root: Whether to remove the file from the bootflash root + """ + try: + filename = self.opts[option] + if filename is None: + return # Nothing to clean up + + if bootflash_root: + path = "/bootflash" + else: + path = self.dest_path + + self.remove_file(os.path.join(path, filename)) + self.remove_file(os.path.join(path, "%s.tmp" % filename)) + except KeyError: + # Option doesn't exist, nothing to clean up + pass + + def sigterm_handler(self, signum, stack): + """ + A signal handler for the SIGTERM signal. Cleans up and exits + + :param signum: The signal number + :param stack: The stack trace + """ + self.abort("SIGTERM signal received") + + def process_cfg_file(self): + """ + Processes the downloaded switch configuration file. + Copies the config to the scheduled config file for bootstrap replay. + """ + self._log("Processing Config file") + + # Copy config directly to scheduled-config + self._log("Command: copy %s scheduled-config" % self.opts["dest_path"]) + cli("copy %s scheduled-config" % self.opts["dest_path"]) + + self._log("Config processed and prepared for scheduled application") + + def _build_copy_cmd(self, source, dest): + """Build the copy command with all necessary options + + :param source: Source file path on remote server + :param dest: Destination path on local switch + :return: Complete copy command string + """ + # Extract parameters from opts + protocol = self.opts["protocol"] + host = self.opts["hostname"] + user = self.opts.get("username") + password = self.opts.get("password") + vrf = self.opts["vrf"] + ignore_ssl = self.opts["ignore_cert"] + port = self.opts.get("port") + # Build copy command with Cisco NX-OS terminal automation features + parts = [] + + # terminal dont-ask: Auto-answer "yes" to all confirmation prompts + parts.append("terminal dont-ask") + + # Determine if authentication is needed based on protocol and credentials + auth_needed = protocol in ["scp"] or ( + protocol in ["http", "https"] and user and password + ) + + if auth_needed: + if password: + # terminal password: Pre-store password for automatic authentication + # The password will be used automatically for any auth prompts during copy + parts.append("terminal password %s" % password) + + # Build the copy URL - only include user@ if authentication is needed + url = "%s://" % protocol + if auth_needed and user: + url += "%s@" % user + + # Add hostname with optional port + url += host + if port: + url += ":%s" % port + + # Ensure source path starts with / for proper URL construction + if not source.startswith("/"): + source = "/" + source + url += source + + # Build the copy command with URL and destination + cmd = "copy %s %s" % (url, dest) + + # Add ignore-certificate if needed + if protocol == "https" and ignore_ssl: + cmd += " ignore-certificate" + + # Add VRF + cmd += " vrf %s" % vrf + + # Add the complete copy command to the parts + parts.append(cmd) + + # Join all command parts with semicolon separator + return " ; ".join(parts) + + def copy(self, source, dest): + """Copies the files + + Copy the provided file from source to destination via network transfer. + + :param source: The source file to copy + :param dest: The destination file to copy + """ + self._log("Copying %s to %s" % (source, dest)) + + # Build the copy command using the dedicated method + cmd = self._build_copy_cmd(source, dest) + self._log("Copy command: %s" % cmd) + + try: + cli(cmd) + except Exception as e: + exc_type, exc_value, exc_tb = sys.exc_info() + traceback_str = "".join( + traceback.format_exception(exc_type, exc_value, exc_tb) + ) + self._log("Copy failed, Traceback: %s" % traceback_str) + self.abort("Copy failed: %s" % str(e)) + + self._log("Copy completed successfully") + + def get_currently_booted_image_filename(self): + match = None + try: + output = cli("show version") + except Exception as e: + self.abort("Show version failed: %s" % str(e)) + + match = re.search("NXOS image file is:\s+(.+)", output) + + if match: + directory, image = os.path.split(match.group(1)) + return image.strip() + + def install_nxos(self): + """Install the NXOS image on the switch + + Assume the current image is the one we want to install. + """ + boot_image = self.get_currently_booted_image_filename() + if not boot_image: + self.abort("No image found to install") + + # Build the install command + parts = [] + parts.append("terminal dont-ask ") + parts.append("install all nxos %s no-reload non-interruptive" % boot_image) + parts.append("terminal dont-ask") + parts.append("write erase") + cmd = " ; ".join(parts) + self._log("Install command: %s" % cmd) + + try: + cli(cmd) + time.sleep(5) + except Exception as e: + self._log("Failed to ISSU to image %s" % boot_image) + self.abort(str(e)) + + def run_with_logging(self): + """Execute the POAP provisioning process with proper log file management""" + log_file = get_log_file_path() + + with open(log_file, "w+") as log_file_handler: + self.log_file_handler = log_file_handler + self._log("Logfile name: %s" % log_file) + + try: + self.run() + finally: + # Ensure log_file_handler is reset when context exits + self.log_file_handler = None + + def run(self): + """Execute the POAP provisioning process""" + + # Set dynamic VRF value from environment + self.opts.setdefault("vrf", os.environ.get("POAP_VRF", "management")) + + # # create the directory structure needed, if any + # self.create_dest_dirs() + + # Copy config + self.copy(self.opts["cfg_path"], self.opts["dest_path"]) + self.process_cfg_file() + + # Install the NXOS image + self.install_nxos() + + +def main(): + """Main entry point for the POAP script""" + poap_handler = POAPHandler() + poap_handler.run_with_logging() + + +if __name__ == "__main__": + try: + main() + except Exception: + exc_type, exc_value, exc_tb = sys.exc_info() + # Create a temporary handler just for error logging + handler = POAPHandler() + + # Log the full traceback as a single formatted string + traceback_str = "".join(traceback.format_exception(exc_type, exc_value, exc_tb)) + handler._log("Exception occurred:\n%s" % traceback_str) + + handler.abort() diff --git a/scenarios/networking-lab/devstack-nxsw-vxlan/spine02-poap.cfg b/scenarios/networking-lab/devstack-nxsw-vxlan/spine02-poap.cfg new file mode 100644 index 00000000..1bb9968b --- /dev/null +++ b/scenarios/networking-lab/devstack-nxsw-vxlan/spine02-poap.cfg @@ -0,0 +1,86 @@ +hostname spine02.netlab.example.com +vdc spine02 id 1 + limit-resource vlan minimum 16 maximum 4094 + limit-resource vrf minimum 2 maximum 4096 + limit-resource port-channel minimum 0 maximum 511 + limit-resource m4route-mem minimum 58 maximum 58 + limit-resource m6route-mem minimum 8 maximum 8 + +feature netconf +feature lacp +feature ospf +feature bgp +feature nv overlay + +nv overlay evpn + +no password strength-check +username admin password 5 $5$LFGHKE$ND3U7npkwgMUzxLjjgDQxCx8JFJ5.ZlFyQTTt1vZgA5 role network-admin +ssh key rsa 2048 force + +system default switchport + +vrf context management + ip name-server 192.168.32.254 + ip route 0.0.0.0/0 192.168.32.1 + +interface mgmt0 + description "Management interface" + ip address dhcp + vrf member management + +interface Ethernet1/1 + description "Link to Spine01" + no switchport + mtu 1442 + ip address 10.1.1.2/30 + ip ospf network point-to-point + ip router ospf 1 area 0.0.0.0 + no shutdown + +interface Ethernet1/2 + description "Link to Leaf01" + no switchport + mtu 1442 + ip address 10.1.1.9/30 + ip ospf network point-to-point + ip router ospf 1 area 0.0.0.0 + no shutdown + +interface Ethernet1/3 + description "Link to Leaf02" + no switchport + mtu 1442 + ip address 10.1.1.17/30 + ip ospf network point-to-point + ip router ospf 1 area 0.0.0.0 + no shutdown + +interface loopback0 + description "Router ID and VTEP Source" + ip address 10.255.255.2/32 + ip router ospf 1 area 0.0.0.0 + +router ospf 1 + router-id 10.255.255.2 + +router bgp 65001 + router-id 10.255.255.2 + address-family l2vpn evpn + neighbor 10.255.255.3 remote-as 65001 + description leaf01 + update-source loopback0 + address-family l2vpn evpn + send-community + send-community extended + route-reflector-client + neighbor 10.255.255.4 remote-as 65001 + description leaf02 + update-source loopback0 + address-family l2vpn evpn + send-community + send-community extended + route-reflector-client + +line console +line vty diff --git a/scenarios/networking-lab/devstack-nxsw-vxlan/spine02-poap.py b/scenarios/networking-lab/devstack-nxsw-vxlan/spine02-poap.py new file mode 100644 index 00000000..f580327f --- /dev/null +++ b/scenarios/networking-lab/devstack-nxsw-vxlan/spine02-poap.py @@ -0,0 +1,437 @@ +#!/bin/env python3 +# md5sum="ecb6b3fe4fcb947a69cd567bcc2c238e" + +# If any changes are made to this script, please run the below command +# in bash shell to update the above md5sum. This is used for integrity check. +# f=poap.py ; sed '/^# *md5sum/d' "$f" > "$f.md5" ; sed -i \ +# "s/^# *md5sum=.*/#md5sum=\"$(md5sum $f.md5 | sed 's/ .*//')\"/" $f + +# Protocol Authentication Support: +# - SCP: Always requires username and password +# - HTTP/HTTPS: Authentication optional (anonymous or username/password) +# - TFTP: No authentication support (anonymous only) +# +# Authentication parameters (username/password) are only required for SCP. +# For HTTP/HTTPS, if you provide username, you must also provide password. +# +# Example usage: +# - Anonymous HTTP: protocol="http", hostname="server.com" (no auth needed) +# - Authenticated HTTP: protocol="http", hostname="server.com", port=8080, username="user", password="pass" +# - Anonymous TFTP: protocol="tftp", hostname="server.com" (no auth supported) +# - Authenticated SCP: protocol="scp", hostname="server.com", username="user", password="pass" (auth required) + +import os +import re +import signal +import sys +import syslog +import traceback +import time + +try: + from cisco import cli +except ImportError: + from cli import * + +# Default configuration options +DEFAULT_OPTS = { + "hostname": "192.168.32.254", + "protocol": "tftp", + "cfg_path": "/spine02-poap.cfg", + "dest_path": "bootflash:poap.cfg", + "ignore_cert": True, + # username, password and port are optional - only needed for SCP or authenticated HTTP/HTTPS + # port is optional for custom ports on any protocol (e.g., SCP:2222, HTTP:8080, HTTPS:8088, TFTP:6969) +} + +# Valid configuration options +VALID_OPTS = { + "username", + "password", + "hostname", + "protocol", + "port", + "cfg_path", + "dest_path", + "ignore_cert", + "vrf", +} + +# Required configuration parameters (always required) +REQUIRED_OPTS = {"hostname"} + +# Logging prefix for syslog messages +SYSLOG_PREFIX = "POAPHandler" + + +def get_log_file_path(): + """Generate the log file path with timestamp and PID""" + return "/bootflash/%s_poap_%s_script.log" % ( + time.strftime("%Y%m%d%H%M%S", time.gmtime()), + os.environ["POAP_PID"], + ) + + +class POAPHandler: + """ + POAP (Power-On Auto Provisioning) handler for Cisco NXOS switches. + Handles configuration download and application for switch bootstrap. + """ + + def __init__(self): + """Initialize the POAP handler with default settings.""" + + # Set up signal handler + signal.signal(signal.SIGTERM, self.sigterm_handler) + + # Initialize logging + self.syslog_prefix = SYSLOG_PREFIX + self.log_file_handler = None # Will be set when using context manager + + self.opts = DEFAULT_OPTS.copy() + + # Validate required parameters + self._validate_required_opts() + + # Check that options are valid + self.validate_opts() + + @property + def dest_path(self): + """Normalized destination path without trailing slashes""" + return self.opts["dest_path"].rstrip("/") + + @property + def cfg_path(self): + """Normalized config path without trailing slashes""" + return self.opts["cfg_path"].rstrip("/") + + def _validate_required_opts(self): + """Validates that required options are provided""" + missing_params = REQUIRED_OPTS.difference(self.opts.keys()) + + if missing_params: + self._log("Required parameters are missing:") + self.abort("Missing %s" % ", ".join(missing_params)) + + # Protocol-specific validation for authentication parameters + protocol = self.opts.get("protocol", "http") + + # SCP always requires authentication + if protocol == "scp": + username = self.opts.get("username") + password = self.opts.get("password") + if not username or not password: + self.abort("SCP protocol requires both username and password") + + # HTTP/HTTPS with authentication requires both username and password + if protocol in ["http", "https"]: + username = self.opts.get("username") + password = self.opts.get("password") + if (username and not password) or (password and not username): + self.abort( + "HTTP/HTTPS authentication requires both username and password" + ) + + # Validate port if provided + port = self.opts.get("port") + if port is not None: + try: + port_int = int(port) + if not (1 <= port_int <= 65535): + self.abort("Port must be between 1 and 65535") + except (ValueError, TypeError): + self.abort("Port must be a valid integer") + + def validate_opts(self): + """ + Validates that the options provided by the user are valid. + Aborts the script if they are not. + """ + # Find any invalid options (ones not in VALID_OPTS) + invalid_opts = set(self.opts.keys()) - VALID_OPTS + if invalid_opts: + self._log( + "Invalid options detected: %s (check spelling, capitalization, and underscores)" + % ", ".join(invalid_opts) + ) + self.abort() + + def abort(self, msg=None): + """Aborts the POAP script + + :param msg: The message to log before aborting + """ + if msg: + self._log(msg) + + # Destination config + self.cleanup_file_from_option("dest_cfg") + + # Log file will be closed by context manager + exit(1) + + def _redact_passwords(self, message): + """Redacts passwords from log messages for security + + :param message: The log message to redact passwords from + :return: The message with passwords replaced with '' + """ + parts = re.split("\s+", message.strip()) + for index, part in enumerate(parts): + # blank out the password after the password keyword (terminal password *****, etc.) + if part == "password" and len(parts) >= index + 2: + parts[index + 1] = "" + + return " ".join(parts) + + def _log(self, info): + """ + Log the trace into console and poap_script log file in bootflash + + :param info: The information that needs to be logged. + """ + # Redact sensitive information before logging + info = self._redact_passwords(info) + + # Add syslog prefix + info = "%s - %s" % (self.syslog_prefix, info) + + syslog.syslog(9, info) + if self.log_file_handler is not None: + print(info, file=self.log_file_handler, flush=True) + + def remove_file(self, filename): + """Removes a file if it exists and it's not a directory. + + :param filename: The file to remove + """ + if os.path.isfile(filename): + try: + os.remove(filename) + except (IOError, OSError) as e: + self._log("Failed to remove %s: %s" % (filename, str(e))) + + def cleanup_file_from_option(self, option, bootflash_root=False): + """Removes a file indicated by the option in the POAP opts and removes it if it exists. + + Handle the cases where the variable is unused or not set yet. + + :param option: The option to remove + :param bootflash_root: Whether to remove the file from the bootflash root + """ + try: + filename = self.opts[option] + if filename is None: + return # Nothing to clean up + + if bootflash_root: + path = "/bootflash" + else: + path = self.dest_path + + self.remove_file(os.path.join(path, filename)) + self.remove_file(os.path.join(path, "%s.tmp" % filename)) + except KeyError: + # Option doesn't exist, nothing to clean up + pass + + def sigterm_handler(self, signum, stack): + """ + A signal handler for the SIGTERM signal. Cleans up and exits + + :param signum: The signal number + :param stack: The stack trace + """ + self.abort("SIGTERM signal received") + + def process_cfg_file(self): + """ + Processes the downloaded switch configuration file. + Copies the config to the scheduled config file for bootstrap replay. + """ + self._log("Processing Config file") + + # Copy config directly to scheduled-config + self._log("Command: copy %s scheduled-config" % self.opts["dest_path"]) + cli("copy %s scheduled-config" % self.opts["dest_path"]) + + self._log("Config processed and prepared for scheduled application") + + def _build_copy_cmd(self, source, dest): + """Build the copy command with all necessary options + + :param source: Source file path on remote server + :param dest: Destination path on local switch + :return: Complete copy command string + """ + # Extract parameters from opts + protocol = self.opts["protocol"] + host = self.opts["hostname"] + user = self.opts.get("username") + password = self.opts.get("password") + vrf = self.opts["vrf"] + ignore_ssl = self.opts["ignore_cert"] + port = self.opts.get("port") + # Build copy command with Cisco NX-OS terminal automation features + parts = [] + + # terminal dont-ask: Auto-answer "yes" to all confirmation prompts + parts.append("terminal dont-ask") + + # Determine if authentication is needed based on protocol and credentials + auth_needed = protocol in ["scp"] or ( + protocol in ["http", "https"] and user and password + ) + + if auth_needed: + if password: + # terminal password: Pre-store password for automatic authentication + # The password will be used automatically for any auth prompts during copy + parts.append("terminal password %s" % password) + + # Build the copy URL - only include user@ if authentication is needed + url = "%s://" % protocol + if auth_needed and user: + url += "%s@" % user + + # Add hostname with optional port + url += host + if port: + url += ":%s" % port + + # Ensure source path starts with / for proper URL construction + if not source.startswith("/"): + source = "/" + source + url += source + + # Build the copy command with URL and destination + cmd = "copy %s %s" % (url, dest) + + # Add ignore-certificate if needed + if protocol == "https" and ignore_ssl: + cmd += " ignore-certificate" + + # Add VRF + cmd += " vrf %s" % vrf + + # Add the complete copy command to the parts + parts.append(cmd) + + # Join all command parts with semicolon separator + return " ; ".join(parts) + + def copy(self, source, dest): + """Copies the files + + Copy the provided file from source to destination via network transfer. + + :param source: The source file to copy + :param dest: The destination file to copy + """ + self._log("Copying %s to %s" % (source, dest)) + + # Build the copy command using the dedicated method + cmd = self._build_copy_cmd(source, dest) + self._log("Copy command: %s" % cmd) + + try: + cli(cmd) + except Exception as e: + exc_type, exc_value, exc_tb = sys.exc_info() + traceback_str = "".join( + traceback.format_exception(exc_type, exc_value, exc_tb) + ) + self._log("Copy failed, Traceback: %s" % traceback_str) + self.abort("Copy failed: %s" % str(e)) + + self._log("Copy completed successfully") + + def get_currently_booted_image_filename(self): + match = None + try: + output = cli("show version") + except Exception as e: + self.abort("Show version failed: %s" % str(e)) + + match = re.search("NXOS image file is:\s+(.+)", output) + + if match: + directory, image = os.path.split(match.group(1)) + return image.strip() + + def install_nxos(self): + """Install the NXOS image on the switch + + Assume the current image is the one we want to install. + """ + boot_image = self.get_currently_booted_image_filename() + if not boot_image: + self.abort("No image found to install") + + # Build the install command + parts = [] + parts.append("terminal dont-ask ") + parts.append("install all nxos %s no-reload non-interruptive" % boot_image) + parts.append("terminal dont-ask") + parts.append("write erase") + cmd = " ; ".join(parts) + self._log("Install command: %s" % cmd) + + try: + cli(cmd) + time.sleep(5) + except Exception as e: + self._log("Failed to ISSU to image %s" % boot_image) + self.abort(str(e)) + + def run_with_logging(self): + """Execute the POAP provisioning process with proper log file management""" + log_file = get_log_file_path() + + with open(log_file, "w+") as log_file_handler: + self.log_file_handler = log_file_handler + self._log("Logfile name: %s" % log_file) + + try: + self.run() + finally: + # Ensure log_file_handler is reset when context exits + self.log_file_handler = None + + def run(self): + """Execute the POAP provisioning process""" + + # Set dynamic VRF value from environment + self.opts.setdefault("vrf", os.environ.get("POAP_VRF", "management")) + + # # create the directory structure needed, if any + # self.create_dest_dirs() + + # Copy config + self.copy(self.opts["cfg_path"], self.opts["dest_path"]) + self.process_cfg_file() + + # Install the NXOS image + self.install_nxos() + + +def main(): + """Main entry point for the POAP script""" + poap_handler = POAPHandler() + poap_handler.run_with_logging() + + +if __name__ == "__main__": + try: + main() + except Exception: + exc_type, exc_value, exc_tb = sys.exc_info() + # Create a temporary handler just for error logging + handler = POAPHandler() + + # Log the full traceback as a single formatted string + traceback_str = "".join(traceback.format_exception(exc_type, exc_value, exc_tb)) + handler._log("Exception occurred:\n%s" % traceback_str) + + handler.abort() diff --git a/scenarios/networking-lab/devstack-nxsw-vxlan/topology-diagram.svg b/scenarios/networking-lab/devstack-nxsw-vxlan/topology-diagram.svg new file mode 100644 index 00000000..30619b75 --- /dev/null +++ b/scenarios/networking-lab/devstack-nxsw-vxlan/topology-diagram.svg @@ -0,0 +1,132 @@ + + + + + + + + + Spine-and-Leaf Topology (BGP AS 65001) + + + + + Management Network (192.168.32.0/24) + + + + controller + 192.168.32.254 + + + + + spine01 + (RR) + 10.255.255.1 + + + + spine02 + (RR) + 10.255.255.2 + + + + 10.1.1.0/30 + + + + + leaf01 + (RRC, NVE1) + 10.255.255.3 + + + + leaf02 + (RRC, NVE1) + 10.255.255.4 + + + + 10.1.1.4/30 + + + + 10.1.1.12/30 + + + + 10.1.1.8/30 + + + + 10.1.1.16/30 + + + + + devstack + 192.168.32.20 + + + + ironic0 + BM Node + + + + ironic1 + BM Node + + + + + Eth1/5 + + + + Eth1/4 + + + + Eth1/4 + + + + Legend + + Switch + + Server + + Controller + + P2P Link + + Management + + BGP EVPN + + + + + BGP EVPN: AS 65001 iBGP | RR=Route Reflector | RRC=Route Reflector Client | NVE1=VXLAN VTEP + + + Leaf Ethernet1/3: Trunk for VLANs 100,103-105 | Eth1/4: Ironic | Eth1/5: Devstack (Leaf01 only) + +