diff --git a/tests/network/bgp/evpn/conftest.py b/tests/network/bgp/evpn/conftest.py index 680ecfa655..250aee4daf 100644 --- a/tests/network/bgp/evpn/conftest.py +++ b/tests/network/bgp/evpn/conftest.py @@ -21,9 +21,11 @@ cudn_evpn_subnets, deploy_evpn_bridge, deploy_evpn_l2_endpoint, + deploy_evpn_l3_endpoint, evpn_workloads_active_connections, teardown_evpn_bridge, teardown_evpn_l2_endpoint, + teardown_evpn_l3_endpoint, ) from tests.network.libs import cluster_user_defined_network as libcudn from tests.network.libs.bgp import ( @@ -41,6 +43,10 @@ CUDN_EVPN_BGP_LABEL: Final[dict] = {"cudn-bgp": "evpn"} EXTERNAL_L2_ENDPOINT_IPV4: Final[str] = f"{random_ipv4_address(net_seed=5, host_address=250)}/24" EXTERNAL_L2_ENDPOINT_IPV6: Final[str] = f"{random_ipv6_address(net_seed=5, host_address=250)}/64" +EXTERNAL_L3_ENDPOINT_IPV4: Final[str] = "192.168.100.100/24" +EXTERNAL_L3_ENDPOINT_IPV6: Final[str] = "fd01:1234:5678::64/64" +EXTERNAL_L3_GATEWAY_IPV4: Final[str] = "192.168.100.1/24" +EXTERNAL_L3_GATEWAY_IPV6: Final[str] = "fd01:1234:5678::1/64" EVPN_MAC_VRF_VNI: Final[int] = 10100 EVPN_IP_VRF_VNI: Final[int] = 20102 @@ -232,6 +238,21 @@ def external_l2_endpoint( teardown_evpn_l2_endpoint(pod=frr_external_pod.pod) +@pytest.fixture(scope="module") +def external_l3_endpoint( + evpn_bridge: None, + frr_external_pod: ExternalFrrPodInfo, +) -> Generator[EvpnEndpoint]: + endpoint = deploy_evpn_l3_endpoint( + pod=frr_external_pod.pod, + vni=EVPN_IP_VRF_VNI, + endpoint_ips=[EXTERNAL_L3_ENDPOINT_IPV4, EXTERNAL_L3_ENDPOINT_IPV6], + gateway_ips=[EXTERNAL_L3_GATEWAY_IPV4, EXTERNAL_L3_GATEWAY_IPV6], + ) + yield endpoint + teardown_evpn_l3_endpoint(pod=frr_external_pod.pod) + + @pytest.fixture() def evpn_stretched_l2_active_connections( external_l2_endpoint: EvpnEndpoint, @@ -239,3 +260,12 @@ def evpn_stretched_l2_active_connections( ) -> Generator[list[tuple[EndpointTcpClient, TcpServer]]]: with evpn_workloads_active_connections(endpoint=external_l2_endpoint, vm=vm_evpn_target) as connections: yield connections + + +@pytest.fixture() +def evpn_routed_l3_active_connections( + external_l3_endpoint: EvpnEndpoint, + vm_evpn_target: BaseVirtualMachine, +) -> Generator[list[tuple[EndpointTcpClient, TcpServer]]]: + with evpn_workloads_active_connections(endpoint=external_l3_endpoint, vm=vm_evpn_target) as connections: + yield connections diff --git a/tests/network/bgp/evpn/libevpn.py b/tests/network/bgp/evpn/libevpn.py index 575a5bdcce..44090cb091 100644 --- a/tests/network/bgp/evpn/libevpn.py +++ b/tests/network/bgp/evpn/libevpn.py @@ -5,13 +5,14 @@ from dataclasses import dataclass from ocp_resources.pod import Pod +from timeout_sampler import retry from libs.net.cluster import ipv4_supported_cluster, ipv6_supported_cluster from libs.net.ip import filter_link_local_addresses, random_ipv4_address, random_ipv6_address from libs.net.traffic_generator import IPERF_SERVER_PORT, PodTcpClient, TcpServer from libs.net.vmspec import lookup_iface_status, lookup_primary_network from libs.vm.vm import BaseVirtualMachine -from tests.network.libs.bgp import NET_TOOLS_CONTAINER_NAME +from tests.network.libs.bgp import CLUSTER_FRR_ASN, EXTERNAL_FRR_ASN, NET_TOOLS_CONTAINER_NAME LOGGER = logging.getLogger(__name__) @@ -27,6 +28,13 @@ _L2_VETH_POD_SIDE: str = "veth-l2-frr" _L2_VETH_EP_SIDE: str = "veth-l2-ep" +_L3_VID: int = 200 +_L3_VRF_NAME: str = "vrf-blue" +_L3_SVI_NAME: str = f"{_BRIDGE_NAME}.{_L3_VID}" +_L3_ENDPOINT_NETNS: str = "l3-ep" +_L3_VETH_POD_SIDE: str = "veth-l3-frr" +_L3_VETH_EP_SIDE: str = "veth-l3-ep" + @dataclass class EvpnEndpoint: @@ -182,6 +190,129 @@ def _build_l2_endpoint_commands(vni: int, endpoint_ips: list[str]) -> list[str]: ] +def deploy_evpn_l3_endpoint( + pod: Pod, + vni: int, + endpoint_ips: list[str], + gateway_ips: list[str], +) -> EvpnEndpoint: + """Creates a routed L3 endpoint on the external FRR pod. + + Deploys Linux infra (VRF, SVI, VLAN/VNI, veth, netns), configures + external FRR BGP VRF for Type-5 routes, and waits for OVN-K routes. + + Data path: VM -> OVN L3 lookup -> VXLAN (IP-VRF VNI) -> vxlan0 -> br0 -> SVI -> VRF -> veth -> l3-ep. + + Args: + pod: The external FRR pod. + vni: IP-VRF VNI (must match UDN's ipVRF VNI). + endpoint_ips: IPs with prefix on a different subnet than CUDN (e.g. ["192.168.100.100/24"]). + gateway_ips: Gateway IPs with prefix for the VRF veth side (e.g. ["192.168.100.1/24"]). + + Returns: + EvpnEndpoint. + """ + commands = _build_l3_endpoint_commands(vni=vni, endpoint_ips=endpoint_ips, gateway_ips=gateway_ips) + for command in commands: + pod.execute(command=shlex.split(command), container=NET_TOOLS_CONTAINER_NAME) + + _configure_external_frr_l3_vrf(pod=pod, vni=vni) + + bare_ips = [ip.split("/")[0] for ip in endpoint_ips] + LOGGER.info(f"EVPN L3 endpoint deployed: {bare_ips} in namespace {_L3_ENDPOINT_NETNS}") + + return EvpnEndpoint(pod=pod, ip_addresses=bare_ips, netns_name=_L3_ENDPOINT_NETNS) + + +def teardown_evpn_l3_endpoint(pod: Pod) -> None: + """Removes the EVPN L3 endpoint and VRF from the external FRR pod.""" + for cmd in [ + f"ip netns delete {_L3_ENDPOINT_NETNS}", + f"ip link delete {_L3_VETH_POD_SIDE}", + f"ip link delete {_L3_SVI_NAME}", + f"ip link delete {_L3_VRF_NAME}", + ]: + pod.execute(command=shlex.split(cmd), container=NET_TOOLS_CONTAINER_NAME, ignore_rc=True) + + pod.execute( + command=["vtysh", "-c", "configure terminal", "-c", f"no router bgp {EXTERNAL_FRR_ASN} vrf {_L3_VRF_NAME}"], + ignore_rc=True, + ) + + LOGGER.info(f"EVPN L3 endpoint removed: namespace={_L3_ENDPOINT_NETNS}, VRF={_L3_VRF_NAME}") + + +def _build_l3_endpoint_commands( + vni: int, + endpoint_ips: list[str], + gateway_ips: list[str], +) -> list[str]: + return [ + "sysctl -w net.ipv4.ip_forward=1", + "sysctl -w net.ipv6.conf.all.forwarding=1", + f"ip link add {_L3_VRF_NAME} type vrf table {vni}", + f"ip link set {_L3_VRF_NAME} up", + f"bridge vlan add dev {_BRIDGE_NAME} vid {_L3_VID} self", + f"bridge vlan add dev {_VXLAN_NAME} vid {_L3_VID}", + f"bridge vni add dev {_VXLAN_NAME} vni {vni}", + f"bridge vlan add dev {_VXLAN_NAME} vid {_L3_VID} tunnel_info id {vni}", + f"ip link add {_L3_SVI_NAME} link {_BRIDGE_NAME} type vlan id {_L3_VID}", + f"ip link set {_L3_SVI_NAME} master {_L3_VRF_NAME}", + f"ip link set {_L3_SVI_NAME} up", + f"ip link add {_L3_VETH_POD_SIDE} type veth peer name {_L3_VETH_EP_SIDE}", + f"ip link set {_L3_VETH_POD_SIDE} master {_L3_VRF_NAME}", + *(f"ip addr add {ip} dev {_L3_VETH_POD_SIDE}" for ip in gateway_ips), + f"ip link set {_L3_VETH_POD_SIDE} up", + f"ip netns add {_L3_ENDPOINT_NETNS}", + f"ip link set {_L3_VETH_EP_SIDE} netns {_L3_ENDPOINT_NETNS}", + *(f"ip netns exec {_L3_ENDPOINT_NETNS} ip addr add {ip} dev {_L3_VETH_EP_SIDE}" for ip in endpoint_ips), + f"ip netns exec {_L3_ENDPOINT_NETNS} ip link set {_L3_VETH_EP_SIDE} up", + f"ip netns exec {_L3_ENDPOINT_NETNS} ip link set lo up", + *( + f"ip netns exec {_L3_ENDPOINT_NETNS} ip {'-6' if ':' in ip else ''} route add default" + f" via {ip.split('/')[0]}" + for ip in gateway_ips + ), + ] + + +def _configure_external_frr_l3_vrf(pod: Pod, vni: int) -> None: + config = "\n".join([ + f"vrf {_L3_VRF_NAME}", + f" vni {vni}", + "exit-vrf", + f"router bgp {EXTERNAL_FRR_ASN} vrf {_L3_VRF_NAME}", + " address-family ipv4 unicast", + " redistribute connected", + " exit-address-family", + " address-family ipv6 unicast", + " redistribute connected", + " exit-address-family", + " address-family l2vpn evpn", + f" rd {EXTERNAL_FRR_ASN}:{vni}", + f" route-target import {CLUSTER_FRR_ASN}:{vni}", + f" route-target export {CLUSTER_FRR_ASN}:{vni}", + " advertise ipv4 unicast", + " advertise ipv6 unicast", + " exit-address-family", + ]) + pod.execute(command=["vtysh", "-c", "configure terminal", "-c", config]) + _wait_for_l3_vrf_routes(pod=pod) + + LOGGER.info(f"External FRR L3 VRF configured: {_L3_VRF_NAME} VNI {vni}") + + +@retry(wait_timeout=60, sleep=5, exceptions_dict={RuntimeError: []}) +def _wait_for_l3_vrf_routes(pod: Pod) -> bool: + output = pod.execute( + command=shlex.split(f"ip route show vrf {_L3_VRF_NAME} proto bgp"), + container=NET_TOOLS_CONTAINER_NAME, + ) + if not output.strip(): + raise RuntimeError(f"VRF {_L3_VRF_NAME} has no BGP routes") + return True + + @contextlib.contextmanager def evpn_workloads_active_connections( endpoint: EvpnEndpoint, diff --git a/tests/network/bgp/evpn/test_evpn_connectivity.py b/tests/network/bgp/evpn/test_evpn_connectivity.py index b719aef902..c1f233dbc6 100644 --- a/tests/network/bgp/evpn/test_evpn_connectivity.py +++ b/tests/network/bgp/evpn/test_evpn_connectivity.py @@ -100,7 +100,7 @@ def test_stretched_l2_connectivity_is_preserved_over_live_migration( @pytest.mark.polarion("CNV-15230") -def test_routed_l3_connectivity_udn_vm_and_external_provider(): +def test_routed_l3_connectivity_udn_vm_and_external_provider(external_l3_endpoint, vm_evpn_target, subtests): """ Preconditions: - External Source Provider L3 endpoint. @@ -112,13 +112,18 @@ def test_routed_l3_connectivity_udn_vm_and_external_provider(): Expected: - The VM successfully communicates with the external L3 endpoint. """ - - -test_routed_l3_connectivity_udn_vm_and_external_provider.__test__ = False + with evpn_workloads_active_connections(endpoint=external_l3_endpoint, vm=vm_evpn_target) as connections: + for client, server in connections: + with subtests.test(f"IPv{ipaddress.ip_address(client.server_ip).version}"): + assert is_tcp_connection(server=server, client=client) @pytest.mark.polarion("CNV-15231") -def test_routed_l3_connectivity_is_preserved_over_live_migration(): +def test_routed_l3_connectivity_is_preserved_over_live_migration( + evpn_routed_l3_active_connections, + vm_evpn_target, + subtests, +): """ Preconditions: - External Source Provider L3 endpoint. @@ -126,14 +131,15 @@ def test_routed_l3_connectivity_is_preserved_over_live_migration(): - Established TCP connectivity between the target under-test VM and the external L3 endpoint. Steps: - 1. Live-migrate UDN VM and wait for completion. + 1. Live-migrate the target under-test VM and wait for completion. Expected: - The initial TCP connection is preserved (no disconnection). """ - - -test_routed_l3_connectivity_is_preserved_over_live_migration.__test__ = False + migrate_vm_and_verify(vm=vm_evpn_target) + for client, server in evpn_routed_l3_active_connections: + with subtests.test(f"IPv{ipaddress.ip_address(client.server_ip).version}"): + assert is_tcp_connection(server=server, client=client) @pytest.mark.polarion("CNV-15232") diff --git a/tests/network/libs/bgp.py b/tests/network/libs/bgp.py index de72b6999b..518f3f0bf5 100644 --- a/tests/network/libs/bgp.py +++ b/tests/network/libs/bgp.py @@ -21,10 +21,10 @@ from utilities.constants import NET_UTIL_CONTAINER_IMAGE, NamespacesNames from utilities.infra import get_resources_by_name_prefix -_CLUSTER_FRR_ASN: Final[int] = 64512 -_EXTERNAL_FRR_ASN: Final[int] = 64000 _EXTERNAL_FRR_IMAGE: Final[str] = "quay.io/frrouting/frr:10.6.0" _FRR_DEPLOYMENT_NAME: Final[str] = "frr-k8s-statuscleaner" +CLUSTER_FRR_ASN: Final[int] = 64512 +EXTERNAL_FRR_ASN: Final[int] = 64000 POD_SECONDARY_IFACE_NAME: Final[str] = "net1" NET_TOOLS_CONTAINER_NAME: Final[str] = "net-tools" EXTERNAL_FRR_POD_LABEL: Final[dict] = {"role": "frr-external"} @@ -130,11 +130,11 @@ def create_frr_configuration( bgp_config = { "routers": [ { - "asn": _CLUSTER_FRR_ASN, + "asn": CLUSTER_FRR_ASN, "neighbors": [ { "address": frr_pod_ipv4, - "asn": _EXTERNAL_FRR_ASN, + "asn": EXTERNAL_FRR_ASN, "disableMP": True, "toReceive": {"allowed": {"mode": "filtered", "prefixes": [{"prefix": external_subnet_ipv4}]}}, } @@ -163,11 +163,11 @@ def create_evpn_frr_configuration( bgp_config = { "routers": [ { - "asn": _CLUSTER_FRR_ASN, + "asn": CLUSTER_FRR_ASN, "neighbors": [ { "address": frr_pod_ipv4, - "asn": _EXTERNAL_FRR_ASN, + "asn": EXTERNAL_FRR_ASN, } ], } @@ -199,7 +199,7 @@ def generate_frr_conf( # Route-map: strips cluster ASN from AS_PATH for eBGP EVPN re-advertisement lines = [ f"route-map {evpn_route_map} permit 10", - f" set as-path exclude {_CLUSTER_FRR_ASN}", + f" set as-path exclude {CLUSTER_FRR_ASN}", " set ip next-hop unchanged", "exit", "", @@ -207,13 +207,13 @@ def generate_frr_conf( # BGP router and neighbor definitions lines.extend([ - f"router bgp {_EXTERNAL_FRR_ASN}", + f"router bgp {EXTERNAL_FRR_ASN}", " no bgp ebgp-requires-policy", " no bgp default ipv4-unicast", " no bgp network import-check", "", ]) - lines.extend([f" neighbor {ip} remote-as {_CLUSTER_FRR_ASN}" for ip in nodes_ipv4_list]) + lines.extend([f" neighbor {ip} remote-as {CLUSTER_FRR_ASN}" for ip in nodes_ipv4_list]) lines.append("") # IPv4 unicast: advertise external subnet, activate neighbors