IPoE Subscriber is being stuck in terminated state after monkey test #330
Description
Describe the bug
IPoE Subscriber is being stuck in terminated state after monkey test in HA A/A Scenario. Out of 24 subscribers one subscriber is in Terminating state.Issue is very inconsistent. Not seen always.But After 3-4 retries it occurs.
To Reproduce
Version (bngblaster -v):
supervisor@SN-STD-27-2810:~ $ bngblaster -v
Version: 0.9.23
Compiler: GNU (11.4.0)
IO Modes: packet_mmap_raw (default), packet_mmap, raw
JSON configuration:
{
"interfaces": {
"tx-interval": 1,
"rx-interval": 1,
"lag": [
{
"interface": "lag1",
"lacp": true,
"lacp-timeout-short": true
},
{
"interface": "lag11",
"lacp": true,
"lacp-timeout-short": true
}
],
"links": [
{
"interface": "SN-5-C1",
"lag-interface": "lag1"
},
{
"interface": "SN-6-C1",
"lag-interface": "lag1"
},
{
"interface": "SN-7-C1",
"lag-interface": "lag1"
},
{
"interface": "SN-11-C2",
"lag-interface": "lag1"
},
{
"interface": "SN-12-C2",
"lag-interface": "lag1"
},
{
"interface": "SN-13-C2",
"lag-interface": "lag1"
},
{
"interface": "SN-20-C1",
"lag-interface": "lag11"
},
{
"interface": "SN-21-C1",
"lag-interface": "lag11"
},
{
"interface": "SN-22-C1",
"lag-interface": "lag11"
},
{
"interface": "SN-23-C2",
"lag-interface": "lag11"
},
{
"interface": "SN-24-C2",
"lag-interface": "lag11"
},
{
"interface": "SN-25-C2",
"lag-interface": "lag11"
}
],
"network": [
{
"interface": "SN-19-RR",
"address": "131.0.0.2/24",
"gateway": "131.0.0.1",
"address-ipv6": "fc66:1337:7331::2",
"gateway-ipv6": "fc66:1337:7331::1",
"isis-instance-id": 1,
"isis-level": 1,
"ldp-instance-id": 1
}
],
"access": [
{
"interface": "lag1",
"type": "ipoe",
"agent-remote-id": "CBNG1.RTBRICK.{session-global}",
"outer-vlan-min": 1001,
"outer-vlan-max": 2000,
"inner-vlan-min": 1001,
"inner-vlan-max": 1100,
"monkey": true
},
{
"interface": "lag11",
"type": "ipoe",
"agent-remote-id": "CBNG2.RTBRICK.{session-global}",
"outer-vlan-min": 2001,
"outer-vlan-max": 3000,
"inner-vlan-min": 1001,
"inner-vlan-max": 1100,
"monkey": true
}
]
},
"sessions": {
"count": 24,
"max-outstanding": 800,
"start-rate": 400,
"stop-rate": 400,
"monkey-autostart": false,
"reconnect": true
},
"access-line": {
"agent-remote-id": "DEU.RTBRICK.{session-global}",
"agent-circuit-id": "0.0.0.0/0.0.0.0 eth 0:{session-global}",
"rate-up": 2000,
"rate-down": 16384,
"dsl-type": 5
},
"dhcp": {
"enable": true
},
"dhcpv6": {
"ldra": true,
"enable": true
},
"session-traffic": {
"ipv4-pps": 1,
"ipv6-pps": 1,
"ipv6pd-pps": 1,
"autostart": false
},
"ldp": [
{
"instance-id": 1,
"lsr-id": "192.1.0.6",
"keepalive-time": 30,
"raw-update-file": "/home/supervisor/ldp_1000.ldp"
}
],
"isis": [
{
"instance-id": 1,
"area": [
"49.0002/24"
],
"system-id": "0204.0000.0001",
"router-id": "192.1.0.6",
"hostname": "BBL-RR",
"hello-padding": true,
"teardown-time": 30,
"external": {
"mrt-file": "/home/supervisor/isis_1000.mrt",
"connections": [
{
"system-id": "0300.1001.0001",
"l1-metric": 10
}
]
}
}
],
"bgp": [
{
"__comment__": "RR1-C1-IPv4",
"network-interface": "SN-19-RR",
"local-ipv4-address": "192.168.0.10",
"peer-ipv4-address": "192.1.0.1",
"local-as": 4200000001,
"peer-as": 4200000001
},
{
"__comment__": "RR1-C1-IPv6",
"network-interface": "SN-19-RR",
"local-ipv4-address": "192.168.0.20",
"peer-ipv4-address": "192.1.0.1",
"local-as": 4200000001,
"peer-as": 4200000001
},
{
"__comment__": "RR1-C2-IPv4",
"network-interface": "SN-19-RR",
"local-ipv4-address": "192.168.0.10",
"peer-ipv4-address": "192.1.0.2",
"local-as": 4200000001,
"peer-as": 4200000001
},
{
"__comment__": "RR1-C2-IPv6",
"network-interface": "SN-19-RR",
"local-ipv4-address": "192.168.0.20",
"peer-ipv4-address": "192.1.0.2",
"local-as": 4200000001,
"peer-as": 4200000001
},
{
"__comment__": "RR2-C1-IPv4",
"network-interface": "SN-19-RR",
"local-ipv4-address": "192.168.0.30",
"peer-ipv4-address": "192.1.0.1",
"local-as": 4200000001,
"peer-as": 4200000001
},
{
"__comment__": "RR2-C1-IPv6",
"network-interface": "SN-19-RR",
"local-ipv4-address": "192.168.0.40",
"peer-ipv4-address": "192.1.0.1",
"local-as": 4200000001,
"peer-as": 4200000001
},
{
"__comment__": "RR2-C2-IPv4",
"network-interface": "SN-19-RR",
"local-ipv4-address": "192.168.0.30",
"peer-ipv4-address": "192.1.0.2",
"local-as": 4200000001,
"peer-as": 4200000001
},
{
"__comment__": "RR2-C2-IPv6",
"network-interface": "SN-19-RR",
"local-ipv4-address": "192.168.0.40",
"peer-ipv4-address": "192.1.0.2",
"local-as": 4200000001,
"peer-as": 4200000001
}
]
}
Steps to reproduce the behavior:
- Start Blaster and ensure subscribers are established.
- Start the monkey test.
- Stop the monkey test.
- Stop the sessions using session-stop command.
- Start the sessions using session-start command.
- One Subscriber is not established state.
Expected behavior
A clear and concise description of what you expected to happen.
Subscriber should be in Established state.
If applicable, add screenshots to help explain your problem.
Additional context
supervisor@SN-STD-27-2810:~ $ sudo bngblaster-cli /var/bngblaster/bng_instance_1/run.sock session-info session-id 22
[sudo] password for supervisor:
{
"status": "ok",
"code": 200,
"session-info": {
"type": "ipoe",
"session-id": 22,
"session-state": "Terminated",
"session-substate": "DHCPv4 pending",
"session-version": 8,
"flapped": 1,
"interface": "lag11",
"outer-vlan": 2001,
"inner-vlan": 1011,
"mac": "02:00:00:00:00:16",
"agent-circuit-id": "0.0.0.0/0.0.0.0 eth 0:22",
"agent-remote-id": "CBNG2.RTBRICK.22",
"ipv4-address": "16.100.128.185",
"ipv4-netmask": "255.255.255.255",
"ipv4-gateway": "192.168.0.100",
"ipv4-dns1": "10.0.0.5",
"ipv4-dns2": "10.0.0.6",
"dhcp-state": "Init",
"dhcp-server": "192.168.0.100",
"dhcp-lease-time": 90,
"dhcp-lease-expire": 0,
"dhcp-lease-expire-t1": 0,
"dhcp-lease-expire-t2": 0,
"dhcp-tx": 8,
"dhcp-rx": 4,
"dhcp-tx-discover": 2,
"dhcp-rx-offer": 1,
"dhcp-tx-request": 3,
"dhcp-rx-ack": 3,
"dhcp-rx-nak": 0,
"dhcp-tx-release": 3,
"dhcpv6-state": "Init",
"dhcpv6-lease-time": 0,
"dhcpv6-lease-expire": 0,
"dhcpv6-lease-expire-t1": 0,
"dhcpv6-lease-expire-t2": 0,
"dhcpv6-tx": 14,
"dhcpv6-rx": 3,
"dhcpv6-tx-solicit": 3,
"dhcpv6-rx-advertise": 0,
"dhcpv6-tx-request": 0,
"dhcpv6-rx-reply": 2,
"dhcpv6-tx-renew": 0,
"dhcpv6-tx-release": 11,
"tx-packets": 27,
"rx-packets": 22,
"rx-fragmented-packets": 0,
"tx-bytes": 7222,
"rx-bytes": 3726,
"tx-accounting-packets": 0,
"rx-accounting-packets": 2,
"tx-accounting-bytes": 0,
"rx-accounting-bytes": 348,
"tx-arp": 0,
"rx-arp": 0,
"tx-igmp": 0,
"rx-igmp": 0,
"tx-icmp": 0,
"rx-icmp": 0,
"tx-icmpv6": 1,
"rx-icmpv6": 10,
"session-traffic": {
"total-flows": 6,
"verified-flows": 0,
"downstream-ipv4-flow-id": 128,
"downstream-ipv4-tx-packets": 0,
"downstream-ipv4-rx-packets": 0,
"downstream-ipv4-rx-first-seq": 0,
"downstream-ipv4-loss": 0,
"downstream-ipv4-wrong-session": 0,
"upstream-ipv4-flow-id": 127,
"upstream-ipv4-tx-packets": 0,
"upstream-ipv4-rx-packets": 0,
"upstream-ipv4-rx-first-seq": 0,
"upstream-ipv4-loss": 0,
"upstream-ipv4-wrong-session": 0,
"downstream-ipv6-flow-id": 130,
"downstream-ipv6-tx-packets": 0,
"downstream-ipv6-rx-packets": 0,
"downstream-ipv6-rx-first-seq": 0,
"downstream-ipv6-loss": 0,
"downstream-ipv6-wrong-session": 0,
"upstream-ipv6-flow-id": 129,
"upstream-ipv6-tx-packets": 0,
"upstream-ipv6-rx-packets": 0,
"upstream-ipv6-rx-first-seq": 0,
"upstream-ipv6-loss": 0,
"upstream-ipv6-wrong-session": 0,
"downstream-ipv6pd-flow-id": 132,
"downstream-ipv6pd-tx-packets": 0,
"downstream-ipv6pd-rx-packets": 0,
"downstream-ipv6pd-rx-first-seq": 0,
"downstream-ipv6pd-loss": 0,
"downstream-ipv6pd-wrong-session": 0,
"upstream-ipv6pd-flow-id": 131,
"upstream-ipv6pd-tx-packets": 0,
"upstream-ipv6pd-rx-packets": 0,
"upstream-ipv6pd-rx-first-seq": 0,
"upstream-ipv6pd-loss": 0,
"upstream-ipv6pd-wrong-session": 0
}
}
}