From a80e7b1ee401c49cf12ea4697c06fdfa50b6638f Mon Sep 17 00:00:00 2001 From: YehiaGewily <79144005+YehiaGewily@users.noreply.github.com> Date: Sun, 19 Jul 2026 03:13:31 +0300 Subject: [PATCH 01/12] chore: ignore local /plans/ working directory Implementation plans consumed by AI agents are local working documents, not part of the shipped repo. --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index d798c9a..15886e4 100644 --- a/.gitignore +++ b/.gitignore @@ -99,6 +99,10 @@ Thumbs.db # --- Agent Workspace Cache --- /.agents/ +# --- Local Planning Documents --- +# Implementation plans consumed by AI agents; not part of the shipped repo. +/plans/ + # --- Demo Recording --- /docs/record-demo.md From 0dee960decd7d6133362c15a420426e0d9bd8551 Mon Sep 17 00:00:00 2001 From: YehiaGewily <79144005+YehiaGewily@users.noreply.github.com> Date: Sun, 19 Jul 2026 03:13:31 +0300 Subject: [PATCH 02/12] build: extract NetProbeCore static library Capture, parsing, and analysis now live in a NetProbeCore static library with no ImGui/GLFW dependency, so the GUI, the test suite, and (next) a headless CLI can all build on the same code. - Replace file(GLOB_RECURSE) with explicit source lists. The glob made it too easy for a new core file to silently miss the test and fuzz targets, which re-list core sources by hand. - Split platform libraries by consumer: ws2_32/iphlpapi back the core (ProcessResolver needs them), while dwmapi/opengl32 stay GUI-only. - NetProbeTests drops its hand-maintained copy of the core source list and links NetProbeCore instead. - NetProbeParserFuzzer keeps its explicit list on purpose: libFuzzer needs every analyzed TU built with coverage instrumentation, which NetProbeCore is not. Documented in place so the duplication is intentional, not stale. --- CMakeLists.txt | 133 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 98 insertions(+), 35 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7eebe23..2b6da84 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,16 +26,19 @@ if(WIN32) message(FATAL_ERROR "Npcap SDK was not found at ${NPCAP_SDK_PATH}. Install the SDK or set NPCAP_SDK_PATH.") endif() - set(NETPROBE_APP_PLATFORM_LIBRARIES ws2_32 iphlpapi dwmapi opengl32) - set(NETPROBE_TEST_PLATFORM_LIBRARIES ws2_32) + # Split by consumer: CORE libraries back the capture/analysis library (and + # therefore also the tests and the headless CLI), while GUI libraries are + # needed only by the windowed executable. + set(NETPROBE_CORE_PLATFORM_LIBRARIES ws2_32 iphlpapi) + set(NETPROBE_GUI_PLATFORM_LIBRARIES dwmapi opengl32) else() find_path(PCAP_INCLUDE_DIR NAMES pcap.h pcap/pcap.h REQUIRED) find_library(PCAP_LIBRARY NAMES pcap REQUIRED) find_package(OpenGL REQUIRED) set(PCAP_LIBRARIES ${PCAP_LIBRARY}) - set(NETPROBE_APP_PLATFORM_LIBRARIES OpenGL::GL) - set(NETPROBE_TEST_PLATFORM_LIBRARIES) + set(NETPROBE_CORE_PLATFORM_LIBRARIES) + set(NETPROBE_GUI_PLATFORM_LIBRARIES OpenGL::GL) endif() # --- 2. FetchContent Dependencies (Gui Stack) --- @@ -123,10 +126,55 @@ if(BUILD_TESTING) endif() # --- 3. Source Management --- -# Gather all source files -file(GLOB_RECURSE APP_SOURCES - "src/*.cpp" - "src/*.hpp" +# Sources are listed explicitly rather than globbed: NetProbeTests and the +# fuzz harnesses select subsets of these files, and a glob silently changing +# what lands in which target is exactly the kind of surprise that costs an +# afternoon. Headers are listed too so they appear in IDE project trees. +set(NETPROBE_CORE_SOURCES + src/capture/CaptureEngine.cpp + src/capture/LibpcapBackend.cpp + src/capture/NpcapBackend.cpp + src/core/DNSParser.cpp + src/core/FlowAggregator.cpp + src/core/GeoIPResolver.cpp + src/core/ProcessResolver.cpp + src/core/ProtocolParser.cpp + src/core/QuicParser.cpp + src/core/QuicTracker.cpp + src/core/TlsReassembler.cpp + + src/capture/CaptureEngine.hpp + src/capture/ICaptureBackend.hpp + src/capture/PcapPlatform.hpp + src/core/DNSParser.hpp + src/core/FlowAggregator.hpp + src/core/GeoIPResolver.hpp + src/core/HostnameCache.hpp + src/core/LinkType.hpp + src/core/NetworkPlatform.hpp + src/core/PacketData.hpp + src/core/PacketQueue.hpp + src/core/ParsedPacket.hpp + src/core/ProcessResolver.hpp + src/core/ProtocolHeaders.hpp + src/core/ProtocolParser.hpp + src/core/QuicParser.hpp + src/core/QuicTracker.hpp + src/core/TlsReassembler.hpp +) + +set(NETPROBE_GUI_SOURCES + src/main.cpp + src/ui/Dashboard.cpp + src/ui/FlowsView.cpp + src/ui/GuiLayer.cpp + src/ui/PacketDetail.cpp + src/ui/PacketView.cpp + src/ui/StatsView.cpp + + src/ui/EmbeddedIcon.hpp + src/ui/GuiLayer.hpp + src/ui/GuiTheme.hpp ) # ImGui Backend Wrapper @@ -152,9 +200,34 @@ set(IMPLOT_SOURCES "${implot_SOURCE_DIR}/implot_demo.cpp" ) -# --- 4. Executable Target --- +# --- 4. Core Library --- +# Capture + parsing + analysis, with no dependency on ImGui/GLFW. The GUI, the +# headless CLI, and the test suite all build on this. +add_library(NetProbeCore STATIC ${NETPROBE_CORE_SOURCES}) + +target_include_directories(NetProbeCore PUBLIC + include + src + ${PCAP_INCLUDE_DIR} +) + +target_link_libraries(NetProbeCore PUBLIC + maxminddb::maxminddb + MbedTLS::mbedcrypto + ${PCAP_LIBRARIES} + ${NETPROBE_CORE_PLATFORM_LIBRARIES} +) + +# PRIVATE: these configure how core's own sources compile; consumers neither +# need nor should inherit them. +target_compile_definitions(NetProbeCore PRIVATE + NETPROBE_GEOIP_DATA_DIR="${CMAKE_SOURCE_DIR}/data" + NETPROBE_VERSION="${PROJECT_VERSION}" +) + +# --- 4b. GUI Executable Target --- add_executable(NetProbe - ${APP_SOURCES} + ${NETPROBE_GUI_SOURCES} ${IMGUI_BACKEND_SOURCES} ${IMGUI_CORE_SOURCES} ${IMPLOT_SOURCES} @@ -169,10 +242,8 @@ if(WIN32 AND EXISTS "${CMAKE_SOURCE_DIR}/resources/netprobe.ico") endif() # --- 5. Include Directories --- +# src/include/pcap come through NetProbeCore's PUBLIC includes. target_include_directories(NetProbe PRIVATE - include - src - ${PCAP_INCLUDE_DIR} ${imgui_SOURCE_DIR} ${imgui_SOURCE_DIR}/backends ${implot_SOURCE_DIR} @@ -180,15 +251,13 @@ target_include_directories(NetProbe PRIVATE # --- 6. Linking --- target_link_libraries(NetProbe PRIVATE + NetProbeCore glfw nfd::nfd - maxminddb::maxminddb - MbedTLS::mbedcrypto - ${PCAP_LIBRARIES} - ${NETPROBE_APP_PLATFORM_LIBRARIES} + ${NETPROBE_GUI_PLATFORM_LIBRARIES} ) target_compile_definitions(NetProbe PRIVATE - NETPROBE_GEOIP_DATA_DIR="${CMAKE_SOURCE_DIR}/data" + NETPROBE_VERSION="${PROJECT_VERSION}" ) install(TARGETS NetProbe @@ -224,28 +293,15 @@ if(BUILD_TESTING) enable_testing() add_executable(NetProbeTests tests/parser_test.cpp - src/core/DNSParser.cpp - src/core/FlowAggregator.cpp - src/core/GeoIPResolver.cpp - src/core/ProtocolParser.cpp - src/core/QuicParser.cpp - src/core/QuicTracker.cpp - src/core/TlsReassembler.cpp - src/capture/CaptureEngine.cpp - src/capture/NpcapBackend.cpp - src/capture/LibpcapBackend.cpp ) + # src/, pcap headers, and the crypto/geo libraries all arrive through + # NetProbeCore's PUBLIC usage requirements. target_include_directories(NetProbeTests PRIVATE - src tests - ${PCAP_INCLUDE_DIR} ) target_link_libraries(NetProbeTests PRIVATE + NetProbeCore GTest::gtest_main - maxminddb::maxminddb - MbedTLS::mbedcrypto - ${PCAP_LIBRARIES} - ${NETPROBE_TEST_PLATFORM_LIBRARIES} ) target_compile_definitions(NetProbeTests PRIVATE LIBMAXMINDDB_TEST_DATA_DIR="${libmaxminddb_SOURCE_DIR}/t/maxmind-db/test-data" @@ -279,6 +335,13 @@ if(BUILD_FUZZERS) target_include_directories(NetProbeFuzzerSeeds PRIVATE tests) if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") + # NOTE: this target deliberately re-lists its sources instead of + # linking NetProbeCore. libFuzzer needs every analyzed translation + # unit built with -fsanitize=fuzzer coverage instrumentation, and + # NetProbeCore is compiled without it. The list is also a curated + # parser-only subset with no libpcap dependency. If you add a parser + # to src/core that handles untrusted bytes, add it here too — see + # NETPROBE_CORE_SOURCES above. add_executable(NetProbeParserFuzzer tests/fuzz/parser_fuzzer.cpp src/core/DNSParser.cpp @@ -293,7 +356,7 @@ if(BUILD_FUZZERS) ) target_link_libraries(NetProbeParserFuzzer PRIVATE MbedTLS::mbedcrypto - ${NETPROBE_TEST_PLATFORM_LIBRARIES} + ${NETPROBE_CORE_PLATFORM_LIBRARIES} ) target_compile_options(NetProbeParserFuzzer PRIVATE -fsanitize=fuzzer,address,undefined From 42f51ac4f587b88a3970a146bd7bab6452241eed Mon Sep 17 00:00:00 2001 From: YehiaGewily <79144005+YehiaGewily@users.noreply.github.com> Date: Sun, 19 Jul 2026 03:19:17 +0300 Subject: [PATCH 03/12] refactor: extract core::AnalysisSession from GuiLayer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-packet pipeline — protocol decode, cross-packet SNI recovery, DNS answer harvesting, hostname attribution, process lookup, and flow aggregation — lived inside GuiLayer::processQueue(), which meant the only way to analyse a capture was to open a window. It now lives in core::AnalysisSession, and GuiLayer keeps just the presentation state layered on top. The pipeline logic moved verbatim; the analysis-owning members (FlowAggregator, HostnameCache, TlsReassembler, QuicTracker, ProcessResolver, DNS counters) moved with it. GeoIPResolver stays in the GUI since it is display enrichment, not analysis. AnalysisSession can skip process resolution, which is pure overhead when replaying a capture file whose sockets are long gone. Adds tests covering the cross-packet behaviour the split risked breaking: a DNS answer naming a later flow, SNI recovered only once a split ClientHello's second segment arrives, and clear() resetting every piece of state. 63/63 tests pass; GUI verified to start and render. --- CMakeLists.txt | 3 + src/core/AnalysisSession.cpp | 119 +++++++++++++++++++++++++++ src/core/AnalysisSession.hpp | 77 +++++++++++++++++ src/ui/FlowsView.cpp | 4 +- src/ui/GuiLayer.cpp | 101 ++--------------------- src/ui/GuiLayer.hpp | 22 ++--- src/ui/PacketDetail.cpp | 4 +- src/ui/PacketView.cpp | 4 +- src/ui/StatsView.cpp | 12 +-- tests/analysis_session_test.cpp | 141 ++++++++++++++++++++++++++++++++ 10 files changed, 368 insertions(+), 119 deletions(-) create mode 100644 src/core/AnalysisSession.cpp create mode 100644 src/core/AnalysisSession.hpp create mode 100644 tests/analysis_session_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 2b6da84..6ec6883 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -134,6 +134,7 @@ set(NETPROBE_CORE_SOURCES src/capture/CaptureEngine.cpp src/capture/LibpcapBackend.cpp src/capture/NpcapBackend.cpp + src/core/AnalysisSession.cpp src/core/DNSParser.cpp src/core/FlowAggregator.cpp src/core/GeoIPResolver.cpp @@ -146,6 +147,7 @@ set(NETPROBE_CORE_SOURCES src/capture/CaptureEngine.hpp src/capture/ICaptureBackend.hpp src/capture/PcapPlatform.hpp + src/core/AnalysisSession.hpp src/core/DNSParser.hpp src/core/FlowAggregator.hpp src/core/GeoIPResolver.hpp @@ -293,6 +295,7 @@ if(BUILD_TESTING) enable_testing() add_executable(NetProbeTests tests/parser_test.cpp + tests/analysis_session_test.cpp ) # src/, pcap headers, and the crypto/geo libraries all arrive through # NetProbeCore's PUBLIC usage requirements. diff --git a/src/core/AnalysisSession.cpp b/src/core/AnalysisSession.cpp new file mode 100644 index 0000000..82f7835 --- /dev/null +++ b/src/core/AnalysisSession.cpp @@ -0,0 +1,119 @@ +#include "core/AnalysisSession.hpp" + +#include "core/DNSParser.hpp" +#include "core/ProtocolParser.hpp" +#include "core/QuicParser.hpp" + +#include + +namespace core { + + AnalysisSession::AnalysisSession(bool resolveProcesses) + : m_resolveProcesses(resolveProcesses) {} + + AnalysisSession::Result AnalysisSession::feed(const PacketData& packet) { + Result result; + result.parsed = ProtocolParser::parse(packet); + ParsedPacket& parsed = result.parsed; + + // Recover an SNI that the single-packet parse could not see: a + // ClientHello split across TCP segments, or a QUIC ClientHello split + // across Initial packets. Both are routine now that post-quantum key + // shares have pushed the message past one MTU. + if (parsed.sni.empty() && parsed.payloadLength > 0 + && parsed.payloadOffset < packet.payload.size()) { + const uint8_t* payload = packet.payload.data() + parsed.payloadOffset; + const size_t payloadLength = std::min(parsed.payloadLength, + packet.payload.size() - parsed.payloadOffset); + + std::optional recovered; + if (parsed.protocol == "TCP") { + recovered = m_tlsReassembler.feed(parsed, payload, payloadLength); + } else if (parsed.protocol == "UDP" + && QuicParser::looksLikeLongHeader(payload, payloadLength)) { + recovered = m_quicTracker.feed(payload, payloadLength, parsed.timestamp); + } + if (recovered && !recovered->empty()) { + parsed.sni = *recovered; + if (const std::string named = ProtocolParser::identifyService(*recovered); + !named.empty()) { + parsed.service = named; + } + } + } + + if (const auto dnsResponse = DNSParser::parseResponse(packet, parsed)) { + ++m_plaintextDnsResponses; + if (dnsResponse->encryptedClientHelloAdvertised) m_echAdvertised = true; + for (const auto& answer : dnsResponse->answers) { + switch (answer.type) { + case DNSRecordType::A: + case DNSRecordType::AAAA: { + const std::string& hostname = dnsResponse->queryName.empty() + ? answer.name + : dnsResponse->queryName; + m_hostnameCache.store(answer.value, hostname); + m_flowAggregator.setHostnameForAddress(answer.value, hostname); + break; + } + case DNSRecordType::PTR: { + // Reverse answers name devices that never appear in a + // forward lookup — printers, NAS boxes, the router. + if (const auto address = DNSParser::reverseNameToAddress(answer.name)) { + m_hostnameCache.store(*address, answer.value); + m_flowAggregator.setHostnameForAddress(*address, answer.value); + } + break; + } + default: + break; + } + } + } + + if (parsed.service == "DNS-over-HTTPS" || parsed.service == "DNS-over-TLS" + || parsed.service == "DNS-over-QUIC") { + ++m_encryptedDnsPackets; + } + + if (!parsed.sni.empty()) { + result.hostname = parsed.sni; + } else if (!parsed.hostname.empty()) { + result.hostname = parsed.hostname; + } else if (const auto destinationHostname = m_hostnameCache.lookup(parsed.dstIP)) { + result.hostname = *destinationHostname; + } else if (const auto sourceHostname = m_hostnameCache.lookup(parsed.srcIP)) { + result.hostname = *sourceHostname; + } + + // Resolve the owning process now, while the socket is almost + // certainly still in the OS table; both the flow and the caller's + // packet record cache it so the name survives after a short-lived + // connection closes. + if (m_resolveProcesses) { + result.process = m_processResolver.lookup(parsed); + } + m_flowAggregator.update(parsed, result.hostname, result.process); + + return result; + } + + std::vector AnalysisSession::flows(int64_t nowMicroseconds) const { + return m_flowAggregator.snapshot(nowMicroseconds); + } + + void AnalysisSession::clear() { + m_hostnameCache.clear(); + m_flowAggregator.clear(); + m_tlsReassembler.clear(); + m_quicTracker.clear(); + m_plaintextDnsResponses = 0; + m_encryptedDnsPackets = 0; + m_echAdvertised = false; + } + + std::optional AnalysisSession::lookupHostname(const std::string& ip) const { + return m_hostnameCache.lookup(ip); + } + +} // namespace core diff --git a/src/core/AnalysisSession.hpp b/src/core/AnalysisSession.hpp new file mode 100644 index 0000000..b7355d0 --- /dev/null +++ b/src/core/AnalysisSession.hpp @@ -0,0 +1,77 @@ +#pragma once + +#include "core/FlowAggregator.hpp" +#include "core/HostnameCache.hpp" +#include "core/PacketData.hpp" +#include "core/ParsedPacket.hpp" +#include "core/ProcessResolver.hpp" +#include "core/QuicTracker.hpp" +#include "core/TlsReassembler.hpp" + +#include +#include +#include +#include + +namespace core { + + // Owns the per-packet analysis pipeline end to end: protocol decode, + // cross-packet SNI recovery (TLS segment reassembly and QUIC Initial + // decryption), DNS answer harvesting, hostname attribution, owning-process + // lookup, and flow aggregation. + // + // This deliberately holds no presentation state. The GUI and the headless + // CLI drive the same session and differ only in what they do with the + // results, which is what keeps the two from drifting apart. + class AnalysisSession { + public: + // What one packet turned into. `parsed` carries any SNI recovered from + // earlier packets in the same stream, so it can differ from what a + // stateless parse of these bytes alone would yield. + struct Result { + ParsedPacket parsed; + // Best hostname known for the peer at ingest time: SNI, then an + // HTTP Host header, then a DNS-derived name for either endpoint. + std::string hostname; + // Owning local process, resolved while the socket is still likely + // to be in the OS table. Empty when disabled or unresolvable. + std::string process; + }; + + // resolveProcesses: the OS socket-table walk is worthwhile for an + // interactive session but pure overhead when replaying a capture file, + // where the sockets are long gone. + explicit AnalysisSession(bool resolveProcesses = true); + + Result feed(const PacketData& packet); + + std::vector flows(int64_t nowMicroseconds) const; + + // Resets all analysis state: flows, hostnames, half-built handshakes, + // and the encrypted-DNS counters. + void clear(); + + // Encrypted-DNS visibility. When resolution moves to DoH/DoT/DoQ the + // hostname cache stops filling, and a consumer should say so rather + // than silently showing bare addresses. + uint64_t plaintextDnsResponses() const { return m_plaintextDnsResponses; } + uint64_t encryptedDnsPackets() const { return m_encryptedDnsPackets; } + bool echAdvertised() const { return m_echAdvertised; } + + // Display-time lookup for addresses that never anchored a flow. + std::optional lookupHostname(const std::string& ip) const; + + private: + FlowAggregator m_flowAggregator; + HostnameCache m_hostnameCache; + TlsReassembler m_tlsReassembler; + QuicTracker m_quicTracker; + ProcessResolver m_processResolver; + + bool m_resolveProcesses = true; + uint64_t m_plaintextDnsResponses = 0; + uint64_t m_encryptedDnsPackets = 0; + bool m_echAdvertised = false; + }; + +} // namespace core diff --git a/src/ui/FlowsView.cpp b/src/ui/FlowsView.cpp index f6cf658..d8f802e 100644 --- a/src/ui/FlowsView.cpp +++ b/src/ui/FlowsView.cpp @@ -55,7 +55,7 @@ namespace ui { } bool GuiLayer::exportFlowsCsv(const std::string& path, std::string& error) { - const auto flows = m_flowAggregator.snapshot(currentUnixTimeMicroseconds()); + const auto flows = m_session.flows(currentUnixTimeMicroseconds()); std::ofstream file(path, std::ios::trunc); if (!file) { error = "Unable to open the destination file for writing."; @@ -102,7 +102,7 @@ namespace ui { if (m_lastFlowsRefresh >= 0.0 && now - m_lastFlowsRefresh < 0.5) return; m_lastFlowsRefresh = now; - m_flowsCache = m_flowAggregator.snapshot(currentUnixTimeMicroseconds()); + m_flowsCache = m_session.flows(currentUnixTimeMicroseconds()); m_activeFlowCount = m_flowsCache.size(); // The geo cache persists across refreshes so each destination IP is diff --git a/src/ui/GuiLayer.cpp b/src/ui/GuiLayer.cpp index df7151e..95f2e97 100644 --- a/src/ui/GuiLayer.cpp +++ b/src/ui/GuiLayer.cpp @@ -1,9 +1,6 @@ #include "ui/GuiLayer.hpp" #include "ui/GuiTheme.hpp" #include "ui/EmbeddedIcon.hpp" -#include "core/DNSParser.hpp" -#include "core/ProtocolParser.hpp" -#include "core/QuicParser.hpp" #include "imgui.h" #include "imgui_internal.h" #include "imgui_impl_glfw.h" @@ -457,91 +454,17 @@ namespace ui { auto packetOpt = m_queue.try_pop(); if (!packetOpt) break; - auto parsed = core::ProtocolParser::parse(*packetOpt); - - // Recover an SNI that the single-packet parse could not see: a - // ClientHello split across TCP segments, or a QUIC ClientHello split - // across Initial packets. Both are routine now that post-quantum key - // shares have pushed the message past one MTU. - if (parsed.sni.empty() && parsed.payloadLength > 0 - && parsed.payloadOffset < packetOpt->payload.size()) { - const uint8_t* payload = packetOpt->payload.data() + parsed.payloadOffset; - const size_t payloadLength = std::min(parsed.payloadLength, - packetOpt->payload.size() - parsed.payloadOffset); - - std::optional recovered; - if (parsed.protocol == "TCP") { - recovered = m_tlsReassembler.feed(parsed, payload, payloadLength); - } else if (parsed.protocol == "UDP" - && core::QuicParser::looksLikeLongHeader(payload, payloadLength)) { - recovered = m_quicTracker.feed(payload, payloadLength, parsed.timestamp); - } - if (recovered && !recovered->empty()) { - parsed.sni = *recovered; - if (const std::string named = core::ProtocolParser::identifyService(*recovered); - !named.empty()) { - parsed.service = named; - } - } - } - - if (const auto dnsResponse = core::DNSParser::parseResponse(*packetOpt, parsed)) { - ++m_plaintextDnsResponses; - if (dnsResponse->encryptedClientHelloAdvertised) m_echAdvertised = true; - for (const auto& answer : dnsResponse->answers) { - switch (answer.type) { - case core::DNSRecordType::A: - case core::DNSRecordType::AAAA: { - const std::string& hostname = dnsResponse->queryName.empty() - ? answer.name - : dnsResponse->queryName; - m_hostnameCache.store(answer.value, hostname); - m_flowAggregator.setHostnameForAddress(answer.value, hostname); - break; - } - case core::DNSRecordType::PTR: { - // Reverse answers name devices that never appear in a - // forward lookup — printers, NAS boxes, the router. - if (const auto address = core::DNSParser::reverseNameToAddress(answer.name)) { - m_hostnameCache.store(*address, answer.value); - m_flowAggregator.setHostnameForAddress(*address, answer.value); - } - break; - } - default: - break; - } - } - } - - if (parsed.service == "DNS-over-HTTPS" || parsed.service == "DNS-over-TLS" - || parsed.service == "DNS-over-QUIC") { - ++m_encryptedDnsPackets; - } - - std::string hostname; - if (!parsed.sni.empty()) { - hostname = parsed.sni; - } else if (!parsed.hostname.empty()) { - hostname = parsed.hostname; - } else if (const auto destinationHostname = m_hostnameCache.lookup(parsed.dstIP)) { - hostname = *destinationHostname; - } else if (const auto sourceHostname = m_hostnameCache.lookup(parsed.srcIP)) { - hostname = *sourceHostname; - } - // Resolve the owning process now, while the socket is almost - // certainly still in the OS table; both the flow and the packet - // record cache it so the name survives after a short-lived - // connection closes. - std::string process = m_processResolver.lookup(parsed); - m_flowAggregator.update(parsed, hostname, process); + // Decode, recover cross-packet SNI, harvest DNS answers, attribute + // the owning process, and fold the packet into its flow. + auto analyzed = m_session.feed(*packetOpt); // Keep the visual history bounded while retaining all-time counters separately. if (m_packetHistory.size() >= maxPacketHistory) { m_packetHistory.pop_front(); if (m_selectedPacketIndex >= 0) --m_selectedPacketIndex; } - m_packetHistory.push_back({*packetOpt, std::move(parsed), std::move(process)}); + m_packetHistory.push_back({*packetOpt, std::move(analyzed.parsed), + std::move(analyzed.process)}); ++m_packetHistoryVersion; const core::ParsedPacket& stored = m_packetHistory.back().parsed; @@ -688,13 +611,7 @@ namespace ui { m_appCounts.clear(); m_protocolCounts.clear(); m_protocolBytes.clear(); - m_hostnameCache.clear(); - m_flowAggregator.clear(); - m_tlsReassembler.clear(); - m_quicTracker.clear(); - m_plaintextDnsResponses = 0; - m_encryptedDnsPackets = 0; - m_echAdvertised = false; + m_session.clear(); m_packetFlowFilter.reset(); m_flowRateHistory.Erase(); m_flowRateKey.reset(); @@ -972,8 +889,8 @@ namespace ui { // ClientHello SNI, and browsers reuse one connection for every lookup, // so a handful of sightings can already mean all resolution is hidden. constexpr uint64_t minimumEvidence = 3; - return m_encryptedDnsPackets >= minimumEvidence - && m_encryptedDnsPackets > m_plaintextDnsResponses * 2; + return m_session.encryptedDnsPackets() >= minimumEvidence + && m_session.encryptedDnsPackets() > m_session.plaintextDnsResponses() * 2; } void GuiLayer::renderEncryptedDnsNotice() { @@ -1001,7 +918,7 @@ namespace ui { ImGui::SameLine(); ImGui::TextColored(kText1, "Encrypted DNS in use."); ImGui::SameLine(); - ImGui::TextColored(kText2, m_echAdvertised + ImGui::TextColored(kText2, m_session.echAdvertised() ? "Name resolution is hidden, and Encrypted Client Hello was advertised, so some hostnames will show only an IP." : "Name resolution is not visible on the wire, so hostnames come only from TLS/QUIC SNI and may be incomplete."); diff --git a/src/ui/GuiLayer.hpp b/src/ui/GuiLayer.hpp index 6cd8ca7..c6c3189 100644 --- a/src/ui/GuiLayer.hpp +++ b/src/ui/GuiLayer.hpp @@ -1,14 +1,11 @@ #pragma once +#include "core/AnalysisSession.hpp" #include "core/PacketData.hpp" #include "core/PacketQueue.hpp" #include "core/ParsedPacket.hpp" -#include "core/HostnameCache.hpp" #include "core/FlowAggregator.hpp" #include "core/GeoIPResolver.hpp" -#include "core/ProcessResolver.hpp" -#include "core/QuicTracker.hpp" -#include "core/TlsReassembler.hpp" #include #include #include @@ -144,19 +141,12 @@ namespace ui { std::map m_appCounts; std::map m_protocolCounts; std::map m_protocolBytes; - core::HostnameCache m_hostnameCache; - core::FlowAggregator m_flowAggregator; + // Capture analysis: parsing, SNI recovery, DNS, flows, and process + // attribution. The GUI owns only presentation state on top of it. + core::AnalysisSession m_session; + // Display-only enrichment, so it stays out of the shared session. core::GeoIPResolver m_geoIPResolver; - core::ProcessResolver m_processResolver; - core::TlsReassembler m_tlsReassembler; - core::QuicTracker m_quicTracker; - - // Encrypted-DNS visibility. When resolution moves to DoH/DoT/DoQ the - // hostname cache stops filling, and the UI should say so rather than - // silently showing bare IP addresses. - uint64_t m_plaintextDnsResponses = 0; - uint64_t m_encryptedDnsPackets = 0; - bool m_echAdvertised = false; + bool m_dnsNoticeDismissed = false; ScrollingBuffer m_bandwidthData; ScrollingBuffer m_tcpBandwidth; diff --git a/src/ui/PacketDetail.cpp b/src/ui/PacketDetail.cpp index 95583c2..9e621a7 100644 --- a/src/ui/PacketDetail.cpp +++ b/src/ui/PacketDetail.cpp @@ -207,9 +207,9 @@ namespace ui { if (!p.hostname.empty()) { kv("Host header", p.hostname); } - if (auto host = m_hostnameCache.lookup(p.dstIP)) { + if (auto host = m_session.lookupHostname(p.dstIP)) { kv("Hostname", *host); - } else if (auto host2 = m_hostnameCache.lookup(p.srcIP)) { + } else if (auto host2 = m_session.lookupHostname(p.srcIP)) { kv("Hostname", *host2); } else { kv("Hostname", ""); diff --git a/src/ui/PacketView.cpp b/src/ui/PacketView.cpp index 6081729..08412f8 100644 --- a/src/ui/PacketView.cpp +++ b/src/ui/PacketView.cpp @@ -52,7 +52,7 @@ namespace ui { if (containsCI(packet.tunnel, needle)) return true; if (packet.srcPort != 0 && containsCI(std::format("{}", packet.srcPort), needle)) return true; if (packet.dstPort != 0 && containsCI(std::format("{}", packet.dstPort), needle)) return true; - if (const auto hostname = m_hostnameCache.lookup(packet.dstIP); + if (const auto hostname = m_session.lookupHostname(packet.dstIP); hostname && containsCI(*hostname, needle)) return true; return false; } @@ -266,7 +266,7 @@ namespace ui { ImGui::TextColored(kText2, "%s", p.sni.c_str()); } else if (!p.info.empty()) { ImGui::TextColored(kText2, "%s", p.info.c_str()); - } else if (const auto hostname = m_hostnameCache.lookup(p.dstIP)) { + } else if (const auto hostname = m_session.lookupHostname(p.dstIP)) { ImGui::TextColored(kText3, "host"); ImGui::SameLine(); ImGui::TextColored(kText2, "%s", hostname->c_str()); diff --git a/src/ui/StatsView.cpp b/src/ui/StatsView.cpp index 39d63a5..08c6cfe 100644 --- a/src/ui/StatsView.cpp +++ b/src/ui/StatsView.cpp @@ -67,21 +67,23 @@ namespace ui { // this view can be labelled at all, so it goes first. ImGui::TextColored(kText3, "NAME RESOLUTION"); ImGui::Dummy(ImVec2(0, 4)); - if (m_plaintextDnsResponses == 0 && m_encryptedDnsPackets == 0) { + const uint64_t plaintextDns = m_session.plaintextDnsResponses(); + const uint64_t encryptedDns = m_session.encryptedDnsPackets(); + if (plaintextDns == 0 && encryptedDns == 0) { ImGui::TextColored(kText3, "No DNS observed yet."); } else { ImGui::TextColored(kText2, "Plaintext DNS responses"); ImGui::SameLine(220.0f); - ImGui::TextColored(kText1, "%s", formatCount(m_plaintextDnsResponses).c_str()); + ImGui::TextColored(kText1, "%s", formatCount(plaintextDns).c_str()); ImGui::TextColored(kText2, "Encrypted DNS sightings"); ImGui::SameLine(220.0f); - ImGui::TextColored(m_encryptedDnsPackets > 0 ? kWarning : kText1, - "%s", formatCount(m_encryptedDnsPackets).c_str()); + ImGui::TextColored(encryptedDns > 0 ? kWarning : kText1, + "%s", formatCount(encryptedDns).c_str()); ImGui::TextColored(kText2, "Encrypted Client Hello"); ImGui::SameLine(220.0f); - if (m_echAdvertised) { + if (m_session.echAdvertised()) { ImGui::TextColored(kWarning, "advertised"); if (ImGui::IsItemHovered(ImGuiHoveredFlags_ForTooltip)) { ImGui::SetTooltip( diff --git a/tests/analysis_session_test.cpp b/tests/analysis_session_test.cpp new file mode 100644 index 0000000..0b814f4 --- /dev/null +++ b/tests/analysis_session_test.cpp @@ -0,0 +1,141 @@ +#include "PacketBuilders.hpp" + +#include "core/AnalysisSession.hpp" +#include "core/PacketData.hpp" + +#include + +#include +#include +#include +#include + +// Exercises the pipeline as a whole rather than one parser at a time: these +// pin the cross-packet behaviour (a DNS answer naming a later flow, a +// ClientHello that only yields SNI once its second segment arrives) that the +// GUI and the CLI both depend on. +namespace { + + using namespace test; + + constexpr int64_t kBaseTimestamp = 1'700'000'000'000'000; + + const std::vector kClient = {192, 168, 1, 10}; + const std::vector kServer = {93, 184, 216, 34}; + const std::vector kResolver = {8, 8, 8, 8}; + + core::PacketData toPacketData(const std::vector& bytes, int64_t timestamp) { + core::PacketData packet(timestamp, static_cast(bytes.size()), + static_cast(bytes.size()), bytes.data()); + return packet; + } + + std::vector makeTcpPacket(const std::vector& source, + const std::vector& destination, uint16_t sourcePort, uint16_t destinationPort, + uint32_t sequence, uint8_t flags, const std::vector& payload) { + std::vector packet; + appendEthernetHeader(packet, 0x0800); + appendIPv4Header(packet, static_cast(20 + 20 + payload.size()), 6, + source, destination); + appendTcpHeader(packet, sourcePort, destinationPort, sequence, flags); + packet.insert(packet.end(), payload.begin(), payload.end()); + return packet; + } + + // A minimal DNS response: one question for `name`, one A answer. + std::vector makeDnsResponsePacket(const std::string& name, + const std::vector& address) { + std::vector dns; + appendU16(dns, 0x1234); // transaction ID + appendU16(dns, 0x8180); // standard response + appendU16(dns, 1); // questions + appendU16(dns, 1); // answers + appendU16(dns, 0); + appendU16(dns, 0); + appendDnsName(dns, name); + appendU16(dns, 1); // QTYPE A + appendU16(dns, 1); // QCLASS IN + + appendDnsName(dns, name); + appendU16(dns, 1); + appendU16(dns, 1); + appendU32BE(dns, 60); + appendU16(dns, static_cast(address.size())); + dns.insert(dns.end(), address.begin(), address.end()); + + std::vector packet; + appendEthernetHeader(packet, 0x0800); + appendIPv4Header(packet, static_cast(20 + 8 + dns.size()), 17, + kResolver, kClient); + appendUdpHeader(packet, 53, 53000, static_cast(dns.size())); + packet.insert(packet.end(), dns.begin(), dns.end()); + return packet; + } + + // Process resolution walks the OS socket tables, which is meaningless for + // synthetic packets, so every test drives the session with it disabled. + core::AnalysisSession makeSession() { return core::AnalysisSession(false); } + +} // namespace + +TEST(AnalysisSessionTest, DnsAnswerNamesASubsequentFlow) { + auto session = makeSession(); + + const auto dnsPacket = makeDnsResponsePacket("www.example.com", kServer); + session.feed(toPacketData(dnsPacket, kBaseTimestamp)); + + const auto tcpPacket = makeTcpPacket(kClient, kServer, 50000, 443, 1000, 0x10, {}); + const auto analyzed = session.feed(toPacketData(tcpPacket, kBaseTimestamp + 1000)); + + EXPECT_EQ(analyzed.hostname, "www.example.com"); + + const auto flows = session.flows(kBaseTimestamp + 2000); + const auto named = std::find_if(flows.begin(), flows.end(), [](const core::Flow& flow) { + return flow.hostname == "www.example.com"; + }); + ASSERT_NE(named, flows.end()) << "DNS answer did not name the flow it resolved"; + EXPECT_EQ(named->key.protocol, "TCP"); +} + +TEST(AnalysisSessionTest, RecoversSniFromClientHelloSplitAcrossSegments) { + auto session = makeSession(); + + const auto clientHello = makeTlsClientHello("split.example.com"); + const size_t firstHalf = clientHello.size() / 2; + + const std::vector firstPayload(clientHello.begin(), + clientHello.begin() + static_cast(firstHalf)); + const std::vector secondPayload(clientHello.begin() + static_cast(firstHalf), + clientHello.end()); + + const auto first = makeTcpPacket(kClient, kServer, 50000, 443, 1000, 0x18, firstPayload); + const auto firstResult = session.feed(toPacketData(first, kBaseTimestamp)); + EXPECT_TRUE(firstResult.parsed.sni.empty()) + << "half a ClientHello should not yield an SNI"; + + const auto second = makeTcpPacket(kClient, kServer, 50000, 443, + static_cast(1000 + firstHalf), 0x18, secondPayload); + const auto secondResult = session.feed(toPacketData(second, kBaseTimestamp + 1000)); + EXPECT_EQ(secondResult.parsed.sni, "split.example.com"); + EXPECT_EQ(secondResult.hostname, "split.example.com"); +} + +TEST(AnalysisSessionTest, CountsPlaintextDnsAndClearsAllState) { + auto session = makeSession(); + + session.feed(toPacketData(makeDnsResponsePacket("www.example.com", kServer), kBaseTimestamp)); + const auto tcpPacket = makeTcpPacket(kClient, kServer, 50000, 443, 1000, 0x10, {}); + session.feed(toPacketData(tcpPacket, kBaseTimestamp + 1000)); + + EXPECT_EQ(session.plaintextDnsResponses(), 1u); + EXPECT_FALSE(session.flows(kBaseTimestamp + 2000).empty()); + EXPECT_TRUE(session.lookupHostname("93.184.216.34").has_value()); + + session.clear(); + + EXPECT_EQ(session.plaintextDnsResponses(), 0u); + EXPECT_EQ(session.encryptedDnsPackets(), 0u); + EXPECT_FALSE(session.echAdvertised()); + EXPECT_TRUE(session.flows(kBaseTimestamp + 2000).empty()); + EXPECT_FALSE(session.lookupHostname("93.184.216.34").has_value()); +} From 0dec59bd8bf132b9e8ce4f58ae1f83b3437f0676 Mon Sep 17 00:00:00 2001 From: YehiaGewily <79144005+YehiaGewily@users.noreply.github.com> Date: Sun, 19 Jul 2026 03:23:35 +0300 Subject: [PATCH 04/12] feat: add core::FlowExporter with CSV and JSON output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flow export moves out of the GUI so the headless CLI can share it, and gains a JSON format for pipelines (Elastic, Splunk, jq) that cannot read the spreadsheet-oriented CSV. The CSV column set and ordering are unchanged; GuiLayer::exportFlowsCsv is now a one-line delegation, so the file dialog flow is untouched. JSON strings are written by hand rather than via a JSON library. The one real correctness concern is that hostnames and SNI arrive as raw bytes off the wire and are not guaranteed to be valid UTF-8, so the writer decodes each multi-byte sequence and substitutes U+FFFD for anything malformed — truncated sequences, overlong encodings, surrogate halves, out-of-range code points, stray continuation bytes. Tests cover each of those, since a malformed SNI silently producing an unparseable document is exactly the failure a network tool must not have. organizationLabel() moves to core so the flow table and the exported files cannot disagree about how a network is named. 69/69 tests pass. --- CMakeLists.txt | 3 + src/core/FlowExporter.cpp | 245 +++++++++++++++++++++++++++++++++++ src/core/FlowExporter.hpp | 40 ++++++ src/core/GeoIPResolver.cpp | 9 ++ src/core/GeoIPResolver.hpp | 5 + src/ui/FlowsView.cpp | 66 ++-------- tests/flow_exporter_test.cpp | 158 ++++++++++++++++++++++ 7 files changed, 468 insertions(+), 58 deletions(-) create mode 100644 src/core/FlowExporter.cpp create mode 100644 src/core/FlowExporter.hpp create mode 100644 tests/flow_exporter_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ec6883..ac50358 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -137,6 +137,7 @@ set(NETPROBE_CORE_SOURCES src/core/AnalysisSession.cpp src/core/DNSParser.cpp src/core/FlowAggregator.cpp + src/core/FlowExporter.cpp src/core/GeoIPResolver.cpp src/core/ProcessResolver.cpp src/core/ProtocolParser.cpp @@ -150,6 +151,7 @@ set(NETPROBE_CORE_SOURCES src/core/AnalysisSession.hpp src/core/DNSParser.hpp src/core/FlowAggregator.hpp + src/core/FlowExporter.hpp src/core/GeoIPResolver.hpp src/core/HostnameCache.hpp src/core/LinkType.hpp @@ -296,6 +298,7 @@ if(BUILD_TESTING) add_executable(NetProbeTests tests/parser_test.cpp tests/analysis_session_test.cpp + tests/flow_exporter_test.cpp ) # src/, pcap headers, and the crypto/geo libraries all arrive through # NetProbeCore's PUBLIC usage requirements. diff --git a/src/core/FlowExporter.cpp b/src/core/FlowExporter.cpp new file mode 100644 index 0000000..8737cef --- /dev/null +++ b/src/core/FlowExporter.cpp @@ -0,0 +1,245 @@ +#include "core/FlowExporter.hpp" + +#include +#include +#include +#include +#include +#include + +#ifndef NETPROBE_VERSION +#define NETPROBE_VERSION "unknown" +#endif + +namespace core { + + namespace { + + std::string csvEscape(const std::string& value) { + if (value.find_first_of(",\"\n") == std::string::npos) return value; + std::string escaped = "\""; + for (char c : value) { + if (c == '"') escaped += "\"\""; + else escaped += c; + } + escaped += '"'; + return escaped; + } + + GeoIPInfo lookupGeo(GeoIPResolver* geo, const std::string& ip) { + return geo ? geo->lookup(ip) : GeoIPInfo{}; + } + + double rttMillisecondsOf(const Flow& flow) { + return flow.initialRttMicroseconds > 0 + ? static_cast(flow.initialRttMicroseconds) / 1000.0 + : 0.0; + } + + int64_t durationSecondsOf(const Flow& flow) { + return std::max(0, (flow.lastSeen - flow.firstSeen) / 1'000'000); + } + + int64_t currentUnixTimeMicroseconds() { + return std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count(); + } + + } // namespace + + void FlowExporter::appendJsonString(std::string& out, const std::string& value) { + static constexpr char kHex[] = "0123456789abcdef"; + + out += '"'; + const auto* bytes = reinterpret_cast(value.data()); + const size_t size = value.size(); + + for (size_t i = 0; i < size;) { + const unsigned char c = bytes[i]; + + if (c < 0x80) { + switch (c) { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\b': out += "\\b"; break; + case '\f': out += "\\f"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + default: + if (c < 0x20) { + out += "\\u00"; + out += kHex[(c >> 4) & 0x0F]; + out += kHex[c & 0x0F]; + } else { + out += static_cast(c); + } + break; + } + ++i; + continue; + } + + // Multi-byte sequence: decode it so malformed input can be caught + // rather than copied through into an unparseable document. + size_t length = 0; + uint32_t codepoint = 0; + if ((c & 0xE0) == 0xC0) { length = 2; codepoint = c & 0x1FU; } + else if ((c & 0xF0) == 0xE0) { length = 3; codepoint = c & 0x0FU; } + else if ((c & 0xF8) == 0xF0) { length = 4; codepoint = c & 0x07U; } + + bool valid = length != 0 && i + length <= size; + if (valid) { + for (size_t k = 1; k < length; ++k) { + const unsigned char continuation = bytes[i + k]; + if ((continuation & 0xC0) != 0x80) { valid = false; break; } + codepoint = (codepoint << 6) | (continuation & 0x3FU); + } + } + // Reject overlong encodings, UTF-16 surrogate halves, and + // anything past the Unicode range — all of which are ill-formed + // even though their lead bytes look plausible. + if (valid) { + const bool overlong = (length == 2 && codepoint < 0x80) + || (length == 3 && codepoint < 0x800) + || (length == 4 && codepoint < 0x10000); + const bool surrogate = codepoint >= 0xD800 && codepoint <= 0xDFFF; + if (overlong || surrogate || codepoint > 0x10FFFF) valid = false; + } + + if (!valid) { + out += "\xEF\xBF\xBD"; // U+FFFD REPLACEMENT CHARACTER + ++i; // resynchronize one byte at a time + continue; + } + + out.append(reinterpret_cast(bytes + i), length); + i += length; + } + + out += '"'; + } + + void FlowExporter::writeCsv(const std::vector& flows, std::ostream& out, + GeoIPResolver* geo) { + out << "host,src_ip,src_port,dst_ip,dst_port,protocol,service,process,country,organization," + "packets,bytes_up,bytes_down,rate_bytes_per_sec,initial_rtt_ms,duration_sec\n"; + for (const auto& flow : flows) { + const auto info = lookupGeo(geo, flow.key.dstIP); + const std::string& host = flow.hostname.empty() ? flow.key.dstIP : flow.hostname; + out << csvEscape(host) << ',' + << flow.key.srcIP << ',' + << flow.key.srcPort << ',' + << flow.key.dstIP << ',' + << flow.key.dstPort << ',' + << flow.key.protocol << ',' + << csvEscape(flow.service) << ',' + << csvEscape(flow.process) << ',' + << csvEscape(info.country) << ',' + << csvEscape(organizationLabel(info)) << ',' + << flow.packets << ',' + << flow.bytesUp << ',' + << flow.bytesDown << ',' + << std::format("{:.1f}", flow.rateBytesPerSecond) << ',' + << std::format("{:.2f}", rttMillisecondsOf(flow)) << ',' + << durationSecondsOf(flow) << '\n'; + } + } + + void FlowExporter::writeJson(const std::vector& flows, std::ostream& out, + GeoIPResolver* geo) { + std::string document; + document.reserve(flows.size() * 320 + 128); + + document += "{\n \"netprobe_version\": "; + appendJsonString(document, NETPROBE_VERSION); + document += ",\n \"generated_unix_us\": "; + document += std::format("{}", currentUnixTimeMicroseconds()); + document += ",\n \"flow_count\": "; + document += std::format("{}", flows.size()); + document += ",\n \"flows\": ["; + + bool first = true; + for (const auto& flow : flows) { + const auto info = lookupGeo(geo, flow.key.dstIP); + document += first ? "\n {" : ",\n {"; + first = false; + + document += "\n \"src_ip\": "; + appendJsonString(document, flow.key.srcIP); + document += ",\n \"src_port\": "; + document += std::format("{}", flow.key.srcPort); + document += ",\n \"dst_ip\": "; + appendJsonString(document, flow.key.dstIP); + document += ",\n \"dst_port\": "; + document += std::format("{}", flow.key.dstPort); + document += ",\n \"protocol\": "; + appendJsonString(document, flow.key.protocol); + document += ",\n \"service\": "; + appendJsonString(document, flow.service); + document += ",\n \"hostname\": "; + appendJsonString(document, flow.hostname); + document += ",\n \"process\": "; + appendJsonString(document, flow.process); + document += ",\n \"country\": "; + appendJsonString(document, info.country); + document += ",\n \"asn\": "; + document += std::format("{}", info.asn); + document += ",\n \"organization\": "; + appendJsonString(document, info.organization); + document += ",\n \"packets\": "; + document += std::format("{}", flow.packets); + document += ",\n \"bytes_up\": "; + document += std::format("{}", flow.bytesUp); + document += ",\n \"bytes_down\": "; + document += std::format("{}", flow.bytesDown); + document += ",\n \"first_seen_us\": "; + document += std::format("{}", flow.firstSeen); + document += ",\n \"last_seen_us\": "; + document += std::format("{}", flow.lastSeen); + document += ",\n \"duration_sec\": "; + document += std::format("{}", durationSecondsOf(flow)); + document += ",\n \"rate_bytes_per_sec\": "; + document += std::format("{:.1f}", flow.rateBytesPerSecond); + document += ",\n \"initial_rtt_ms\": "; + document += std::format("{:.2f}", rttMillisecondsOf(flow)); + document += ",\n \"encrypted_tunnel\": "; + document += flow.encryptedTunnel ? "true" : "false"; + document += "\n }"; + } + + document += flows.empty() ? "]\n}\n" : "\n ]\n}\n"; + out << document; + } + + bool FlowExporter::writeCsv(const std::vector& flows, const std::string& path, + std::string& error, GeoIPResolver* geo) { + std::ofstream file(path, std::ios::trunc); + if (!file) { + error = "Unable to open the destination file for writing."; + return false; + } + writeCsv(flows, file, geo); + if (!file.good()) { + error = "Writing the CSV file failed."; + return false; + } + return true; + } + + bool FlowExporter::writeJson(const std::vector& flows, const std::string& path, + std::string& error, GeoIPResolver* geo) { + std::ofstream file(path, std::ios::trunc | std::ios::binary); + if (!file) { + error = "Unable to open the destination file for writing."; + return false; + } + writeJson(flows, file, geo); + if (!file.good()) { + error = "Writing the JSON file failed."; + return false; + } + return true; + } + +} // namespace core diff --git a/src/core/FlowExporter.hpp b/src/core/FlowExporter.hpp new file mode 100644 index 0000000..9bf2081 --- /dev/null +++ b/src/core/FlowExporter.hpp @@ -0,0 +1,40 @@ +#pragma once + +#include "core/FlowAggregator.hpp" +#include "core/GeoIPResolver.hpp" + +#include +#include +#include + +namespace core { + + // Serializes a flow snapshot for consumption outside NetProbe: a + // spreadsheet (CSV) or a pipeline such as Elastic, Splunk, or jq (JSON). + // + // `geo` is optional. When null — or when no GeoLite2 database is + // installed — the country and organization fields are written empty, + // which is the honest representation of "not known" rather than a + // structural difference in the output. + class FlowExporter { + public: + static void writeCsv(const std::vector& flows, std::ostream& out, + GeoIPResolver* geo = nullptr); + static void writeJson(const std::vector& flows, std::ostream& out, + GeoIPResolver* geo = nullptr); + + // File variants. Return false and set `error` when the destination + // cannot be opened or the write fails part way through. + static bool writeCsv(const std::vector& flows, const std::string& path, + std::string& error, GeoIPResolver* geo = nullptr); + static bool writeJson(const std::vector& flows, const std::string& path, + std::string& error, GeoIPResolver* geo = nullptr); + + // Exposed for testing: appends `value` to `out` as a quoted JSON + // string. Hostnames and SNI arrive as raw bytes off the wire and are + // not guaranteed to be valid UTF-8, so anything malformed is replaced + // with U+FFFD rather than emitted verbatim into the document. + static void appendJsonString(std::string& out, const std::string& value); + }; + +} // namespace core diff --git a/src/core/GeoIPResolver.cpp b/src/core/GeoIPResolver.cpp index b4e5088..18a380a 100644 --- a/src/core/GeoIPResolver.cpp +++ b/src/core/GeoIPResolver.cpp @@ -1,9 +1,18 @@ #include "core/GeoIPResolver.hpp" #include +#include #include namespace core { + + std::string organizationLabel(const GeoIPInfo& info) { + if (info.organization.empty()) return std::string{"-"}; + return info.asn == 0 + ? info.organization + : std::format("AS{} {}", info.asn, info.organization); + } + namespace { std::filesystem::path geoIpDataDirectory() { #ifdef NETPROBE_GEOIP_DATA_DIR diff --git a/src/core/GeoIPResolver.hpp b/src/core/GeoIPResolver.hpp index 822db92..460e03d 100644 --- a/src/core/GeoIPResolver.hpp +++ b/src/core/GeoIPResolver.hpp @@ -17,6 +17,11 @@ namespace core { std::string organization; }; + // Human-readable owner label, e.g. "AS15169 Google LLC", or "-" when the + // organization is unknown. Shared so the flow table and the exported files + // cannot disagree about how a network is named. + std::string organizationLabel(const GeoIPInfo& info); + class GeoIPResolver { public: explicit GeoIPResolver(size_t cacheCapacity = 4'096); diff --git a/src/ui/FlowsView.cpp b/src/ui/FlowsView.cpp index d8f802e..3aaa90f 100644 --- a/src/ui/FlowsView.cpp +++ b/src/ui/FlowsView.cpp @@ -1,6 +1,7 @@ #include "ui/GuiLayer.hpp" #include "ui/GuiTheme.hpp" #include "core/FlowAggregator.hpp" +#include "core/FlowExporter.hpp" #include "core/GeoIPResolver.hpp" #include "imgui.h" #include "implot.h" @@ -18,23 +19,7 @@ namespace ui { namespace { - std::string csvEscape(const std::string& value) { - if (value.find_first_of(",\"\n") == std::string::npos) return value; - std::string escaped = "\""; - for (char c : value) { - if (c == '"') escaped += "\"\""; - else escaped += c; - } - escaped += '"'; - return escaped; - } - - std::string organizationLabelFor(const core::GeoIPInfo& info) { - if (info.organization.empty()) return std::string{"-"}; - return info.asn == 0 - ? info.organization - : std::format("AS{} {}", info.asn, info.organization); - } + using core::organizationLabel; // Two-column key/value row used by the flow detail pane. void detailRow(const char* key, const std::string& value) { @@ -55,43 +40,8 @@ namespace ui { } bool GuiLayer::exportFlowsCsv(const std::string& path, std::string& error) { - const auto flows = m_session.flows(currentUnixTimeMicroseconds()); - std::ofstream file(path, std::ios::trunc); - if (!file) { - error = "Unable to open the destination file for writing."; - return false; - } - file << "host,src_ip,src_port,dst_ip,dst_port,protocol,service,process,country,organization," - "packets,bytes_up,bytes_down,rate_bytes_per_sec,initial_rtt_ms,duration_sec\n"; - for (const auto& flow : flows) { - const auto geo = m_geoIPResolver.lookup(flow.key.dstIP); - const std::string& host = flow.hostname.empty() ? flow.key.dstIP : flow.hostname; - const double rttMs = flow.initialRttMicroseconds > 0 - ? static_cast(flow.initialRttMicroseconds) / 1000.0 - : 0.0; - const int64_t durationSec = std::max(0, (flow.lastSeen - flow.firstSeen) / 1'000'000); - file << csvEscape(host) << ',' - << flow.key.srcIP << ',' - << flow.key.srcPort << ',' - << flow.key.dstIP << ',' - << flow.key.dstPort << ',' - << flow.key.protocol << ',' - << csvEscape(flow.service) << ',' - << csvEscape(flow.process) << ',' - << csvEscape(geo.country) << ',' - << csvEscape(organizationLabelFor(geo)) << ',' - << flow.packets << ',' - << flow.bytesUp << ',' - << flow.bytesDown << ',' - << std::format("{:.1f}", flow.rateBytesPerSecond) << ',' - << std::format("{:.2f}", rttMs) << ',' - << durationSec << '\n'; - } - if (!file.good()) { - error = "Writing the CSV file failed."; - return false; - } - return true; + return core::FlowExporter::writeCsv(m_session.flows(currentUnixTimeMicroseconds()), + path, error, &m_geoIPResolver); } // Refresh the shared flow snapshot on a 0.5s cadence. Copying every flow, @@ -132,8 +82,8 @@ namespace ui { const auto& rightGeo = geoFor(right.key.dstIP); const std::string leftCountry = leftGeo.country.empty() ? "-" : leftGeo.country; const std::string rightCountry = rightGeo.country.empty() ? "-" : rightGeo.country; - const std::string leftOrganization = organizationLabelFor(leftGeo); - const std::string rightOrganization = organizationLabelFor(rightGeo); + const std::string leftOrganization = organizationLabel(leftGeo); + const std::string rightOrganization = organizationLabel(rightGeo); const uint64_t leftBytes = left.bytesUp + left.bytesDown; const uint64_t rightBytes = right.bytesUp + right.bytesDown; @@ -205,7 +155,7 @@ namespace ui { detailRow("Protocol", flow.key.protocol); detailRow("Process", flow.process); detailRow("Country", geo.country.empty() ? "-" : geo.country); - detailRow("Org", organizationLabelFor(geo)); + detailRow("Org", organizationLabel(geo)); // Name the direction by its endpoint rather than "up"/"down": for a // peer-to-peer flow neither side is the client. detailRow(std::format("{} sent", flow.key.srcIP).c_str(), formatBytes(flow.bytesUp)); @@ -338,7 +288,7 @@ namespace ui { const std::string& host = flow.hostname.empty() ? flow.key.dstIP : flow.hostname; const auto& geoInfo = geoFor(flow.key.dstIP); const std::string country = geoInfo.country.empty() ? "-" : geoInfo.country; - const std::string organization = organizationLabelFor(geoInfo); + const std::string organization = organizationLabel(geoInfo); const uint64_t totalBytes = flow.bytesUp + flow.bytesDown; const bool selected = m_packetFlowFilter && *m_packetFlowFilter == flow.key; diff --git a/tests/flow_exporter_test.cpp b/tests/flow_exporter_test.cpp new file mode 100644 index 0000000..f41d563 --- /dev/null +++ b/tests/flow_exporter_test.cpp @@ -0,0 +1,158 @@ +#include "core/FlowExporter.hpp" + +#include + +#include +#include +#include +#include + +// The exported files are the project's machine-readable contract with other +// tools, so these tests pin both the shape of the output and — more +// importantly — its robustness to hostile field values. Hostnames and SNI +// arrive as raw bytes off the wire; a naive writer turns a malformed one into +// a document no consumer can parse. +namespace { + + core::Flow makeFlow(const std::string& hostname = "example.com") { + core::Flow flow; + flow.key.srcIP = "192.168.1.10"; + flow.key.srcPort = 50000; + flow.key.dstIP = "93.184.216.34"; + flow.key.dstPort = 443; + flow.key.protocol = "TCP"; + flow.hostname = hostname; + flow.service = "HTTPS"; + flow.process = "firefox"; + flow.bytesUp = 1024; + flow.bytesDown = 8192; + flow.packets = 12; + flow.firstSeen = 1'700'000'000'000'000; + flow.lastSeen = 1'700'000'005'000'000; + flow.rateBytesPerSecond = 1536.0; + flow.initialRttMicroseconds = 24'500; + return flow; + } + + std::string toJson(const std::vector& flows) { + std::ostringstream out; + core::FlowExporter::writeJson(flows, out); + return out.str(); + } + + std::string toCsv(const std::vector& flows) { + std::ostringstream out; + core::FlowExporter::writeCsv(flows, out); + return out.str(); + } + + std::string jsonStringOf(const std::string& value) { + std::string encoded; + core::FlowExporter::appendJsonString(encoded, value); + return encoded; + } + + // Structural check that every brace/bracket is balanced and every quote + // is closed, honouring escapes. Enough to catch a field value that has + // broken out of its string, which is the failure mode that matters. + bool jsonIsStructurallyBalanced(const std::string& document) { + int depth = 0; + bool inString = false; + bool escaped = false; + for (const char c : document) { + if (inString) { + if (escaped) escaped = false; + else if (c == '\\') escaped = true; + else if (c == '"') inString = false; + else if (static_cast(c) < 0x20) return false; + continue; + } + if (c == '"') inString = true; + else if (c == '{' || c == '[') ++depth; + else if (c == '}' || c == ']') { + --depth; + if (depth < 0) return false; + } + } + return depth == 0 && !inString; + } + +} // namespace + +TEST(FlowExporterTest, JsonContainsExpectedFieldsAndValues) { + const auto document = toJson({makeFlow()}); + + EXPECT_TRUE(jsonIsStructurallyBalanced(document)) << document; + EXPECT_NE(document.find("\"flow_count\": 1"), std::string::npos); + EXPECT_NE(document.find("\"src_ip\": \"192.168.1.10\""), std::string::npos); + EXPECT_NE(document.find("\"dst_port\": 443"), std::string::npos); + EXPECT_NE(document.find("\"protocol\": \"TCP\""), std::string::npos); + EXPECT_NE(document.find("\"hostname\": \"example.com\""), std::string::npos); + EXPECT_NE(document.find("\"bytes_down\": 8192"), std::string::npos); + EXPECT_NE(document.find("\"packets\": 12"), std::string::npos); + EXPECT_NE(document.find("\"initial_rtt_ms\": 24.50"), std::string::npos); + EXPECT_NE(document.find("\"duration_sec\": 5"), std::string::npos); + EXPECT_NE(document.find("\"encrypted_tunnel\": false"), std::string::npos); +} + +TEST(FlowExporterTest, JsonWithNoFlowsIsStillValid) { + const auto document = toJson({}); + + EXPECT_TRUE(jsonIsStructurallyBalanced(document)) << document; + EXPECT_NE(document.find("\"flow_count\": 0"), std::string::npos); + EXPECT_NE(document.find("\"flows\": []"), std::string::npos); +} + +TEST(FlowExporterTest, JsonEscapesQuotesBackslashesAndControlCharacters) { + EXPECT_EQ(jsonStringOf("say \"hi\""), "\"say \\\"hi\\\"\""); + EXPECT_EQ(jsonStringOf("back\\slash"), "\"back\\\\slash\""); + EXPECT_EQ(jsonStringOf("line\nbreak"), "\"line\\nbreak\""); + EXPECT_EQ(jsonStringOf("tab\there"), "\"tab\\there\""); + EXPECT_EQ(jsonStringOf(std::string("nul\0byte", 8)), "\"nul\\u0000byte\""); + EXPECT_EQ(jsonStringOf("bell\x07"), "\"bell\\u0007\""); +} + +TEST(FlowExporterTest, JsonPreservesValidMultiByteUtf8) { + // U+00E9 (2-byte), U+4E2D (3-byte), U+1F600 (4-byte) must survive intact. + EXPECT_EQ(jsonStringOf("caf\xC3\xA9"), "\"caf\xC3\xA9\""); + EXPECT_EQ(jsonStringOf("\xE4\xB8\xAD"), "\"\xE4\xB8\xAD\""); + EXPECT_EQ(jsonStringOf("\xF0\x9F\x98\x80"), "\"\xF0\x9F\x98\x80\""); +} + +TEST(FlowExporterTest, JsonReplacesMalformedUtf8RatherThanEmittingIt) { + // Each of these is a hostname an attacker could put in an SNI field. + const std::vector malformed = { + "\xFF\xFE", // never-valid lead bytes + "\xC3", // truncated 2-byte sequence + "\xE4\xB8", // truncated 3-byte sequence + "\xC0\xAF", // overlong encoding of '/' + "\xED\xA0\x80", // UTF-16 surrogate half + "\xF5\x80\x80\x80", // beyond U+10FFFF + "host\x80name", // stray continuation byte + }; + + for (const auto& value : malformed) { + const auto encoded = jsonStringOf(value); + EXPECT_TRUE(jsonIsStructurallyBalanced(encoded)) + << "malformed input produced unbalanced JSON: " << encoded; + // U+FFFD, not the raw bytes. + EXPECT_NE(encoded.find("\xEF\xBF\xBD"), std::string::npos) + << "expected a replacement character for: " << encoded; + } + + auto flow = makeFlow("\xFF\xFE evil.example.com"); + EXPECT_TRUE(jsonIsStructurallyBalanced(toJson({flow}))); +} + +TEST(FlowExporterTest, CsvKeepsItsHeaderAndQuotesEmbeddedCommas) { + const auto csv = toCsv({makeFlow("a,b\"c")}); + + ASSERT_FALSE(csv.empty()); + const auto headerEnd = csv.find('\n'); + ASSERT_NE(headerEnd, std::string::npos); + EXPECT_EQ(csv.substr(0, headerEnd), + "host,src_ip,src_port,dst_ip,dst_port,protocol,service,process,country,organization," + "packets,bytes_up,bytes_down,rate_bytes_per_sec,initial_rtt_ms,duration_sec"); + // The comma and quote must be escaped, not passed through raw. + EXPECT_NE(csv.find("\"a,b\"\"c\""), std::string::npos) << csv; +} From 56cdb29ccd12d6c7f5456c0c5df3ae9576c2dc72 Mon Sep 17 00:00:00 2001 From: YehiaGewily <79144005+YehiaGewily@users.noreply.github.com> Date: Sun, 19 Jul 2026 03:29:23 +0300 Subject: [PATCH 05/12] feat: add netprobe-cli headless executable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Runs the full capture and analysis pipeline with no window, so NetProbe can be used over SSH, on servers, and in CI. Links NetProbeCore only — no ImGui, GLFW, or OpenGL. netprobe-cli --list-devices netprobe-cli -r capture.pcap -o flows.json netprobe-cli -i eth0 -f "tcp port 443" --duration 60 -o flows.csv netprobe-cli -r capture.pcap -o - | jq '.flows[]' Adds CaptureEngine::replayFile(), which streams an offline capture packet by packet to a callback instead of routing it through the bounded PacketQueue. openFile() pushes the entire file into a 50k-entry queue that evicts its oldest entries when full, so any capture larger than the queue silently loses its beginning. That is survivable for a live UI showing recent traffic; it is not survivable when the output is an exported flow table that is supposed to describe the whole file. Live capture keeps using the queue, and reports queue drops on stderr rather than quietly undercounting. Ctrl+C stops a capture and still writes the output. Exit codes separate usage errors (1) from runtime failures (2) so scripts can branch on them, and all diagnostics go to stderr so `-o -` stays pipeable. Adds four CTest cases driving the real binary over the generated sample capture — the only coverage of the complete chain from pcap file through parsing and attribution to an exported file. Both failure paths of the check script were verified to fail as intended. 73/73 tests pass. --- CMakeLists.txt | 34 ++- README.md | 31 +++ src/capture/CaptureEngine.cpp | 36 +++ src/capture/CaptureEngine.hpp | 20 ++ src/cli/main.cpp | 422 ++++++++++++++++++++++++++++++++++ tests/check_cli_export.cmake | 63 +++++ 6 files changed, 605 insertions(+), 1 deletion(-) create mode 100644 src/cli/main.cpp create mode 100644 tests/check_cli_export.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index ac50358..793b27e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -264,7 +264,17 @@ target_compile_definitions(NetProbe PRIVATE NETPROBE_VERSION="${PROJECT_VERSION}" ) -install(TARGETS NetProbe +# --- 6b. Headless CLI --- +# Links NetProbeCore only: no ImGui, GLFW, or OpenGL, so it builds and runs on +# a machine with no display. +add_executable(NetProbeCli src/cli/main.cpp) +target_link_libraries(NetProbeCli PRIVATE NetProbeCore) +target_compile_definitions(NetProbeCli PRIVATE + NETPROBE_VERSION="${PROJECT_VERSION}" +) +set_target_properties(NetProbeCli PROPERTIES OUTPUT_NAME "netprobe-cli") + +install(TARGETS NetProbe NetProbeCli RUNTIME DESTINATION . BUNDLE DESTINATION . ) @@ -312,6 +322,28 @@ if(BUILD_TESTING) target_compile_definitions(NetProbeTests PRIVATE LIBMAXMINDDB_TEST_DATA_DIR="${libmaxminddb_SOURCE_DIR}/t/maxmind-db/test-data" ) + # End-to-end CLI checks. These drive the real binary over the generated + # sample capture, which is the only coverage of the complete chain from + # pcap file through to an exported file. + foreach(_netprobe_cli_format json csv) + add_test(NAME CliExportsSampleFlowsAs${_netprobe_cli_format} + COMMAND ${CMAKE_COMMAND} + -DCLI=$ + -DPCAP=${CMAKE_SOURCE_DIR}/data/sample.pcap + -DOUT=${CMAKE_CURRENT_BINARY_DIR}/cli-flows.${_netprobe_cli_format} + -DFORMAT=${_netprobe_cli_format} + -P ${CMAKE_SOURCE_DIR}/tests/check_cli_export.cmake) + endforeach() + + # Usage errors must be distinguishable from runtime failures by exit code, + # since that is what a calling script branches on. + add_test(NAME CliRejectsMissingSource COMMAND NetProbeCli --output flows.json) + set_tests_properties(CliRejectsMissingSource PROPERTIES WILL_FAIL TRUE) + + add_test(NAME CliShowsHelp COMMAND NetProbeCli --help) + set_tests_properties(CliShowsHelp PROPERTIES + PASS_REGULAR_EXPRESSION "Usage:") + include(GoogleTest) # PRE_TEST defers the exe enumeration from build-time to ctest-time. On # Windows this matters because the test executable links wpcap.dll diff --git a/README.md b/README.md index 0d4129f..dfec07b 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,37 @@ cpack --config build/CPackConfig.cmake -C Release -G ZIP -B package For GeoIP and ASN columns, place `GeoLite2-Country.mmdb` and `GeoLite2-ASN.mmdb` in `data/`. See [data/README.md](data/README.md). +## Headless CLI + +`netprobe-cli` runs the same capture and analysis pipeline with no window, for servers, CI, and SSH sessions. It ships alongside the GUI in every release ZIP. + +```bash +# List the adapters available for live capture +netprobe-cli --list-devices + +# Replay a capture file and export its flows as JSON +netprobe-cli -r capture.pcap -o flows.json + +# Capture live for 60 seconds, filtered, exporting CSV +netprobe-cli -i eth0 -f "tcp port 443" --duration 60 -o flows.csv + +# Pipe JSON straight into jq +netprobe-cli -r capture.pcap -o - | jq '.flows[] | select(.bytes_down > 100000)' +``` + +| Flag | Purpose | +| --- | --- | +| `-r, --read ` | Replay an offline capture | +| `-i, --device ` | Capture live from an adapter (needs privileges) | +| `-f, --filter ` | BPF capture filter, live capture only | +| `-o, --output ` | Destination file, or `-` for stdout | +| `--format ` | Override the format inferred from the extension | +| `--duration ` | Stop after this many seconds | +| `--packet-count ` | Stop after this many packets | +| `--no-process` | Skip the owning-process lookup | + +Ctrl+C stops a live capture and still writes the output. Exit codes are `0` success, `1` usage error, `2` runtime failure, so scripts can tell a bad invocation from a failed capture. Diagnostics go to stderr, which keeps `-o -` output clean for piping. + ## Architecture NetProbe is built around a bounded **producer-consumer** pipeline. Capture runs on a backend-owned path, while the Dear ImGui frame loop drains packets, enriches them, and updates view models without blocking the capture thread. diff --git a/src/capture/CaptureEngine.cpp b/src/capture/CaptureEngine.cpp index 0f5945b..377d2c9 100644 --- a/src/capture/CaptureEngine.cpp +++ b/src/capture/CaptureEngine.cpp @@ -83,6 +83,42 @@ namespace capture { } } + bool CaptureEngine::replayFile(const std::string& path, + const std::function& onPacket, std::string& error) { + stopCapture(); + m_queue.clear(); + clearSession(); + m_currentDevice.clear(); + + if (!m_backend->openFile(path, error)) return false; + + m_sessionLinkType.store(m_backend->linkType(), std::memory_order_release); + if (m_backend->linkType() == core::LinkType::Unsupported) { + error = "the capture uses an unsupported link-layer type; packets cannot be decoded"; + m_backend->close(); + return false; + } + + while (true) { + core::PacketData packet; + std::string readError; + switch (m_backend->nextPacket(packet, readError)) { + case PacketReadStatus::Packet: + onPacket(packet); + break; + case PacketReadStatus::Timeout: + break; + case PacketReadStatus::EndOfFile: + m_backend->close(); + return true; + case PacketReadStatus::Error: + error = readError; + m_backend->close(); + return false; + } + } + } + bool CaptureEngine::setFilter(const std::string& filter, std::string& error) { if (!m_captureThread.joinable() || !m_backend->isOpen()) { error = "A live capture must be running before a BPF filter can be applied."; diff --git a/src/capture/CaptureEngine.hpp b/src/capture/CaptureEngine.hpp index da41134..588d182 100644 --- a/src/capture/CaptureEngine.hpp +++ b/src/capture/CaptureEngine.hpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -25,6 +26,25 @@ namespace capture { std::vector getAvailableDevices() const; void startCapture(const std::string& deviceName); bool openFile(const std::string& path); + + // True once a live capture has been started successfully. Note that a + // capture which started and then ended (adapter unplugged, end of a + // replayed source) still reads as true; this answers "did we get off + // the ground", not "is traffic still arriving". + bool isCapturing() const { return m_captureThread.joinable(); } + + // Reads an offline capture start to finish, handing each packet to + // `onPacket` as it is read. + // + // Unlike openFile(), packets never pass through the bounded + // PacketQueue, which drops its oldest entries once full. A capture + // with more packets than the queue holds would otherwise lose the + // beginning of the file before anything could consume it — silently + // wrong totals, which is unacceptable when the output is an exported + // flow table. + bool replayFile(const std::string& path, + const std::function& onPacket, + std::string& error); bool setFilter(const std::string& filter, std::string& error); bool exportSession(const std::string& path, std::string& error) const; void stopCapture(); diff --git a/src/cli/main.cpp b/src/cli/main.cpp new file mode 100644 index 0000000..4e2d57c --- /dev/null +++ b/src/cli/main.cpp @@ -0,0 +1,422 @@ +// NetProbe headless CLI. +// +// Runs the same capture and analysis pipeline as the GUI with no window, so +// captures can be taken and flows exported over SSH, in CI, or on a server. +// Deliberately small: replay a file or capture for a bounded time, then write +// the flow table as JSON or CSV. + +#include "capture/CaptureEngine.hpp" +#include "core/AnalysisSession.hpp" +#include "core/FlowExporter.hpp" +#include "core/PacketQueue.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef NETPROBE_VERSION +#define NETPROBE_VERSION "unknown" +#endif + +namespace { + + // Exit codes are part of the CLI's contract with scripts that call it. + constexpr int kExitSuccess = 0; + constexpr int kExitUsage = 1; + constexpr int kExitRuntime = 2; + + // Set from the signal handler; only ever transitions false -> true. + std::atomic g_stopRequested{false}; + + extern "C" void handleInterrupt(int) { + g_stopRequested.store(true, std::memory_order_release); + } + + enum class OutputFormat { Json, Csv }; + + struct Options { + bool listDevices = false; + bool showHelp = false; + bool showVersion = false; + std::string readPath; // offline capture to replay + std::string device; // live adapter to capture from + std::string bpfFilter; + std::string outputPath; // "-" means stdout + std::optional format; + std::optional durationSeconds; + std::optional packetLimit; + bool resolveProcesses = true; + }; + + void printUsage(std::ostream& out) { + out << "NetProbe CLI " NETPROBE_VERSION " - headless packet capture and flow export\n" + "\n" + "Usage:\n" + " netprobe-cli --list-devices\n" + " netprobe-cli -r -o \n" + " netprobe-cli -i [-f ] [--duration ] -o \n" + "\n" + "Source (exactly one required):\n" + " -r, --read Replay an offline capture file\n" + " -i, --device Capture live from an adapter (needs privileges)\n" + "\n" + "Output:\n" + " -o, --output Destination file, or '-' for stdout\n" + " --format Override the format inferred from the extension\n" + "\n" + "Capture limits (live capture only):\n" + " --duration Stop after this many seconds\n" + " --packet-count Stop after this many packets\n" + "\n" + "Other:\n" + " --no-process Skip owning-process lookup\n" + " --list-devices List capture adapters and exit\n" + " -h, --help Show this help and exit\n" + " -V, --version Show the version and exit\n" + "\n" + "Interrupting with Ctrl+C stops the capture and still writes the output.\n"; + } + + // Returns false when `text` is not a complete non-negative integer. + template + bool parseNonNegative(std::string_view text, T& value) { + if (text.empty()) return false; + const char* begin = text.data(); + const char* end = begin + text.size(); + const auto result = std::from_chars(begin, end, value); + return result.ec == std::errc{} && result.ptr == end; + } + + bool endsWith(const std::string& value, std::string_view suffix) { + return value.size() >= suffix.size() + && value.compare(value.size() - suffix.size(), suffix.size(), suffix) == 0; + } + + // Parses argv. On failure prints the reason to stderr and returns nullopt. + std::optional parseArguments(int argc, char** argv) { + Options options; + + // Every option below takes exactly one value; centralising the + // "is there a next argument" check keeps the loop readable. + const auto takeValue = [&](int& index, std::string_view flag) -> const char* { + if (index + 1 >= argc) { + std::cerr << "netprobe-cli: " << flag << " requires a value\n"; + return nullptr; + } + return argv[++index]; + }; + + for (int i = 1; i < argc; ++i) { + const std::string_view argument = argv[i]; + + if (argument == "-h" || argument == "--help") { + options.showHelp = true; + return options; + } + if (argument == "-V" || argument == "--version") { + options.showVersion = true; + return options; + } + if (argument == "--list-devices") { + options.listDevices = true; + continue; + } + if (argument == "--no-process") { + options.resolveProcesses = false; + continue; + } + if (argument == "-r" || argument == "--read") { + const char* value = takeValue(i, argument); + if (!value) return std::nullopt; + options.readPath = value; + continue; + } + if (argument == "-i" || argument == "--device") { + const char* value = takeValue(i, argument); + if (!value) return std::nullopt; + options.device = value; + continue; + } + if (argument == "-f" || argument == "--filter") { + const char* value = takeValue(i, argument); + if (!value) return std::nullopt; + options.bpfFilter = value; + continue; + } + if (argument == "-o" || argument == "--output") { + const char* value = takeValue(i, argument); + if (!value) return std::nullopt; + options.outputPath = value; + continue; + } + if (argument == "--format") { + const char* value = takeValue(i, argument); + if (!value) return std::nullopt; + const std::string_view format = value; + if (format == "json") options.format = OutputFormat::Json; + else if (format == "csv") options.format = OutputFormat::Csv; + else { + std::cerr << "netprobe-cli: unknown format '" << format + << "' (expected 'json' or 'csv')\n"; + return std::nullopt; + } + continue; + } + if (argument == "--duration") { + const char* value = takeValue(i, argument); + if (!value) return std::nullopt; + int seconds = 0; + if (!parseNonNegative(value, seconds) || seconds <= 0) { + std::cerr << "netprobe-cli: --duration expects a positive number of seconds\n"; + return std::nullopt; + } + options.durationSeconds = seconds; + continue; + } + if (argument == "--packet-count") { + const char* value = takeValue(i, argument); + if (!value) return std::nullopt; + uint64_t count = 0; + if (!parseNonNegative(value, count) || count == 0) { + std::cerr << "netprobe-cli: --packet-count expects a positive number\n"; + return std::nullopt; + } + options.packetLimit = count; + continue; + } + + std::cerr << "netprobe-cli: unrecognized option '" << argument << "'\n"; + return std::nullopt; + } + + return options; + } + + // Validates the combination of options, not just their individual syntax. + bool validate(const Options& options) { + if (options.listDevices) return true; + + const bool hasFile = !options.readPath.empty(); + const bool hasDevice = !options.device.empty(); + + if (!hasFile && !hasDevice) { + std::cerr << "netprobe-cli: nothing to do - pass --read or --device \n" + "Try 'netprobe-cli --help'.\n"; + return false; + } + if (hasFile && hasDevice) { + std::cerr << "netprobe-cli: --read and --device are mutually exclusive\n"; + return false; + } + if (options.outputPath.empty()) { + std::cerr << "netprobe-cli: no destination - pass --output (or '-' for stdout)\n"; + return false; + } + // Silently ignoring a flag that cannot apply is how people end up + // trusting a limit that was never enforced. + if (hasFile && (options.durationSeconds || options.packetLimit)) { + std::cerr << "netprobe-cli: --duration and --packet-count apply to live capture only\n"; + return false; + } + if (hasFile && !options.bpfFilter.empty()) { + std::cerr << "netprobe-cli: --filter applies to live capture only\n"; + return false; + } + return true; + } + + OutputFormat resolveFormat(const Options& options) { + if (options.format) return *options.format; + if (endsWith(options.outputPath, ".csv")) return OutputFormat::Csv; + return OutputFormat::Json; + } + + int64_t currentUnixTimeMicroseconds() { + return std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count(); + } + + int listDevices(const capture::CaptureEngine& engine) { + const auto devices = engine.getAvailableDevices(); + if (devices.empty()) { + std::cerr << "No capture devices found. On most systems live capture needs " + "administrator or root privileges.\n"; + return kExitRuntime; + } + for (const auto& device : devices) { + std::cout << device.name; + if (!device.description.empty()) std::cout << " (" << device.description << ")"; + std::cout << '\n'; + } + return kExitSuccess; + } + + // Replays a capture file. Packets are handed over as they are read, so a + // file of any size is processed without loss. + bool runOffline(capture::CaptureEngine& engine, core::AnalysisSession& session, + const Options& options, uint64_t& packetsAnalyzed) { + std::string error; + const bool ok = engine.replayFile(options.readPath, + [&](const core::PacketData& packet) { + session.feed(packet); + ++packetsAnalyzed; + }, error); + + if (!ok) { + std::cerr << "netprobe-cli: unable to read '" << options.readPath << "': " + << error << '\n'; + return false; + } + return true; + } + + // Captures live until the duration elapses, the packet limit is reached, + // or the user interrupts. + bool runLive(capture::CaptureEngine& engine, core::PacketQueue& queue, + core::AnalysisSession& session, const Options& options, uint64_t& packetsAnalyzed) { + engine.startCapture(options.device); + if (!engine.isCapturing()) { + std::cerr << "netprobe-cli: unable to open device '" << options.device + << "'. Live capture usually requires administrator or root privileges.\n" + "Run --list-devices to see the available adapters.\n"; + return false; + } + + if (!options.bpfFilter.empty()) { + std::string error; + if (!engine.setFilter(options.bpfFilter, error)) { + std::cerr << "netprobe-cli: invalid capture filter '" << options.bpfFilter + << "': " << error << '\n'; + engine.stopCapture(); + return false; + } + } + + const auto deadline = options.durationSeconds + ? std::optional{std::chrono::steady_clock::now() + + std::chrono::seconds(*options.durationSeconds)} + : std::nullopt; + + std::cerr << "Capturing on " << options.device; + if (options.durationSeconds) std::cerr << " for " << *options.durationSeconds << "s"; + if (options.packetLimit) std::cerr << ", up to " << *options.packetLimit << " packets"; + std::cerr << ". Press Ctrl+C to stop early.\n"; + + while (!g_stopRequested.load(std::memory_order_acquire)) { + if (deadline && std::chrono::steady_clock::now() >= *deadline) break; + if (options.packetLimit && packetsAnalyzed >= *options.packetLimit) break; + + auto packet = queue.try_pop(); + if (!packet) { + // Nothing buffered: yield briefly rather than spin a core. + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + continue; + } + session.feed(*packet); + ++packetsAnalyzed; + } + + engine.stopCapture(); + + // The capture thread has stopped, but packets it already queued are + // still pending; analysing them costs nothing and makes the totals + // match what was actually captured. + while (auto packet = queue.try_pop()) { + if (options.packetLimit && packetsAnalyzed >= *options.packetLimit) break; + session.feed(*packet); + ++packetsAnalyzed; + } + + if (const size_t dropped = queue.droppedPackets(); dropped > 0) { + std::cerr << "Warning: " << dropped << " packets were dropped because analysis " + "could not keep up with the capture; flow totals undercount by that much.\n"; + } + return true; + } + + bool writeOutput(const std::vector& flows, const Options& options, + OutputFormat format) { + if (options.outputPath == "-") { + if (format == OutputFormat::Csv) core::FlowExporter::writeCsv(flows, std::cout); + else core::FlowExporter::writeJson(flows, std::cout); + std::cout.flush(); + if (!std::cout.good()) { + std::cerr << "netprobe-cli: writing to stdout failed\n"; + return false; + } + return true; + } + + std::string error; + const bool ok = format == OutputFormat::Csv + ? core::FlowExporter::writeCsv(flows, options.outputPath, error) + : core::FlowExporter::writeJson(flows, options.outputPath, error); + if (!ok) { + std::cerr << "netprobe-cli: unable to write '" << options.outputPath << "': " + << error << '\n'; + return false; + } + return true; + } + +} // namespace + +int main(int argc, char** argv) { + const auto parsed = parseArguments(argc, argv); + if (!parsed) { + std::cerr << "Try 'netprobe-cli --help'.\n"; + return kExitUsage; + } + const Options options = *parsed; + + if (options.showHelp) { + printUsage(std::cout); + return kExitSuccess; + } + if (options.showVersion) { + std::cout << "netprobe-cli " NETPROBE_VERSION "\n"; + return kExitSuccess; + } + if (!validate(options)) return kExitUsage; + + core::PacketQueue queue; + capture::CaptureEngine engine(queue); + + if (options.listDevices) return listDevices(engine); + + std::signal(SIGINT, handleInterrupt); +#ifdef SIGTERM + std::signal(SIGTERM, handleInterrupt); +#endif + + // Replaying a file means the sockets that owned those packets are long + // gone, so the socket-table walk is pure overhead with nothing to find. + const bool resolveProcesses = options.resolveProcesses && options.readPath.empty(); + core::AnalysisSession session(resolveProcesses); + + uint64_t packetsAnalyzed = 0; + const bool captured = options.readPath.empty() + ? runLive(engine, queue, session, options, packetsAnalyzed) + : runOffline(engine, session, options, packetsAnalyzed); + if (!captured) return kExitRuntime; + + const auto flows = session.flows(currentUnixTimeMicroseconds()); + const OutputFormat format = resolveFormat(options); + if (!writeOutput(flows, options, format)) return kExitRuntime; + + std::cerr << "Analyzed " << packetsAnalyzed << " packets into " << flows.size() + << " flows; wrote " + << (options.outputPath == "-" ? std::string{"stdout"} : options.outputPath) + << " as " << (format == OutputFormat::Csv ? "CSV" : "JSON") << ".\n"; + return kExitSuccess; +} diff --git a/tests/check_cli_export.cmake b/tests/check_cli_export.cmake new file mode 100644 index 0000000..4528c27 --- /dev/null +++ b/tests/check_cli_export.cmake @@ -0,0 +1,63 @@ +# End-to-end check for the headless CLI, driven by CTest. +# +# This is the only test that exercises the whole chain in one go — pcap file → +# capture backend → parser → SNI/DNS attribution → flow aggregation → export — +# in the process a user actually runs. The unit tests each cover one stage. +# +# Invoked as: +# cmake -DCLI= -DPCAP= -DOUT= -DFORMAT= +# -P check_cli_export.cmake + +foreach(required CLI PCAP OUT FORMAT) + if(NOT DEFINED ${required}) + message(FATAL_ERROR "check_cli_export.cmake: -D${required}=... is required") + endif() +endforeach() + +if(NOT EXISTS "${PCAP}") + message(FATAL_ERROR "sample capture is missing: ${PCAP}\n" + "It is generated by the NetProbeSamplePcap target during a normal build.") +endif() + +# Start from a clean slate so a stale file from an earlier run cannot make a +# failing export look like a passing one. +file(REMOVE "${OUT}") + +execute_process( + COMMAND "${CLI}" --read "${PCAP}" --output "${OUT}" --format "${FORMAT}" + RESULT_VARIABLE exit_status + OUTPUT_VARIABLE standard_output + ERROR_VARIABLE standard_error +) + +if(NOT exit_status EQUAL 0) + message(FATAL_ERROR "netprobe-cli exited with ${exit_status}\n" + "stdout:\n${standard_output}\nstderr:\n${standard_error}") +endif() + +if(NOT EXISTS "${OUT}") + message(FATAL_ERROR "netprobe-cli reported success but wrote no file to ${OUT}") +endif() + +file(READ "${OUT}" exported) +if(exported STREQUAL "") + message(FATAL_ERROR "netprobe-cli wrote an empty file to ${OUT}") +endif() + +if(FORMAT STREQUAL "json") + # Structural keys, plus a value that can only appear if hostname + # attribution survived the whole pipeline rather than just the bytes. + set(expected "\"flows\"" "\"src_ip\"" "\"protocol\": \"TCP\"" "\"protocol\": \"UDP\"" "cdn.netprobe.test") +else() + set(expected "host,src_ip,src_port" "cdn.netprobe.test" "TCP" "UDP") +endif() + +foreach(needle IN LISTS expected) + string(FIND "${exported}" "${needle}" position) + if(position EQUAL -1) + message(FATAL_ERROR "exported ${FORMAT} is missing '${needle}'\n" + "----- contents -----\n${exported}") + endif() +endforeach() + +message(STATUS "netprobe-cli exported ${FORMAT} with the expected flows") From cb46122eed9a516cb2a55ae480cd1c62b7e7662e Mon Sep 17 00:00:00 2001 From: YehiaGewily <79144005+YehiaGewily@users.noreply.github.com> Date: Sun, 19 Jul 2026 03:36:38 +0300 Subject: [PATCH 06/12] feat: per-flow TCP retransmission and out-of-order detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The flow table reported how much a connection moved and how fast, but not whether it was healthy. Each direction of a TCP flow now tracks the highest sequence number seen and classifies data-bearing segments against it: segments landing behind it are retransmissions, segments skipping ahead are out of order. All sequence comparisons use signed 32-bit differences (RFC 1982 serial arithmetic). A plain integer comparison passes every ordinary test and then reports a flood of phantom loss the moment a long-lived stream wraps past 2^32; two tests cover the wrap in both directions. Bare ACKs are excluded — they carry no bytes and repeat sequence numbers by design, so counting them would report constant duplicates. A flow whose first segment arrives mid-stream only establishes a baseline, and an RST clears tracking so a reused port pair does not read as a huge backwards jump. Surfaced as a sortable, color-coded Loss column in the flows table (a rate, since 20 retransmits out of 50 packets is a broken connection and out of 500,000 is noise), with absolute per-direction counts in the detail pane and four new fields in both exports. Also fixes the bundled sample capture, which this feature exposed: all three ClientHellos were addressed to the same server with the same sequence number regardless of the hostname they advertised, contradicting the DNS answers in the same file. That collapsed them into one flow whose repeated sequence numbers correctly read as retransmissions. Each site now gets its own address, client port, and full three-way handshake, so the sample shows four flows with matching hostnames, real RTTs, and no loss. 83/83 tests pass; GUI verified to render the widened table. --- CMakeLists.txt | 1 + README.md | 6 +- src/core/FlowAggregator.cpp | 56 ++++++++++ src/core/FlowAggregator.hpp | 22 ++++ src/core/FlowExporter.cpp | 19 +++- src/ui/FlowsView.cpp | 58 ++++++++++- tests/flow_exporter_test.cpp | 8 +- tests/flow_loss_test.cpp | 193 +++++++++++++++++++++++++++++++++++ tools/make_sample_pcap.cpp | 104 +++++++++++++------ 9 files changed, 424 insertions(+), 43 deletions(-) create mode 100644 tests/flow_loss_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 793b27e..9cedac5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -309,6 +309,7 @@ if(BUILD_TESTING) tests/parser_test.cpp tests/analysis_session_test.cpp tests/flow_exporter_test.cpp + tests/flow_loss_test.cpp ) # src/, pcap headers, and the crypto/geo libraries all arrive through # NetProbeCore's PUBLIC usage requirements. diff --git a/README.md b/README.md index dfec07b..b6de66a 100644 --- a/README.md +++ b/README.md @@ -21,10 +21,10 @@ Download the ZIP for your platform from [GitHub Releases](https://github.com/Yeh - **DNS/mDNS**: A/AAAA/CNAME, plus **PTR** (names LAN devices that never appear in a forward lookup), SRV, TXT, and **HTTPS/SVCB** records. - **Encrypted-DNS visibility**: detects DoH/DoT/DoQ and **Encrypted Client Hello**, and tells you when name resolution has moved off the wire rather than silently showing bare IPs. - **Packet detail pane**: click any packet to see a structured Frame → Tunnel → Network → Transport → Application decode plus an `xxd`-style hex dump of the raw bytes. -- **Flows view**: sortable per-connection byte totals, one-second rate, **initial TCP RTT** (from the SYN / SYN-ACK delta, color-coded by latency), duration, service, hostname, owning process, Country, and ASN/organization, with a detail pane and live rate sparkline. Flow keys are canonical, so both directions of a peer-to-peer conversation collapse into a single row. +- **Flows view**: sortable per-connection byte totals, one-second rate, **initial TCP RTT** (from the SYN / SYN-ACK delta, color-coded by latency), duration, **per-flow retransmission and out-of-order rates**, service, hostname, owning process, Country, and ASN/organization, with a detail pane and live rate sparkline. Flow keys are canonical, so both directions of a peer-to-peer conversation collapse into a single row. - **Statistics view**: protocol hierarchy, top talkers by bytes, and name-resolution health. - **Process resolution**: the *App* column in both the packet list and the flows table shows the owning process on Windows (iphlpapi), Linux (`/proc/net/*` + `/proc//fd/`), and macOS (`proc_pidfdinfo`). Flows capture the process while the socket is live, so the name persists after a short-lived connection closes. Running elevated widens coverage to other users' processes. -- **Filtering and export**: live BPF capture filters such as `tcp port 443`, a separate display filter over captured packets, PCAP export, and flow export to CSV. +- **Filtering and export**: live BPF capture filters such as `tcp port 443`, a separate display filter over captured packets, PCAP export, and flow export to CSV or JSON. - **Light and dark themes**, adjustable UI scale, and keyboard shortcuts — all persisted between runs. - **Cross-platform backends**: Npcap on Windows and libpcap on Linux/macOS. @@ -93,7 +93,7 @@ cpack --config build/CPackConfig.cmake -C Release -G ZIP -B package ## Usage 1. Run the executable from the extracted release ZIP or build output directory. -2. **No admin? Try the sample first.** The build emits `data/sample.pcap` (mixed ARP / DNS / TLS-SNI traffic with three resolvable hostnames). Open it via **File → Open PCAP…** to exercise the full UI without elevation. +2. **No admin? Try the sample first.** The build emits `data/sample.pcap` (mixed ARP / DNS / TCP traffic: three resolvable hostnames, each with a full TLS handshake and a measurable RTT). Open it via **File → Open PCAP…** to exercise the full UI without elevation. 3. **Live capture** usually requires elevated privileges. On Windows: ```powershell diff --git a/src/core/FlowAggregator.cpp b/src/core/FlowAggregator.cpp index 54f6a60..0b7e6a5 100644 --- a/src/core/FlowAggregator.cpp +++ b/src/core/FlowAggregator.cpp @@ -115,6 +115,62 @@ namespace core { } } + // Delivery problems, judged per direction against the highest sequence + // number seen so far. Only data-bearing segments participate: a bare + // ACK carries no bytes and repeats the sequence number of the next + // byte it expects, so counting them would report constant duplicates. + // + // All comparisons go through a signed 32-bit difference, which is + // RFC 1982 serial arithmetic — the only way these stay correct when a + // long-lived stream wraps past 2^32. + if (packet.protocol == "TCP" && packet.payloadLength > 0) { + DirectionSequence& sequence = + downstream ? state.downSequence : state.upSequence; + uint64_t& retransmissions = + downstream ? state.flow.retransmissionsDown : state.flow.retransmissionsUp; + uint64_t& outOfOrder = + downstream ? state.flow.outOfOrderDown : state.flow.outOfOrderUp; + + const uint32_t segmentEnd = + packet.tcpSeq + static_cast(packet.payloadLength); + + if (!sequence.valid) { + // First data we have seen in this direction establishes the + // baseline; a flow that began before the capture started must + // not be reported as one giant reordering. + sequence.nextExpected = segmentEnd; + sequence.valid = true; + } else { + const int32_t offsetFromExpected = + static_cast(packet.tcpSeq - sequence.nextExpected); + + if (offsetFromExpected == 0) { + sequence.nextExpected = segmentEnd; + } else if (offsetFromExpected < 0) { + // Starts in territory already covered: a retransmission, + // whole or partial. + ++retransmissions; + if (static_cast(segmentEnd - sequence.nextExpected) > 0) { + sequence.nextExpected = segmentEnd; // partial overlap carried new bytes + } + } else { + // Skips ahead of what we expected, so something earlier is + // missing or reordered. When the gap is filled later that + // segment lands in the branch above, which is why a single + // reordering shows as one of each. + ++outOfOrder; + sequence.nextExpected = segmentEnd; + } + } + } + + // A reset ends the byte stream; sequence numbers after it belong to a + // new connection that may reuse this address/port pair. + if (packet.tcpRst) { + state.upSequence = {}; + state.downSequence = {}; + } + if (!hostname.empty()) state.flow.hostname = hostname; if (!packet.sni.empty()) state.flow.hostname = packet.sni; else if (!packet.hostname.empty()) state.flow.hostname = packet.hostname; diff --git a/src/core/FlowAggregator.hpp b/src/core/FlowAggregator.hpp index 9f5c90c..e1e0db9 100644 --- a/src/core/FlowAggregator.hpp +++ b/src/core/FlowAggregator.hpp @@ -51,6 +51,20 @@ namespace core { // True when the transport is an encrypted tunnel (ESP, WireGuard, // OpenVPN) and the inner conversation can never be resolved. bool encryptedTunnel = false; + + // TCP delivery problems seen from this capture point, split by + // direction (Up = sent by the key's `src`). Always zero for non-TCP. + // + // These are capture-point counts, in the same sense as Wireshark's: + // a segment carrying bytes we have already seen is counted as a + // retransmission, and one that skips ahead of the expected sequence + // as out-of-order. Capturing on the sending host makes them a good + // proxy for real loss; capturing mid-path only shows what reached + // this vantage point. + uint64_t retransmissionsUp = 0; + uint64_t retransmissionsDown = 0; + uint64_t outOfOrderUp = 0; + uint64_t outOfOrderDown = 0; }; class FlowAggregator { @@ -78,9 +92,17 @@ namespace core { uint32_t bytes; }; + // Highest sequence number seen so far in one direction of a stream. + struct DirectionSequence { + uint32_t nextExpected = 0; + bool valid = false; // false until the first data segment arrives + }; + struct FlowState { Flow flow; std::deque recentTraffic; + DirectionSequence upSequence; + DirectionSequence downSequence; // Timestamps used to compute the initial three-way-handshake RTT. // synTimestamp is the first client→server SYN we saw; syn-ack // arrival in the other direction yields the RTT delta. diff --git a/src/core/FlowExporter.cpp b/src/core/FlowExporter.cpp index 8737cef..ef5656a 100644 --- a/src/core/FlowExporter.cpp +++ b/src/core/FlowExporter.cpp @@ -122,8 +122,11 @@ namespace core { void FlowExporter::writeCsv(const std::vector& flows, std::ostream& out, GeoIPResolver* geo) { + // New columns are appended rather than inserted so existing consumers + // that index by position keep working. out << "host,src_ip,src_port,dst_ip,dst_port,protocol,service,process,country,organization," - "packets,bytes_up,bytes_down,rate_bytes_per_sec,initial_rtt_ms,duration_sec\n"; + "packets,bytes_up,bytes_down,rate_bytes_per_sec,initial_rtt_ms,duration_sec," + "retransmissions_up,retransmissions_down,out_of_order_up,out_of_order_down\n"; for (const auto& flow : flows) { const auto info = lookupGeo(geo, flow.key.dstIP); const std::string& host = flow.hostname.empty() ? flow.key.dstIP : flow.hostname; @@ -142,7 +145,11 @@ namespace core { << flow.bytesDown << ',' << std::format("{:.1f}", flow.rateBytesPerSecond) << ',' << std::format("{:.2f}", rttMillisecondsOf(flow)) << ',' - << durationSecondsOf(flow) << '\n'; + << durationSecondsOf(flow) << ',' + << flow.retransmissionsUp << ',' + << flow.retransmissionsDown << ',' + << flow.outOfOrderUp << ',' + << flow.outOfOrderDown << '\n'; } } @@ -203,6 +210,14 @@ namespace core { document += std::format("{:.1f}", flow.rateBytesPerSecond); document += ",\n \"initial_rtt_ms\": "; document += std::format("{:.2f}", rttMillisecondsOf(flow)); + document += ",\n \"retransmissions_up\": "; + document += std::format("{}", flow.retransmissionsUp); + document += ",\n \"retransmissions_down\": "; + document += std::format("{}", flow.retransmissionsDown); + document += ",\n \"out_of_order_up\": "; + document += std::format("{}", flow.outOfOrderUp); + document += ",\n \"out_of_order_down\": "; + document += std::format("{}", flow.outOfOrderDown); document += ",\n \"encrypted_tunnel\": "; document += flow.encryptedTunnel ? "true" : "false"; document += "\n }"; diff --git a/src/ui/FlowsView.cpp b/src/ui/FlowsView.cpp index 3aaa90f..23e36bf 100644 --- a/src/ui/FlowsView.cpp +++ b/src/ui/FlowsView.cpp @@ -21,6 +21,20 @@ namespace ui { using core::organizationLabel; + uint64_t deliveryProblemsOf(const core::Flow& flow) { + return flow.retransmissionsUp + flow.retransmissionsDown + + flow.outOfOrderUp + flow.outOfOrderDown; + } + + // Retransmits and reorderings as a share of the flow's packets. A rate + // reads better than a raw count here: 20 retransmits out of 50 packets + // is a broken connection, out of 500,000 it is noise. + double deliveryProblemRateOf(const core::Flow& flow) { + if (flow.packets == 0) return 0.0; + return 100.0 * static_cast(deliveryProblemsOf(flow)) + / static_cast(flow.packets); + } + // Two-column key/value row used by the flow detail pane. void detailRow(const char* key, const std::string& value) { ImGui::TableNextRow(); @@ -107,9 +121,15 @@ namespace ui { comparison = lr < rr ? -1 : lr > rr ? 1 : 0; break; } - case 9: comparison = (left.lastSeen - left.firstSeen) < (right.lastSeen - right.firstSeen) ? -1 + case 9: { + const double leftLoss = deliveryProblemRateOf(left); + const double rightLoss = deliveryProblemRateOf(right); + comparison = leftLoss < rightLoss ? -1 : leftLoss > rightLoss ? 1 : 0; + break; + } + case 10: comparison = (left.lastSeen - left.firstSeen) < (right.lastSeen - right.firstSeen) ? -1 : (left.lastSeen - left.firstSeen) > (right.lastSeen - right.firstSeen) ? 1 : 0; break; - case 10: comparison = left.process.compare(right.process); break; + case 11: comparison = left.process.compare(right.process); break; default: break; } if (comparison == 0) comparison = leftHost.compare(rightHost); @@ -169,6 +189,15 @@ namespace ui { detailRow("Init RTT", ""); } detailRow("Duration", formatDuration(flow.firstSeen, flow.lastSeen)); + if (flow.key.protocol == "TCP") { + // Per direction and absolute here: the table already shows the + // combined rate, and when diagnosing you need to know which + // side is losing packets. + detailRow("Retrans", std::format("{} / {}", + flow.retransmissionsUp, flow.retransmissionsDown)); + detailRow("Reordered", std::format("{} / {}", + flow.outOfOrderUp, flow.outOfOrderDown)); + } ImGui::EndTable(); } @@ -258,7 +287,7 @@ namespace ui { | ImGuiTableFlags_Resizable | ImGuiTableFlags_Sortable; - if (ImGui::BeginTable("FlowsTable", 11, tableFlags)) { + if (ImGui::BeginTable("FlowsTable", 12, tableFlags)) { ImGui::TableSetupColumn("Host", ImGuiTableColumnFlags_WidthStretch); ImGui::TableSetupColumn("Country", ImGuiTableColumnFlags_WidthFixed, scaled(65.0f)); ImGui::TableSetupColumn("Org", ImGuiTableColumnFlags_WidthFixed, scaled(180.0f)); @@ -268,6 +297,7 @@ namespace ui { ImGui::TableSetupColumn("Bytes", ImGuiTableColumnFlags_WidthFixed, scaled(90.0f)); ImGui::TableSetupColumn("Rate", ImGuiTableColumnFlags_WidthFixed | ImGuiTableColumnFlags_DefaultSort | ImGuiTableColumnFlags_PreferSortDescending, scaled(90.0f)); ImGui::TableSetupColumn("RTT", ImGuiTableColumnFlags_WidthFixed, scaled(85.0f)); + ImGui::TableSetupColumn("Loss", ImGuiTableColumnFlags_WidthFixed | ImGuiTableColumnFlags_PreferSortDescending, scaled(70.0f)); ImGui::TableSetupColumn("Dur", ImGuiTableColumnFlags_WidthFixed, scaled(70.0f)); ImGui::TableSetupColumn("App", ImGuiTableColumnFlags_WidthFixed, scaled(130.0f)); ImGui::TableHeadersRow(); @@ -366,9 +396,29 @@ namespace ui { ImGui::TextDisabled("--"); } ImGui::TableSetColumnIndex(9); + if (flow.key.protocol != "TCP") { + // Only TCP carries the sequence numbers this is derived from. + ImGui::TextDisabled("--"); + } else { + const uint64_t problems = deliveryProblemsOf(flow); + const double lossRate = deliveryProblemRateOf(flow); + const ImVec4& lossColor = lossRate < 1.0 ? kSuccess + : lossRate < 3.0 ? kWarning : kDanger; + ImGui::TextColored(lossColor, "%.1f%%", lossRate); + if (problems > 0 && ImGui::IsItemHovered(ImGuiHoveredFlags_ForTooltip)) { + ImGui::SetTooltip( + "%llu retransmitted, %llu out of order, of %llu packets", + static_cast( + flow.retransmissionsUp + flow.retransmissionsDown), + static_cast( + flow.outOfOrderUp + flow.outOfOrderDown), + static_cast(flow.packets)); + } + } + ImGui::TableSetColumnIndex(10); const std::string duration = formatDuration(flow.firstSeen, flow.lastSeen); ImGui::TextUnformatted(duration.c_str()); - ImGui::TableSetColumnIndex(10); + ImGui::TableSetColumnIndex(11); if (flow.process.empty()) { ImGui::TextDisabled("--"); } else { diff --git a/tests/flow_exporter_test.cpp b/tests/flow_exporter_test.cpp index f41d563..362f51f 100644 --- a/tests/flow_exporter_test.cpp +++ b/tests/flow_exporter_test.cpp @@ -31,6 +31,8 @@ namespace { flow.lastSeen = 1'700'000'005'000'000; flow.rateBytesPerSecond = 1536.0; flow.initialRttMicroseconds = 24'500; + flow.retransmissionsUp = 3; + flow.outOfOrderDown = 2; return flow; } @@ -93,6 +95,9 @@ TEST(FlowExporterTest, JsonContainsExpectedFieldsAndValues) { EXPECT_NE(document.find("\"initial_rtt_ms\": 24.50"), std::string::npos); EXPECT_NE(document.find("\"duration_sec\": 5"), std::string::npos); EXPECT_NE(document.find("\"encrypted_tunnel\": false"), std::string::npos); + EXPECT_NE(document.find("\"retransmissions_up\": 3"), std::string::npos); + EXPECT_NE(document.find("\"retransmissions_down\": 0"), std::string::npos); + EXPECT_NE(document.find("\"out_of_order_down\": 2"), std::string::npos); } TEST(FlowExporterTest, JsonWithNoFlowsIsStillValid) { @@ -152,7 +157,8 @@ TEST(FlowExporterTest, CsvKeepsItsHeaderAndQuotesEmbeddedCommas) { ASSERT_NE(headerEnd, std::string::npos); EXPECT_EQ(csv.substr(0, headerEnd), "host,src_ip,src_port,dst_ip,dst_port,protocol,service,process,country,organization," - "packets,bytes_up,bytes_down,rate_bytes_per_sec,initial_rtt_ms,duration_sec"); + "packets,bytes_up,bytes_down,rate_bytes_per_sec,initial_rtt_ms,duration_sec," + "retransmissions_up,retransmissions_down,out_of_order_up,out_of_order_down"); // The comma and quote must be escaped, not passed through raw. EXPECT_NE(csv.find("\"a,b\"\"c\""), std::string::npos) << csv; } diff --git a/tests/flow_loss_test.cpp b/tests/flow_loss_test.cpp new file mode 100644 index 0000000..39032c0 --- /dev/null +++ b/tests/flow_loss_test.cpp @@ -0,0 +1,193 @@ +#include "core/FlowAggregator.hpp" + +#include + +#include +#include +#include + +// Retransmission and out-of-order accounting. The sequence-space arithmetic +// here is the kind that looks obviously correct and is not: these pin the +// wraparound and partial-overlap cases that a plain integer comparison gets +// wrong only after a stream has moved four gigabytes. +namespace { + + constexpr int64_t kBaseTimestamp = 1'700'000'000'000'000; + + core::ParsedPacket makeSegment(uint32_t sequence, uint32_t payloadLength, + bool fromClient = true, int64_t timestamp = kBaseTimestamp) { + core::ParsedPacket packet; + packet.protocol = "TCP"; + packet.timestamp = timestamp; + packet.srcIP = fromClient ? "192.168.1.10" : "93.184.216.34"; + packet.dstIP = fromClient ? "93.184.216.34" : "192.168.1.10"; + packet.srcPort = fromClient ? 50000 : 443; + packet.dstPort = fromClient ? 443 : 50000; + packet.tcpSeq = sequence; + packet.tcpAck = true; + packet.payloadOffset = 54; + packet.payloadLength = payloadLength; + packet.length = 54 + payloadLength; + return packet; + } + + core::Flow onlyFlow(const core::FlowAggregator& aggregator) { + const auto flows = aggregator.snapshot(kBaseTimestamp + 1'000'000); + EXPECT_EQ(flows.size(), 1u); + return flows.empty() ? core::Flow{} : flows.front(); + } + +} // namespace + +TEST(FlowLossTest, InOrderStreamReportsNoProblems) { + core::FlowAggregator aggregator; + aggregator.update(makeSegment(1000, 100)); + aggregator.update(makeSegment(1100, 100)); + aggregator.update(makeSegment(1200, 100)); + + const auto flow = onlyFlow(aggregator); + EXPECT_EQ(flow.retransmissionsUp, 0u); + EXPECT_EQ(flow.outOfOrderUp, 0u); + EXPECT_EQ(flow.packets, 3u); +} + +TEST(FlowLossTest, ExactDuplicateCountsAsRetransmission) { + core::FlowAggregator aggregator; + aggregator.update(makeSegment(1000, 100)); + aggregator.update(makeSegment(1100, 100)); + aggregator.update(makeSegment(1100, 100)); // the wire repeats itself + + const auto flow = onlyFlow(aggregator); + EXPECT_EQ(flow.retransmissionsUp, 1u); + EXPECT_EQ(flow.outOfOrderUp, 0u); +} + +TEST(FlowLossTest, PartialOverlapCountsOnceAndStillAdvances) { + core::FlowAggregator aggregator; + aggregator.update(makeSegment(1000, 200)); // covers 1000..1200 + // Rewinds 100 bytes but carries 100 genuinely new ones (1100..1300). + aggregator.update(makeSegment(1100, 200)); + // The next in-order segment must not itself look like a problem, which it + // would if the overlap had failed to advance the expected sequence. + aggregator.update(makeSegment(1300, 100)); + + const auto flow = onlyFlow(aggregator); + EXPECT_EQ(flow.retransmissionsUp, 1u); + EXPECT_EQ(flow.outOfOrderUp, 0u); +} + +TEST(FlowLossTest, GapThenFillCountsOneOutOfOrderAndOneRetransmission) { + core::FlowAggregator aggregator; + aggregator.update(makeSegment(1000, 100)); // A + aggregator.update(makeSegment(1200, 100)); // C, skipping B + aggregator.update(makeSegment(1100, 100)); // B arrives late + + const auto flow = onlyFlow(aggregator); + EXPECT_EQ(flow.outOfOrderUp, 1u); // observed at C + EXPECT_EQ(flow.retransmissionsUp, 1u); // B lands behind the high-water mark +} + +TEST(FlowLossTest, SequenceWraparoundIsNotMistakenForLoss) { + core::FlowAggregator aggregator; + // Runs off the end of the 32-bit sequence space and back around. Every + // segment is strictly in order, so nothing may be reported. + aggregator.update(makeSegment(0xFFFFFF00u, 100)); // ends at 0xFFFFFF64 + aggregator.update(makeSegment(0xFFFFFF64u, 200)); // ends at 0x0000002C + aggregator.update(makeSegment(0x0000002Cu, 100)); // ends at 0x00000090 + aggregator.update(makeSegment(0x00000090u, 100)); + + const auto flow = onlyFlow(aggregator); + EXPECT_EQ(flow.retransmissionsUp, 0u) + << "wraparound was misread as a retransmission"; + EXPECT_EQ(flow.outOfOrderUp, 0u) + << "wraparound was misread as reordering"; +} + +TEST(FlowLossTest, RetransmissionAcrossWraparoundIsStillDetected) { + core::FlowAggregator aggregator; + aggregator.update(makeSegment(0xFFFFFF00u, 100)); + aggregator.update(makeSegment(0xFFFFFF64u, 200)); // wraps to 0x2C + aggregator.update(makeSegment(0xFFFFFF64u, 200)); // repeat of the wrapping segment + + const auto flow = onlyFlow(aggregator); + EXPECT_EQ(flow.retransmissionsUp, 1u); + EXPECT_EQ(flow.outOfOrderUp, 0u); +} + +TEST(FlowLossTest, DirectionsAreCountedIndependently) { + core::FlowAggregator aggregator; + aggregator.update(makeSegment(1000, 100, /*fromClient=*/false)); + aggregator.update(makeSegment(1100, 100, /*fromClient=*/false)); + aggregator.update(makeSegment(1100, 100, /*fromClient=*/false)); // server retransmits + aggregator.update(makeSegment(500, 50, /*fromClient=*/true)); + aggregator.update(makeSegment(550, 50, /*fromClient=*/true)); + + const auto flow = onlyFlow(aggregator); + // The flow key is canonical, so establish which side is which rather than + // assuming the client ended up as `src`. + const bool clientIsSource = flow.key.srcPort == 50000; + const uint64_t serverRetransmissions = + clientIsSource ? flow.retransmissionsDown : flow.retransmissionsUp; + const uint64_t clientRetransmissions = + clientIsSource ? flow.retransmissionsUp : flow.retransmissionsDown; + + EXPECT_EQ(serverRetransmissions, 1u); + EXPECT_EQ(clientRetransmissions, 0u) << "a server retransmit leaked into the client direction"; +} + +TEST(FlowLossTest, PureAcksAndFlowsStartingMidStreamReportNothing) { + core::FlowAggregator aggregator; + // A capture that began after the connection did: the first segment we see + // sits at an arbitrary offset and must only establish the baseline. + aggregator.update(makeSegment(987'654'321u, 100)); + // Bare ACKs carry no payload and repeat sequence numbers by design. + core::ParsedPacket bareAck = makeSegment(987'654'421u, 0); + aggregator.update(bareAck); + aggregator.update(bareAck); + aggregator.update(makeSegment(987'654'421u, 100)); + + const auto flow = onlyFlow(aggregator); + EXPECT_EQ(flow.retransmissionsUp, 0u); + EXPECT_EQ(flow.outOfOrderUp, 0u); +} + +TEST(FlowLossTest, ResetClearsSequenceTrackingForAReusedPortPair) { + core::FlowAggregator aggregator; + aggregator.update(makeSegment(1000, 100)); + aggregator.update(makeSegment(1100, 100)); + + core::ParsedPacket reset = makeSegment(1200, 0); + reset.tcpRst = true; + aggregator.update(reset); + + // A new connection reusing the same ports starts a fresh sequence space; + // without the reset this would look like a huge backwards jump. + aggregator.update(makeSegment(50, 100)); + aggregator.update(makeSegment(150, 100)); + + const auto flow = onlyFlow(aggregator); + EXPECT_EQ(flow.retransmissionsUp, 0u); + EXPECT_EQ(flow.outOfOrderUp, 0u); +} + +TEST(FlowLossTest, NonTcpTrafficNeverReportsLoss) { + core::FlowAggregator aggregator; + core::ParsedPacket udp; + udp.protocol = "UDP"; + udp.timestamp = kBaseTimestamp; + udp.srcIP = "192.168.1.10"; + udp.dstIP = "1.1.1.1"; + udp.srcPort = 53000; + udp.dstPort = 53; + udp.tcpSeq = 999'999; // stale field; must be ignored for UDP + udp.payloadLength = 40; + udp.length = 82; + + aggregator.update(udp); + aggregator.update(udp); + + const auto flow = onlyFlow(aggregator); + EXPECT_EQ(flow.retransmissionsUp, 0u); + EXPECT_EQ(flow.retransmissionsDown, 0u); + EXPECT_EQ(flow.outOfOrderUp, 0u); +} diff --git a/tools/make_sample_pcap.cpp b/tools/make_sample_pcap.cpp index ef6c7fa..3ba3c42 100644 --- a/tools/make_sample_pcap.cpp +++ b/tools/make_sample_pcap.cpp @@ -47,17 +47,37 @@ namespace { f.write(reinterpret_cast(out.data()), static_cast(out.size())); } - std::vector tlsClientHelloPacket(const std::string& sni) { - const auto tls = test::makeTlsClientHello(sni); + const std::vector kClientAddress = {192, 168, 1, 42}; + + // 20-byte TCP header. test::appendTcpHeader hardcodes a zero + // acknowledgement number, which would make the SYN-ACKs below wrong if + // anyone opened this capture in Wireshark. + void appendTcpHeader(std::vector& packet, uint16_t sourcePort, + uint16_t destinationPort, uint32_t sequence, uint32_t acknowledgement, uint8_t flags) { + test::appendU16(packet, sourcePort); + test::appendU16(packet, destinationPort); + test::appendU32BE(packet, sequence); + test::appendU32BE(packet, acknowledgement); + packet.push_back(0x50); // data offset = 5 words, no options + packet.push_back(flags); + test::appendU16(packet, 0xFFFF); // window + test::appendU16(packet, 0x0000); // checksum + test::appendU16(packet, 0x0000); // urgent pointer + } + + std::vector tcpSegment(const std::vector& serverAddress, + uint16_t clientPort, uint32_t sequence, uint32_t acknowledgement, uint8_t flags, + bool fromServer, const std::vector& payload = {}) { std::vector packet; test::appendEthernetHeader(packet, 0x0800); - test::appendIPv4Header(packet, static_cast(20 + 20 + tls.size()), 6, - {192, 168, 1, 42}, {93, 184, 216, 34}); - test::appendU16(packet, 50123); - test::appendU16(packet, 443); - packet.insert(packet.end(), {0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, - 0x50, 0x18, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00}); - packet.insert(packet.end(), tls.begin(), tls.end()); + test::appendIPv4Header(packet, static_cast(20 + 20 + payload.size()), 6, + fromServer ? serverAddress : kClientAddress, + fromServer ? kClientAddress : serverAddress); + appendTcpHeader(packet, + fromServer ? uint16_t{443} : clientPort, + fromServer ? clientPort : uint16_t{443}, + sequence, acknowledgement, flags); + packet.insert(packet.end(), payload.begin(), payload.end()); return packet; } @@ -129,18 +149,6 @@ namespace { return packet; } - std::vector tcpSynPacket() { - std::vector packet; - test::appendEthernetHeader(packet, 0x0800); - test::appendIPv4Header(packet, static_cast(20 + 20), 6, - {192, 168, 1, 42}, {93, 184, 216, 34}); - test::appendU16(packet, 50123); - test::appendU16(packet, 443); - packet.insert(packet.end(), {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x50, 0x02, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00}); - return packet; - } - } int main(int argc, char** argv) { @@ -151,19 +159,49 @@ int main(int argc, char** argv) { const std::filesystem::path output = argv[1]; std::filesystem::create_directories(output.parent_path()); + // One resolvable site per connection. Each gets its own address and client + // port so the capture yields three distinct flows, each matching the DNS + // answer that precedes it — previously every ClientHello was addressed to + // the same server regardless of the name it advertised, which collapsed + // them into a single flow whose repeated sequence numbers read as + // retransmissions. + struct Site { + std::string hostname; + std::vector address; + uint16_t clientPort; + uint32_t roundTripMicroseconds; // gap between SYN and SYN-ACK + }; + const std::vector sites = { + {"www.example.com", {93, 184, 216, 34}, 50123, 12'000}, + {"api.github.com", {140, 82, 121, 6}, 50124, 38'000}, + {"cdn.netprobe.test", {52, 84, 100, 5}, 50125, 6'500}, + }; + + constexpr uint8_t kSyn = 0x02; + constexpr uint8_t kSynAck = 0x12; + constexpr uint8_t kPshAck = 0x18; + std::vector packets; - uint32_t t = 1700000000; - packets.push_back({t, 0, arpRequestPacket()}); - packets.push_back({t, 1500, dnsQueryPacket("www.example.com")}); - packets.push_back({t, 4200, dnsResponsePacket("www.example.com", {93, 184, 216, 34})}); - packets.push_back({t, 5100, tcpSynPacket()}); - packets.push_back({t, 6300, tlsClientHelloPacket("www.example.com")}); - packets.push_back({t, 8700, dnsQueryPacket("api.github.com")}); - packets.push_back({t, 12100, dnsResponsePacket("api.github.com", {140, 82, 121, 6})}); - packets.push_back({t, 13400, tlsClientHelloPacket("api.github.com")}); - packets.push_back({t, 19000, dnsQueryPacket("cdn.netprobe.test")}); - packets.push_back({t, 21300, dnsResponsePacket("cdn.netprobe.test", {52, 84, 100, 5})}); - packets.push_back({t, 22700, tlsClientHelloPacket("cdn.netprobe.test")}); + const uint32_t t = 1700000000; + uint32_t at = 0; + const auto advance = [&at](uint32_t microseconds) { return at += microseconds; }; + + packets.push_back({t, at, arpRequestPacket()}); + + for (const auto& site : sites) { + packets.push_back({t, advance(1'500), dnsQueryPacket(site.hostname)}); + packets.push_back({t, advance(2'700), dnsResponsePacket(site.hostname, site.address)}); + + // A complete three-way handshake, so the flow reports an initial RTT. + // The SYN consumes sequence 0, which is why the ClientHello starts at 1. + packets.push_back({t, advance(900), + tcpSegment(site.address, site.clientPort, 0, 0, kSyn, /*fromServer=*/false)}); + packets.push_back({t, advance(site.roundTripMicroseconds), + tcpSegment(site.address, site.clientPort, 0, 1, kSynAck, /*fromServer=*/true)}); + packets.push_back({t, advance(1'200), + tcpSegment(site.address, site.clientPort, 1, 1, kPshAck, /*fromServer=*/false, + test::makeTlsClientHello(site.hostname))}); + } writePcap(output, packets); std::cout << "Wrote " << packets.size() << " packets to " << output.string() << "\n"; From 0c3af2a0adc782ab3fcac9dc374069c3fc524d7d Mon Sep 17 00:00:00 2001 From: YehiaGewily <79144005+YehiaGewily@users.noreply.github.com> Date: Sun, 19 Jul 2026 03:44:09 +0300 Subject: [PATCH 07/12] test: add adversarial malformed-packet fixtures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 35 named fixtures of the shapes that crash capture tools in production: headers claiming lengths the buffer does not contain, IPv4 IHL and total-length lies, TCP data offsets past the payload, UDP lengths below their own header, IPv6 extension chains that overrun or point at themselves, DNS compression-pointer loops and record counts larger than the payload, truncated TLS records, and QUIC connection-ID and token lengths beyond the packet. Each is checked against the parsers, the stateful reassemblers, and the full AnalysisSession. The assertions are about degradation rather than success: a payload window must never point outside the captured bytes, and recovered hostnames must stay bounded — a parser that walks off the end of a packet and returns a plausible-looking hostname is worse than one that crashes, because the wrong answer is shown to the user as fact. Also checks every prefix of a valid frame exhaustively, since any driver or capture file can hand us a truncated packet, and pins the termination of the IPv6 extension walk, which is an unbounded `while (true)` that is safe only because each step advances at least eight bytes. No defects were found: the existing parsers already handle all of these. The value is the regression barrier — these run under the ASan/UBSan CI job, where an out-of-bounds read fails the build instead of passing silently. The same fixtures now seed the fuzz corpus, taking it from 5 entries to 40, so libFuzzer starts from these shapes instead of rediscovering them byte by byte. 89/89 tests pass. --- CMakeLists.txt | 1 + README.md | 421 +++++++++++++++++----------------- tests/PacketBuilders.hpp | 268 ++++++++++++++++++++++ tests/fuzz/seed_generator.cpp | 15 +- tests/malformed_test.cpp | 189 +++++++++++++++ 5 files changed, 683 insertions(+), 211 deletions(-) create mode 100644 tests/malformed_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 9cedac5..06c1180 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -310,6 +310,7 @@ if(BUILD_TESTING) tests/analysis_session_test.cpp tests/flow_exporter_test.cpp tests/flow_loss_test.cpp + tests/malformed_test.cpp ) # src/, pcap headers, and the crypto/geo libraries all arrive through # NetProbeCore's PUBLIC usage requirements. diff --git a/README.md b/README.md index b6de66a..f7c4516 100644 --- a/README.md +++ b/README.md @@ -1,210 +1,211 @@ -# NetProbe - -A real-time and offline network traffic analyzer built with **Modern C++20**, **Npcap/libpcap**, and **Dear ImGui**. - -![Platform](https://img.shields.io/badge/platform-Windows%20%7C%20Linux%20%7C%20macOS-lightgrey.svg) ![Standard](https://img.shields.io/badge/C%2B%2B-20-blue.svg) ![License](https://img.shields.io/badge/license-MIT-blue.svg) - -## Download - -Download the ZIP for your platform from [GitHub Releases](https://github.com/YehiaGewily/netprobe-cpp/releases). Windows releases require the [Npcap driver](https://npcap.com/#download) for live capture and PCAP support. Linux and macOS releases require their system libpcap runtime. - -## Features - -- **Live and offline capture**: inspect an active adapter or drag a `.pcap` / `.pcapng` into the app (classic libpcap and modern PCAPNG both supported). -- **Link-layer aware**: decodes Ethernet, VLAN/QinQ, Linux cooked captures (`SLL`/`SLL2`, as produced by the `any` device), BSD/OpenBSD loopback, and raw-IP captures. Exported PCAPs preserve the original link type. -- **Protocol insight**: IPv4/IPv6, TCP, UDP, ARP, ICMP/ICMPv6, SCTP, and named non-IP frames (LLDP, EAPOL, PPPoE, MPLS). -- **Tunnel descent**: transparently decodes **GRE, IP-in-IP, 6in4, VXLAN, GENEVE, MPLS, and PPPoE**, so flows are keyed on the real inner endpoints rather than the tunnel. Encrypted tunnels (ESP/IPsec, WireGuard, OpenVPN) are labelled as such instead of being silently misreported. -- **Deep application identification**: - - **TLS SNI on any port** (content-sniffed, not port-gated), with **reassembly of ClientHellos split across TCP segments** — required now that post-quantum key shares push the message past one MSS. - - **QUIC ClientHello SNI** for **v1 (RFC 9000) and v2 (RFC 9369)** on any UDP port, decrypting Initial packets and **reassembling CRYPTO frames across multiple Initials**. - - **Cleartext HTTP**: request line and `Host:` header, which names hosts even when no DNS was observed. - - **DNS/mDNS**: A/AAAA/CNAME, plus **PTR** (names LAN devices that never appear in a forward lookup), SRV, TXT, and **HTTPS/SVCB** records. -- **Encrypted-DNS visibility**: detects DoH/DoT/DoQ and **Encrypted Client Hello**, and tells you when name resolution has moved off the wire rather than silently showing bare IPs. -- **Packet detail pane**: click any packet to see a structured Frame → Tunnel → Network → Transport → Application decode plus an `xxd`-style hex dump of the raw bytes. -- **Flows view**: sortable per-connection byte totals, one-second rate, **initial TCP RTT** (from the SYN / SYN-ACK delta, color-coded by latency), duration, **per-flow retransmission and out-of-order rates**, service, hostname, owning process, Country, and ASN/organization, with a detail pane and live rate sparkline. Flow keys are canonical, so both directions of a peer-to-peer conversation collapse into a single row. -- **Statistics view**: protocol hierarchy, top talkers by bytes, and name-resolution health. -- **Process resolution**: the *App* column in both the packet list and the flows table shows the owning process on Windows (iphlpapi), Linux (`/proc/net/*` + `/proc//fd/`), and macOS (`proc_pidfdinfo`). Flows capture the process while the socket is live, so the name persists after a short-lived connection closes. Running elevated widens coverage to other users' processes. -- **Filtering and export**: live BPF capture filters such as `tcp port 443`, a separate display filter over captured packets, PCAP export, and flow export to CSV or JSON. -- **Light and dark themes**, adjustable UI scale, and keyboard shortcuts — all persisted between runs. -- **Cross-platform backends**: Npcap on Windows and libpcap on Linux/macOS. - -| Live packet analysis | Connection flows | Offline PCAP mode | -| --- | --- | --- | -| Protocol, hostname, and per-packet hex | Rate, service, GeoIP, and ASN | No administrator rights required | - -## Tech Stack - -

- NetProbe technology stack: C++20 core surrounded by its FetchContent-pinned dependencies -

- -- **Language**: C++20 -- **Packet Capture**: [Npcap SDK](https://npcap.com/) (Windows) / libpcap (Linux/macOS) -- **UI Framework**: [Dear ImGui](https://github.com/ocornut/imgui) + [GLFW](https://www.glfw.org/) with docking -- **Plotting**: [ImPlot](https://github.com/epezent/implot) -- **Crypto** (QUIC Initial decryption): [Mbed TLS](https://github.com/Mbed-TLS/mbedtls) — HKDF-SHA256, AES-128-ECB, AES-128-GCM -- **GeoIP/ASN**: [libmaxminddb](https://github.com/maxmind/libmaxminddb) -- **File dialogs**: [Native File Dialog Extended](https://github.com/btzy/nativefiledialog-extended) -- **Build System**: CMake (all third-party deps pinned via `FetchContent`) -- **Tests / fuzzing**: GoogleTest + libFuzzer harness on `ProtocolParser` and `DNSParser` - - -## Prerequisites - -NetProbe uses Npcap on Windows and libpcap on Linux/macOS. - -- **Windows 10/11**: Visual Studio 2022 with C++ Desktop Development, the [Npcap driver](https://npcap.com/#download), and the Npcap SDK extracted to `C:\Npcap-SDK` (`Include` and `Lib` directly inside). -- **Ubuntu/Debian**: `sudo apt install build-essential cmake libpcap-dev libgl1-mesa-dev libgtk-3-dev pkg-config` -- **macOS**: Xcode Command Line Tools and CMake. libpcap and OpenGL are provided by macOS; Native File Dialog uses Cocoa. - -## Build Instructions - -```powershell -# 1. Clone the repository -git clone https://github.com/YehiaGewily/netprobe-cpp.git -cd netprobe-cpp - -# 2. Build via Script (Recommended) -.\build_project.bat -``` - -On Linux or macOS: - -```bash -cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -cmake --build build --parallel -ctest --test-dir build --output-on-failure -``` - -To create the same ZIP layout used by Releases: - -```bash -cpack --config build/CPackConfig.cmake -C Release -G ZIP -B package -``` - -### Optional CMake flags - -| Flag | Default | What it does | -| --- | --- | --- | -| `-DBUILD_TESTING=ON` | `ON` when top-level | Build the GoogleTest suite (`NetProbeTests`). | -| `-DENABLE_SANITIZERS=ON` | `OFF` | Build with `-fsanitize=address,undefined` (Clang/GCC only). | -| `-DBUILD_FUZZERS=ON` | `OFF` | Build the libFuzzer harness for the protocol parsers. Requires `clang` / `clang-cl`. | - -## Usage - -1. Run the executable from the extracted release ZIP or build output directory. -2. **No admin? Try the sample first.** The build emits `data/sample.pcap` (mixed ARP / DNS / TCP traffic: three resolvable hostnames, each with a full TLS handshake and a measurable RTT). Open it via **File → Open PCAP…** to exercise the full UI without elevation. -3. **Live capture** usually requires elevated privileges. On Windows: - - ```powershell - Start-Process ".\build\Release\NetProbe.exe" -Verb RunAs - ``` - -4. Select a live adapter in the menu bar, or choose **File → Open PCAP…** / drop a PCAP file onto the window. -5. Use **Flows** for per-connection activity, or enter a BPF filter such as `tcp port 443` in the menu bar. -6. **Click a packet row** in *Live Packets* to populate the *Packet Detail* pane with the layered decode and hex dump. - -For GeoIP and ASN columns, place `GeoLite2-Country.mmdb` and `GeoLite2-ASN.mmdb` in `data/`. See [data/README.md](data/README.md). - -## Headless CLI - -`netprobe-cli` runs the same capture and analysis pipeline with no window, for servers, CI, and SSH sessions. It ships alongside the GUI in every release ZIP. - -```bash -# List the adapters available for live capture -netprobe-cli --list-devices - -# Replay a capture file and export its flows as JSON -netprobe-cli -r capture.pcap -o flows.json - -# Capture live for 60 seconds, filtered, exporting CSV -netprobe-cli -i eth0 -f "tcp port 443" --duration 60 -o flows.csv - -# Pipe JSON straight into jq -netprobe-cli -r capture.pcap -o - | jq '.flows[] | select(.bytes_down > 100000)' -``` - -| Flag | Purpose | -| --- | --- | -| `-r, --read ` | Replay an offline capture | -| `-i, --device ` | Capture live from an adapter (needs privileges) | -| `-f, --filter ` | BPF capture filter, live capture only | -| `-o, --output ` | Destination file, or `-` for stdout | -| `--format ` | Override the format inferred from the extension | -| `--duration ` | Stop after this many seconds | -| `--packet-count ` | Stop after this many packets | -| `--no-process` | Skip the owning-process lookup | - -Ctrl+C stops a live capture and still writes the output. Exit codes are `0` success, `1` usage error, `2` runtime failure, so scripts can tell a bad invocation from a failed capture. Diagnostics go to stderr, which keeps `-o -` output clean for piping. - -## Architecture - -NetProbe is built around a bounded **producer-consumer** pipeline. Capture runs on a backend-owned path, while the Dear ImGui frame loop drains packets, enriches them, and updates view models without blocking the capture thread. - -

- Producer-consumer threading: capture thread pushes into a bounded queue, UI thread drains it -

- -**Figure 1: Runtime Component Map** - -

- NetProbe runtime architecture: capture, core, and UI layers -

- -**Figure 2: Packet Decode And Enrichment Pipeline** - -

- Packet decode pipeline: link, network, tunnel, transport, and application layers converging into flows -

- -**Figure 3: Live Capture Control Flow** - -

- Live capture sequence: user starts capture, engine opens device and spawns a thread, queue feeds the UI frame loop -

- -**Figure 4: Offline PCAP Path** - -```mermaid -flowchart LR - file[".pcap / .pcapng file"] --> open["CaptureEngine::openFile"] - open --> backend["pcap offline reader"] - backend --> dlt["Snapshot link type
for correct export DLT"] - backend --> packets["Read packets until EOF"] - packets --> session["Retain recent session packets"] - packets --> queue["Push into PacketQueue"] - queue --> ui["GuiLayer frame loop"] - ui --> same["Same parser, DNS/SNI recovery,
flow aggregation, and views as live mode"] -``` - -### Main Modules - -- **Capture backend (`src/capture/`)**: `ICaptureBackend` hides the Npcap and libpcap implementations. `CaptureEngine` owns the backend, live capture thread, BPF application, offline reads, and a capped session buffer for PCAP export. -- **Packet queue (`src/core/PacketQueue.hpp`)**: thread-safe bounded queue (`std::mutex` + `std::condition_variable`). On overflow it drops the oldest packets so the UI stays close to live traffic; the dropped count is surfaced in the control bar. -- **Protocol stack (`src/core/`)**: - - `LinkType` maps libpcap `DLT_*` values to supported encapsulations, so cooked, loopback, raw-IP, and Ethernet-family captures are decoded intentionally. - - `ProtocolParser` is the stateless fast path: link layer -> IPv4/IPv6 -> tunnel descent -> transport -> application hints and service classification. - - `TlsReassembler` buffers split TCP ClientHellos with size and expiry caps; it is owned by the UI layer because it is stream state. - - `QuicParser` decrypts QUIC v1/v2 Initial packets with mbedTLS and extracts TLS ClientHello data from CRYPTO frames. - - `QuicTracker` stitches CRYPTO fragments across multiple Initial packets, keyed by connection id. - - `DNSParser` extracts DNS/mDNS/LLMNR answers, feeds `HostnameCache`, and flags advertised ECH configs. - - `FlowAggregator` collapses both directions of a conversation into one canonical key, tracks rates and byte totals, and measures initial TCP RTT from SYN/SYN-ACK timing. - - `GeoIPResolver` and `ProcessResolver` add country/ASN and process ownership where platform data is available. -- **GUI layer (`src/ui/`)**: `GuiLayer` owns the Dear ImGui dockspace, capture controls, settings, queue drain, stateful reassemblers, DNS name cache, flow aggregator, and bounded packet history. `Dashboard`, `FlowsView`, `PacketView`, `PacketDetail`, and `StatsView` render those derived models. - -## Quality - -- **60 unit and integration tests** (GoogleTest) covering protocol parsing, link-type handling, tunnel descent, TLS/QUIC ClientHello reassembly, HTTP header extraction, DNS record coverage, canonical flow keying, RTT measurement, queue concurrency, GeoIP lookup, QUIC Initial decryption end-to-end, and both classic-PCAP + PCAPNG fixtures — including an end-to-end test that drives a crafted capture file through the same pipeline the UI runs. -- **libFuzzer harness** on `ProtocolParser::parse`, `DNSParser::parseResponse`, and the stateful `TlsReassembler` / `QuicTracker`, across every supported link type, with a deterministic seed corpus. CI fuzzes every push for 60 seconds on Ubuntu + Clang and uploads any crash inputs as build artifacts. -- **AddressSanitizer + UndefinedBehaviorSanitizer** CI job runs the full test suite under ASan/UBSan on every push. -- **Three-platform CI matrix** (Windows / Ubuntu / macOS) builds the app, builds and runs the test suite, and packages release ZIPs via CPack on tag pushes. Windows test runs use a 7-Zip-extracted Npcap user-mode DLL so the kernel-driver install is unnecessary in CI. - -## Contributing - - -Contributions are welcome! Please feel free to submit a Pull Request. - -## License - -This project is licensed under the MIT License. - +# NetProbe + +A real-time and offline network traffic analyzer built with **Modern C++20**, **Npcap/libpcap**, and **Dear ImGui**. + +![Platform](https://img.shields.io/badge/platform-Windows%20%7C%20Linux%20%7C%20macOS-lightgrey.svg) ![Standard](https://img.shields.io/badge/C%2B%2B-20-blue.svg) ![License](https://img.shields.io/badge/license-MIT-blue.svg) + +## Download + +Download the ZIP for your platform from [GitHub Releases](https://github.com/YehiaGewily/netprobe-cpp/releases). Windows releases require the [Npcap driver](https://npcap.com/#download) for live capture and PCAP support. Linux and macOS releases require their system libpcap runtime. + +## Features + +- **Live and offline capture**: inspect an active adapter or drag a `.pcap` / `.pcapng` into the app (classic libpcap and modern PCAPNG both supported). +- **Link-layer aware**: decodes Ethernet, VLAN/QinQ, Linux cooked captures (`SLL`/`SLL2`, as produced by the `any` device), BSD/OpenBSD loopback, and raw-IP captures. Exported PCAPs preserve the original link type. +- **Protocol insight**: IPv4/IPv6, TCP, UDP, ARP, ICMP/ICMPv6, SCTP, and named non-IP frames (LLDP, EAPOL, PPPoE, MPLS). +- **Tunnel descent**: transparently decodes **GRE, IP-in-IP, 6in4, VXLAN, GENEVE, MPLS, and PPPoE**, so flows are keyed on the real inner endpoints rather than the tunnel. Encrypted tunnels (ESP/IPsec, WireGuard, OpenVPN) are labelled as such instead of being silently misreported. +- **Deep application identification**: + - **TLS SNI on any port** (content-sniffed, not port-gated), with **reassembly of ClientHellos split across TCP segments** — required now that post-quantum key shares push the message past one MSS. + - **QUIC ClientHello SNI** for **v1 (RFC 9000) and v2 (RFC 9369)** on any UDP port, decrypting Initial packets and **reassembling CRYPTO frames across multiple Initials**. + - **Cleartext HTTP**: request line and `Host:` header, which names hosts even when no DNS was observed. + - **DNS/mDNS**: A/AAAA/CNAME, plus **PTR** (names LAN devices that never appear in a forward lookup), SRV, TXT, and **HTTPS/SVCB** records. +- **Encrypted-DNS visibility**: detects DoH/DoT/DoQ and **Encrypted Client Hello**, and tells you when name resolution has moved off the wire rather than silently showing bare IPs. +- **Packet detail pane**: click any packet to see a structured Frame → Tunnel → Network → Transport → Application decode plus an `xxd`-style hex dump of the raw bytes. +- **Flows view**: sortable per-connection byte totals, one-second rate, **initial TCP RTT** (from the SYN / SYN-ACK delta, color-coded by latency), duration, **per-flow retransmission and out-of-order rates**, service, hostname, owning process, Country, and ASN/organization, with a detail pane and live rate sparkline. Flow keys are canonical, so both directions of a peer-to-peer conversation collapse into a single row. +- **Statistics view**: protocol hierarchy, top talkers by bytes, and name-resolution health. +- **Process resolution**: the *App* column in both the packet list and the flows table shows the owning process on Windows (iphlpapi), Linux (`/proc/net/*` + `/proc//fd/`), and macOS (`proc_pidfdinfo`). Flows capture the process while the socket is live, so the name persists after a short-lived connection closes. Running elevated widens coverage to other users' processes. +- **Filtering and export**: live BPF capture filters such as `tcp port 443`, a separate display filter over captured packets, PCAP export, and flow export to CSV or JSON. +- **Light and dark themes**, adjustable UI scale, and keyboard shortcuts — all persisted between runs. +- **Cross-platform backends**: Npcap on Windows and libpcap on Linux/macOS. + +| Live packet analysis | Connection flows | Offline PCAP mode | +| --- | --- | --- | +| Protocol, hostname, and per-packet hex | Rate, service, GeoIP, and ASN | No administrator rights required | + +## Tech Stack + +

+ NetProbe technology stack: C++20 core surrounded by its FetchContent-pinned dependencies +

+ +- **Language**: C++20 +- **Packet Capture**: [Npcap SDK](https://npcap.com/) (Windows) / libpcap (Linux/macOS) +- **UI Framework**: [Dear ImGui](https://github.com/ocornut/imgui) + [GLFW](https://www.glfw.org/) with docking +- **Plotting**: [ImPlot](https://github.com/epezent/implot) +- **Crypto** (QUIC Initial decryption): [Mbed TLS](https://github.com/Mbed-TLS/mbedtls) — HKDF-SHA256, AES-128-ECB, AES-128-GCM +- **GeoIP/ASN**: [libmaxminddb](https://github.com/maxmind/libmaxminddb) +- **File dialogs**: [Native File Dialog Extended](https://github.com/btzy/nativefiledialog-extended) +- **Build System**: CMake (all third-party deps pinned via `FetchContent`) +- **Tests / fuzzing**: GoogleTest + libFuzzer harness on `ProtocolParser` and `DNSParser` + + +## Prerequisites + +NetProbe uses Npcap on Windows and libpcap on Linux/macOS. + +- **Windows 10/11**: Visual Studio 2022 with C++ Desktop Development, the [Npcap driver](https://npcap.com/#download), and the Npcap SDK extracted to `C:\Npcap-SDK` (`Include` and `Lib` directly inside). +- **Ubuntu/Debian**: `sudo apt install build-essential cmake libpcap-dev libgl1-mesa-dev libgtk-3-dev pkg-config` +- **macOS**: Xcode Command Line Tools and CMake. libpcap and OpenGL are provided by macOS; Native File Dialog uses Cocoa. + +## Build Instructions + +```powershell +# 1. Clone the repository +git clone https://github.com/YehiaGewily/netprobe-cpp.git +cd netprobe-cpp + +# 2. Build via Script (Recommended) +.\build_project.bat +``` + +On Linux or macOS: + +```bash +cmake -S . -B build -DCMAKE_BUILD_TYPE=Release +cmake --build build --parallel +ctest --test-dir build --output-on-failure +``` + +To create the same ZIP layout used by Releases: + +```bash +cpack --config build/CPackConfig.cmake -C Release -G ZIP -B package +``` + +### Optional CMake flags + +| Flag | Default | What it does | +| --- | --- | --- | +| `-DBUILD_TESTING=ON` | `ON` when top-level | Build the GoogleTest suite (`NetProbeTests`). | +| `-DENABLE_SANITIZERS=ON` | `OFF` | Build with `-fsanitize=address,undefined` (Clang/GCC only). | +| `-DBUILD_FUZZERS=ON` | `OFF` | Build the libFuzzer harness for the protocol parsers. Requires `clang` / `clang-cl`. | + +## Usage + +1. Run the executable from the extracted release ZIP or build output directory. +2. **No admin? Try the sample first.** The build emits `data/sample.pcap` (mixed ARP / DNS / TCP traffic: three resolvable hostnames, each with a full TLS handshake and a measurable RTT). Open it via **File → Open PCAP…** to exercise the full UI without elevation. +3. **Live capture** usually requires elevated privileges. On Windows: + + ```powershell + Start-Process ".\build\Release\NetProbe.exe" -Verb RunAs + ``` + +4. Select a live adapter in the menu bar, or choose **File → Open PCAP…** / drop a PCAP file onto the window. +5. Use **Flows** for per-connection activity, or enter a BPF filter such as `tcp port 443` in the menu bar. +6. **Click a packet row** in *Live Packets* to populate the *Packet Detail* pane with the layered decode and hex dump. + +For GeoIP and ASN columns, place `GeoLite2-Country.mmdb` and `GeoLite2-ASN.mmdb` in `data/`. See [data/README.md](data/README.md). + +## Headless CLI + +`netprobe-cli` runs the same capture and analysis pipeline with no window, for servers, CI, and SSH sessions. It ships alongside the GUI in every release ZIP. + +```bash +# List the adapters available for live capture +netprobe-cli --list-devices + +# Replay a capture file and export its flows as JSON +netprobe-cli -r capture.pcap -o flows.json + +# Capture live for 60 seconds, filtered, exporting CSV +netprobe-cli -i eth0 -f "tcp port 443" --duration 60 -o flows.csv + +# Pipe JSON straight into jq +netprobe-cli -r capture.pcap -o - | jq '.flows[] | select(.bytes_down > 100000)' +``` + +| Flag | Purpose | +| --- | --- | +| `-r, --read ` | Replay an offline capture | +| `-i, --device ` | Capture live from an adapter (needs privileges) | +| `-f, --filter ` | BPF capture filter, live capture only | +| `-o, --output ` | Destination file, or `-` for stdout | +| `--format ` | Override the format inferred from the extension | +| `--duration ` | Stop after this many seconds | +| `--packet-count ` | Stop after this many packets | +| `--no-process` | Skip the owning-process lookup | + +Ctrl+C stops a live capture and still writes the output. Exit codes are `0` success, `1` usage error, `2` runtime failure, so scripts can tell a bad invocation from a failed capture. Diagnostics go to stderr, which keeps `-o -` output clean for piping. + +## Architecture + +NetProbe is built around a bounded **producer-consumer** pipeline. Capture runs on a backend-owned path, while the Dear ImGui frame loop drains packets, enriches them, and updates view models without blocking the capture thread. + +

+ Producer-consumer threading: capture thread pushes into a bounded queue, UI thread drains it +

+ +**Figure 1: Runtime Component Map** + +

+ NetProbe runtime architecture: capture, core, and UI layers +

+ +**Figure 2: Packet Decode And Enrichment Pipeline** + +

+ Packet decode pipeline: link, network, tunnel, transport, and application layers converging into flows +

+ +**Figure 3: Live Capture Control Flow** + +

+ Live capture sequence: user starts capture, engine opens device and spawns a thread, queue feeds the UI frame loop +

+ +**Figure 4: Offline PCAP Path** + +```mermaid +flowchart LR + file[".pcap / .pcapng file"] --> open["CaptureEngine::openFile"] + open --> backend["pcap offline reader"] + backend --> dlt["Snapshot link type
for correct export DLT"] + backend --> packets["Read packets until EOF"] + packets --> session["Retain recent session packets"] + packets --> queue["Push into PacketQueue"] + queue --> ui["GuiLayer frame loop"] + ui --> same["Same parser, DNS/SNI recovery,
flow aggregation, and views as live mode"] +``` + +### Main Modules + +- **Capture backend (`src/capture/`)**: `ICaptureBackend` hides the Npcap and libpcap implementations. `CaptureEngine` owns the backend, live capture thread, BPF application, offline reads, and a capped session buffer for PCAP export. +- **Packet queue (`src/core/PacketQueue.hpp`)**: thread-safe bounded queue (`std::mutex` + `std::condition_variable`). On overflow it drops the oldest packets so the UI stays close to live traffic; the dropped count is surfaced in the control bar. +- **Protocol stack (`src/core/`)**: + - `LinkType` maps libpcap `DLT_*` values to supported encapsulations, so cooked, loopback, raw-IP, and Ethernet-family captures are decoded intentionally. + - `ProtocolParser` is the stateless fast path: link layer -> IPv4/IPv6 -> tunnel descent -> transport -> application hints and service classification. + - `TlsReassembler` buffers split TCP ClientHellos with size and expiry caps; it is owned by the UI layer because it is stream state. + - `QuicParser` decrypts QUIC v1/v2 Initial packets with mbedTLS and extracts TLS ClientHello data from CRYPTO frames. + - `QuicTracker` stitches CRYPTO fragments across multiple Initial packets, keyed by connection id. + - `DNSParser` extracts DNS/mDNS/LLMNR answers, feeds `HostnameCache`, and flags advertised ECH configs. + - `FlowAggregator` collapses both directions of a conversation into one canonical key, tracks rates and byte totals, and measures initial TCP RTT from SYN/SYN-ACK timing. + - `GeoIPResolver` and `ProcessResolver` add country/ASN and process ownership where platform data is available. +- **GUI layer (`src/ui/`)**: `GuiLayer` owns the Dear ImGui dockspace, capture controls, settings, queue drain, stateful reassemblers, DNS name cache, flow aggregator, and bounded packet history. `Dashboard`, `FlowsView`, `PacketView`, `PacketDetail`, and `StatsView` render those derived models. + +## Quality + +- **89 unit and integration tests** (GoogleTest) covering protocol parsing, link-type handling, tunnel descent, TLS/QUIC ClientHello reassembly, HTTP header extraction, DNS record coverage, canonical flow keying, RTT measurement, queue concurrency, GeoIP lookup, QUIC Initial decryption end-to-end, and both classic-PCAP + PCAPNG fixtures — including an end-to-end test that drives a crafted capture file through the same pipeline the UI runs, and end-to-end tests that run the real `netprobe-cli` binary over a capture file and validate the exported JSON and CSV. +- **Adversarial input suite**: 35 named malformed-packet fixtures — headers claiming lengths the buffer does not contain, IPv4 IHL and total-length lies, IPv6 extension-header chains that overrun or self-reference, DNS compression-pointer loops and inflated record counts, truncated TLS records, QUIC connection-ID and token lengths beyond the packet — plus an exhaustive check that every prefix of a valid frame parses without crashing. The same fixtures seed the fuzz corpus. +- **libFuzzer harness** on `ProtocolParser::parse`, `DNSParser::parseResponse`, and the stateful `TlsReassembler` / `QuicTracker`, across every supported link type, with a deterministic seed corpus. CI fuzzes every push for 60 seconds on Ubuntu + Clang and uploads any crash inputs as build artifacts. +- **AddressSanitizer + UndefinedBehaviorSanitizer** CI job runs the full test suite under ASan/UBSan on every push. +- **Three-platform CI matrix** (Windows / Ubuntu / macOS) builds the app, builds and runs the test suite, and packages release ZIPs via CPack on tag pushes. Windows test runs use a 7-Zip-extracted Npcap user-mode DLL so the kernel-driver install is unnecessary in CI. + +## Contributing + + +Contributions are welcome! Please feel free to submit a Pull Request. + +## License + +This project is licensed under the MIT License. + diff --git a/tests/PacketBuilders.hpp b/tests/PacketBuilders.hpp index 84f0baa..224e144 100644 --- a/tests/PacketBuilders.hpp +++ b/tests/PacketBuilders.hpp @@ -128,4 +128,272 @@ namespace test { return record; } + // --------------------------------------------------------------------- + // Adversarial inputs. + // + // Wire-format constructors that lie about their own structure: headers + // claiming lengths the buffer does not contain, chains that run off the + // end, counts that exceed the data present. These are the shapes that + // crash capture tools in production, and they are shared with the fuzz + // seed corpus so libFuzzer starts from them rather than rediscovering + // them byte by byte. + // --------------------------------------------------------------------- + namespace malformed { + + struct Case { + std::string name; + std::vector bytes; + }; + + // Offsets into an Ethernet + IPv4 frame. + inline constexpr size_t kEthernetSize = 14; + inline constexpr size_t kIPv4Offset = kEthernetSize; + inline constexpr size_t kIPv4VersionIhl = kIPv4Offset; + inline constexpr size_t kIPv4TotalLength = kIPv4Offset + 2; + inline constexpr size_t kTcpOffset = kIPv4Offset + 20; + inline constexpr size_t kTcpDataOffset = kTcpOffset + 12; + + inline std::vector validTcpFrame(const std::vector& payload) { + std::vector packet; + appendEthernetHeader(packet, 0x0800); + appendIPv4Header(packet, static_cast(20 + 20 + payload.size()), 6, + {192, 168, 1, 10}, {93, 184, 216, 34}); + appendTcpHeader(packet, 50000, 443, 1000, 0x18); + packet.insert(packet.end(), payload.begin(), payload.end()); + return packet; + } + + inline std::vector validUdpFrame(const std::vector& payload) { + std::vector packet; + appendEthernetHeader(packet, 0x0800); + appendIPv4Header(packet, static_cast(20 + 8 + payload.size()), 17, + {8, 8, 8, 8}, {192, 168, 1, 10}); + appendUdpHeader(packet, 53, 53000, static_cast(payload.size())); + packet.insert(packet.end(), payload.begin(), payload.end()); + return packet; + } + + inline std::vector truncatedTo(std::vector bytes, size_t size) { + if (bytes.size() > size) bytes.resize(size); + return bytes; + } + + // An IPv6 frame whose next-header chain is supplied verbatim, so a + // caller can build chains that overrun or run long. + inline std::vector ipv6WithChain(uint8_t firstNextHeader, + const std::vector& chain) { + std::vector packet; + appendEthernetHeader(packet, 0x86DD); + appendIPv6Header(packet, static_cast(chain.size()), firstNextHeader, + {0x20, 0x01, 0x0D, 0xB8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}, + {0x20, 0x01, 0x0D, 0xB8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}); + packet.insert(packet.end(), chain.begin(), chain.end()); + return packet; + } + + inline std::vector allCases() { + std::vector cases; + const auto add = [&cases](std::string name, std::vector bytes) { + cases.push_back({std::move(name), std::move(bytes)}); + }; + + const auto tlsFrame = validTcpFrame(makeTlsClientHello("truncated.example.com")); + const auto httpFrame = validTcpFrame(makeHttpRequest("truncated.example.com")); + + // --- Truncation at every layer boundary -------------------------- + add("empty", {}); + add("one-byte", {0x00}); + for (const size_t size : {6u, 13u, 14u, 15u, 20u, 33u, 34u, 40u, 45u, 53u}) { + add("tls-frame-truncated-to-" + std::to_string(size), + truncatedTo(tlsFrame, size)); + } + add("http-frame-truncated-mid-payload", truncatedTo(httpFrame, 60)); + + // --- IPv4 headers that lie -------------------------------------- + { + auto packet = tlsFrame; + packet[kIPv4VersionIhl] = 0x40; // IHL = 0, below the 20-byte minimum + add("ipv4-ihl-zero", packet); + } + { + auto packet = tlsFrame; + packet[kIPv4VersionIhl] = 0x44; // IHL = 4 words = 16 bytes + add("ipv4-ihl-below-minimum", packet); + } + { + auto packet = tlsFrame; + packet[kIPv4VersionIhl] = 0x4F; // IHL = 15 words = 60 bytes of options + add("ipv4-ihl-claims-options-past-header", packet); + } + { + auto packet = tlsFrame; + packet[kIPv4TotalLength] = 0xFF; // total length far beyond the buffer + packet[kIPv4TotalLength + 1] = 0xFF; + add("ipv4-total-length-exceeds-buffer", packet); + } + { + auto packet = tlsFrame; + packet[kIPv4TotalLength] = 0x00; // total length below the header size + packet[kIPv4TotalLength + 1] = 0x08; + add("ipv4-total-length-below-header", packet); + } + { + auto packet = tlsFrame; + packet[kIPv4VersionIhl] = 0x65; // version 6 in an IPv4 ethertype frame + add("ipv4-version-mismatch", packet); + } + + // --- TCP headers that lie --------------------------------------- + { + auto packet = tlsFrame; + packet[kTcpDataOffset] = 0xF0; // data offset = 15 words = 60 bytes + add("tcp-data-offset-past-payload", packet); + } + { + auto packet = tlsFrame; + packet[kTcpDataOffset] = 0x00; // data offset = 0, below the 20-byte minimum + add("tcp-data-offset-zero", packet); + } + + // --- UDP headers that lie --------------------------------------- + { + auto packet = validUdpFrame({0xDE, 0xAD, 0xBE, 0xEF}); + packet[kIPv4Offset + 20 + 4] = 0xFF; // UDP length beyond the buffer + packet[kIPv4Offset + 20 + 5] = 0xFF; + add("udp-length-exceeds-buffer", packet); + } + { + auto packet = validUdpFrame({0xDE, 0xAD, 0xBE, 0xEF}); + packet[kIPv4Offset + 20 + 4] = 0x00; // UDP length below its own 8-byte header + packet[kIPv4Offset + 20 + 5] = 0x02; + add("udp-length-below-header", packet); + } + + // --- IPv6 extension header chains ------------------------------- + { + // Hop-by-hop -> routing -> fragment -> TCP, all well formed. + std::vector chain = {43, 0}; // hop-by-hop: next=routing, len=0 (8 bytes) + chain.resize(8, 0x00); + chain.insert(chain.end(), {44, 0}); // routing: next=fragment + chain.resize(16, 0x00); + chain.insert(chain.end(), {6, 0}); // fragment: next=TCP + chain.resize(24, 0x00); + add("ipv6-valid-extension-chain", ipv6WithChain(0, chain)); + } + { + // Hop-by-hop claiming 2048 bytes of options that are not there. + const std::vector chain = {6, 0xFF, 0, 0, 0, 0, 0, 0}; + add("ipv6-extension-length-past-packet", ipv6WithChain(0, chain)); + } + { + // A long but individually valid chain: the walk must terminate + // because every step advances at least 8 bytes. + std::vector chain; + constexpr int kLinks = 64; + for (int link = 0; link < kLinks; ++link) { + chain.push_back(link + 1 == kLinks ? uint8_t{6} : uint8_t{0}); + chain.push_back(0); // length 0 => this header occupies 8 bytes + chain.resize(chain.size() + 6, 0x00); + } + add("ipv6-long-extension-chain", ipv6WithChain(0, chain)); + } + { + // Destination-options header pointing at itself as the next + // header, repeatedly. Terminates only because pos advances. + std::vector chain; + for (int link = 0; link < 16; ++link) { + chain.push_back(60); // next header = destination options again + chain.push_back(0); + chain.resize(chain.size() + 6, 0x00); + } + add("ipv6-self-referential-extension-chain", ipv6WithChain(60, chain)); + } + + // --- DNS payloads that lie -------------------------------------- + { + std::vector dns; + appendU16(dns, 0x1234); + appendU16(dns, 0x8180); + appendU16(dns, 1); + appendU16(dns, 20); // claims 20 answers + appendU16(dns, 0); + appendU16(dns, 0); + appendDnsName(dns, "www.example.com"); + appendU16(dns, 1); + appendU16(dns, 1); + add("dns-answer-count-exceeds-payload", validUdpFrame(dns)); + } + { + // A compression pointer that points back at itself. + std::vector dns; + appendU16(dns, 0x1234); + appendU16(dns, 0x8180); + appendU16(dns, 1); + appendU16(dns, 1); + appendU16(dns, 0); + appendU16(dns, 0); + dns.insert(dns.end(), {0xC0, 0x0C}); // pointer to offset 12 == itself + appendU16(dns, 1); + appendU16(dns, 1); + add("dns-self-referential-compression-pointer", validUdpFrame(dns)); + } + { + // A label whose length runs past the end of the payload. + std::vector dns; + appendU16(dns, 0x1234); + appendU16(dns, 0x8180); + appendU16(dns, 1); + appendU16(dns, 0); + appendU16(dns, 0); + appendU16(dns, 0); + dns.push_back(0x3F); // 63-byte label with 3 bytes following + dns.insert(dns.end(), {'a', 'b', 'c'}); + add("dns-label-runs-past-payload", validUdpFrame(dns)); + } + { + std::vector dns; + appendU16(dns, 0x1234); + appendU16(dns, 0x8180); + add("dns-truncated-mid-header", validUdpFrame(dns)); + } + + // --- TLS records that lie --------------------------------------- + { + // Record header announcing 64 KB with almost nothing behind it. + std::vector tls = {0x16, 0x03, 0x01, 0xFF, 0xFF, 0x01, 0x00, 0xFF, 0xFB}; + add("tls-record-length-exceeds-payload", validTcpFrame(tls)); + } + { + auto tls = makeTlsClientHello("half.example.com"); + tls.resize(tls.size() / 2); + add("tls-client-hello-truncated", validTcpFrame(tls)); + } + + // --- QUIC long headers that lie --------------------------------- + { + // Long header, version 1, destination CID length 255. + std::vector quic = {0xC0, 0x00, 0x00, 0x00, 0x01, 0xFF}; + quic.resize(32, 0x41); + add("quic-connection-id-length-exceeds-packet", validUdpFrame(quic)); + } + { + // Truncated part way through the version field. + const std::vector quic = {0xC0, 0x00, 0x00}; + add("quic-truncated-mid-version", validUdpFrame(quic)); + } + { + // Token length varint claiming an implausible size. + std::vector quic = {0xC0, 0x00, 0x00, 0x00, 0x01, 0x08}; + quic.resize(14, 0x42); // 8-byte destination CID + quic.push_back(0x00); // source CID length 0 + quic.push_back(0xFF); // token length varint, 8-byte form, huge + quic.resize(quic.size() + 3, 0xFF); + add("quic-token-length-exceeds-packet", validUdpFrame(quic)); + } + + return cases; + } + + } // namespace malformed + } // namespace test diff --git a/tests/fuzz/seed_generator.cpp b/tests/fuzz/seed_generator.cpp index 08586a6..83f0f35 100644 --- a/tests/fuzz/seed_generator.cpp +++ b/tests/fuzz/seed_generator.cpp @@ -1,5 +1,6 @@ #include "PacketBuilders.hpp" +#include #include #include #include @@ -129,7 +130,19 @@ int main(int argc, char** argv) { writeSeed(outputDir, "dns_response.bin", dnsResponsePacket()); writeSeed(outputDir, "arp_request.bin", arpRequestPacket()); writeSeed(outputDir, "ipv6_udp.bin", ipv6UdpPacket()); + size_t seedCount = 5; - std::cout << "Wrote 5 seed packets to " << outputDir.string() << "\n"; + // Adversarial shapes shared with tests/malformed_test.cpp. Handing these + // to libFuzzer as starting points is far cheaper than waiting for it to + // rediscover, byte by byte, that a length field can exceed its buffer. + for (const auto& testCase : test::malformed::allCases()) { + if (testCase.bytes.empty()) continue; // an empty file is not a useful seed + std::string filename = "malformed_" + testCase.name + ".bin"; + std::replace(filename.begin(), filename.end(), '-', '_'); + writeSeed(outputDir, filename, testCase.bytes); + ++seedCount; + } + + std::cout << "Wrote " << seedCount << " seed packets to " << outputDir.string() << "\n"; return 0; } diff --git a/tests/malformed_test.cpp b/tests/malformed_test.cpp new file mode 100644 index 0000000..e3bfb43 --- /dev/null +++ b/tests/malformed_test.cpp @@ -0,0 +1,189 @@ +#include "PacketBuilders.hpp" + +#include "core/AnalysisSession.hpp" +#include "core/DNSParser.hpp" +#include "core/PacketData.hpp" +#include "core/ProtocolParser.hpp" +#include "core/QuicParser.hpp" +#include "core/QuicTracker.hpp" +#include "core/TlsReassembler.hpp" + +#include + +#include +#include +#include +#include + +// Deterministic adversarial inputs: headers that lie about their own length, +// chains that run off the end of the buffer, counts larger than the data +// present. Fuzzing finds these shapes eventually; checking them in as named +// cases means a regression is reported as a failing test with a name rather +// than as a crash artifact from a 60-second CI fuzz run. +// +// Under the ASan/UBSan CI job these also assert the absence of out-of-bounds +// reads, which is the failure mode that matters most here — a parser can walk +// off the end of a packet and still return a plausible-looking result. +namespace { + + using namespace test; + + core::PacketData toPacketData(const std::vector& bytes) { + // A zero-length capture still has to be representable: PacketData + // takes a pointer, and passing data() of an empty vector is fine only + // because the length is zero. + static const uint8_t placeholder = 0; + return core::PacketData(1'700'000'000'000'000, + static_cast(bytes.size()), + static_cast(bytes.size()), + bytes.empty() ? &placeholder : bytes.data()); + } + + // Everything a malformed packet is allowed to produce. The point is not + // that parsing succeeds, but that failure is expressed as absent or + // clearly-marked fields rather than as garbage that reads as real. + void expectDegradedButSane(const core::ParsedPacket& parsed, + const std::vector& bytes, const std::string& name) { + // A payload window must never point outside the captured bytes. + if (parsed.payloadLength > 0) { + EXPECT_LE(parsed.payloadOffset, bytes.size()) << name; + EXPECT_LE(parsed.payloadOffset + parsed.payloadLength, bytes.size()) + << name << ": payload window runs past the captured data"; + } + // Addresses are either empty or plausible text, never raw bytes. + for (const std::string& address : {parsed.srcIP, parsed.dstIP}) { + EXPECT_LT(address.size(), 64u) << name; + EXPECT_EQ(address.find('\0'), std::string::npos) + << name << ": address contains a NUL byte"; + } + // A hostname recovered from a malformed packet is the most dangerous + // field to get wrong, since it is shown to the user as fact. + EXPECT_LT(parsed.sni.size(), 1024u) << name; + EXPECT_LT(parsed.hostname.size(), 1024u) << name; + } + +} // namespace + +TEST(MalformedPacketTest, ProtocolParserSurvivesEveryAdversarialCase) { + const auto cases = malformed::allCases(); + ASSERT_FALSE(cases.empty()); + + for (const auto& testCase : cases) { + const auto raw = toPacketData(testCase.bytes); + const auto parsed = core::ProtocolParser::parse(raw); + expectDegradedButSane(parsed, testCase.bytes, testCase.name); + } +} + +TEST(MalformedPacketTest, DnsParserSurvivesEveryAdversarialCase) { + for (const auto& testCase : malformed::allCases()) { + const auto raw = toPacketData(testCase.bytes); + const auto parsed = core::ProtocolParser::parse(raw); + + const auto response = core::DNSParser::parseResponse(raw, parsed); + if (!response) continue; + + // A record count in the header is a claim, not a promise; whatever + // survives parsing must be backed by bytes that were actually there. + EXPECT_LT(response->answers.size(), 256u) << testCase.name; + EXPECT_LT(response->queryName.size(), 1024u) << testCase.name; + for (const auto& answer : response->answers) { + EXPECT_LT(answer.name.size(), 1024u) << testCase.name; + EXPECT_LT(answer.value.size(), 1024u) << testCase.name; + } + } +} + +TEST(MalformedPacketTest, StatefulParsersSurviveEveryAdversarialCase) { + // The reassemblers hold state across packets, so feed the whole corpus + // through single instances: a case that corrupts internal state shows up + // as a later failure rather than an immediate one. + core::TlsReassembler tlsReassembler; + core::QuicTracker quicTracker; + + for (const auto& testCase : malformed::allCases()) { + const auto raw = toPacketData(testCase.bytes); + const auto parsed = core::ProtocolParser::parse(raw); + + if (parsed.payloadLength == 0 || parsed.payloadOffset >= testCase.bytes.size()) continue; + + const uint8_t* payload = testCase.bytes.data() + parsed.payloadOffset; + const size_t payloadLength = std::min(parsed.payloadLength, + testCase.bytes.size() - parsed.payloadOffset); + + if (parsed.protocol == "TCP") { + const auto sni = tlsReassembler.feed(parsed, payload, payloadLength); + if (sni) EXPECT_LT(sni->size(), 1024u) << testCase.name; + } else if (parsed.protocol == "UDP") { + if (core::QuicParser::looksLikeLongHeader(payload, payloadLength)) { + const auto sni = quicTracker.feed(payload, payloadLength, parsed.timestamp); + if (sni) EXPECT_LT(sni->size(), 1024u) << testCase.name; + } + } + } + + // Malformed input must not leave half-built streams pinned forever. + EXPECT_LT(tlsReassembler.trackedStreamCount(), 64u); +} + +TEST(MalformedPacketTest, AnalysisSessionSurvivesEveryAdversarialCase) { + // The full pipeline, including flow aggregation, on the same corpus. + core::AnalysisSession session(/*resolveProcesses=*/false); + + for (const auto& testCase : malformed::allCases()) { + const auto result = session.feed(toPacketData(testCase.bytes)); + expectDegradedButSane(result.parsed, testCase.bytes, testCase.name); + EXPECT_LT(result.hostname.size(), 1024u) << testCase.name; + } + + // Whatever flows were created must be internally consistent. + for (const auto& flow : session.flows(1'700'000'001'000'000)) { + EXPECT_GT(flow.packets, 0u); + EXPECT_LE(flow.firstSeen, flow.lastSeen); + EXPECT_LT(flow.hostname.size(), 1024u); + } +} + +TEST(MalformedPacketTest, TruncatingAValidFrameAtEveryOffsetNeverCrashes) { + // Exhaustive rather than sampled: every prefix of a well-formed packet is + // a packet some driver or capture file can hand us. + const auto frame = malformed::validTcpFrame(makeTlsClientHello("prefix.example.com")); + core::TlsReassembler reassembler; + + for (size_t length = 0; length <= frame.size(); ++length) { + const std::vector prefix(frame.begin(), + frame.begin() + static_cast(length)); + const auto raw = toPacketData(prefix); + const auto parsed = core::ProtocolParser::parse(raw); + expectDegradedButSane(parsed, prefix, "prefix-length-" + std::to_string(length)); + + (void)core::DNSParser::parseResponse(raw, parsed); + if (parsed.protocol == "TCP" && parsed.payloadLength > 0 + && parsed.payloadOffset < prefix.size()) { + (void)reassembler.feed(parsed, prefix.data() + parsed.payloadOffset, + std::min(parsed.payloadLength, prefix.size() - parsed.payloadOffset)); + } + } +} + +TEST(MalformedPacketTest, Ipv6ExtensionChainWalkAlwaysTerminates) { + // The walk is an unbounded `while (true)`; it is safe only because every + // step advances at least eight bytes within a bounded buffer. If that ever + // stops being true this test hangs instead of passing, which is the point. + for (const auto& testCase : malformed::allCases()) { + if (testCase.name.find("ipv6") == std::string::npos) continue; + const auto parsed = core::ProtocolParser::parse(toPacketData(testCase.bytes)); + expectDegradedButSane(parsed, testCase.bytes, testCase.name); + } + + // A chain built entirely of zero-length headers, filling the packet. + std::vector chain; + for (int link = 0; link < 200; ++link) { + chain.push_back(0); // hop-by-hop pointing at another hop-by-hop + chain.push_back(0); + chain.resize(chain.size() + 6, 0x00); + } + const auto packet = malformed::ipv6WithChain(0, chain); + const auto parsed = core::ProtocolParser::parse(toPacketData(packet)); + expectDegradedButSane(parsed, packet, "ipv6-exhaustive-zero-length-chain"); +} From f57309679ad9cbcccc22022cfea1f4ae81bfd21f Mon Sep 17 00:00:00 2001 From: YehiaGewily <79144005+YehiaGewily@users.noreply.github.com> Date: Sun, 19 Jul 2026 04:11:55 +0300 Subject: [PATCH 08/12] ci: add clang-tidy static analysis, clean the tree, badge the README MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Static analysis was the missing third leg next to the existing ASan/UBSan and fuzzing jobs. The check list is curated for signal rather than coverage. Checks that fire constantly on idiomatic code are disabled in .clang-tidy with the reasoning recorded there, rather than silenced case by case at each site — a warning nobody acts on trains everyone to ignore the tool. Notably disabled: enum-size (saves bytes that do not matter, invites surprising conversions) and the padding analyzer (advises reordering ~70 members of a class with exactly one instance, trading a readable header for 48 bytes). Every first-party source was then brought clean, so WarningsAsErrors is on and a new finding fails the build instead of scrolling past in a log. This was verified by introducing a deliberate defect and confirming the gate fails, then confirming it passes once reverted. Fixes worth calling out: - ~GuiLayer could let an exception from saving preferences escape a destructor, which terminates the process during shutdown. Now caught, and reported via fputs rather than iostreams, since a handler that can itself throw defeats the purpose. - Both main() functions now catch on a non-throwing path; previously an escaping exception gave a bare failure code with no diagnostic, and for the GUI meant the window simply vanished. - Explicit widening on the multiplications feeding pointer offsets in ProtocolParser. All four are provably bounded today (header lengths cap at 60 bytes), so this fixes no live defect — but it keeps the warning class meaningful for a future site that is not bounded. - sscanf replaced with istringstream extraction: type-safe and portable, without reaching for the Windows-only sscanf_s. - std::endl to '\n' throughout, nullptr for NULL, and the value-parameter and reserve fixes clang-tidy identified. The CI job drives clang-tidy directly rather than run-clang-tidy, which is packaged inconsistently across distributions, and configures with Ninja since compile_commands.json is what the tool needs. README gains the CI badge and a one-line summary of what every push runs, plus a marked TODO for the screenshots that need a human at a screen. 89/89 tests pass; GUI and CLI both verified to run. --- .clang-tidy | 63 + .github/workflows/ci.yml | 40 + CMakeLists.txt | 5 + README.md | 14 +- src/capture/CaptureEngine.cpp | 440 +++---- src/cli/main.cpp | 865 +++++++------- src/core/DNSParser.cpp | 614 +++++----- src/core/GeoIPResolver.cpp | 258 ++-- src/core/GeoIPResolver.hpp | 100 +- src/core/ProtocolParser.cpp | 2075 +++++++++++++++++---------------- src/core/QuicTracker.hpp | 108 +- src/core/TlsReassembler.hpp | 132 +-- src/main.cpp | 172 +-- src/ui/GuiLayer.cpp | 1888 +++++++++++++++--------------- src/ui/GuiLayer.hpp | 4 +- 15 files changed, 3477 insertions(+), 3301 deletions(-) create mode 100644 .clang-tidy diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 0000000..871b113 --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,63 @@ +# Static analysis configuration. +# +# Curated for signal rather than coverage: every check enabled here is one +# whose findings are worth reading. Checks that fire constantly on idiomatic +# code are disabled outright instead of being silenced with NOLINT at each +# site, because a warning nobody acts on trains everyone to ignore the tool. +--- +Checks: > + bugprone-*, + performance-*, + clang-analyzer-*, + misc-*, + modernize-use-nullptr, + modernize-use-override, + modernize-use-emplace, + modernize-loop-convert, + -bugprone-easily-swappable-parameters, + -bugprone-narrowing-conversions, + -bugprone-branch-clone, + -misc-non-private-member-variables-in-classes, + -misc-no-recursion, + -misc-include-cleaner, + -misc-use-anonymous-namespace, + -misc-const-correctness, + -clang-analyzer-optin.core.EnumCastOutOfRange, + -performance-enum-size, + -clang-analyzer-optin.performance.Padding + +# The tree is clean under this configuration, so anything new is a +# regression and should fail the build rather than scroll past in a log. +WarningsAsErrors: '*' + +# Only first-party headers. Without this, findings from the FetchContent'd +# ImGui, Mbed TLS, and libmaxminddb headers drown out our own. +HeaderFilterRegex: '.*[/\\](src|include)[/\\].*' + +# clang-tidy must not rewrite formatting; that belongs to the author. +FormatStyle: none + +# Notes on the disabled checks above: +# performance-enum-size shrinking enum base types saves bytes that +# do not matter here and invites surprising +# implicit conversions. +# clang-analyzer-optin.performance.Padding +# advises reordering members by size. It is +# aimed at objects allocated in bulk; the +# class it fires on has exactly one instance, +# so it would trade a readable, logically +# grouped header for ~48 bytes. +# misc-const-correctness proposes const on nearly every local. +# misc-include-cleaner cannot see through the pinned third-party +# headers pulled in by FetchContent. + +CheckOptions: + # NetProbe uses lowerCamelCase for functions and locals, m_ for members, + # and UpperCamelCase for types. Naming checks are not enabled above, but + # recording the convention here documents it for anyone who turns them on. + - key: readability-identifier-naming.ClassCase + value: CamelCase + - key: readability-identifier-naming.FunctionCase + value: camelBack + - key: readability-identifier-naming.PrivateMemberPrefix + value: m_ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b3d340..8a5df39 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -148,6 +148,46 @@ jobs: UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1 run: ctest --test-dir build-asan -C RelWithDebInfo --output-on-failure + clang-tidy: + name: clang-tidy (static analysis) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y clang clang-tidy libpcap-dev libgl1-mesa-dev libgtk-3-dev pkg-config + + # Ninja rather than the default generator: clang-tidy needs + # compile_commands.json, and configuring is enough to produce it + # (FetchContent populates its dependencies at configure time). + - name: Configure + env: + CC: clang + CXX: clang++ + run: cmake -S . -B build-tidy -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF + + # Analyse first-party sources only. Third-party code arrives through + # FetchContent and is not ours to fix; .clang-tidy's HeaderFilterRegex + # keeps their headers out of the report as well. + # Driving clang-tidy directly rather than through run-clang-tidy, which + # is packaged inconsistently across distributions. .clang-tidy sets + # WarningsAsErrors, so any finding makes the invocation fail. + - name: Run clang-tidy + run: | + status=0 + while IFS= read -r file; do + echo "--- $file" + clang-tidy -p build-tidy --quiet "$file" || status=1 + done < <(find src -name '*.cpp' | sort) + if [ "$status" -ne 0 ]; then + echo "::error::clang-tidy reported findings (see the log above)" + fi + exit $status + shell: bash + fuzz: name: Fuzz parser (60s) runs-on: ubuntu-latest diff --git a/CMakeLists.txt b/CMakeLists.txt index 06c1180..5b30663 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,11 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) include(CTest) +# Emit compile_commands.json for clang-tidy and editor tooling. Note that the +# Visual Studio generator ignores this; use Ninja when you need the database +# (see the clang-tidy CI job). +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + # AddressSanitizer + UndefinedBehaviorSanitizer for local debug runs and CI. # Clang/GCC only; MSVC ASan has compatibility caveats with the third-party # CMake projects we pull in, so we restrict this to non-MSVC compilers. diff --git a/README.md b/README.md index f7c4516..c4280b5 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,16 @@ A real-time and offline network traffic analyzer built with **Modern C++20**, **Npcap/libpcap**, and **Dear ImGui**. -![Platform](https://img.shields.io/badge/platform-Windows%20%7C%20Linux%20%7C%20macOS-lightgrey.svg) ![Standard](https://img.shields.io/badge/C%2B%2B-20-blue.svg) ![License](https://img.shields.io/badge/license-MIT-blue.svg) +[![CI](https://github.com/YehiaGewily/netprobe-cpp/actions/workflows/ci.yml/badge.svg)](https://github.com/YehiaGewily/netprobe-cpp/actions/workflows/ci.yml) ![Platform](https://img.shields.io/badge/platform-Windows%20%7C%20Linux%20%7C%20macOS-lightgrey.svg) ![Standard](https://img.shields.io/badge/C%2B%2B-20-blue.svg) ![License](https://img.shields.io/badge/license-MIT-blue.svg) + +> Every push is built on Windows, Linux, and macOS, run under AddressSanitizer and UndefinedBehaviorSanitizer, checked with clang-tidy, and fuzzed with libFuzzer. + + + ## Download @@ -46,7 +55,7 @@ Download the ZIP for your platform from [GitHub Releases](https://github.com/Yeh - **GeoIP/ASN**: [libmaxminddb](https://github.com/maxmind/libmaxminddb) - **File dialogs**: [Native File Dialog Extended](https://github.com/btzy/nativefiledialog-extended) - **Build System**: CMake (all third-party deps pinned via `FetchContent`) -- **Tests / fuzzing**: GoogleTest + libFuzzer harness on `ProtocolParser` and `DNSParser` +- **Tests / fuzzing / analysis**: GoogleTest + libFuzzer harness on the protocol parsers + clang-tidy ## Prerequisites @@ -198,6 +207,7 @@ flowchart LR - **Adversarial input suite**: 35 named malformed-packet fixtures — headers claiming lengths the buffer does not contain, IPv4 IHL and total-length lies, IPv6 extension-header chains that overrun or self-reference, DNS compression-pointer loops and inflated record counts, truncated TLS records, QUIC connection-ID and token lengths beyond the packet — plus an exhaustive check that every prefix of a valid frame parses without crashing. The same fixtures seed the fuzz corpus. - **libFuzzer harness** on `ProtocolParser::parse`, `DNSParser::parseResponse`, and the stateful `TlsReassembler` / `QuicTracker`, across every supported link type, with a deterministic seed corpus. CI fuzzes every push for 60 seconds on Ubuntu + Clang and uploads any crash inputs as build artifacts. - **AddressSanitizer + UndefinedBehaviorSanitizer** CI job runs the full test suite under ASan/UBSan on every push. +- **clang-tidy static analysis** in CI over every first-party source, with `WarningsAsErrors` enabled: the tree is clean, so a new finding fails the build rather than scrolling past in a log. Checks are curated for signal — those that fire constantly on idiomatic code are disabled in `.clang-tidy` with the reasoning recorded there, rather than silenced case by case. - **Three-platform CI matrix** (Windows / Ubuntu / macOS) builds the app, builds and runs the test suite, and packages release ZIPs via CPack on tag pushes. Windows test runs use a 7-Zip-extracted Npcap user-mode DLL so the kernel-driver install is unnecessary in CI. ## Contributing diff --git a/src/capture/CaptureEngine.cpp b/src/capture/CaptureEngine.cpp index 377d2c9..85de355 100644 --- a/src/capture/CaptureEngine.cpp +++ b/src/capture/CaptureEngine.cpp @@ -1,220 +1,220 @@ -#include "capture/CaptureEngine.hpp" -#include "capture/PcapPlatform.hpp" -#include "core/PacketQueue.hpp" - -#include -#include - -namespace capture { - - CaptureEngine::CaptureEngine(core::PacketQueue& queue) - : m_queue(queue) - , m_backend(createPlatformCaptureBackend()) {} - - CaptureEngine::~CaptureEngine() { - stopCapture(); - } - - std::vector CaptureEngine::getAvailableDevices() const { - std::string error; - auto devices = m_backend->listDevices(error); - if (!error.empty()) std::cerr << "Unable to list capture devices: " << error << std::endl; - return devices; - } - - void CaptureEngine::startCapture(const std::string& deviceName) { - stopCapture(); - m_queue.clear(); - clearSession(); - m_currentDevice = deviceName; - - std::string error; - if (!m_backend->open(deviceName, error)) { - std::cerr << "Unable to open adapter " << deviceName << ": " << error << std::endl; - return; - } - if (!m_activeFilter.empty() && !m_backend->setFilter(m_activeFilter, error)) { - std::cerr << "Unable to apply BPF filter '" << m_activeFilter << "': " << error << std::endl; - m_backend->close(); - return; - } - m_sessionLinkType.store(m_backend->linkType(), std::memory_order_release); - - m_stopRequested.store(false, std::memory_order_release); - m_captureThread = std::thread([this] { captureLoop(); }); - } - - bool CaptureEngine::openFile(const std::string& path) { - stopCapture(); - m_queue.clear(); - clearSession(); - m_currentDevice.clear(); - - std::string error; - if (!m_backend->openFile(path, error)) { - std::cerr << "Unable to open capture file '" << path << "': " << error << std::endl; - return false; - } - m_sessionLinkType.store(m_backend->linkType(), std::memory_order_release); - if (m_backend->linkType() == core::LinkType::Unsupported) { - std::cerr << "Capture file '" << path - << "' uses an unsupported link-layer type; packets cannot be decoded." - << std::endl; - } - - bool succeeded = true; - while (true) { - core::PacketData packet; - switch (m_backend->nextPacket(packet, error)) { - case PacketReadStatus::Packet: - consumePacket(std::move(packet)); - break; - case PacketReadStatus::Timeout: - break; - case PacketReadStatus::EndOfFile: - m_backend->close(); - return succeeded; - case PacketReadStatus::Error: - std::cerr << "Error while reading capture file '" << path << "': " << error << std::endl; - succeeded = false; - m_backend->close(); - return succeeded; - } - } - } - - bool CaptureEngine::replayFile(const std::string& path, - const std::function& onPacket, std::string& error) { - stopCapture(); - m_queue.clear(); - clearSession(); - m_currentDevice.clear(); - - if (!m_backend->openFile(path, error)) return false; - - m_sessionLinkType.store(m_backend->linkType(), std::memory_order_release); - if (m_backend->linkType() == core::LinkType::Unsupported) { - error = "the capture uses an unsupported link-layer type; packets cannot be decoded"; - m_backend->close(); - return false; - } - - while (true) { - core::PacketData packet; - std::string readError; - switch (m_backend->nextPacket(packet, readError)) { - case PacketReadStatus::Packet: - onPacket(packet); - break; - case PacketReadStatus::Timeout: - break; - case PacketReadStatus::EndOfFile: - m_backend->close(); - return true; - case PacketReadStatus::Error: - error = readError; - m_backend->close(); - return false; - } - } - } - - bool CaptureEngine::setFilter(const std::string& filter, std::string& error) { - if (!m_captureThread.joinable() || !m_backend->isOpen()) { - error = "A live capture must be running before a BPF filter can be applied."; - return false; - } - if (!m_backend->setFilter(filter, error)) return false; - m_activeFilter = filter; - return true; - } - - bool CaptureEngine::exportSession(const std::string& path, std::string& error) const { - std::vector packets; - { - std::lock_guard lock(m_sessionMutex); - packets.assign(m_sessionPackets.begin(), m_sessionPackets.end()); - } - - // Write the DLT the packets were actually captured with — writing - // Ethernet for a loopback or cooked capture would produce a PCAP that - // Wireshark decodes as garbage. - pcap_t* deadHandle = pcap_open_dead( - core::dltFromLinkType(m_sessionLinkType.load(std::memory_order_acquire)), 65536); - if (!deadHandle) { - error = "Unable to create a PCAP writer."; - return false; - } - pcap_dumper_t* dumper = pcap_dump_open(deadHandle, path.c_str()); - if (!dumper) { - error = pcap_geterr(deadHandle); - pcap_close(deadHandle); - return false; - } - - static const uint8_t emptyPacket = 0; - for (const auto& packet : packets) { - pcap_pkthdr header{}; - header.ts.tv_sec = static_cast(packet.timestamp / 1'000'000); - header.ts.tv_usec = static_cast(packet.timestamp % 1'000'000); - header.caplen = static_cast(packet.payload.size()); - header.len = static_cast(packet.length); - const auto* bytes = packet.payload.empty() ? &emptyPacket : packet.payload.data(); - pcap_dump(reinterpret_cast(dumper), &header, bytes); - } - - pcap_dump_close(dumper); - pcap_close(deadHandle); - return true; - } - - core::LinkType CaptureEngine::linkType() const { - return m_sessionLinkType.load(std::memory_order_acquire); - } - - void CaptureEngine::stopCapture() { - if (m_captureThread.joinable()) { - m_stopRequested.store(true, std::memory_order_release); - m_captureThread.join(); - } - m_backend->close(); - } - - void CaptureEngine::captureLoop() { - while (!m_stopRequested.load(std::memory_order_acquire)) { - core::PacketData packet; - std::string error; - switch (m_backend->nextPacket(packet, error)) { - case PacketReadStatus::Packet: - consumePacket(std::move(packet)); - break; - case PacketReadStatus::Timeout: - break; - case PacketReadStatus::EndOfFile: - return; - case PacketReadStatus::Error: - if (!m_stopRequested.load(std::memory_order_acquire)) { - std::cerr << "Capture error: " << error << std::endl; - } - return; - } - } - } - - void CaptureEngine::clearSession() { - std::lock_guard lock(m_sessionMutex); - m_sessionPackets.clear(); - } - - void CaptureEngine::retainPacket(const core::PacketData& packet) { - std::lock_guard lock(m_sessionMutex); - if (m_sessionPackets.size() >= maxSessionPackets) m_sessionPackets.pop_front(); - m_sessionPackets.push_back(packet); - } - - void CaptureEngine::consumePacket(core::PacketData&& packet) { - retainPacket(packet); - m_queue.push(std::move(packet)); - } - -} // namespace capture +#include "capture/CaptureEngine.hpp" +#include "capture/PcapPlatform.hpp" +#include "core/PacketQueue.hpp" + +#include +#include + +namespace capture { + + CaptureEngine::CaptureEngine(core::PacketQueue& queue) + : m_queue(queue) + , m_backend(createPlatformCaptureBackend()) {} + + CaptureEngine::~CaptureEngine() { + stopCapture(); + } + + std::vector CaptureEngine::getAvailableDevices() const { + std::string error; + auto devices = m_backend->listDevices(error); + if (!error.empty()) std::cerr << "Unable to list capture devices: " << error << '\n'; + return devices; + } + + void CaptureEngine::startCapture(const std::string& deviceName) { + stopCapture(); + m_queue.clear(); + clearSession(); + m_currentDevice = deviceName; + + std::string error; + if (!m_backend->open(deviceName, error)) { + std::cerr << "Unable to open adapter " << deviceName << ": " << error << '\n'; + return; + } + if (!m_activeFilter.empty() && !m_backend->setFilter(m_activeFilter, error)) { + std::cerr << "Unable to apply BPF filter '" << m_activeFilter << "': " << error << '\n'; + m_backend->close(); + return; + } + m_sessionLinkType.store(m_backend->linkType(), std::memory_order_release); + + m_stopRequested.store(false, std::memory_order_release); + m_captureThread = std::thread([this] { captureLoop(); }); + } + + bool CaptureEngine::openFile(const std::string& path) { + stopCapture(); + m_queue.clear(); + clearSession(); + m_currentDevice.clear(); + + std::string error; + if (!m_backend->openFile(path, error)) { + std::cerr << "Unable to open capture file '" << path << "': " << error << '\n'; + return false; + } + m_sessionLinkType.store(m_backend->linkType(), std::memory_order_release); + if (m_backend->linkType() == core::LinkType::Unsupported) { + std::cerr << "Capture file '" << path + << "' uses an unsupported link-layer type; packets cannot be decoded." + << '\n'; + } + + bool succeeded = true; + while (true) { + core::PacketData packet; + switch (m_backend->nextPacket(packet, error)) { + case PacketReadStatus::Packet: + consumePacket(std::move(packet)); + break; + case PacketReadStatus::Timeout: + break; + case PacketReadStatus::EndOfFile: + m_backend->close(); + return succeeded; + case PacketReadStatus::Error: + std::cerr << "Error while reading capture file '" << path << "': " << error << '\n'; + succeeded = false; + m_backend->close(); + return succeeded; + } + } + } + + bool CaptureEngine::replayFile(const std::string& path, + const std::function& onPacket, std::string& error) { + stopCapture(); + m_queue.clear(); + clearSession(); + m_currentDevice.clear(); + + if (!m_backend->openFile(path, error)) return false; + + m_sessionLinkType.store(m_backend->linkType(), std::memory_order_release); + if (m_backend->linkType() == core::LinkType::Unsupported) { + error = "the capture uses an unsupported link-layer type; packets cannot be decoded"; + m_backend->close(); + return false; + } + + while (true) { + core::PacketData packet; + std::string readError; + switch (m_backend->nextPacket(packet, readError)) { + case PacketReadStatus::Packet: + onPacket(packet); + break; + case PacketReadStatus::Timeout: + break; + case PacketReadStatus::EndOfFile: + m_backend->close(); + return true; + case PacketReadStatus::Error: + error = readError; + m_backend->close(); + return false; + } + } + } + + bool CaptureEngine::setFilter(const std::string& filter, std::string& error) { + if (!m_captureThread.joinable() || !m_backend->isOpen()) { + error = "A live capture must be running before a BPF filter can be applied."; + return false; + } + if (!m_backend->setFilter(filter, error)) return false; + m_activeFilter = filter; + return true; + } + + bool CaptureEngine::exportSession(const std::string& path, std::string& error) const { + std::vector packets; + { + std::lock_guard lock(m_sessionMutex); + packets.assign(m_sessionPackets.begin(), m_sessionPackets.end()); + } + + // Write the DLT the packets were actually captured with — writing + // Ethernet for a loopback or cooked capture would produce a PCAP that + // Wireshark decodes as garbage. + pcap_t* deadHandle = pcap_open_dead( + core::dltFromLinkType(m_sessionLinkType.load(std::memory_order_acquire)), 65536); + if (!deadHandle) { + error = "Unable to create a PCAP writer."; + return false; + } + pcap_dumper_t* dumper = pcap_dump_open(deadHandle, path.c_str()); + if (!dumper) { + error = pcap_geterr(deadHandle); + pcap_close(deadHandle); + return false; + } + + static const uint8_t emptyPacket = 0; + for (const auto& packet : packets) { + pcap_pkthdr header{}; + header.ts.tv_sec = static_cast(packet.timestamp / 1'000'000); + header.ts.tv_usec = static_cast(packet.timestamp % 1'000'000); + header.caplen = static_cast(packet.payload.size()); + header.len = static_cast(packet.length); + const auto* bytes = packet.payload.empty() ? &emptyPacket : packet.payload.data(); + pcap_dump(reinterpret_cast(dumper), &header, bytes); + } + + pcap_dump_close(dumper); + pcap_close(deadHandle); + return true; + } + + core::LinkType CaptureEngine::linkType() const { + return m_sessionLinkType.load(std::memory_order_acquire); + } + + void CaptureEngine::stopCapture() { + if (m_captureThread.joinable()) { + m_stopRequested.store(true, std::memory_order_release); + m_captureThread.join(); + } + m_backend->close(); + } + + void CaptureEngine::captureLoop() { + while (!m_stopRequested.load(std::memory_order_acquire)) { + core::PacketData packet; + std::string error; + switch (m_backend->nextPacket(packet, error)) { + case PacketReadStatus::Packet: + consumePacket(std::move(packet)); + break; + case PacketReadStatus::Timeout: + break; + case PacketReadStatus::EndOfFile: + return; + case PacketReadStatus::Error: + if (!m_stopRequested.load(std::memory_order_acquire)) { + std::cerr << "Capture error: " << error << '\n'; + } + return; + } + } + } + + void CaptureEngine::clearSession() { + std::lock_guard lock(m_sessionMutex); + m_sessionPackets.clear(); + } + + void CaptureEngine::retainPacket(const core::PacketData& packet) { + std::lock_guard lock(m_sessionMutex); + if (m_sessionPackets.size() >= maxSessionPackets) m_sessionPackets.pop_front(); + m_sessionPackets.push_back(packet); + } + + void CaptureEngine::consumePacket(core::PacketData&& packet) { + retainPacket(packet); + m_queue.push(std::move(packet)); + } + +} // namespace capture diff --git a/src/cli/main.cpp b/src/cli/main.cpp index 4e2d57c..d6ac347 100644 --- a/src/cli/main.cpp +++ b/src/cli/main.cpp @@ -1,422 +1,443 @@ -// NetProbe headless CLI. -// -// Runs the same capture and analysis pipeline as the GUI with no window, so -// captures can be taken and flows exported over SSH, in CI, or on a server. -// Deliberately small: replay a file or capture for a bounded time, then write -// the flow table as JSON or CSV. - -#include "capture/CaptureEngine.hpp" -#include "core/AnalysisSession.hpp" -#include "core/FlowExporter.hpp" -#include "core/PacketQueue.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifndef NETPROBE_VERSION -#define NETPROBE_VERSION "unknown" -#endif - -namespace { - - // Exit codes are part of the CLI's contract with scripts that call it. - constexpr int kExitSuccess = 0; - constexpr int kExitUsage = 1; - constexpr int kExitRuntime = 2; - - // Set from the signal handler; only ever transitions false -> true. - std::atomic g_stopRequested{false}; - - extern "C" void handleInterrupt(int) { - g_stopRequested.store(true, std::memory_order_release); - } - - enum class OutputFormat { Json, Csv }; - - struct Options { - bool listDevices = false; - bool showHelp = false; - bool showVersion = false; - std::string readPath; // offline capture to replay - std::string device; // live adapter to capture from - std::string bpfFilter; - std::string outputPath; // "-" means stdout - std::optional format; - std::optional durationSeconds; - std::optional packetLimit; - bool resolveProcesses = true; - }; - - void printUsage(std::ostream& out) { - out << "NetProbe CLI " NETPROBE_VERSION " - headless packet capture and flow export\n" - "\n" - "Usage:\n" - " netprobe-cli --list-devices\n" - " netprobe-cli -r -o \n" - " netprobe-cli -i [-f ] [--duration ] -o \n" - "\n" - "Source (exactly one required):\n" - " -r, --read Replay an offline capture file\n" - " -i, --device Capture live from an adapter (needs privileges)\n" - "\n" - "Output:\n" - " -o, --output Destination file, or '-' for stdout\n" - " --format Override the format inferred from the extension\n" - "\n" - "Capture limits (live capture only):\n" - " --duration Stop after this many seconds\n" - " --packet-count Stop after this many packets\n" - "\n" - "Other:\n" - " --no-process Skip owning-process lookup\n" - " --list-devices List capture adapters and exit\n" - " -h, --help Show this help and exit\n" - " -V, --version Show the version and exit\n" - "\n" - "Interrupting with Ctrl+C stops the capture and still writes the output.\n"; - } - - // Returns false when `text` is not a complete non-negative integer. - template - bool parseNonNegative(std::string_view text, T& value) { - if (text.empty()) return false; - const char* begin = text.data(); - const char* end = begin + text.size(); - const auto result = std::from_chars(begin, end, value); - return result.ec == std::errc{} && result.ptr == end; - } - - bool endsWith(const std::string& value, std::string_view suffix) { - return value.size() >= suffix.size() - && value.compare(value.size() - suffix.size(), suffix.size(), suffix) == 0; - } - - // Parses argv. On failure prints the reason to stderr and returns nullopt. - std::optional parseArguments(int argc, char** argv) { - Options options; - - // Every option below takes exactly one value; centralising the - // "is there a next argument" check keeps the loop readable. - const auto takeValue = [&](int& index, std::string_view flag) -> const char* { - if (index + 1 >= argc) { - std::cerr << "netprobe-cli: " << flag << " requires a value\n"; - return nullptr; - } - return argv[++index]; - }; - - for (int i = 1; i < argc; ++i) { - const std::string_view argument = argv[i]; - - if (argument == "-h" || argument == "--help") { - options.showHelp = true; - return options; - } - if (argument == "-V" || argument == "--version") { - options.showVersion = true; - return options; - } - if (argument == "--list-devices") { - options.listDevices = true; - continue; - } - if (argument == "--no-process") { - options.resolveProcesses = false; - continue; - } - if (argument == "-r" || argument == "--read") { - const char* value = takeValue(i, argument); - if (!value) return std::nullopt; - options.readPath = value; - continue; - } - if (argument == "-i" || argument == "--device") { - const char* value = takeValue(i, argument); - if (!value) return std::nullopt; - options.device = value; - continue; - } - if (argument == "-f" || argument == "--filter") { - const char* value = takeValue(i, argument); - if (!value) return std::nullopt; - options.bpfFilter = value; - continue; - } - if (argument == "-o" || argument == "--output") { - const char* value = takeValue(i, argument); - if (!value) return std::nullopt; - options.outputPath = value; - continue; - } - if (argument == "--format") { - const char* value = takeValue(i, argument); - if (!value) return std::nullopt; - const std::string_view format = value; - if (format == "json") options.format = OutputFormat::Json; - else if (format == "csv") options.format = OutputFormat::Csv; - else { - std::cerr << "netprobe-cli: unknown format '" << format - << "' (expected 'json' or 'csv')\n"; - return std::nullopt; - } - continue; - } - if (argument == "--duration") { - const char* value = takeValue(i, argument); - if (!value) return std::nullopt; - int seconds = 0; - if (!parseNonNegative(value, seconds) || seconds <= 0) { - std::cerr << "netprobe-cli: --duration expects a positive number of seconds\n"; - return std::nullopt; - } - options.durationSeconds = seconds; - continue; - } - if (argument == "--packet-count") { - const char* value = takeValue(i, argument); - if (!value) return std::nullopt; - uint64_t count = 0; - if (!parseNonNegative(value, count) || count == 0) { - std::cerr << "netprobe-cli: --packet-count expects a positive number\n"; - return std::nullopt; - } - options.packetLimit = count; - continue; - } - - std::cerr << "netprobe-cli: unrecognized option '" << argument << "'\n"; - return std::nullopt; - } - - return options; - } - - // Validates the combination of options, not just their individual syntax. - bool validate(const Options& options) { - if (options.listDevices) return true; - - const bool hasFile = !options.readPath.empty(); - const bool hasDevice = !options.device.empty(); - - if (!hasFile && !hasDevice) { - std::cerr << "netprobe-cli: nothing to do - pass --read or --device \n" - "Try 'netprobe-cli --help'.\n"; - return false; - } - if (hasFile && hasDevice) { - std::cerr << "netprobe-cli: --read and --device are mutually exclusive\n"; - return false; - } - if (options.outputPath.empty()) { - std::cerr << "netprobe-cli: no destination - pass --output (or '-' for stdout)\n"; - return false; - } - // Silently ignoring a flag that cannot apply is how people end up - // trusting a limit that was never enforced. - if (hasFile && (options.durationSeconds || options.packetLimit)) { - std::cerr << "netprobe-cli: --duration and --packet-count apply to live capture only\n"; - return false; - } - if (hasFile && !options.bpfFilter.empty()) { - std::cerr << "netprobe-cli: --filter applies to live capture only\n"; - return false; - } - return true; - } - - OutputFormat resolveFormat(const Options& options) { - if (options.format) return *options.format; - if (endsWith(options.outputPath, ".csv")) return OutputFormat::Csv; - return OutputFormat::Json; - } - - int64_t currentUnixTimeMicroseconds() { - return std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()).count(); - } - - int listDevices(const capture::CaptureEngine& engine) { - const auto devices = engine.getAvailableDevices(); - if (devices.empty()) { - std::cerr << "No capture devices found. On most systems live capture needs " - "administrator or root privileges.\n"; - return kExitRuntime; - } - for (const auto& device : devices) { - std::cout << device.name; - if (!device.description.empty()) std::cout << " (" << device.description << ")"; - std::cout << '\n'; - } - return kExitSuccess; - } - - // Replays a capture file. Packets are handed over as they are read, so a - // file of any size is processed without loss. - bool runOffline(capture::CaptureEngine& engine, core::AnalysisSession& session, - const Options& options, uint64_t& packetsAnalyzed) { - std::string error; - const bool ok = engine.replayFile(options.readPath, - [&](const core::PacketData& packet) { - session.feed(packet); - ++packetsAnalyzed; - }, error); - - if (!ok) { - std::cerr << "netprobe-cli: unable to read '" << options.readPath << "': " - << error << '\n'; - return false; - } - return true; - } - - // Captures live until the duration elapses, the packet limit is reached, - // or the user interrupts. - bool runLive(capture::CaptureEngine& engine, core::PacketQueue& queue, - core::AnalysisSession& session, const Options& options, uint64_t& packetsAnalyzed) { - engine.startCapture(options.device); - if (!engine.isCapturing()) { - std::cerr << "netprobe-cli: unable to open device '" << options.device - << "'. Live capture usually requires administrator or root privileges.\n" - "Run --list-devices to see the available adapters.\n"; - return false; - } - - if (!options.bpfFilter.empty()) { - std::string error; - if (!engine.setFilter(options.bpfFilter, error)) { - std::cerr << "netprobe-cli: invalid capture filter '" << options.bpfFilter - << "': " << error << '\n'; - engine.stopCapture(); - return false; - } - } - - const auto deadline = options.durationSeconds - ? std::optional{std::chrono::steady_clock::now() - + std::chrono::seconds(*options.durationSeconds)} - : std::nullopt; - - std::cerr << "Capturing on " << options.device; - if (options.durationSeconds) std::cerr << " for " << *options.durationSeconds << "s"; - if (options.packetLimit) std::cerr << ", up to " << *options.packetLimit << " packets"; - std::cerr << ". Press Ctrl+C to stop early.\n"; - - while (!g_stopRequested.load(std::memory_order_acquire)) { - if (deadline && std::chrono::steady_clock::now() >= *deadline) break; - if (options.packetLimit && packetsAnalyzed >= *options.packetLimit) break; - - auto packet = queue.try_pop(); - if (!packet) { - // Nothing buffered: yield briefly rather than spin a core. - std::this_thread::sleep_for(std::chrono::milliseconds(5)); - continue; - } - session.feed(*packet); - ++packetsAnalyzed; - } - - engine.stopCapture(); - - // The capture thread has stopped, but packets it already queued are - // still pending; analysing them costs nothing and makes the totals - // match what was actually captured. - while (auto packet = queue.try_pop()) { - if (options.packetLimit && packetsAnalyzed >= *options.packetLimit) break; - session.feed(*packet); - ++packetsAnalyzed; - } - - if (const size_t dropped = queue.droppedPackets(); dropped > 0) { - std::cerr << "Warning: " << dropped << " packets were dropped because analysis " - "could not keep up with the capture; flow totals undercount by that much.\n"; - } - return true; - } - - bool writeOutput(const std::vector& flows, const Options& options, - OutputFormat format) { - if (options.outputPath == "-") { - if (format == OutputFormat::Csv) core::FlowExporter::writeCsv(flows, std::cout); - else core::FlowExporter::writeJson(flows, std::cout); - std::cout.flush(); - if (!std::cout.good()) { - std::cerr << "netprobe-cli: writing to stdout failed\n"; - return false; - } - return true; - } - - std::string error; - const bool ok = format == OutputFormat::Csv - ? core::FlowExporter::writeCsv(flows, options.outputPath, error) - : core::FlowExporter::writeJson(flows, options.outputPath, error); - if (!ok) { - std::cerr << "netprobe-cli: unable to write '" << options.outputPath << "': " - << error << '\n'; - return false; - } - return true; - } - -} // namespace - -int main(int argc, char** argv) { - const auto parsed = parseArguments(argc, argv); - if (!parsed) { - std::cerr << "Try 'netprobe-cli --help'.\n"; - return kExitUsage; - } - const Options options = *parsed; - - if (options.showHelp) { - printUsage(std::cout); - return kExitSuccess; - } - if (options.showVersion) { - std::cout << "netprobe-cli " NETPROBE_VERSION "\n"; - return kExitSuccess; - } - if (!validate(options)) return kExitUsage; - - core::PacketQueue queue; - capture::CaptureEngine engine(queue); - - if (options.listDevices) return listDevices(engine); - - std::signal(SIGINT, handleInterrupt); -#ifdef SIGTERM - std::signal(SIGTERM, handleInterrupt); -#endif - - // Replaying a file means the sockets that owned those packets are long - // gone, so the socket-table walk is pure overhead with nothing to find. - const bool resolveProcesses = options.resolveProcesses && options.readPath.empty(); - core::AnalysisSession session(resolveProcesses); - - uint64_t packetsAnalyzed = 0; - const bool captured = options.readPath.empty() - ? runLive(engine, queue, session, options, packetsAnalyzed) - : runOffline(engine, session, options, packetsAnalyzed); - if (!captured) return kExitRuntime; - - const auto flows = session.flows(currentUnixTimeMicroseconds()); - const OutputFormat format = resolveFormat(options); - if (!writeOutput(flows, options, format)) return kExitRuntime; - - std::cerr << "Analyzed " << packetsAnalyzed << " packets into " << flows.size() - << " flows; wrote " - << (options.outputPath == "-" ? std::string{"stdout"} : options.outputPath) - << " as " << (format == OutputFormat::Csv ? "CSV" : "JSON") << ".\n"; - return kExitSuccess; -} +// NetProbe headless CLI. +// +// Runs the same capture and analysis pipeline as the GUI with no window, so +// captures can be taken and flows exported over SSH, in CI, or on a server. +// Deliberately small: replay a file or capture for a bounded time, then write +// the flow table as JSON or CSV. + +#include "capture/CaptureEngine.hpp" +#include "core/AnalysisSession.hpp" +#include "core/FlowExporter.hpp" +#include "core/PacketQueue.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef NETPROBE_VERSION +#define NETPROBE_VERSION "unknown" +#endif + +namespace { + + // Exit codes are part of the CLI's contract with scripts that call it. + constexpr int kExitSuccess = 0; + constexpr int kExitUsage = 1; + constexpr int kExitRuntime = 2; + + // Set from the signal handler; only ever transitions false -> true. + std::atomic g_stopRequested{false}; + + extern "C" void handleInterrupt(int) { + g_stopRequested.store(true, std::memory_order_release); + } + + enum class OutputFormat { Json, Csv }; + + struct Options { + bool listDevices = false; + bool showHelp = false; + bool showVersion = false; + std::string readPath; // offline capture to replay + std::string device; // live adapter to capture from + std::string bpfFilter; + std::string outputPath; // "-" means stdout + std::optional format; + std::optional durationSeconds; + std::optional packetLimit; + bool resolveProcesses = true; + }; + + void printUsage(std::ostream& out) { + out << "NetProbe CLI " NETPROBE_VERSION " - headless packet capture and flow export\n" + "\n" + "Usage:\n" + " netprobe-cli --list-devices\n" + " netprobe-cli -r -o \n" + " netprobe-cli -i [-f ] [--duration ] -o \n" + "\n" + "Source (exactly one required):\n" + " -r, --read Replay an offline capture file\n" + " -i, --device Capture live from an adapter (needs privileges)\n" + "\n" + "Output:\n" + " -o, --output Destination file, or '-' for stdout\n" + " --format Override the format inferred from the extension\n" + "\n" + "Capture limits (live capture only):\n" + " --duration Stop after this many seconds\n" + " --packet-count Stop after this many packets\n" + "\n" + "Other:\n" + " --no-process Skip owning-process lookup\n" + " --list-devices List capture adapters and exit\n" + " -h, --help Show this help and exit\n" + " -V, --version Show the version and exit\n" + "\n" + "Interrupting with Ctrl+C stops the capture and still writes the output.\n"; + } + + // Returns false when `text` is not a complete non-negative integer. + template + bool parseNonNegative(std::string_view text, T& value) { + if (text.empty()) return false; + const char* begin = text.data(); + const char* end = begin + text.size(); + const auto result = std::from_chars(begin, end, value); + return result.ec == std::errc{} && result.ptr == end; + } + + bool endsWith(const std::string& value, std::string_view suffix) { + return value.size() >= suffix.size() + && value.compare(value.size() - suffix.size(), suffix.size(), suffix) == 0; + } + + // Parses argv. On failure prints the reason to stderr and returns nullopt. + std::optional parseArguments(int argc, char** argv) { + Options options; + + // Every option below takes exactly one value; centralising the + // "is there a next argument" check keeps the loop readable. + const auto takeValue = [&](int& index, std::string_view flag) -> const char* { + if (index + 1 >= argc) { + std::cerr << "netprobe-cli: " << flag << " requires a value\n"; + return nullptr; + } + return argv[++index]; + }; + + for (int i = 1; i < argc; ++i) { + const std::string_view argument = argv[i]; + + if (argument == "-h" || argument == "--help") { + options.showHelp = true; + return options; + } + if (argument == "-V" || argument == "--version") { + options.showVersion = true; + return options; + } + if (argument == "--list-devices") { + options.listDevices = true; + continue; + } + if (argument == "--no-process") { + options.resolveProcesses = false; + continue; + } + if (argument == "-r" || argument == "--read") { + const char* value = takeValue(i, argument); + if (!value) return std::nullopt; + options.readPath = value; + continue; + } + if (argument == "-i" || argument == "--device") { + const char* value = takeValue(i, argument); + if (!value) return std::nullopt; + options.device = value; + continue; + } + if (argument == "-f" || argument == "--filter") { + const char* value = takeValue(i, argument); + if (!value) return std::nullopt; + options.bpfFilter = value; + continue; + } + if (argument == "-o" || argument == "--output") { + const char* value = takeValue(i, argument); + if (!value) return std::nullopt; + options.outputPath = value; + continue; + } + if (argument == "--format") { + const char* value = takeValue(i, argument); + if (!value) return std::nullopt; + const std::string_view format = value; + if (format == "json") options.format = OutputFormat::Json; + else if (format == "csv") options.format = OutputFormat::Csv; + else { + std::cerr << "netprobe-cli: unknown format '" << format + << "' (expected 'json' or 'csv')\n"; + return std::nullopt; + } + continue; + } + if (argument == "--duration") { + const char* value = takeValue(i, argument); + if (!value) return std::nullopt; + int seconds = 0; + if (!parseNonNegative(value, seconds) || seconds <= 0) { + std::cerr << "netprobe-cli: --duration expects a positive number of seconds\n"; + return std::nullopt; + } + options.durationSeconds = seconds; + continue; + } + if (argument == "--packet-count") { + const char* value = takeValue(i, argument); + if (!value) return std::nullopt; + uint64_t count = 0; + if (!parseNonNegative(value, count) || count == 0) { + std::cerr << "netprobe-cli: --packet-count expects a positive number\n"; + return std::nullopt; + } + options.packetLimit = count; + continue; + } + + std::cerr << "netprobe-cli: unrecognized option '" << argument << "'\n"; + return std::nullopt; + } + + return options; + } + + // Validates the combination of options, not just their individual syntax. + bool validate(const Options& options) { + if (options.listDevices) return true; + + const bool hasFile = !options.readPath.empty(); + const bool hasDevice = !options.device.empty(); + + if (!hasFile && !hasDevice) { + std::cerr << "netprobe-cli: nothing to do - pass --read or --device \n" + "Try 'netprobe-cli --help'.\n"; + return false; + } + if (hasFile && hasDevice) { + std::cerr << "netprobe-cli: --read and --device are mutually exclusive\n"; + return false; + } + if (options.outputPath.empty()) { + std::cerr << "netprobe-cli: no destination - pass --output (or '-' for stdout)\n"; + return false; + } + // Silently ignoring a flag that cannot apply is how people end up + // trusting a limit that was never enforced. + if (hasFile && (options.durationSeconds || options.packetLimit)) { + std::cerr << "netprobe-cli: --duration and --packet-count apply to live capture only\n"; + return false; + } + if (hasFile && !options.bpfFilter.empty()) { + std::cerr << "netprobe-cli: --filter applies to live capture only\n"; + return false; + } + return true; + } + + OutputFormat resolveFormat(const Options& options) { + if (options.format) return *options.format; + if (endsWith(options.outputPath, ".csv")) return OutputFormat::Csv; + return OutputFormat::Json; + } + + int64_t currentUnixTimeMicroseconds() { + return std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count(); + } + + int listDevices(const capture::CaptureEngine& engine) { + const auto devices = engine.getAvailableDevices(); + if (devices.empty()) { + std::cerr << "No capture devices found. On most systems live capture needs " + "administrator or root privileges.\n"; + return kExitRuntime; + } + for (const auto& device : devices) { + std::cout << device.name; + if (!device.description.empty()) std::cout << " (" << device.description << ")"; + std::cout << '\n'; + } + return kExitSuccess; + } + + // Replays a capture file. Packets are handed over as they are read, so a + // file of any size is processed without loss. + bool runOffline(capture::CaptureEngine& engine, core::AnalysisSession& session, + const Options& options, uint64_t& packetsAnalyzed) { + std::string error; + const bool ok = engine.replayFile(options.readPath, + [&](const core::PacketData& packet) { + session.feed(packet); + ++packetsAnalyzed; + }, error); + + if (!ok) { + std::cerr << "netprobe-cli: unable to read '" << options.readPath << "': " + << error << '\n'; + return false; + } + return true; + } + + // Captures live until the duration elapses, the packet limit is reached, + // or the user interrupts. + bool runLive(capture::CaptureEngine& engine, core::PacketQueue& queue, + core::AnalysisSession& session, const Options& options, uint64_t& packetsAnalyzed) { + engine.startCapture(options.device); + if (!engine.isCapturing()) { + std::cerr << "netprobe-cli: unable to open device '" << options.device + << "'. Live capture usually requires administrator or root privileges.\n" + "Run --list-devices to see the available adapters.\n"; + return false; + } + + if (!options.bpfFilter.empty()) { + std::string error; + if (!engine.setFilter(options.bpfFilter, error)) { + std::cerr << "netprobe-cli: invalid capture filter '" << options.bpfFilter + << "': " << error << '\n'; + engine.stopCapture(); + return false; + } + } + + const auto deadline = options.durationSeconds + ? std::optional{std::chrono::steady_clock::now() + + std::chrono::seconds(*options.durationSeconds)} + : std::nullopt; + + std::cerr << "Capturing on " << options.device; + if (options.durationSeconds) std::cerr << " for " << *options.durationSeconds << "s"; + if (options.packetLimit) std::cerr << ", up to " << *options.packetLimit << " packets"; + std::cerr << ". Press Ctrl+C to stop early.\n"; + + while (!g_stopRequested.load(std::memory_order_acquire)) { + if (deadline && std::chrono::steady_clock::now() >= *deadline) break; + if (options.packetLimit && packetsAnalyzed >= *options.packetLimit) break; + + auto packet = queue.try_pop(); + if (!packet) { + // Nothing buffered: yield briefly rather than spin a core. + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + continue; + } + session.feed(*packet); + ++packetsAnalyzed; + } + + engine.stopCapture(); + + // The capture thread has stopped, but packets it already queued are + // still pending; analysing them costs nothing and makes the totals + // match what was actually captured. + while (auto packet = queue.try_pop()) { + if (options.packetLimit && packetsAnalyzed >= *options.packetLimit) break; + session.feed(*packet); + ++packetsAnalyzed; + } + + if (const size_t dropped = queue.droppedPackets(); dropped > 0) { + std::cerr << "Warning: " << dropped << " packets were dropped because analysis " + "could not keep up with the capture; flow totals undercount by that much.\n"; + } + return true; + } + + bool writeOutput(const std::vector& flows, const Options& options, + OutputFormat format) { + if (options.outputPath == "-") { + if (format == OutputFormat::Csv) core::FlowExporter::writeCsv(flows, std::cout); + else core::FlowExporter::writeJson(flows, std::cout); + std::cout.flush(); + if (!std::cout.good()) { + std::cerr << "netprobe-cli: writing to stdout failed\n"; + return false; + } + return true; + } + + std::string error; + const bool ok = format == OutputFormat::Csv + ? core::FlowExporter::writeCsv(flows, options.outputPath, error) + : core::FlowExporter::writeJson(flows, options.outputPath, error); + if (!ok) { + std::cerr << "netprobe-cli: unable to write '" << options.outputPath << "': " + << error << '\n'; + return false; + } + return true; + } + +} // namespace + +static int run(int argc, char** argv) { + const auto parsed = parseArguments(argc, argv); + if (!parsed) { + std::cerr << "Try 'netprobe-cli --help'.\n"; + return kExitUsage; + } + const Options& options = *parsed; + + if (options.showHelp) { + printUsage(std::cout); + return kExitSuccess; + } + if (options.showVersion) { + std::cout << "netprobe-cli " NETPROBE_VERSION "\n"; + return kExitSuccess; + } + if (!validate(options)) return kExitUsage; + + core::PacketQueue queue; + capture::CaptureEngine engine(queue); + + if (options.listDevices) return listDevices(engine); + + std::signal(SIGINT, handleInterrupt); +#ifdef SIGTERM + std::signal(SIGTERM, handleInterrupt); +#endif + + // Replaying a file means the sockets that owned those packets are long + // gone, so the socket-table walk is pure overhead with nothing to find. + const bool resolveProcesses = options.resolveProcesses && options.readPath.empty(); + core::AnalysisSession session(resolveProcesses); + + uint64_t packetsAnalyzed = 0; + const bool captured = options.readPath.empty() + ? runLive(engine, queue, session, options, packetsAnalyzed) + : runOffline(engine, session, options, packetsAnalyzed); + if (!captured) return kExitRuntime; + + const auto flows = session.flows(currentUnixTimeMicroseconds()); + const OutputFormat format = resolveFormat(options); + if (!writeOutput(flows, options, format)) return kExitRuntime; + + std::cerr << "Analyzed " << packetsAnalyzed << " packets into " << flows.size() + << " flows; wrote " + << (options.outputPath == "-" ? std::string{"stdout"} : options.outputPath) + << " as " << (format == OutputFormat::Csv ? "CSV" : "JSON") << ".\n"; + return kExitSuccess; +} + +int main(int argc, char** argv) { + // An exception escaping main terminates with no diagnostic at all, which + // for a tool run from a script means a bare failure code and no clue why. + try { + return run(argc, argv); + } catch (const std::exception& error) { + // Reported without iostreams deliberately: a handler that can itself + // throw would let the exception escape after all, which is the whole + // thing this is here to prevent. std::fputs does not throw. + std::fputs("netprobe-cli: unexpected failure: ", stderr); + std::fputs(error.what(), stderr); + std::fputs("\n", stderr); + return kExitRuntime; + } catch (...) { + std::fputs("netprobe-cli: unexpected failure\n", stderr); + return kExitRuntime; + } +} diff --git a/src/core/DNSParser.cpp b/src/core/DNSParser.cpp index f375990..673ddf2 100644 --- a/src/core/DNSParser.cpp +++ b/src/core/DNSParser.cpp @@ -1,307 +1,307 @@ -#include "core/DNSParser.hpp" -#include "core/NetworkPlatform.hpp" -#include "core/ProtocolParser.hpp" - -#include -#include -#include -#include -#include - -namespace core { - namespace { - constexpr uint16_t dnsPort = 53; - constexpr uint16_t mdnsPort = 5353; - constexpr uint16_t llmnrPort = 5355; - constexpr uint16_t dnsClassInternet = 1; - - // RR type codes. - constexpr uint16_t rrA = 1; - constexpr uint16_t rrCname = 5; - constexpr uint16_t rrPtr = 12; - constexpr uint16_t rrTxt = 16; - constexpr uint16_t rrAaaa = 28; - constexpr uint16_t rrSrv = 33; - constexpr uint16_t rrSvcb = 64; - constexpr uint16_t rrHttps = 65; - - // SVCB parameter keys (RFC 9460 §14.3.2). - constexpr uint16_t svcParamEch = 5; - - constexpr size_t kMaxTxtBytes = 255; - - uint16_t readU16(const uint8_t* bytes) { - return static_cast(bytes[0]) << 8 | bytes[1]; - } - - std::optional readName(const uint8_t* data, size_t size, size_t& position) { - std::string name; - size_t cursor = position; - size_t pointerCount = 0; - bool followedPointer = false; - - while (cursor < size) { - const uint8_t labelLength = data[cursor]; - if (labelLength == 0) { - ++cursor; - if (!followedPointer) position = cursor; - return name; - } - - if ((labelLength & 0xC0) == 0xC0) { - if (cursor + 1 >= size || ++pointerCount > size) return std::nullopt; - const size_t pointer = (static_cast(labelLength & 0x3F) << 8) | data[cursor + 1]; - if (pointer >= size) return std::nullopt; - if (!followedPointer) position = cursor + 2; - cursor = pointer; - followedPointer = true; - continue; - } - - if ((labelLength & 0xC0) != 0 || labelLength > 63 || cursor + 1 + labelLength > size) { - return std::nullopt; - } - - if (!name.empty()) name += '.'; - name.append(reinterpret_cast(data + cursor + 1), labelLength); - cursor += 1 + labelLength; - } - - return std::nullopt; - } - - std::string addressToString(int family, const uint8_t* address) { - char text[INET6_ADDRSTRLEN]{}; - return inet_ntop(family, address, text, sizeof(text)) ? text : ""; - } - - std::vector splitLabels(const std::string& name) { - std::vector labels; - size_t start = 0; - while (start <= name.size()) { - const size_t dot = name.find('.', start); - if (dot == std::string::npos) { - labels.push_back(name.substr(start)); - break; - } - labels.push_back(name.substr(start, dot - start)); - start = dot + 1; - } - return labels; - } - - std::string toLower(std::string value) { - std::transform(value.begin(), value.end(), value.begin(), - [](unsigned char c) { return static_cast(std::tolower(c)); }); - return value; - } - - // Walk a SVCB/HTTPS RDATA blob, reporting whether it advertises ECH. - // Layout: priority(2), TargetName, then zero or more {key(2), len(2), value}. - bool svcbAdvertisesEch(const uint8_t* message, size_t size, size_t position, size_t recordEnd) { - if (recordEnd - position < 2) return false; - position += 2; // priority - - size_t targetPosition = position; - if (!readName(message, size, targetPosition)) return false; - position = targetPosition; - - while (position + 4 <= recordEnd) { - const uint16_t key = readU16(message + position); - const uint16_t valueLength = readU16(message + position + 2); - position += 4; - if (valueLength > recordEnd - position) return false; - if (key == svcParamEch && valueLength > 0) return true; - position += valueLength; - } - return false; - } - - // "1.1.168.192.in-addr.arpa" -> "192.168.1.1" - std::optional reverseIpv4(const std::vector& labels) { - if (labels.size() != 6) return std::nullopt; - std::string address; - for (int i = 3; i >= 0; --i) { - const std::string& octet = labels[static_cast(i)]; - if (octet.empty() || octet.size() > 3) return std::nullopt; - for (char c : octet) { - if (!std::isdigit(static_cast(c))) return std::nullopt; - } - if (std::stoi(octet) > 255) return std::nullopt; - if (!address.empty()) address += '.'; - address += octet; - } - return address; - } - - // 32 reversed nibble labels + "ip6.arpa" -> a canonical IPv6 string. - std::optional reverseIpv6(const std::vector& labels) { - if (labels.size() != 34) return std::nullopt; - uint8_t bytes[16]{}; - for (size_t i = 0; i < 32; ++i) { - const std::string& nibble = labels[31 - i]; - if (nibble.size() != 1 || !std::isxdigit(static_cast(nibble[0]))) { - return std::nullopt; - } - const char c = static_cast(std::tolower(static_cast(nibble[0]))); - const uint8_t value = static_cast(c <= '9' ? c - '0' : c - 'a' + 10); - if (i % 2 == 0) bytes[i / 2] = static_cast(value << 4); - else bytes[i / 2] = static_cast(bytes[i / 2] | value); - } - const std::string text = addressToString(AF_INET6, bytes); - if (text.empty()) return std::nullopt; - return text; - } - } - - std::optional DNSParser::reverseNameToAddress(const std::string& name) { - const auto labels = splitLabels(toLower(name)); - if (labels.size() >= 3) { - const std::string& tld = labels[labels.size() - 1]; - const std::string& sld = labels[labels.size() - 2]; - if (tld == "arpa" && sld == "in-addr") return reverseIpv4(labels); - if (tld == "arpa" && sld == "ip6") return reverseIpv6(labels); - } - return std::nullopt; - } - - std::optional DNSParser::parseMessage(const uint8_t* message, size_t size) { - if (!message || size < 12) return std::nullopt; - - const uint16_t flags = readU16(message + 2); - if ((flags & 0x8000) == 0) return std::nullopt; // Queries do not populate the cache. - - const uint16_t questionCount = readU16(message + 4); - const uint16_t answerCount = readU16(message + 6); - size_t position = 12; - DNSResponse response; - - for (uint16_t question = 0; question < questionCount; ++question) { - const auto name = readName(message, size, position); - if (!name || size - position < 4) return std::nullopt; - if (question == 0) response.queryName = *name; - position += 4; // QTYPE + QCLASS - } - - for (uint16_t answer = 0; answer < answerCount; ++answer) { - const auto name = readName(message, size, position); - if (!name || size - position < 10) return std::nullopt; - - const uint16_t type = readU16(message + position); - // mDNS reuses the top class bit as a cache-flush flag, so compare - // only the low 15 bits. - const uint16_t recordClass = readU16(message + position + 2) & 0x7FFF; - const size_t dataLength = readU16(message + position + 8); - position += 10; - if (dataLength > size - position) return std::nullopt; - const size_t recordEnd = position + dataLength; - - if (recordClass == dnsClassInternet) { - switch (type) { - case rrA: - if (dataLength == 4) { - response.answers.push_back({DNSRecordType::A, *name, - addressToString(AF_INET, message + position), 0}); - } - break; - case rrAaaa: - if (dataLength == 16) { - response.answers.push_back({DNSRecordType::AAAA, *name, - addressToString(AF_INET6, message + position), 0}); - } - break; - case rrCname: { - size_t cursor = position; - const auto canonicalName = readName(message, size, cursor); - if (canonicalName && cursor <= recordEnd) { - response.answers.push_back({DNSRecordType::CNAME, *name, *canonicalName, 0}); - } - break; - } - case rrPtr: { - size_t cursor = position; - const auto target = readName(message, size, cursor); - if (target && cursor <= recordEnd && !target->empty()) { - response.answers.push_back({DNSRecordType::PTR, *name, *target, 0}); - } - break; - } - case rrSrv: { - if (dataLength < 7) break; - const uint16_t priority = readU16(message + position); - const uint16_t port = readU16(message + position + 4); - size_t cursor = position + 6; - const auto target = readName(message, size, cursor); - if (target && cursor <= recordEnd && !target->empty()) { - response.answers.push_back({DNSRecordType::SRV, *name, - *target + ":" + std::to_string(port), priority}); - } - break; - } - case rrTxt: { - // Character-strings, each length-prefixed. Join with spaces. - std::string text; - size_t cursor = position; - while (cursor < recordEnd && text.size() < kMaxTxtBytes) { - const size_t chunkLength = message[cursor++]; - if (chunkLength > recordEnd - cursor) break; - if (!text.empty()) text += ' '; - text.append(reinterpret_cast(message + cursor), - std::min(chunkLength, kMaxTxtBytes - text.size())); - cursor += chunkLength; - } - if (!text.empty()) { - response.answers.push_back({DNSRecordType::TXT, *name, text, 0}); - } - break; - } - case rrSvcb: - case rrHttps: { - if (dataLength < 2) break; - const uint16_t priority = readU16(message + position); - size_t cursor = position + 2; - const auto target = readName(message, size, cursor); - const bool ech = svcbAdvertisesEch(message, size, position, recordEnd); - if (ech) response.encryptedClientHelloAdvertised = true; - response.answers.push_back({ - type == rrHttps ? DNSRecordType::HTTPS : DNSRecordType::SVCB, - *name, - target && !target->empty() ? *target : *name, - priority}); - break; - } - default: - break; - } - } - - position = recordEnd; - } - - return response; - } - - std::optional DNSParser::parseResponse(const PacketData& rawData, - const ParsedPacket& parsed) { - - if (parsed.protocol != "UDP" || parsed.payloadLength == 0) return std::nullopt; - - const bool isDns = parsed.srcPort == dnsPort || parsed.dstPort == dnsPort - || parsed.srcPort == mdnsPort || parsed.dstPort == mdnsPort - || parsed.srcPort == llmnrPort || parsed.dstPort == llmnrPort; - if (!isDns) return std::nullopt; - - if (parsed.payloadOffset >= rawData.payload.size()) return std::nullopt; - const size_t available = std::min(parsed.payloadLength, - rawData.payload.size() - parsed.payloadOffset); - return parseMessage(rawData.payload.data() + parsed.payloadOffset, available); - } - - std::optional DNSParser::parseResponse(const PacketData& rawData) { - // Locating the UDP payload means walking the link, network, and tunnel - // layers — exactly what ProtocolParser already does, including for - // loopback and cooked captures. - return parseResponse(rawData, ProtocolParser::parse(rawData)); - } - -} // namespace core +#include "core/DNSParser.hpp" +#include "core/NetworkPlatform.hpp" +#include "core/ProtocolParser.hpp" + +#include +#include +#include +#include +#include + +namespace core { + namespace { + constexpr uint16_t dnsPort = 53; + constexpr uint16_t mdnsPort = 5353; + constexpr uint16_t llmnrPort = 5355; + constexpr uint16_t dnsClassInternet = 1; + + // RR type codes. + constexpr uint16_t rrA = 1; + constexpr uint16_t rrCname = 5; + constexpr uint16_t rrPtr = 12; + constexpr uint16_t rrTxt = 16; + constexpr uint16_t rrAaaa = 28; + constexpr uint16_t rrSrv = 33; + constexpr uint16_t rrSvcb = 64; + constexpr uint16_t rrHttps = 65; + + // SVCB parameter keys (RFC 9460 §14.3.2). + constexpr uint16_t svcParamEch = 5; + + constexpr size_t kMaxTxtBytes = 255; + + uint16_t readU16(const uint8_t* bytes) { + return static_cast(bytes[0]) << 8 | bytes[1]; + } + + std::optional readName(const uint8_t* data, size_t size, size_t& position) { + std::string name; + size_t cursor = position; + size_t pointerCount = 0; + bool followedPointer = false; + + while (cursor < size) { + const uint8_t labelLength = data[cursor]; + if (labelLength == 0) { + ++cursor; + if (!followedPointer) position = cursor; + return name; + } + + if ((labelLength & 0xC0) == 0xC0) { + if (cursor + 1 >= size || ++pointerCount > size) return std::nullopt; + const size_t pointer = (static_cast(labelLength & 0x3F) << 8) | data[cursor + 1]; + if (pointer >= size) return std::nullopt; + if (!followedPointer) position = cursor + 2; + cursor = pointer; + followedPointer = true; + continue; + } + + if ((labelLength & 0xC0) != 0 || labelLength > 63 || cursor + 1 + labelLength > size) { + return std::nullopt; + } + + if (!name.empty()) name += '.'; + name.append(reinterpret_cast(data + cursor + 1), labelLength); + cursor += 1 + labelLength; + } + + return std::nullopt; + } + + std::string addressToString(int family, const uint8_t* address) { + char text[INET6_ADDRSTRLEN]{}; + return inet_ntop(family, address, text, sizeof(text)) ? text : ""; + } + + std::vector splitLabels(const std::string& name) { + std::vector labels; + size_t start = 0; + while (start <= name.size()) { + const size_t dot = name.find('.', start); + if (dot == std::string::npos) { + labels.push_back(name.substr(start)); + break; + } + labels.push_back(name.substr(start, dot - start)); + start = dot + 1; + } + return labels; + } + + std::string toLower(std::string value) { + std::transform(value.begin(), value.end(), value.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + return value; + } + + // Walk a SVCB/HTTPS RDATA blob, reporting whether it advertises ECH. + // Layout: priority(2), TargetName, then zero or more {key(2), len(2), value}. + bool svcbAdvertisesEch(const uint8_t* message, size_t size, size_t position, size_t recordEnd) { + if (recordEnd - position < 2) return false; + position += 2; // priority + + size_t targetPosition = position; + if (!readName(message, size, targetPosition)) return false; + position = targetPosition; + + while (position + 4 <= recordEnd) { + const uint16_t key = readU16(message + position); + const uint16_t valueLength = readU16(message + position + 2); + position += 4; + if (valueLength > recordEnd - position) return false; + if (key == svcParamEch && valueLength > 0) return true; + position += valueLength; + } + return false; + } + + // "1.1.168.192.in-addr.arpa" -> "192.168.1.1" + std::optional reverseIpv4(const std::vector& labels) { + if (labels.size() != 6) return std::nullopt; + std::string address; + for (int i = 3; i >= 0; --i) { + const std::string& octet = labels[static_cast(i)]; + if (octet.empty() || octet.size() > 3) return std::nullopt; + for (char c : octet) { + if (!std::isdigit(static_cast(c))) return std::nullopt; + } + if (std::stoi(octet) > 255) return std::nullopt; + if (!address.empty()) address += '.'; + address += octet; + } + return address; + } + + // 32 reversed nibble labels + "ip6.arpa" -> a canonical IPv6 string. + std::optional reverseIpv6(const std::vector& labels) { + if (labels.size() != 34) return std::nullopt; + uint8_t bytes[16]{}; + for (size_t i = 0; i < 32; ++i) { + const std::string& nibble = labels[31 - i]; + if (nibble.size() != 1 || !std::isxdigit(static_cast(nibble[0]))) { + return std::nullopt; + } + const char c = static_cast(std::tolower(static_cast(nibble[0]))); + const uint8_t value = static_cast(c <= '9' ? c - '0' : c - 'a' + 10); + if (i % 2 == 0) bytes[i / 2] = static_cast(value << 4); + else bytes[i / 2] = static_cast(bytes[i / 2] | value); + } + std::string text = addressToString(AF_INET6, bytes); + if (text.empty()) return std::nullopt; + return text; + } + } + + std::optional DNSParser::reverseNameToAddress(const std::string& name) { + const auto labels = splitLabels(toLower(name)); + if (labels.size() >= 3) { + const std::string& tld = labels[labels.size() - 1]; + const std::string& sld = labels[labels.size() - 2]; + if (tld == "arpa" && sld == "in-addr") return reverseIpv4(labels); + if (tld == "arpa" && sld == "ip6") return reverseIpv6(labels); + } + return std::nullopt; + } + + std::optional DNSParser::parseMessage(const uint8_t* message, size_t size) { + if (!message || size < 12) return std::nullopt; + + const uint16_t flags = readU16(message + 2); + if ((flags & 0x8000) == 0) return std::nullopt; // Queries do not populate the cache. + + const uint16_t questionCount = readU16(message + 4); + const uint16_t answerCount = readU16(message + 6); + size_t position = 12; + DNSResponse response; + + for (uint16_t question = 0; question < questionCount; ++question) { + const auto name = readName(message, size, position); + if (!name || size - position < 4) return std::nullopt; + if (question == 0) response.queryName = *name; + position += 4; // QTYPE + QCLASS + } + + for (uint16_t answer = 0; answer < answerCount; ++answer) { + const auto name = readName(message, size, position); + if (!name || size - position < 10) return std::nullopt; + + const uint16_t type = readU16(message + position); + // mDNS reuses the top class bit as a cache-flush flag, so compare + // only the low 15 bits. + const uint16_t recordClass = readU16(message + position + 2) & 0x7FFF; + const size_t dataLength = readU16(message + position + 8); + position += 10; + if (dataLength > size - position) return std::nullopt; + const size_t recordEnd = position + dataLength; + + if (recordClass == dnsClassInternet) { + switch (type) { + case rrA: + if (dataLength == 4) { + response.answers.push_back({DNSRecordType::A, *name, + addressToString(AF_INET, message + position), 0}); + } + break; + case rrAaaa: + if (dataLength == 16) { + response.answers.push_back({DNSRecordType::AAAA, *name, + addressToString(AF_INET6, message + position), 0}); + } + break; + case rrCname: { + size_t cursor = position; + const auto canonicalName = readName(message, size, cursor); + if (canonicalName && cursor <= recordEnd) { + response.answers.push_back({DNSRecordType::CNAME, *name, *canonicalName, 0}); + } + break; + } + case rrPtr: { + size_t cursor = position; + const auto target = readName(message, size, cursor); + if (target && cursor <= recordEnd && !target->empty()) { + response.answers.push_back({DNSRecordType::PTR, *name, *target, 0}); + } + break; + } + case rrSrv: { + if (dataLength < 7) break; + const uint16_t priority = readU16(message + position); + const uint16_t port = readU16(message + position + 4); + size_t cursor = position + 6; + const auto target = readName(message, size, cursor); + if (target && cursor <= recordEnd && !target->empty()) { + response.answers.push_back({DNSRecordType::SRV, *name, + *target + ":" + std::to_string(port), priority}); + } + break; + } + case rrTxt: { + // Character-strings, each length-prefixed. Join with spaces. + std::string text; + size_t cursor = position; + while (cursor < recordEnd && text.size() < kMaxTxtBytes) { + const size_t chunkLength = message[cursor++]; + if (chunkLength > recordEnd - cursor) break; + if (!text.empty()) text += ' '; + text.append(reinterpret_cast(message + cursor), + std::min(chunkLength, kMaxTxtBytes - text.size())); + cursor += chunkLength; + } + if (!text.empty()) { + response.answers.push_back({DNSRecordType::TXT, *name, text, 0}); + } + break; + } + case rrSvcb: + case rrHttps: { + if (dataLength < 2) break; + const uint16_t priority = readU16(message + position); + size_t cursor = position + 2; + const auto target = readName(message, size, cursor); + const bool ech = svcbAdvertisesEch(message, size, position, recordEnd); + if (ech) response.encryptedClientHelloAdvertised = true; + response.answers.push_back({ + type == rrHttps ? DNSRecordType::HTTPS : DNSRecordType::SVCB, + *name, + target && !target->empty() ? *target : *name, + priority}); + break; + } + default: + break; + } + } + + position = recordEnd; + } + + return response; + } + + std::optional DNSParser::parseResponse(const PacketData& rawData, + const ParsedPacket& parsed) { + + if (parsed.protocol != "UDP" || parsed.payloadLength == 0) return std::nullopt; + + const bool isDns = parsed.srcPort == dnsPort || parsed.dstPort == dnsPort + || parsed.srcPort == mdnsPort || parsed.dstPort == mdnsPort + || parsed.srcPort == llmnrPort || parsed.dstPort == llmnrPort; + if (!isDns) return std::nullopt; + + if (parsed.payloadOffset >= rawData.payload.size()) return std::nullopt; + const size_t available = std::min(parsed.payloadLength, + rawData.payload.size() - parsed.payloadOffset); + return parseMessage(rawData.payload.data() + parsed.payloadOffset, available); + } + + std::optional DNSParser::parseResponse(const PacketData& rawData) { + // Locating the UDP payload means walking the link, network, and tunnel + // layers — exactly what ProtocolParser already does, including for + // loopback and cooked captures. + return parseResponse(rawData, ProtocolParser::parse(rawData)); + } + +} // namespace core diff --git a/src/core/GeoIPResolver.cpp b/src/core/GeoIPResolver.cpp index 18a380a..e104ce7 100644 --- a/src/core/GeoIPResolver.cpp +++ b/src/core/GeoIPResolver.cpp @@ -1,129 +1,129 @@ -#include "core/GeoIPResolver.hpp" - -#include -#include -#include - -namespace core { - - std::string organizationLabel(const GeoIPInfo& info) { - if (info.organization.empty()) return std::string{"-"}; - return info.asn == 0 - ? info.organization - : std::format("AS{} {}", info.asn, info.organization); - } - - namespace { - std::filesystem::path geoIpDataDirectory() { -#ifdef NETPROBE_GEOIP_DATA_DIR - return NETPROBE_GEOIP_DATA_DIR; -#else - return std::filesystem::current_path() / "data"; -#endif - } - - std::string getUtf8Value(const MMDB_lookup_result_s& result, const char* firstPath, const char* secondPath) { - MMDB_entry_data_s data{}; - if (MMDB_get_value(const_cast(&result.entry), &data, firstPath, secondPath, - static_cast(nullptr)) != MMDB_SUCCESS - || !data.has_data || data.type != MMDB_DATA_TYPE_UTF8_STRING) { - return {}; - } - return std::string(reinterpret_cast(data.utf8_string), data.data_size); - } - - uint32_t getUint32Value(const MMDB_lookup_result_s& result, const char* path) { - MMDB_entry_data_s data{}; - if (MMDB_get_value(const_cast(&result.entry), &data, path, - static_cast(nullptr)) != MMDB_SUCCESS - || !data.has_data || data.type != MMDB_DATA_TYPE_UINT32) { - return 0; - } - return data.uint32; - } - } - - struct GeoIPResolver::Database { - MMDB_s country{}; - MMDB_s asn{}; - bool countryOpen = false; - bool asnOpen = false; - std::string status; - - Database(const std::filesystem::path& countryPath, const std::filesystem::path& asnPath) { - countryOpen = MMDB_open(countryPath.string().c_str(), MMDB_MODE_MMAP, &country) == MMDB_SUCCESS; - asnOpen = MMDB_open(asnPath.string().c_str(), MMDB_MODE_MMAP, &asn) == MMDB_SUCCESS; - if (countryOpen || asnOpen) { - status = "GeoLite2 enrichment enabled."; - } else { - status = "GeoLite2 databases were not found at " + countryPath.string() + " and " + asnPath.string(); - } - } - - ~Database() { - if (countryOpen) MMDB_close(&country); - if (asnOpen) MMDB_close(&asn); - } - }; - - GeoIPResolver::GeoIPResolver(size_t cacheCapacity) - : GeoIPResolver(geoIpDataDirectory() / "GeoLite2-Country.mmdb", - geoIpDataDirectory() / "GeoLite2-ASN.mmdb", cacheCapacity) {} - - GeoIPResolver::GeoIPResolver(std::filesystem::path countryDatabasePath, - std::filesystem::path asnDatabasePath, size_t cacheCapacity) - : m_database(std::make_unique(countryDatabasePath, asnDatabasePath)) - , m_cacheCapacity(cacheCapacity > 0 ? cacheCapacity : 1) {} - - GeoIPResolver::~GeoIPResolver() = default; - - GeoIPInfo GeoIPResolver::lookup(const std::string& ip) { - if (ip.empty()) return {}; - - std::lock_guard lock(m_mutex); - if (const auto cached = m_cacheIndex.find(ip); cached != m_cacheIndex.end()) { - m_cache.splice(m_cache.begin(), m_cache, cached->second); - return cached->second->second; - } - - GeoIPInfo info; - int gaiError = 0; - int mmdbError = MMDB_SUCCESS; - - if (m_database->countryOpen) { - const auto result = MMDB_lookup_string(&m_database->country, ip.c_str(), &gaiError, &mmdbError); - if (gaiError == 0 && mmdbError == MMDB_SUCCESS && result.found_entry) { - info.country = getUtf8Value(result, "country", "iso_code"); - } - } - - gaiError = 0; - mmdbError = MMDB_SUCCESS; - if (m_database->asnOpen) { - const auto result = MMDB_lookup_string(&m_database->asn, ip.c_str(), &gaiError, &mmdbError); - if (gaiError == 0 && mmdbError == MMDB_SUCCESS && result.found_entry) { - info.asn = getUint32Value(result, "autonomous_system_number"); - info.organization = getUtf8Value(result, "autonomous_system_organization", nullptr); - } - } - - m_cache.emplace_front(ip, info); - m_cacheIndex[ip] = m_cache.begin(); - if (m_cache.size() > m_cacheCapacity) { - m_cacheIndex.erase(m_cache.back().first); - m_cache.pop_back(); - } - return info; - } - - bool GeoIPResolver::isAvailable() const { - std::lock_guard lock(m_mutex); - return m_database->countryOpen || m_database->asnOpen; - } - - std::string GeoIPResolver::status() const { - std::lock_guard lock(m_mutex); - return m_database->status; - } - -} // namespace core +#include "core/GeoIPResolver.hpp" + +#include +#include +#include + +namespace core { + + std::string organizationLabel(const GeoIPInfo& info) { + if (info.organization.empty()) return std::string{"-"}; + return info.asn == 0 + ? info.organization + : std::format("AS{} {}", info.asn, info.organization); + } + + namespace { + std::filesystem::path geoIpDataDirectory() { +#ifdef NETPROBE_GEOIP_DATA_DIR + return NETPROBE_GEOIP_DATA_DIR; +#else + return std::filesystem::current_path() / "data"; +#endif + } + + std::string getUtf8Value(const MMDB_lookup_result_s& result, const char* firstPath, const char* secondPath) { + MMDB_entry_data_s data{}; + if (MMDB_get_value(const_cast(&result.entry), &data, firstPath, secondPath, + static_cast(nullptr)) != MMDB_SUCCESS + || !data.has_data || data.type != MMDB_DATA_TYPE_UTF8_STRING) { + return {}; + } + return std::string(reinterpret_cast(data.utf8_string), data.data_size); + } + + uint32_t getUint32Value(const MMDB_lookup_result_s& result, const char* path) { + MMDB_entry_data_s data{}; + if (MMDB_get_value(const_cast(&result.entry), &data, path, + static_cast(nullptr)) != MMDB_SUCCESS + || !data.has_data || data.type != MMDB_DATA_TYPE_UINT32) { + return 0; + } + return data.uint32; + } + } + + struct GeoIPResolver::Database { + MMDB_s country{}; + MMDB_s asn{}; + bool countryOpen = false; + bool asnOpen = false; + std::string status; + + Database(const std::filesystem::path& countryPath, const std::filesystem::path& asnPath) { + countryOpen = MMDB_open(countryPath.string().c_str(), MMDB_MODE_MMAP, &country) == MMDB_SUCCESS; + asnOpen = MMDB_open(asnPath.string().c_str(), MMDB_MODE_MMAP, &asn) == MMDB_SUCCESS; + if (countryOpen || asnOpen) { + status = "GeoLite2 enrichment enabled."; + } else { + status = "GeoLite2 databases were not found at " + countryPath.string() + " and " + asnPath.string(); + } + } + + ~Database() { + if (countryOpen) MMDB_close(&country); + if (asnOpen) MMDB_close(&asn); + } + }; + + GeoIPResolver::GeoIPResolver(size_t cacheCapacity) + : GeoIPResolver(geoIpDataDirectory() / "GeoLite2-Country.mmdb", + geoIpDataDirectory() / "GeoLite2-ASN.mmdb", cacheCapacity) {} + + GeoIPResolver::GeoIPResolver(const std::filesystem::path& countryDatabasePath, + const std::filesystem::path& asnDatabasePath, size_t cacheCapacity) + : m_database(std::make_unique(countryDatabasePath, asnDatabasePath)) + , m_cacheCapacity(cacheCapacity > 0 ? cacheCapacity : 1) {} + + GeoIPResolver::~GeoIPResolver() = default; + + GeoIPInfo GeoIPResolver::lookup(const std::string& ip) { + if (ip.empty()) return {}; + + std::lock_guard lock(m_mutex); + if (const auto cached = m_cacheIndex.find(ip); cached != m_cacheIndex.end()) { + m_cache.splice(m_cache.begin(), m_cache, cached->second); + return cached->second->second; + } + + GeoIPInfo info; + int gaiError = 0; + int mmdbError = MMDB_SUCCESS; + + if (m_database->countryOpen) { + const auto result = MMDB_lookup_string(&m_database->country, ip.c_str(), &gaiError, &mmdbError); + if (gaiError == 0 && mmdbError == MMDB_SUCCESS && result.found_entry) { + info.country = getUtf8Value(result, "country", "iso_code"); + } + } + + gaiError = 0; + mmdbError = MMDB_SUCCESS; + if (m_database->asnOpen) { + const auto result = MMDB_lookup_string(&m_database->asn, ip.c_str(), &gaiError, &mmdbError); + if (gaiError == 0 && mmdbError == MMDB_SUCCESS && result.found_entry) { + info.asn = getUint32Value(result, "autonomous_system_number"); + info.organization = getUtf8Value(result, "autonomous_system_organization", nullptr); + } + } + + m_cache.emplace_front(ip, info); + m_cacheIndex[ip] = m_cache.begin(); + if (m_cache.size() > m_cacheCapacity) { + m_cacheIndex.erase(m_cache.back().first); + m_cache.pop_back(); + } + return info; + } + + bool GeoIPResolver::isAvailable() const { + std::lock_guard lock(m_mutex); + return m_database->countryOpen || m_database->asnOpen; + } + + std::string GeoIPResolver::status() const { + std::lock_guard lock(m_mutex); + return m_database->status; + } + +} // namespace core diff --git a/src/core/GeoIPResolver.hpp b/src/core/GeoIPResolver.hpp index 460e03d..939c40b 100644 --- a/src/core/GeoIPResolver.hpp +++ b/src/core/GeoIPResolver.hpp @@ -1,50 +1,50 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace core { - - struct GeoIPInfo { - std::string country; - uint32_t asn = 0; - std::string organization; - }; - - // Human-readable owner label, e.g. "AS15169 Google LLC", or "-" when the - // organization is unknown. Shared so the flow table and the exported files - // cannot disagree about how a network is named. - std::string organizationLabel(const GeoIPInfo& info); - - class GeoIPResolver { - public: - explicit GeoIPResolver(size_t cacheCapacity = 4'096); - GeoIPResolver(std::filesystem::path countryDatabasePath, - std::filesystem::path asnDatabasePath, size_t cacheCapacity = 4'096); - ~GeoIPResolver(); - - GeoIPResolver(const GeoIPResolver&) = delete; - GeoIPResolver& operator=(const GeoIPResolver&) = delete; - - GeoIPInfo lookup(const std::string& ip); - bool isAvailable() const; - std::string status() const; - - private: - struct Database; - using CacheEntry = std::pair; - - std::unique_ptr m_database; - size_t m_cacheCapacity; - std::list m_cache; - std::unordered_map::iterator> m_cacheIndex; - mutable std::mutex m_mutex; - }; - -} // namespace core +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace core { + + struct GeoIPInfo { + std::string country; + uint32_t asn = 0; + std::string organization; + }; + + // Human-readable owner label, e.g. "AS15169 Google LLC", or "-" when the + // organization is unknown. Shared so the flow table and the exported files + // cannot disagree about how a network is named. + std::string organizationLabel(const GeoIPInfo& info); + + class GeoIPResolver { + public: + explicit GeoIPResolver(size_t cacheCapacity = 4'096); + GeoIPResolver(const std::filesystem::path& countryDatabasePath, + const std::filesystem::path& asnDatabasePath, size_t cacheCapacity = 4'096); + ~GeoIPResolver(); + + GeoIPResolver(const GeoIPResolver&) = delete; + GeoIPResolver& operator=(const GeoIPResolver&) = delete; + + GeoIPInfo lookup(const std::string& ip); + bool isAvailable() const; + std::string status() const; + + private: + struct Database; + using CacheEntry = std::pair; + + std::unique_ptr m_database; + size_t m_cacheCapacity; + std::list m_cache; + std::unordered_map::iterator> m_cacheIndex; + mutable std::mutex m_mutex; + }; + +} // namespace core diff --git a/src/core/ProtocolParser.cpp b/src/core/ProtocolParser.cpp index d7e80d0..39cf0fa 100644 --- a/src/core/ProtocolParser.cpp +++ b/src/core/ProtocolParser.cpp @@ -1,1037 +1,1038 @@ -#include "core/ProtocolParser.hpp" -#include "core/NetworkPlatform.hpp" -#include "core/QuicParser.hpp" -#include -#include -#include -#include -#include -#include - -namespace core { - - namespace { - - constexpr uint16_t kEthIPv4 = 0x0800; - constexpr uint16_t kEthARP = 0x0806; - constexpr uint16_t kEthIPv6 = 0x86DD; - constexpr uint16_t kEthMplsUni = 0x8847; - constexpr uint16_t kEthMplsMul = 0x8848; - constexpr uint16_t kEthPppoeSess = 0x8864; - constexpr uint16_t kEthTrBridge = 0x6558; // Transparent Ethernet Bridging (GENEVE/GRE) - - // Descending through more than a few layers of encapsulation means - // either an exotic capture or a malicious packet trying to make us spin. - constexpr int kMaxTunnelDepth = 4; - - uint16_t readU16(const uint8_t* bytes) { - return static_cast(bytes[0]) << 8 | bytes[1]; - } - - uint32_t readU32(const uint8_t* bytes) { - return (static_cast(bytes[0]) << 24) - | (static_cast(bytes[1]) << 16) - | (static_cast(bytes[2]) << 8) - | static_cast(bytes[3]); - } - - std::string ipv4ToString(const uint8_t* bytes) { - uint32_t address; - std::memcpy(&address, bytes, sizeof(address)); - char buf[INET_ADDRSTRLEN]{}; - return inet_ntop(AF_INET, &address, buf, sizeof(buf)) ? buf : ""; - } - - std::string ipv6ToString(const uint8_t* bytes) { - char buf[INET6_ADDRSTRLEN]{}; - return inet_ntop(AF_INET6, bytes, buf, sizeof(buf)) ? buf : ""; - } - - // Map an L4 (or "next header") protocol number to a short label. - // Returns nullptr for protocols the caller decodes itself (TCP / UDP / - // ICMP / tunnels). - const char* ipProtocolName(uint8_t proto) { - switch (proto) { - case 2: return "IGMP"; - case 33: return "DCCP"; - case 50: return "ESP"; - case 51: return "AH"; - case 89: return "OSPF"; - case 103: return "PIM"; - case 112: return "VRRP"; - case 132: return "SCTP"; - default: return nullptr; - } - } - - // Frames that never carry IP. Naming them beats a bare "Non-IP" bucket, - // since these are constant background noise on any real LAN. - const char* etherTypeName(uint16_t etherType) { - switch (etherType) { - case 0x88CC: return "LLDP"; - case 0x888E: return "EAPOL"; - case 0x8863: return "PPPoE Discovery"; - case 0x8100: - case 0x88A8: - case 0x9100: return "VLAN"; - case 0x8137: return "IPX"; - case 0x88F7: return "PTP"; - case 0x22EA: return "SRP"; - case 0x8035: return "RARP"; - case 0x9000: return "Ethernet Loopback"; - case 0x88A4: return "EtherCAT"; - case 0x8892: return "PROFINET"; - default: return nullptr; - } - } - - // IPv6 extension header? If so, advances pos by header length and - // returns the new next-header value via outNext. Returns true if the - // current value was an extension header. The caller stops walking when - // this returns false (i.e. we've reached the upper-layer protocol). - bool walkV6Extension(uint8_t nh, const uint8_t* buffer, size_t size, - size_t& pos, uint8_t& outNext) { - switch (nh) { - case 0: // Hop-by-hop - case 43: // Routing - case 60: // Destination options - case 135: // Mobility - { - if (size < pos + 2) return false; - const uint8_t newNh = buffer[pos]; - const size_t hdrLen = (static_cast(buffer[pos + 1]) + 1) * 8; - if (size < pos + hdrLen) return false; - pos += hdrLen; - outNext = newNh; - return true; - } - case 44: // Fragment — fixed 8 bytes; the fragment offset matters but - // for live DPI we just keep walking on first-fragment. - { - if (size < pos + 8) return false; - outNext = buffer[pos]; - pos += 8; - return true; - } - case 51: // Authentication Header. Integrity-only, so the payload - // behind it is still readable — walk straight through. - { - if (size < pos + 2) return false; - outNext = buffer[pos]; - const size_t hdrLen = (static_cast(buffer[pos + 1]) + 2) * 4; - if (size < pos + hdrLen) return false; - pos += hdrLen; - return true; - } - default: - return false; - } - } - - // Lower-case in place for case-insensitive substring matches. - std::string toLowerCopy(std::string_view s) { - std::string out(s); - std::transform(out.begin(), out.end(), out.begin(), - [](unsigned char c){ return static_cast(std::tolower(c)); }); - return out; - } - - // Record a layer of encapsulation. The first call also snapshots the - // carrier addresses so the UI can still show who was tunnelling. - void pushTunnel(ParsedPacket& p, const char* name) { - if (p.outerSrcIP.empty() && !p.srcIP.empty()) { - p.outerSrcIP = p.srcIP; - p.outerDstIP = p.dstIP; - } - if (p.tunnel.empty()) p.tunnel = name; - else p.tunnel += std::string("/") + name; - } - - // Resolve the start of the network layer for a given link encapsulation. - // Returns false when the header is truncated or the link type carries no - // IP payload we can locate. `outEtherType` is synthesized for link types - // that do not carry a real EtherType (raw IP, loopback). - bool decodeLinkLayer(const uint8_t* buffer, size_t size, LinkType link, - size_t& outOffset, uint16_t& outEtherType) { - const auto etherTypeForIpVersion = [&](size_t at) -> bool { - if (at >= size) return false; - const uint8_t version = buffer[at] >> 4; - if (version == 4) { outEtherType = kEthIPv4; return true; } - if (version == 6) { outEtherType = kEthIPv6; return true; } - return false; - }; - - switch (link) { - case LinkType::Ethernet: { - if (size < sizeof(EthernetHeader)) return false; - outOffset = sizeof(EthernetHeader); - outEtherType = readU16(buffer + 12); - // 802.1Q / QinQ tags stack ahead of the real EtherType. - while (outEtherType == 0x8100 || outEtherType == 0x88A8 || outEtherType == 0x9100) { - if (size < outOffset + 4) return false; - outEtherType = readU16(buffer + outOffset + 2); - outOffset += 4; - } - return true; - } - case LinkType::LinuxSLL: { - // 16-byte cooked header; EtherType is the last field. - if (size < 16) return false; - outOffset = 16; - outEtherType = readU16(buffer + 14); - return true; - } - case LinkType::LinuxSLL2: { - // 20-byte cooked v2 header; EtherType comes first. - if (size < 20) return false; - outOffset = 20; - outEtherType = readU16(buffer); - return true; - } - case LinkType::NullLoopback: { - // 4-byte address family in the *writing host's* byte order, so - // try both before giving up. AF_INET is 2 everywhere; AF_INET6 - // differs across the BSDs (24/28/30) and Linux (10). - if (size < 4) return false; - outOffset = 4; - const uint32_t le = static_cast(buffer[0]) - | (static_cast(buffer[1]) << 8) - | (static_cast(buffer[2]) << 16) - | (static_cast(buffer[3]) << 24); - const uint32_t be = readU32(buffer); - for (uint32_t af : {le, be}) { - if (af == 2) { outEtherType = kEthIPv4; return true; } - if (af == 10 || af == 23 || af == 24 || af == 28 || af == 30) { - outEtherType = kEthIPv6; - return true; - } - } - // Unknown AF — fall back to sniffing the IP version nibble. - return etherTypeForIpVersion(4); - } - case LinkType::Loop: { - if (size < 4) return false; - outOffset = 4; - const uint32_t af = readU32(buffer); // always big-endian - if (af == 2) { outEtherType = kEthIPv4; return true; } - if (af == 10 || af == 23 || af == 24 || af == 28 || af == 30) { - outEtherType = kEthIPv6; - return true; - } - return etherTypeForIpVersion(4); - } - case LinkType::RawIP: - outOffset = 0; - return etherTypeForIpVersion(0); - case LinkType::IPv4: - outOffset = 0; - outEtherType = kEthIPv4; - return true; - case LinkType::IPv6: - outOffset = 0; - outEtherType = kEthIPv6; - return true; - case LinkType::Unsupported: - return false; - } - return false; - } - - // Forward declaration: the tunnel decoders recurse back into the network - // layer for the encapsulated packet. - void decodeNetwork(const uint8_t* buffer, size_t end, size_t offset, - uint16_t etherType, ParsedPacket& p, int depth); - - // Re-enter at an inner Ethernet frame (VXLAN, GENEVE with TEB, GRE-TEB). - void decodeInnerEthernet(const uint8_t* buffer, size_t end, size_t offset, - ParsedPacket& p, int depth) { - size_t innerOffset = 0; - uint16_t innerEtherType = 0; - if (offset >= end) return; - if (!decodeLinkLayer(buffer + offset, end - offset, LinkType::Ethernet, - innerOffset, innerEtherType)) { - return; - } - decodeNetwork(buffer, end, offset + innerOffset, innerEtherType, p, depth + 1); - } - - // GRE (RFC 2784 + key/sequence extensions). Only version 0 carries a - // plain encapsulated protocol; version 1 is PPTP and its payload is a - // PPP frame we do not decode. - void decodeGre(const uint8_t* buffer, size_t end, size_t offset, - ParsedPacket& p, int depth) { - if (end < offset + 4) return; - const uint16_t flags = readU16(buffer + offset); - const uint16_t innerType = readU16(buffer + offset + 2); - if ((flags & 0x0007) != 0) return; // not version 0 - - size_t headerLen = 4; - if (flags & 0x8000) headerLen += 4; // checksum + reserved1 - if (flags & 0x2000) headerLen += 4; // key - if (flags & 0x1000) headerLen += 4; // sequence - if (end < offset + headerLen) return; - - pushTunnel(p, "GRE"); - const size_t innerOffset = offset + headerLen; - if (innerType == kEthTrBridge) { - decodeInnerEthernet(buffer, end, innerOffset, p, depth); - } else if (innerType == kEthIPv4 || innerType == kEthIPv6) { - decodeNetwork(buffer, end, innerOffset, innerType, p, depth + 1); - } - } - - // VXLAN (RFC 7348): 8-byte header, then a complete inner Ethernet frame. - void decodeVxlan(const uint8_t* buffer, size_t end, size_t offset, - ParsedPacket& p, int depth) { - constexpr size_t vxlanHeader = 8; - if (end < offset + vxlanHeader) return; - if ((buffer[offset] & 0x08) == 0) return; // 'I' flag must be set - pushTunnel(p, "VXLAN"); - decodeInnerEthernet(buffer, end, offset + vxlanHeader, p, depth); - } - - // GENEVE (RFC 8926): 8-byte base header plus variable options, then a - // frame identified by the protocol_type EtherType. - void decodeGeneve(const uint8_t* buffer, size_t end, size_t offset, - ParsedPacket& p, int depth) { - constexpr size_t geneveBase = 8; - if (end < offset + geneveBase) return; - const uint8_t verOptLen = buffer[offset]; - if ((verOptLen >> 6) != 0) return; // version must be 0 - const size_t optionsLen = static_cast(verOptLen & 0x3F) * 4; - const uint16_t innerType = readU16(buffer + offset + 2); - const size_t innerOffset = offset + geneveBase + optionsLen; - if (end < innerOffset) return; - - pushTunnel(p, "GENEVE"); - if (innerType == kEthTrBridge) { - decodeInnerEthernet(buffer, end, innerOffset, p, depth); - } else if (innerType == kEthIPv4 || innerType == kEthIPv6) { - decodeNetwork(buffer, end, innerOffset, innerType, p, depth + 1); - } - } - - // MPLS label stack: 4 bytes per label, bottom-of-stack bit in byte 2. - // Underneath is bare IP (no EtherType), so sniff the version nibble. - void decodeMpls(const uint8_t* buffer, size_t end, size_t offset, - ParsedPacket& p, int depth) { - size_t pos = offset; - for (int label = 0; label < 8; ++label) { - if (end < pos + 4) return; - const bool bottomOfStack = (buffer[pos + 2] & 0x01) != 0; - pos += 4; - if (!bottomOfStack) continue; - - if (pos >= end) return; - const uint8_t version = buffer[pos] >> 4; - if (version != 4 && version != 6) return; - pushTunnel(p, "MPLS"); - decodeNetwork(buffer, end, pos, version == 4 ? kEthIPv4 : kEthIPv6, p, depth + 1); - return; - } - } - - // PPPoE session stage: 6-byte header, then a 2-byte PPP protocol id. - void decodePppoe(const uint8_t* buffer, size_t end, size_t offset, - ParsedPacket& p, int depth) { - constexpr size_t pppoeHeader = 6; - if (end < offset + pppoeHeader + 2) return; - const uint16_t pppProtocol = readU16(buffer + offset + pppoeHeader); - const size_t innerOffset = offset + pppoeHeader + 2; - uint16_t innerType = 0; - if (pppProtocol == 0x0021) innerType = kEthIPv4; - else if (pppProtocol == 0x0057) innerType = kEthIPv6; - else return; - pushTunnel(p, "PPPoE"); - decodeNetwork(buffer, end, innerOffset, innerType, p, depth + 1); - } - - void classifyTcpPort(uint16_t port, ParsedPacket& p); - void classifyUdpPort(uint16_t port, ParsedPacket& p); - - void decodeTcp(const uint8_t* buffer, size_t end, size_t offset, ParsedPacket& p) { - p.protocol = "TCP"; - if (end < offset + sizeof(TCPHeader)) return; - p.srcPort = readU16(buffer + offset); - p.dstPort = readU16(buffer + offset + 2); - p.tcpSeq = readU32(buffer + offset + 4); - - const uint8_t tcpFlags = buffer[offset + 13]; - p.tcpFin = (tcpFlags & 0x01) != 0; - p.tcpSyn = (tcpFlags & 0x02) != 0; - p.tcpRst = (tcpFlags & 0x04) != 0; - p.tcpAck = (tcpFlags & 0x10) != 0; - - const size_t tcpHeaderLen = (buffer[offset + 12] >> 4) * 4; - if (tcpHeaderLen < sizeof(TCPHeader) || end < offset + tcpHeaderLen) return; - - // Application-layer DPI happens in parse(), once, on the innermost - // transport payload — recording where it lives is enough here. - const size_t payloadOffset = offset + tcpHeaderLen; - if (payloadOffset < end) { - p.payloadOffset = payloadOffset; - p.payloadLength = end - payloadOffset; - } - } - - void decodeUdp(const uint8_t* buffer, size_t end, size_t offset, - ParsedPacket& p, int depth) { - p.protocol = "UDP"; - if (end < offset + sizeof(UDPHeader)) return; - p.srcPort = readU16(buffer + offset); - p.dstPort = readU16(buffer + offset + 2); - - // UDP length bounds the payload, but a truncated capture may cut it - // short — trust whichever is smaller. - const uint16_t udpLength = readU16(buffer + offset + 4); - const size_t declaredEnd = udpLength >= sizeof(UDPHeader) - ? offset + udpLength - : end; - const size_t udpEnd = std::min(end, declaredEnd); - const size_t payloadOffset = offset + sizeof(UDPHeader); - if (payloadOffset >= udpEnd) return; - - p.payloadOffset = payloadOffset; - p.payloadLength = udpEnd - payloadOffset; - - // UDP-borne tunnels carry a whole inner frame; descend before we - // classify this as ordinary UDP traffic. - if (depth < kMaxTunnelDepth) { - if (p.dstPort == 4789 || p.srcPort == 4789) { - decodeVxlan(buffer, udpEnd, payloadOffset, p, depth); - return; - } - if (p.dstPort == 6081 || p.srcPort == 6081) { - decodeGeneve(buffer, udpEnd, payloadOffset, p, depth); - return; - } - } - } - - void decodeIcmp(const uint8_t* buffer, size_t end, size_t offset, ParsedPacket& p) { - p.protocol = "ICMP"; - if (end < offset + 2) return; - switch (buffer[offset]) { - case 0: p.service = "Echo reply"; break; - case 8: p.service = "Echo request"; break; - case 3: p.service = "Destination unreachable"; break; - case 11: p.service = "Time exceeded"; break; - case 5: p.service = "Redirect"; break; - default: p.service = "ICMP type " + std::to_string(buffer[offset]); break; - } - } - - void decodeIcmpv6(const uint8_t* buffer, size_t end, size_t offset, ParsedPacket& p) { - p.protocol = "ICMPv6"; - if (end < offset + 2) return; - switch (buffer[offset]) { - case 128: p.service = "Echo request"; break; - case 129: p.service = "Echo reply"; break; - case 133: p.service = "Router solicitation"; break; - case 134: p.service = "Router advertisement"; break; - case 135: p.service = "Neighbor solicitation"; break; - case 136: p.service = "Neighbor advertisement"; break; - case 1: p.service = "Destination unreachable"; break; - case 3: p.service = "Time exceeded"; break; - default: p.service = "ICMPv6 type " + std::to_string(buffer[offset]); break; - } - } - - void decodeTransport(const uint8_t* buffer, size_t end, size_t offset, - uint8_t l4Proto, ParsedPacket& p, int depth) { - switch (l4Proto) { - case 6: decodeTcp(buffer, end, offset, p); return; - case 17: decodeUdp(buffer, end, offset, p, depth); return; - case 1: decodeIcmp(buffer, end, offset, p); return; - case 58: decodeIcmpv6(buffer, end, offset, p); return; - - // Unencrypted IP-in-IP tunnels: the inner packet is right there. - case 4: - if (depth < kMaxTunnelDepth) { - pushTunnel(p, "IP-in-IP"); - decodeNetwork(buffer, end, offset, kEthIPv4, p, depth + 1); - return; - } - break; - case 41: - if (depth < kMaxTunnelDepth) { - pushTunnel(p, "6in4"); - decodeNetwork(buffer, end, offset, kEthIPv6, p, depth + 1); - return; - } - break; - case 47: - if (depth < kMaxTunnelDepth) { - decodeGre(buffer, end, offset, p, depth); - if (p.protocol.empty() || p.protocol == "Unknown") p.protocol = "GRE"; - return; - } - break; - - // Encrypted tunnels: we can see that they exist and how much they - // carry, but never what is inside. Say so rather than pretending. - case 50: - p.protocol = "ESP"; - p.service = "IPsec (encrypted)"; - p.encryptedTunnel = true; - return; - default: - break; - } - - if (const char* name = ipProtocolName(l4Proto)) { - p.protocol = name; - } else { - p.protocol = "IP proto " + std::to_string(l4Proto); - } - } - - void decodeNetwork(const uint8_t* buffer, size_t end, size_t offset, - uint16_t etherType, ParsedPacket& p, int depth) { - if (depth > kMaxTunnelDepth) return; - - if (etherType == kEthMplsUni || etherType == kEthMplsMul) { - decodeMpls(buffer, end, offset, p, depth); - if (p.protocol == "Unknown") p.protocol = "MPLS"; - return; - } - if (etherType == kEthPppoeSess) { - decodePppoe(buffer, end, offset, p, depth); - if (p.protocol == "Unknown") p.protocol = "PPPoE"; - return; - } - - if (etherType == kEthIPv4) { - if (end < offset + sizeof(IPv4Header)) return; - const uint8_t versionHlen = buffer[offset]; - const size_t ihl = (versionHlen & 0x0F) * 4; - if ((versionHlen >> 4) != 4 || ihl < sizeof(IPv4Header) || end < offset + ihl) return; - - const uint16_t totalLength = readU16(buffer + offset + 2); - if (totalLength < ihl) return; - const size_t packetEnd = std::min(end, offset + static_cast(totalLength)); - - p.srcIP = ipv4ToString(buffer + offset + 12); - p.dstIP = ipv4ToString(buffer + offset + 16); - - // Continuation fragments carry no transport header, so there is - // nothing further to decode. The first fragment (offset 0) falls - // through and yields ports and any in-line DPI as usual. - const uint16_t flagsFragOff = readU16(buffer + offset + 6); - if ((flagsFragOff & 0x1FFF) != 0) { - p.protocol = "IPv4 Fragment"; - return; - } - decodeTransport(buffer, packetEnd, offset + ihl, buffer[offset + 9], p, depth); - return; - } - - if (etherType == kEthIPv6) { - constexpr size_t v6FixedHeader = 40; - if (end < offset + v6FixedHeader) return; - const uint16_t payloadLen = readU16(buffer + offset + 4); - const size_t packetEnd = std::min(end, offset + v6FixedHeader + static_cast(payloadLen)); - - p.srcIP = ipv6ToString(buffer + offset + 8); - p.dstIP = ipv6ToString(buffer + offset + 24); - - uint8_t nextHeader = buffer[offset + 6]; - size_t pos = offset + v6FixedHeader; - while (true) { - uint8_t newNext = nextHeader; - if (!walkV6Extension(nextHeader, buffer, packetEnd, pos, newNext)) break; - nextHeader = newNext; - } - decodeTransport(buffer, packetEnd, pos, nextHeader, p, depth); - return; - } - - if (etherType == kEthARP) { - p.protocol = "ARP"; - // ARP frame: HTYPE(2) PTYPE(2) HLEN(1) PLEN(1) OPER(2) - // SHA(HLEN) SPA(PLEN) THA(HLEN) TPA(PLEN) - if (end < offset + 8) return; - const uint16_t ptype = readU16(buffer + offset + 2); - const uint8_t hlen = buffer[offset + 4]; - const uint8_t plen = buffer[offset + 5]; - const uint16_t oper = readU16(buffer + offset + 6); - const size_t arpBody = offset + 8; - if (ptype == kEthIPv4 && plen == 4 && end >= arpBody + 2u * (hlen + plen)) { - p.srcIP = ipv4ToString(buffer + arpBody + hlen); - p.dstIP = ipv4ToString(buffer + arpBody + 2u * hlen + plen); - } - p.service = (oper == 1) ? "ARP request" : (oper == 2) ? "ARP reply" : ""; - return; - } - - if (const char* name = etherTypeName(etherType)) { - p.protocol = name; - return; - } - p.protocol = "Non-IP"; - } - - // Service labels keyed on the port that looks like the server's. Both - // directions of a conversation resolve to the same label. - void classifyTcpPort(uint16_t port, ParsedPacket& p) { - if (!p.service.empty()) return; - switch (port) { - case 21: p.service = "FTP"; break; - case 22: p.service = "SSH"; break; - case 23: p.service = "Telnet"; break; - case 25: p.service = "SMTP"; break; - case 53: p.service = "DNS (TCP)"; break; - case 80: p.service = "HTTP"; break; - case 110: p.service = "POP3"; break; - case 143: p.service = "IMAP"; break; - case 179: p.service = "BGP"; break; - case 389: p.service = "LDAP"; break; - case 443: p.service = "HTTPS"; break; - case 445: p.service = "SMB"; break; - case 465: - case 587: p.service = "SMTP+TLS"; break; - case 514: p.service = "Syslog"; break; - case 636: p.service = "LDAPS"; break; - case 853: p.service = "DNS-over-TLS"; break; - case 993: p.service = "IMAP+TLS"; break; - case 995: p.service = "POP3+TLS"; break; - case 1433: p.service = "MSSQL"; break; - case 1883: p.service = "MQTT"; break; - case 3306: p.service = "MySQL"; break; - case 3389: p.service = "RDP"; break; - case 5060: - case 5061: p.service = "SIP"; break; - case 5222: p.service = "XMPP"; break; - case 5432: p.service = "PostgreSQL"; break; - case 5900: p.service = "VNC"; break; - case 6379: p.service = "Redis"; break; - case 6443: p.service = "Kubernetes API"; break; - case 6667: p.service = "IRC"; break; - case 8000: - case 8080: - case 8888: p.service = "HTTP (alt)"; break; - case 8443: p.service = "HTTPS (alt)"; break; - case 9092: p.service = "Kafka"; break; - case 9200: p.service = "Elasticsearch"; break; - case 9418: p.service = "Git"; break; - case 25565: p.service = "Minecraft"; break; - case 27017: p.service = "MongoDB"; break; - default: break; - } - } - - void classifyUdpPort(uint16_t port, ParsedPacket& p) { - if (!p.service.empty()) return; - switch (port) { - case 53: p.service = "DNS"; break; - case 67: - case 68: p.service = "DHCP"; break; - case 69: p.service = "TFTP"; break; - case 123: p.service = "NTP"; break; - case 137: - case 138: - case 139: p.service = "NetBIOS"; break; - case 161: - case 162: p.service = "SNMP"; break; - case 443: p.service = "QUIC"; break; - case 500: - case 4500: p.service = "IPsec"; break; - case 514: p.service = "Syslog"; break; - case 546: - case 547: p.service = "DHCPv6"; break; - case 784: - case 8853: p.service = "DNS-over-QUIC"; break; - case 853: p.service = "DNS-over-QUIC"; break; - case 1194: p.service = "OpenVPN"; break; - case 1900: p.service = "SSDP"; break; - case 3478: - case 3479: - case 5349: p.service = "STUN/TURN"; break; - case 4789: p.service = "VXLAN"; break; - case 5353: p.service = "mDNS"; break; - case 5355: p.service = "LLMNR"; break; - case 6081: p.service = "GENEVE"; break; - case 51820: p.service = "WireGuard"; break; - default: break; - } - } - - bool isHttpMethod(const uint8_t* p, size_t len, size_t& methodLen) { - static constexpr std::string_view methods[] = { - "GET", "POST", "HEAD", "PUT", "DELETE", "OPTIONS", - "PATCH", "TRACE", "CONNECT", - }; - for (std::string_view method : methods) { - if (len > method.size() + 1 - && std::memcmp(p, method.data(), method.size()) == 0 - && p[method.size()] == ' ') { - methodLen = method.size(); - return true; - } - } - return false; - } - - } // namespace - - bool ProtocolParser::looksLikeTlsHandshake(const uint8_t* payload, size_t length) { - // Handshake record (0x16), TLS major version 3, minor 0..4. - return length >= 5 && payload[0] == 0x16 && payload[1] == 0x03 && payload[2] <= 0x04; - } - - ParsedPacket ProtocolParser::parse(const PacketData& rawData) { - ParsedPacket parsed; - parsed.timestamp = rawData.timestamp; - parsed.length = rawData.length; - parsed.protocol = "Unknown"; - - const uint8_t* buffer = rawData.payload.data(); - const size_t size = rawData.payload.size(); - if (buffer == nullptr || size == 0) return parsed; - - size_t offset = 0; - uint16_t etherType = 0; - if (!decodeLinkLayer(buffer, size, rawData.linkType, offset, etherType)) { - parsed.protocol = rawData.linkType == LinkType::Unsupported - ? "Unsupported link type" - : "Truncated"; - return parsed; - } - - decodeNetwork(buffer, size, offset, etherType, parsed, 0); - - // Application-layer DPI runs once, on the innermost transport payload. - if (parsed.payloadLength > 0 && parsed.payloadOffset < size) { - const uint8_t* payload = buffer + parsed.payloadOffset; - const size_t payloadLen = std::min(parsed.payloadLength, size - parsed.payloadOffset); - - if (parsed.protocol == "TCP") { - if (looksLikeTlsHandshake(payload, payloadLen)) { - parseTLS(payload, payloadLen, parsed); - } else { - parseHTTP(payload, payloadLen, parsed); - } - classifyTcpPort(parsed.dstPort, parsed); - classifyTcpPort(parsed.srcPort, parsed); - } else if (parsed.protocol == "UDP") { - // QUIC runs on whatever port the deployment chose; identify it - // by the long-header + fixed-bit shape and a known version. - if (QuicParser::looksLikeLongHeader(payload, payloadLen)) { - if (auto sni = QuicParser::extractInitialSni(payload, payloadLen)) { - parsed.sni = *sni; - if (parsed.service.empty()) { - parsed.service = identifyService(*sni); - } - } - if (parsed.service.empty()) parsed.service = "QUIC"; - } - classifyUdpPort(parsed.dstPort, parsed); - classifyUdpPort(parsed.srcPort, parsed); - } - } else if (parsed.protocol == "TCP") { - classifyTcpPort(parsed.dstPort, parsed); - classifyTcpPort(parsed.srcPort, parsed); - } else if (parsed.protocol == "UDP") { - classifyUdpPort(parsed.dstPort, parsed); - classifyUdpPort(parsed.srcPort, parsed); - } - - // Encrypted tunnels that ride on UDP are only recognisable by port. - if (parsed.service == "WireGuard" || parsed.service == "OpenVPN" - || parsed.service == "IPsec") { - parsed.encryptedTunnel = true; - } - - return parsed; - } - - bool ProtocolParser::parseTlsClientHello(const uint8_t* payload, size_t len, std::string& outSni) { - constexpr size_t recordHeaderLength = 5; - constexpr size_t handshakeHeaderLength = 4; - constexpr size_t clientHelloFixedLength = 34; // Version + random - - if (!looksLikeTlsHandshake(payload, len)) return false; - - // A truncated record is not a parse failure — the rest of the - // ClientHello is in the next TCP segment. Report "not yet". - const size_t recordLength = readU16(payload + 3); - if (recordHeaderLength + recordLength > len) return false; - const size_t recordEnd = recordHeaderLength + recordLength; - - size_t pos = recordHeaderLength; - if (recordEnd < pos + handshakeHeaderLength || payload[pos] != 0x01) return false; // ClientHello - - const size_t handshakeLength = (static_cast(payload[pos + 1]) << 16) - | (static_cast(payload[pos + 2]) << 8) | payload[pos + 3]; - pos += handshakeHeaderLength; - if (pos + handshakeLength > recordEnd) return false; - const size_t handshakeEnd = pos + handshakeLength; - if (handshakeEnd < pos + clientHelloFixedLength) return false; - pos += clientHelloFixedLength; - - if (pos >= handshakeEnd) return false; - const size_t sessionIdLength = payload[pos++]; - if (sessionIdLength > handshakeEnd - pos) return false; - pos += sessionIdLength; - - if (handshakeEnd - pos < 2) return false; - const size_t cipherSuitesLength = readU16(payload + pos); - pos += 2; - if (cipherSuitesLength > handshakeEnd - pos) return false; - pos += cipherSuitesLength; - - if (pos >= handshakeEnd) return false; - const size_t compressionMethodsLength = payload[pos++]; - if (compressionMethodsLength > handshakeEnd - pos) return false; - pos += compressionMethodsLength; - - if (handshakeEnd - pos < 2) return false; - const size_t extensionsLength = readU16(payload + pos); - pos += 2; - if (extensionsLength > handshakeEnd - pos) return false; - const size_t extensionsEnd = pos + extensionsLength; - - while (extensionsEnd - pos >= 4) { - const uint16_t extensionType = readU16(payload + pos); - const size_t extensionLength = readU16(payload + pos + 2); - pos += 4; - if (extensionLength > extensionsEnd - pos) return false; - const size_t extensionEnd = pos + extensionLength; - - if (extensionType == 0x0000) { // Server Name Indication - if (extensionEnd - pos < 2) return false; - const size_t nameListLength = readU16(payload + pos); - pos += 2; - if (nameListLength > extensionEnd - pos) return false; - const size_t nameListEnd = pos + nameListLength; - - while (nameListEnd - pos >= 3) { - const uint8_t nameType = payload[pos++]; - const size_t nameLength = readU16(payload + pos); - pos += 2; - if (nameLength > nameListEnd - pos) return false; - if (nameType == 0x00) { - outSni.assign(reinterpret_cast(payload + pos), nameLength); - return true; - } - pos += nameLength; - } - return false; - } - - pos = extensionEnd; - } - return false; - } - - void ProtocolParser::parseTLS(const uint8_t* payload, size_t len, ParsedPacket& outPacket) { - std::string sni; - if (parseTlsClientHello(payload, len, sni) && !sni.empty()) { - outPacket.sni = sni; - outPacket.service = identifyService(sni); - } - } - - void ProtocolParser::parseHTTP(const uint8_t* payload, size_t len, ParsedPacket& outPacket) { - // Only the head of the message matters, and a header block that has not - // arrived within 2 KiB is not one we need. - const size_t scanLen = std::min(len, 2048); - - const auto lineEnd = [&](size_t from) { - for (size_t i = from; i + 1 < scanLen; ++i) { - if (payload[i] == '\r' && payload[i + 1] == '\n') return i; - } - return scanLen; - }; - - if (scanLen >= 12 && std::memcmp(payload, "HTTP/1.", 7) == 0) { - const size_t end = lineEnd(0); - outPacket.info.assign(reinterpret_cast(payload), end); - if (outPacket.service.empty()) outPacket.service = "HTTP"; - return; - } - - size_t methodLen = 0; - if (!isHttpMethod(payload, scanLen, methodLen)) return; - - // Request line: METHOD SP request-target SP HTTP/1.x - const size_t requestLineEnd = lineEnd(0); - outPacket.info.assign(reinterpret_cast(payload), - std::min(requestLineEnd, static_cast(120))); - if (outPacket.service.empty()) outPacket.service = "HTTP"; - - // Walk header lines looking for Host. - size_t pos = requestLineEnd + 2; - while (pos < scanLen) { - const size_t end = lineEnd(pos); - if (end == pos) break; // blank line: end of headers - if (end == scanLen) break; - - constexpr std::string_view hostPrefix = "host:"; - if (end - pos > hostPrefix.size()) { - bool isHost = true; - for (size_t i = 0; i < hostPrefix.size(); ++i) { - if (std::tolower(static_cast(payload[pos + i])) != hostPrefix[i]) { - isHost = false; - break; - } - } - if (isHost) { - size_t valueStart = pos + hostPrefix.size(); - while (valueStart < end && payload[valueStart] == ' ') ++valueStart; - if (valueStart < end) { - outPacket.hostname.assign( - reinterpret_cast(payload + valueStart), end - valueStart); - if (const std::string named = identifyService(outPacket.hostname); !named.empty()) { - outPacket.service = named; - } - } - return; - } - } - pos = end + 2; - } - } - - std::string ProtocolParser::identifyService(const std::string& hostname) { - // Substring match against a curated catalog of major services. - // The map is ordered by specificity — more specific patterns first. - struct Pattern { std::string_view needle; std::string_view label; }; - static constexpr std::array catalog = std::to_array({ - // Encrypted DNS resolvers. Checked first: these hosts would - // otherwise match their operator ("Google", "Cloudflare") and hide - // the fact that name resolution has moved off the wire. - {"mozilla.cloudflare-dns.com", "DNS-over-HTTPS"}, - {"cloudflare-dns.com", "DNS-over-HTTPS"}, - {"one.one.one.one", "DNS-over-HTTPS"}, - {"dns.google", "DNS-over-HTTPS"}, - {"dns.quad9.net", "DNS-over-HTTPS"}, - {"doh.opendns.com", "DNS-over-HTTPS"}, - {"dns.nextdns.io", "DNS-over-HTTPS"}, - {"doh.cleanbrowsing.org","DNS-over-HTTPS"}, - {"dns.adguard", "DNS-over-HTTPS"}, - {"doh.dns.sb", "DNS-over-HTTPS"}, - - // Google ecosystem - {"youtube", "YouTube"}, - {"googlevideo", "YouTube"}, - {"ytimg", "YouTube"}, - {"gstatic", "Google"}, - {"googleusercontent","Google"}, - {"googleapis", "Google"}, - {"google.", "Google"}, - {"doubleclick", "Google Ads"}, - - // Streaming / video - {"netflix", "Netflix"}, - {"nflxvideo", "Netflix"}, - {"twitch", "Twitch"}, - {"ttvnw", "Twitch"}, - {"spotify", "Spotify"}, - {"scdn.co", "Spotify"}, - {"hulu", "Hulu"}, - {"disney", "Disney+"}, - {"primevideo", "Prime Video"}, - - // Social / messaging - {"discord", "Discord"}, - {"discordapp", "Discord"}, - {"slack.com", "Slack"}, - {"slack-edge", "Slack"}, - {"telegram", "Telegram"}, - {"whatsapp", "WhatsApp"}, - {"facebook", "Facebook"}, - {"fbcdn", "Facebook"}, - {"instagram", "Instagram"}, - {"cdninstagram", "Instagram"}, - {"twitter", "X (Twitter)"}, - {"twimg", "X (Twitter)"}, - {"x.com", "X (Twitter)"}, - {"tiktok", "TikTok"}, - {"tiktokcdn", "TikTok"}, - {"reddit", "Reddit"}, - {"redditstatic", "Reddit"}, - {"linkedin", "LinkedIn"}, - {"signal.org", "Signal"}, - - // Dev / cloud - {"github", "GitHub"}, - {"githubusercontent","GitHub"}, - {"gitlab", "GitLab"}, - {"bitbucket", "Bitbucket"}, - {"npmjs", "npm"}, - {"jsdelivr", "jsDelivr"}, - {"cloudflare", "Cloudflare"}, - {"akamai", "Akamai"}, - {"fastly", "Fastly"}, - {"amazonaws", "AWS"}, - {"awsstatic", "AWS"}, - {"azureedge", "Azure"}, - {"windows.net", "Azure"}, - {"azure.com", "Azure"}, - {"digitalocean", "DigitalOcean"}, - {"vercel", "Vercel"}, - {"netlify", "Netlify"}, - {"anthropic", "Anthropic"}, - {"openai", "OpenAI"}, - - // Microsoft - {"microsoft", "Microsoft"}, - {"msftncsi", "Microsoft"}, - {"office.com", "Microsoft 365"}, - {"office365", "Microsoft 365"}, - {"outlook", "Outlook"}, - {"live.com", "Microsoft"}, - {"bing.com", "Bing"}, - {"xboxlive", "Xbox Live"}, - {"xbox.com", "Xbox Live"}, - {"skype", "Skype"}, - {"teams.microsoft", "Microsoft Teams"}, - - // Apple - {"apple.com", "Apple"}, - {"icloud", "iCloud"}, - {"itunes.apple", "iTunes"}, - {"mzstatic", "Apple"}, - - // Gaming - {"steampowered", "Steam"}, - {"steamstatic", "Steam"}, - {"steamcommunity", "Steam"}, - {"epicgames", "Epic Games"}, - {"riotgames", "Riot Games"}, - {"battle.net", "Battle.net"}, - {"playstation", "PlayStation"}, - {"nintendo", "Nintendo"}, - - // Shopping - {"amazon", "Amazon"}, - {"ebay", "eBay"}, - {"shopify", "Shopify"}, - - // Communication / video - {"zoom.us", "Zoom"}, - {"zoomgov", "Zoom"}, - {"webex", "Webex"}, - {"meet.google", "Google Meet"}, - - // News / misc - {"wikipedia", "Wikipedia"}, - {"wikimedia", "Wikipedia"}, - - // Mozilla / privacy - {"mozilla", "Mozilla"}, - {"firefox", "Firefox"}, - {"duckduckgo", "DuckDuckGo"}, - {"protonmail", "Proton"}, - {"proton.me", "Proton"}, - }); - - const std::string lower = toLowerCopy(hostname); - for (const auto& pat : catalog) { - if (lower.find(pat.needle) != std::string::npos) return std::string(pat.label); - } - return {}; - } - -} // namespace core +#include "core/ProtocolParser.hpp" +#include "core/NetworkPlatform.hpp" +#include "core/QuicParser.hpp" +#include +#include +#include +#include +#include +#include + +namespace core { + + namespace { + + constexpr uint16_t kEthIPv4 = 0x0800; + constexpr uint16_t kEthARP = 0x0806; + constexpr uint16_t kEthIPv6 = 0x86DD; + constexpr uint16_t kEthMplsUni = 0x8847; + constexpr uint16_t kEthMplsMul = 0x8848; + constexpr uint16_t kEthPppoeSess = 0x8864; + constexpr uint16_t kEthTrBridge = 0x6558; // Transparent Ethernet Bridging (GENEVE/GRE) + + // Descending through more than a few layers of encapsulation means + // either an exotic capture or a malicious packet trying to make us spin. + constexpr int kMaxTunnelDepth = 4; + + uint16_t readU16(const uint8_t* bytes) { + return static_cast(bytes[0]) << 8 | bytes[1]; + } + + uint32_t readU32(const uint8_t* bytes) { + return (static_cast(bytes[0]) << 24) + | (static_cast(bytes[1]) << 16) + | (static_cast(bytes[2]) << 8) + | static_cast(bytes[3]); + } + + std::string ipv4ToString(const uint8_t* bytes) { + uint32_t address; + std::memcpy(&address, bytes, sizeof(address)); + char buf[INET_ADDRSTRLEN]{}; + return inet_ntop(AF_INET, &address, buf, sizeof(buf)) ? buf : ""; + } + + std::string ipv6ToString(const uint8_t* bytes) { + char buf[INET6_ADDRSTRLEN]{}; + return inet_ntop(AF_INET6, bytes, buf, sizeof(buf)) ? buf : ""; + } + + // Map an L4 (or "next header") protocol number to a short label. + // Returns nullptr for protocols the caller decodes itself (TCP / UDP / + // ICMP / tunnels). + const char* ipProtocolName(uint8_t proto) { + switch (proto) { + case 2: return "IGMP"; + case 33: return "DCCP"; + case 50: return "ESP"; + case 51: return "AH"; + case 89: return "OSPF"; + case 103: return "PIM"; + case 112: return "VRRP"; + case 132: return "SCTP"; + default: return nullptr; + } + } + + // Frames that never carry IP. Naming them beats a bare "Non-IP" bucket, + // since these are constant background noise on any real LAN. + const char* etherTypeName(uint16_t etherType) { + switch (etherType) { + case 0x88CC: return "LLDP"; + case 0x888E: return "EAPOL"; + case 0x8863: return "PPPoE Discovery"; + case 0x8100: + case 0x88A8: + case 0x9100: return "VLAN"; + case 0x8137: return "IPX"; + case 0x88F7: return "PTP"; + case 0x22EA: return "SRP"; + case 0x8035: return "RARP"; + case 0x9000: return "Ethernet Loopback"; + case 0x88A4: return "EtherCAT"; + case 0x8892: return "PROFINET"; + default: return nullptr; + } + } + + // IPv6 extension header? If so, advances pos by header length and + // returns the new next-header value via outNext. Returns true if the + // current value was an extension header. The caller stops walking when + // this returns false (i.e. we've reached the upper-layer protocol). + bool walkV6Extension(uint8_t nh, const uint8_t* buffer, size_t size, + size_t& pos, uint8_t& outNext) { + switch (nh) { + case 0: // Hop-by-hop + case 43: // Routing + case 60: // Destination options + case 135: // Mobility + { + if (size < pos + 2) return false; + const uint8_t newNh = buffer[pos]; + const size_t hdrLen = (static_cast(buffer[pos + 1]) + 1) * 8; + if (size < pos + hdrLen) return false; + pos += hdrLen; + outNext = newNh; + return true; + } + case 44: // Fragment — fixed 8 bytes; the fragment offset matters but + // for live DPI we just keep walking on first-fragment. + { + if (size < pos + 8) return false; + outNext = buffer[pos]; + pos += 8; + return true; + } + case 51: // Authentication Header. Integrity-only, so the payload + // behind it is still readable — walk straight through. + { + if (size < pos + 2) return false; + outNext = buffer[pos]; + const size_t hdrLen = (static_cast(buffer[pos + 1]) + 2) * 4; + if (size < pos + hdrLen) return false; + pos += hdrLen; + return true; + } + default: + return false; + } + } + + // Lower-case in place for case-insensitive substring matches. + std::string toLowerCopy(std::string_view s) { + std::string out(s); + std::transform(out.begin(), out.end(), out.begin(), + [](unsigned char c){ return static_cast(std::tolower(c)); }); + return out; + } + + // Record a layer of encapsulation. The first call also snapshots the + // carrier addresses so the UI can still show who was tunnelling. + void pushTunnel(ParsedPacket& p, const char* name) { + if (p.outerSrcIP.empty() && !p.srcIP.empty()) { + p.outerSrcIP = p.srcIP; + p.outerDstIP = p.dstIP; + } + if (p.tunnel.empty()) p.tunnel = name; + else p.tunnel += std::string("/") + name; + } + + // Resolve the start of the network layer for a given link encapsulation. + // Returns false when the header is truncated or the link type carries no + // IP payload we can locate. `outEtherType` is synthesized for link types + // that do not carry a real EtherType (raw IP, loopback). + bool decodeLinkLayer(const uint8_t* buffer, size_t size, LinkType link, + size_t& outOffset, uint16_t& outEtherType) { + const auto etherTypeForIpVersion = [&](size_t at) -> bool { + if (at >= size) return false; + const uint8_t version = buffer[at] >> 4; + if (version == 4) { outEtherType = kEthIPv4; return true; } + if (version == 6) { outEtherType = kEthIPv6; return true; } + return false; + }; + + switch (link) { + case LinkType::Ethernet: { + if (size < sizeof(EthernetHeader)) return false; + outOffset = sizeof(EthernetHeader); + outEtherType = readU16(buffer + 12); + // 802.1Q / QinQ tags stack ahead of the real EtherType. + while (outEtherType == 0x8100 || outEtherType == 0x88A8 || outEtherType == 0x9100) { + if (size < outOffset + 4) return false; + outEtherType = readU16(buffer + outOffset + 2); + outOffset += 4; + } + return true; + } + case LinkType::LinuxSLL: { + // 16-byte cooked header; EtherType is the last field. + if (size < 16) return false; + outOffset = 16; + outEtherType = readU16(buffer + 14); + return true; + } + case LinkType::LinuxSLL2: { + // 20-byte cooked v2 header; EtherType comes first. + if (size < 20) return false; + outOffset = 20; + outEtherType = readU16(buffer); + return true; + } + case LinkType::NullLoopback: { + // 4-byte address family in the *writing host's* byte order, so + // try both before giving up. AF_INET is 2 everywhere; AF_INET6 + // differs across the BSDs (24/28/30) and Linux (10). + if (size < 4) return false; + outOffset = 4; + const uint32_t le = static_cast(buffer[0]) + | (static_cast(buffer[1]) << 8) + | (static_cast(buffer[2]) << 16) + | (static_cast(buffer[3]) << 24); + const uint32_t be = readU32(buffer); + for (uint32_t af : {le, be}) { + if (af == 2) { outEtherType = kEthIPv4; return true; } + if (af == 10 || af == 23 || af == 24 || af == 28 || af == 30) { + outEtherType = kEthIPv6; + return true; + } + } + // Unknown AF — fall back to sniffing the IP version nibble. + return etherTypeForIpVersion(4); + } + case LinkType::Loop: { + if (size < 4) return false; + outOffset = 4; + const uint32_t af = readU32(buffer); // always big-endian + if (af == 2) { outEtherType = kEthIPv4; return true; } + if (af == 10 || af == 23 || af == 24 || af == 28 || af == 30) { + outEtherType = kEthIPv6; + return true; + } + return etherTypeForIpVersion(4); + } + case LinkType::RawIP: + outOffset = 0; + return etherTypeForIpVersion(0); + case LinkType::IPv4: + outOffset = 0; + outEtherType = kEthIPv4; + return true; + case LinkType::IPv6: + outOffset = 0; + outEtherType = kEthIPv6; + return true; + case LinkType::Unsupported: + return false; + } + return false; + } + + // Forward declaration: the tunnel decoders recurse back into the network + // layer for the encapsulated packet. + void decodeNetwork(const uint8_t* buffer, size_t end, size_t offset, + uint16_t etherType, ParsedPacket& p, int depth); + + // Re-enter at an inner Ethernet frame (VXLAN, GENEVE with TEB, GRE-TEB). + void decodeInnerEthernet(const uint8_t* buffer, size_t end, size_t offset, + ParsedPacket& p, int depth) { + size_t innerOffset = 0; + uint16_t innerEtherType = 0; + if (offset >= end) return; + if (!decodeLinkLayer(buffer + offset, end - offset, LinkType::Ethernet, + innerOffset, innerEtherType)) { + return; + } + decodeNetwork(buffer, end, offset + innerOffset, innerEtherType, p, depth + 1); + } + + // GRE (RFC 2784 + key/sequence extensions). Only version 0 carries a + // plain encapsulated protocol; version 1 is PPTP and its payload is a + // PPP frame we do not decode. + void decodeGre(const uint8_t* buffer, size_t end, size_t offset, + ParsedPacket& p, int depth) { + if (end < offset + 4) return; + const uint16_t flags = readU16(buffer + offset); + const uint16_t innerType = readU16(buffer + offset + 2); + if ((flags & 0x0007) != 0) return; // not version 0 + + size_t headerLen = 4; + if (flags & 0x8000) headerLen += 4; // checksum + reserved1 + if (flags & 0x2000) headerLen += 4; // key + if (flags & 0x1000) headerLen += 4; // sequence + if (end < offset + headerLen) return; + + pushTunnel(p, "GRE"); + const size_t innerOffset = offset + headerLen; + if (innerType == kEthTrBridge) { + decodeInnerEthernet(buffer, end, innerOffset, p, depth); + } else if (innerType == kEthIPv4 || innerType == kEthIPv6) { + decodeNetwork(buffer, end, innerOffset, innerType, p, depth + 1); + } + } + + // VXLAN (RFC 7348): 8-byte header, then a complete inner Ethernet frame. + void decodeVxlan(const uint8_t* buffer, size_t end, size_t offset, + ParsedPacket& p, int depth) { + constexpr size_t vxlanHeader = 8; + if (end < offset + vxlanHeader) return; + if ((buffer[offset] & 0x08) == 0) return; // 'I' flag must be set + pushTunnel(p, "VXLAN"); + decodeInnerEthernet(buffer, end, offset + vxlanHeader, p, depth); + } + + // GENEVE (RFC 8926): 8-byte base header plus variable options, then a + // frame identified by the protocol_type EtherType. + void decodeGeneve(const uint8_t* buffer, size_t end, size_t offset, + ParsedPacket& p, int depth) { + constexpr size_t geneveBase = 8; + if (end < offset + geneveBase) return; + const uint8_t verOptLen = buffer[offset]; + if ((verOptLen >> 6) != 0) return; // version must be 0 + const size_t optionsLen = static_cast(verOptLen & 0x3F) * 4; + const uint16_t innerType = readU16(buffer + offset + 2); + const size_t innerOffset = offset + geneveBase + optionsLen; + if (end < innerOffset) return; + + pushTunnel(p, "GENEVE"); + if (innerType == kEthTrBridge) { + decodeInnerEthernet(buffer, end, innerOffset, p, depth); + } else if (innerType == kEthIPv4 || innerType == kEthIPv6) { + decodeNetwork(buffer, end, innerOffset, innerType, p, depth + 1); + } + } + + // MPLS label stack: 4 bytes per label, bottom-of-stack bit in byte 2. + // Underneath is bare IP (no EtherType), so sniff the version nibble. + void decodeMpls(const uint8_t* buffer, size_t end, size_t offset, + ParsedPacket& p, int depth) { + size_t pos = offset; + for (int label = 0; label < 8; ++label) { + if (end < pos + 4) return; + const bool bottomOfStack = (buffer[pos + 2] & 0x01) != 0; + pos += 4; + if (!bottomOfStack) continue; + + if (pos >= end) return; + const uint8_t version = buffer[pos] >> 4; + if (version != 4 && version != 6) return; + pushTunnel(p, "MPLS"); + decodeNetwork(buffer, end, pos, version == 4 ? kEthIPv4 : kEthIPv6, p, depth + 1); + return; + } + } + + // PPPoE session stage: 6-byte header, then a 2-byte PPP protocol id. + void decodePppoe(const uint8_t* buffer, size_t end, size_t offset, + ParsedPacket& p, int depth) { + constexpr size_t pppoeHeader = 6; + if (end < offset + pppoeHeader + 2) return; + const uint16_t pppProtocol = readU16(buffer + offset + pppoeHeader); + const size_t innerOffset = offset + pppoeHeader + 2; + uint16_t innerType = 0; + if (pppProtocol == 0x0021) innerType = kEthIPv4; + else if (pppProtocol == 0x0057) innerType = kEthIPv6; + else return; + pushTunnel(p, "PPPoE"); + decodeNetwork(buffer, end, innerOffset, innerType, p, depth + 1); + } + + void classifyTcpPort(uint16_t port, ParsedPacket& p); + void classifyUdpPort(uint16_t port, ParsedPacket& p); + + void decodeTcp(const uint8_t* buffer, size_t end, size_t offset, ParsedPacket& p) { + p.protocol = "TCP"; + if (end < offset + sizeof(TCPHeader)) return; + p.srcPort = readU16(buffer + offset); + p.dstPort = readU16(buffer + offset + 2); + p.tcpSeq = readU32(buffer + offset + 4); + + const uint8_t tcpFlags = buffer[offset + 13]; + p.tcpFin = (tcpFlags & 0x01) != 0; + p.tcpSyn = (tcpFlags & 0x02) != 0; + p.tcpRst = (tcpFlags & 0x04) != 0; + p.tcpAck = (tcpFlags & 0x10) != 0; + + const size_t tcpHeaderLen = static_cast(buffer[offset + 12] >> 4) * 4; + if (tcpHeaderLen < sizeof(TCPHeader) || end < offset + tcpHeaderLen) return; + + // Application-layer DPI happens in parse(), once, on the innermost + // transport payload — recording where it lives is enough here. + const size_t payloadOffset = offset + tcpHeaderLen; + if (payloadOffset < end) { + p.payloadOffset = payloadOffset; + p.payloadLength = end - payloadOffset; + } + } + + void decodeUdp(const uint8_t* buffer, size_t end, size_t offset, + ParsedPacket& p, int depth) { + p.protocol = "UDP"; + if (end < offset + sizeof(UDPHeader)) return; + p.srcPort = readU16(buffer + offset); + p.dstPort = readU16(buffer + offset + 2); + + // UDP length bounds the payload, but a truncated capture may cut it + // short — trust whichever is smaller. + const uint16_t udpLength = readU16(buffer + offset + 4); + const size_t declaredEnd = udpLength >= sizeof(UDPHeader) + ? offset + udpLength + : end; + const size_t udpEnd = std::min(end, declaredEnd); + const size_t payloadOffset = offset + sizeof(UDPHeader); + if (payloadOffset >= udpEnd) return; + + p.payloadOffset = payloadOffset; + p.payloadLength = udpEnd - payloadOffset; + + // UDP-borne tunnels carry a whole inner frame; descend before we + // classify this as ordinary UDP traffic. + if (depth < kMaxTunnelDepth) { + if (p.dstPort == 4789 || p.srcPort == 4789) { + decodeVxlan(buffer, udpEnd, payloadOffset, p, depth); + return; + } + if (p.dstPort == 6081 || p.srcPort == 6081) { + decodeGeneve(buffer, udpEnd, payloadOffset, p, depth); + return; + } + } + } + + void decodeIcmp(const uint8_t* buffer, size_t end, size_t offset, ParsedPacket& p) { + p.protocol = "ICMP"; + if (end < offset + 2) return; + switch (buffer[offset]) { + case 0: p.service = "Echo reply"; break; + case 8: p.service = "Echo request"; break; + case 3: p.service = "Destination unreachable"; break; + case 11: p.service = "Time exceeded"; break; + case 5: p.service = "Redirect"; break; + default: p.service = "ICMP type " + std::to_string(buffer[offset]); break; + } + } + + void decodeIcmpv6(const uint8_t* buffer, size_t end, size_t offset, ParsedPacket& p) { + p.protocol = "ICMPv6"; + if (end < offset + 2) return; + switch (buffer[offset]) { + case 128: p.service = "Echo request"; break; + case 129: p.service = "Echo reply"; break; + case 133: p.service = "Router solicitation"; break; + case 134: p.service = "Router advertisement"; break; + case 135: p.service = "Neighbor solicitation"; break; + case 136: p.service = "Neighbor advertisement"; break; + case 1: p.service = "Destination unreachable"; break; + case 3: p.service = "Time exceeded"; break; + default: p.service = "ICMPv6 type " + std::to_string(buffer[offset]); break; + } + } + + void decodeTransport(const uint8_t* buffer, size_t end, size_t offset, + uint8_t l4Proto, ParsedPacket& p, int depth) { + switch (l4Proto) { + case 6: decodeTcp(buffer, end, offset, p); return; + case 17: decodeUdp(buffer, end, offset, p, depth); return; + case 1: decodeIcmp(buffer, end, offset, p); return; + case 58: decodeIcmpv6(buffer, end, offset, p); return; + + // Unencrypted IP-in-IP tunnels: the inner packet is right there. + case 4: + if (depth < kMaxTunnelDepth) { + pushTunnel(p, "IP-in-IP"); + decodeNetwork(buffer, end, offset, kEthIPv4, p, depth + 1); + return; + } + break; + case 41: + if (depth < kMaxTunnelDepth) { + pushTunnel(p, "6in4"); + decodeNetwork(buffer, end, offset, kEthIPv6, p, depth + 1); + return; + } + break; + case 47: + if (depth < kMaxTunnelDepth) { + decodeGre(buffer, end, offset, p, depth); + if (p.protocol.empty() || p.protocol == "Unknown") p.protocol = "GRE"; + return; + } + break; + + // Encrypted tunnels: we can see that they exist and how much they + // carry, but never what is inside. Say so rather than pretending. + case 50: + p.protocol = "ESP"; + p.service = "IPsec (encrypted)"; + p.encryptedTunnel = true; + return; + default: + break; + } + + if (const char* name = ipProtocolName(l4Proto)) { + p.protocol = name; + } else { + p.protocol = "IP proto " + std::to_string(l4Proto); + } + } + + void decodeNetwork(const uint8_t* buffer, size_t end, size_t offset, + uint16_t etherType, ParsedPacket& p, int depth) { + if (depth > kMaxTunnelDepth) return; + + if (etherType == kEthMplsUni || etherType == kEthMplsMul) { + decodeMpls(buffer, end, offset, p, depth); + if (p.protocol == "Unknown") p.protocol = "MPLS"; + return; + } + if (etherType == kEthPppoeSess) { + decodePppoe(buffer, end, offset, p, depth); + if (p.protocol == "Unknown") p.protocol = "PPPoE"; + return; + } + + if (etherType == kEthIPv4) { + if (end < offset + sizeof(IPv4Header)) return; + const uint8_t versionHlen = buffer[offset]; + const size_t ihl = static_cast(versionHlen & 0x0F) * 4; + if ((versionHlen >> 4) != 4 || ihl < sizeof(IPv4Header) || end < offset + ihl) return; + + const uint16_t totalLength = readU16(buffer + offset + 2); + if (totalLength < ihl) return; + const size_t packetEnd = std::min(end, offset + static_cast(totalLength)); + + p.srcIP = ipv4ToString(buffer + offset + 12); + p.dstIP = ipv4ToString(buffer + offset + 16); + + // Continuation fragments carry no transport header, so there is + // nothing further to decode. The first fragment (offset 0) falls + // through and yields ports and any in-line DPI as usual. + const uint16_t flagsFragOff = readU16(buffer + offset + 6); + if ((flagsFragOff & 0x1FFF) != 0) { + p.protocol = "IPv4 Fragment"; + return; + } + decodeTransport(buffer, packetEnd, offset + ihl, buffer[offset + 9], p, depth); + return; + } + + if (etherType == kEthIPv6) { + constexpr size_t v6FixedHeader = 40; + if (end < offset + v6FixedHeader) return; + const uint16_t payloadLen = readU16(buffer + offset + 4); + const size_t packetEnd = std::min(end, offset + v6FixedHeader + static_cast(payloadLen)); + + p.srcIP = ipv6ToString(buffer + offset + 8); + p.dstIP = ipv6ToString(buffer + offset + 24); + + uint8_t nextHeader = buffer[offset + 6]; + size_t pos = offset + v6FixedHeader; + while (true) { + uint8_t newNext = nextHeader; + if (!walkV6Extension(nextHeader, buffer, packetEnd, pos, newNext)) break; + nextHeader = newNext; + } + decodeTransport(buffer, packetEnd, pos, nextHeader, p, depth); + return; + } + + if (etherType == kEthARP) { + p.protocol = "ARP"; + // ARP frame: HTYPE(2) PTYPE(2) HLEN(1) PLEN(1) OPER(2) + // SHA(HLEN) SPA(PLEN) THA(HLEN) TPA(PLEN) + if (end < offset + 8) return; + const uint16_t ptype = readU16(buffer + offset + 2); + const uint8_t hlen = buffer[offset + 4]; + const uint8_t plen = buffer[offset + 5]; + const uint16_t oper = readU16(buffer + offset + 6); + const size_t arpBody = offset + 8; + if (ptype == kEthIPv4 && plen == 4 + && end >= arpBody + 2 * (static_cast(hlen) + plen)) { + p.srcIP = ipv4ToString(buffer + arpBody + hlen); + p.dstIP = ipv4ToString(buffer + arpBody + 2 * static_cast(hlen) + plen); + } + p.service = (oper == 1) ? "ARP request" : (oper == 2) ? "ARP reply" : ""; + return; + } + + if (const char* name = etherTypeName(etherType)) { + p.protocol = name; + return; + } + p.protocol = "Non-IP"; + } + + // Service labels keyed on the port that looks like the server's. Both + // directions of a conversation resolve to the same label. + void classifyTcpPort(uint16_t port, ParsedPacket& p) { + if (!p.service.empty()) return; + switch (port) { + case 21: p.service = "FTP"; break; + case 22: p.service = "SSH"; break; + case 23: p.service = "Telnet"; break; + case 25: p.service = "SMTP"; break; + case 53: p.service = "DNS (TCP)"; break; + case 80: p.service = "HTTP"; break; + case 110: p.service = "POP3"; break; + case 143: p.service = "IMAP"; break; + case 179: p.service = "BGP"; break; + case 389: p.service = "LDAP"; break; + case 443: p.service = "HTTPS"; break; + case 445: p.service = "SMB"; break; + case 465: + case 587: p.service = "SMTP+TLS"; break; + case 514: p.service = "Syslog"; break; + case 636: p.service = "LDAPS"; break; + case 853: p.service = "DNS-over-TLS"; break; + case 993: p.service = "IMAP+TLS"; break; + case 995: p.service = "POP3+TLS"; break; + case 1433: p.service = "MSSQL"; break; + case 1883: p.service = "MQTT"; break; + case 3306: p.service = "MySQL"; break; + case 3389: p.service = "RDP"; break; + case 5060: + case 5061: p.service = "SIP"; break; + case 5222: p.service = "XMPP"; break; + case 5432: p.service = "PostgreSQL"; break; + case 5900: p.service = "VNC"; break; + case 6379: p.service = "Redis"; break; + case 6443: p.service = "Kubernetes API"; break; + case 6667: p.service = "IRC"; break; + case 8000: + case 8080: + case 8888: p.service = "HTTP (alt)"; break; + case 8443: p.service = "HTTPS (alt)"; break; + case 9092: p.service = "Kafka"; break; + case 9200: p.service = "Elasticsearch"; break; + case 9418: p.service = "Git"; break; + case 25565: p.service = "Minecraft"; break; + case 27017: p.service = "MongoDB"; break; + default: break; + } + } + + void classifyUdpPort(uint16_t port, ParsedPacket& p) { + if (!p.service.empty()) return; + switch (port) { + case 53: p.service = "DNS"; break; + case 67: + case 68: p.service = "DHCP"; break; + case 69: p.service = "TFTP"; break; + case 123: p.service = "NTP"; break; + case 137: + case 138: + case 139: p.service = "NetBIOS"; break; + case 161: + case 162: p.service = "SNMP"; break; + case 443: p.service = "QUIC"; break; + case 500: + case 4500: p.service = "IPsec"; break; + case 514: p.service = "Syslog"; break; + case 546: + case 547: p.service = "DHCPv6"; break; + case 784: + case 8853: p.service = "DNS-over-QUIC"; break; + case 853: p.service = "DNS-over-QUIC"; break; + case 1194: p.service = "OpenVPN"; break; + case 1900: p.service = "SSDP"; break; + case 3478: + case 3479: + case 5349: p.service = "STUN/TURN"; break; + case 4789: p.service = "VXLAN"; break; + case 5353: p.service = "mDNS"; break; + case 5355: p.service = "LLMNR"; break; + case 6081: p.service = "GENEVE"; break; + case 51820: p.service = "WireGuard"; break; + default: break; + } + } + + bool isHttpMethod(const uint8_t* p, size_t len, size_t& methodLen) { + static constexpr std::string_view methods[] = { + "GET", "POST", "HEAD", "PUT", "DELETE", "OPTIONS", + "PATCH", "TRACE", "CONNECT", + }; + for (std::string_view method : methods) { + if (len > method.size() + 1 + && std::memcmp(p, method.data(), method.size()) == 0 + && p[method.size()] == ' ') { + methodLen = method.size(); + return true; + } + } + return false; + } + + } // namespace + + bool ProtocolParser::looksLikeTlsHandshake(const uint8_t* payload, size_t length) { + // Handshake record (0x16), TLS major version 3, minor 0..4. + return length >= 5 && payload[0] == 0x16 && payload[1] == 0x03 && payload[2] <= 0x04; + } + + ParsedPacket ProtocolParser::parse(const PacketData& rawData) { + ParsedPacket parsed; + parsed.timestamp = rawData.timestamp; + parsed.length = rawData.length; + parsed.protocol = "Unknown"; + + const uint8_t* buffer = rawData.payload.data(); + const size_t size = rawData.payload.size(); + if (buffer == nullptr || size == 0) return parsed; + + size_t offset = 0; + uint16_t etherType = 0; + if (!decodeLinkLayer(buffer, size, rawData.linkType, offset, etherType)) { + parsed.protocol = rawData.linkType == LinkType::Unsupported + ? "Unsupported link type" + : "Truncated"; + return parsed; + } + + decodeNetwork(buffer, size, offset, etherType, parsed, 0); + + // Application-layer DPI runs once, on the innermost transport payload. + if (parsed.payloadLength > 0 && parsed.payloadOffset < size) { + const uint8_t* payload = buffer + parsed.payloadOffset; + const size_t payloadLen = std::min(parsed.payloadLength, size - parsed.payloadOffset); + + if (parsed.protocol == "TCP") { + if (looksLikeTlsHandshake(payload, payloadLen)) { + parseTLS(payload, payloadLen, parsed); + } else { + parseHTTP(payload, payloadLen, parsed); + } + classifyTcpPort(parsed.dstPort, parsed); + classifyTcpPort(parsed.srcPort, parsed); + } else if (parsed.protocol == "UDP") { + // QUIC runs on whatever port the deployment chose; identify it + // by the long-header + fixed-bit shape and a known version. + if (QuicParser::looksLikeLongHeader(payload, payloadLen)) { + if (auto sni = QuicParser::extractInitialSni(payload, payloadLen)) { + parsed.sni = *sni; + if (parsed.service.empty()) { + parsed.service = identifyService(*sni); + } + } + if (parsed.service.empty()) parsed.service = "QUIC"; + } + classifyUdpPort(parsed.dstPort, parsed); + classifyUdpPort(parsed.srcPort, parsed); + } + } else if (parsed.protocol == "TCP") { + classifyTcpPort(parsed.dstPort, parsed); + classifyTcpPort(parsed.srcPort, parsed); + } else if (parsed.protocol == "UDP") { + classifyUdpPort(parsed.dstPort, parsed); + classifyUdpPort(parsed.srcPort, parsed); + } + + // Encrypted tunnels that ride on UDP are only recognisable by port. + if (parsed.service == "WireGuard" || parsed.service == "OpenVPN" + || parsed.service == "IPsec") { + parsed.encryptedTunnel = true; + } + + return parsed; + } + + bool ProtocolParser::parseTlsClientHello(const uint8_t* payload, size_t len, std::string& outSni) { + constexpr size_t recordHeaderLength = 5; + constexpr size_t handshakeHeaderLength = 4; + constexpr size_t clientHelloFixedLength = 34; // Version + random + + if (!looksLikeTlsHandshake(payload, len)) return false; + + // A truncated record is not a parse failure — the rest of the + // ClientHello is in the next TCP segment. Report "not yet". + const size_t recordLength = readU16(payload + 3); + if (recordHeaderLength + recordLength > len) return false; + const size_t recordEnd = recordHeaderLength + recordLength; + + size_t pos = recordHeaderLength; + if (recordEnd < pos + handshakeHeaderLength || payload[pos] != 0x01) return false; // ClientHello + + const size_t handshakeLength = (static_cast(payload[pos + 1]) << 16) + | (static_cast(payload[pos + 2]) << 8) | payload[pos + 3]; + pos += handshakeHeaderLength; + if (pos + handshakeLength > recordEnd) return false; + const size_t handshakeEnd = pos + handshakeLength; + if (handshakeEnd < pos + clientHelloFixedLength) return false; + pos += clientHelloFixedLength; + + if (pos >= handshakeEnd) return false; + const size_t sessionIdLength = payload[pos++]; + if (sessionIdLength > handshakeEnd - pos) return false; + pos += sessionIdLength; + + if (handshakeEnd - pos < 2) return false; + const size_t cipherSuitesLength = readU16(payload + pos); + pos += 2; + if (cipherSuitesLength > handshakeEnd - pos) return false; + pos += cipherSuitesLength; + + if (pos >= handshakeEnd) return false; + const size_t compressionMethodsLength = payload[pos++]; + if (compressionMethodsLength > handshakeEnd - pos) return false; + pos += compressionMethodsLength; + + if (handshakeEnd - pos < 2) return false; + const size_t extensionsLength = readU16(payload + pos); + pos += 2; + if (extensionsLength > handshakeEnd - pos) return false; + const size_t extensionsEnd = pos + extensionsLength; + + while (extensionsEnd - pos >= 4) { + const uint16_t extensionType = readU16(payload + pos); + const size_t extensionLength = readU16(payload + pos + 2); + pos += 4; + if (extensionLength > extensionsEnd - pos) return false; + const size_t extensionEnd = pos + extensionLength; + + if (extensionType == 0x0000) { // Server Name Indication + if (extensionEnd - pos < 2) return false; + const size_t nameListLength = readU16(payload + pos); + pos += 2; + if (nameListLength > extensionEnd - pos) return false; + const size_t nameListEnd = pos + nameListLength; + + while (nameListEnd - pos >= 3) { + const uint8_t nameType = payload[pos++]; + const size_t nameLength = readU16(payload + pos); + pos += 2; + if (nameLength > nameListEnd - pos) return false; + if (nameType == 0x00) { + outSni.assign(reinterpret_cast(payload + pos), nameLength); + return true; + } + pos += nameLength; + } + return false; + } + + pos = extensionEnd; + } + return false; + } + + void ProtocolParser::parseTLS(const uint8_t* payload, size_t len, ParsedPacket& outPacket) { + std::string sni; + if (parseTlsClientHello(payload, len, sni) && !sni.empty()) { + outPacket.sni = sni; + outPacket.service = identifyService(sni); + } + } + + void ProtocolParser::parseHTTP(const uint8_t* payload, size_t len, ParsedPacket& outPacket) { + // Only the head of the message matters, and a header block that has not + // arrived within 2 KiB is not one we need. + const size_t scanLen = std::min(len, 2048); + + const auto lineEnd = [&](size_t from) { + for (size_t i = from; i + 1 < scanLen; ++i) { + if (payload[i] == '\r' && payload[i + 1] == '\n') return i; + } + return scanLen; + }; + + if (scanLen >= 12 && std::memcmp(payload, "HTTP/1.", 7) == 0) { + const size_t end = lineEnd(0); + outPacket.info.assign(reinterpret_cast(payload), end); + if (outPacket.service.empty()) outPacket.service = "HTTP"; + return; + } + + size_t methodLen = 0; + if (!isHttpMethod(payload, scanLen, methodLen)) return; + + // Request line: METHOD SP request-target SP HTTP/1.x + const size_t requestLineEnd = lineEnd(0); + outPacket.info.assign(reinterpret_cast(payload), + std::min(requestLineEnd, static_cast(120))); + if (outPacket.service.empty()) outPacket.service = "HTTP"; + + // Walk header lines looking for Host. + size_t pos = requestLineEnd + 2; + while (pos < scanLen) { + const size_t end = lineEnd(pos); + if (end == pos) break; // blank line: end of headers + if (end == scanLen) break; + + constexpr std::string_view hostPrefix = "host:"; + if (end - pos > hostPrefix.size()) { + bool isHost = true; + for (size_t i = 0; i < hostPrefix.size(); ++i) { + if (std::tolower(static_cast(payload[pos + i])) != hostPrefix[i]) { + isHost = false; + break; + } + } + if (isHost) { + size_t valueStart = pos + hostPrefix.size(); + while (valueStart < end && payload[valueStart] == ' ') ++valueStart; + if (valueStart < end) { + outPacket.hostname.assign( + reinterpret_cast(payload + valueStart), end - valueStart); + if (const std::string named = identifyService(outPacket.hostname); !named.empty()) { + outPacket.service = named; + } + } + return; + } + } + pos = end + 2; + } + } + + std::string ProtocolParser::identifyService(const std::string& hostname) { + // Substring match against a curated catalog of major services. + // The map is ordered by specificity — more specific patterns first. + struct Pattern { std::string_view needle; std::string_view label; }; + static constexpr std::array catalog = std::to_array({ + // Encrypted DNS resolvers. Checked first: these hosts would + // otherwise match their operator ("Google", "Cloudflare") and hide + // the fact that name resolution has moved off the wire. + {"mozilla.cloudflare-dns.com", "DNS-over-HTTPS"}, + {"cloudflare-dns.com", "DNS-over-HTTPS"}, + {"one.one.one.one", "DNS-over-HTTPS"}, + {"dns.google", "DNS-over-HTTPS"}, + {"dns.quad9.net", "DNS-over-HTTPS"}, + {"doh.opendns.com", "DNS-over-HTTPS"}, + {"dns.nextdns.io", "DNS-over-HTTPS"}, + {"doh.cleanbrowsing.org","DNS-over-HTTPS"}, + {"dns.adguard", "DNS-over-HTTPS"}, + {"doh.dns.sb", "DNS-over-HTTPS"}, + + // Google ecosystem + {"youtube", "YouTube"}, + {"googlevideo", "YouTube"}, + {"ytimg", "YouTube"}, + {"gstatic", "Google"}, + {"googleusercontent","Google"}, + {"googleapis", "Google"}, + {"google.", "Google"}, + {"doubleclick", "Google Ads"}, + + // Streaming / video + {"netflix", "Netflix"}, + {"nflxvideo", "Netflix"}, + {"twitch", "Twitch"}, + {"ttvnw", "Twitch"}, + {"spotify", "Spotify"}, + {"scdn.co", "Spotify"}, + {"hulu", "Hulu"}, + {"disney", "Disney+"}, + {"primevideo", "Prime Video"}, + + // Social / messaging + {"discord", "Discord"}, + {"discordapp", "Discord"}, + {"slack.com", "Slack"}, + {"slack-edge", "Slack"}, + {"telegram", "Telegram"}, + {"whatsapp", "WhatsApp"}, + {"facebook", "Facebook"}, + {"fbcdn", "Facebook"}, + {"instagram", "Instagram"}, + {"cdninstagram", "Instagram"}, + {"twitter", "X (Twitter)"}, + {"twimg", "X (Twitter)"}, + {"x.com", "X (Twitter)"}, + {"tiktok", "TikTok"}, + {"tiktokcdn", "TikTok"}, + {"reddit", "Reddit"}, + {"redditstatic", "Reddit"}, + {"linkedin", "LinkedIn"}, + {"signal.org", "Signal"}, + + // Dev / cloud + {"github", "GitHub"}, + {"githubusercontent","GitHub"}, + {"gitlab", "GitLab"}, + {"bitbucket", "Bitbucket"}, + {"npmjs", "npm"}, + {"jsdelivr", "jsDelivr"}, + {"cloudflare", "Cloudflare"}, + {"akamai", "Akamai"}, + {"fastly", "Fastly"}, + {"amazonaws", "AWS"}, + {"awsstatic", "AWS"}, + {"azureedge", "Azure"}, + {"windows.net", "Azure"}, + {"azure.com", "Azure"}, + {"digitalocean", "DigitalOcean"}, + {"vercel", "Vercel"}, + {"netlify", "Netlify"}, + {"anthropic", "Anthropic"}, + {"openai", "OpenAI"}, + + // Microsoft + {"microsoft", "Microsoft"}, + {"msftncsi", "Microsoft"}, + {"office.com", "Microsoft 365"}, + {"office365", "Microsoft 365"}, + {"outlook", "Outlook"}, + {"live.com", "Microsoft"}, + {"bing.com", "Bing"}, + {"xboxlive", "Xbox Live"}, + {"xbox.com", "Xbox Live"}, + {"skype", "Skype"}, + {"teams.microsoft", "Microsoft Teams"}, + + // Apple + {"apple.com", "Apple"}, + {"icloud", "iCloud"}, + {"itunes.apple", "iTunes"}, + {"mzstatic", "Apple"}, + + // Gaming + {"steampowered", "Steam"}, + {"steamstatic", "Steam"}, + {"steamcommunity", "Steam"}, + {"epicgames", "Epic Games"}, + {"riotgames", "Riot Games"}, + {"battle.net", "Battle.net"}, + {"playstation", "PlayStation"}, + {"nintendo", "Nintendo"}, + + // Shopping + {"amazon", "Amazon"}, + {"ebay", "eBay"}, + {"shopify", "Shopify"}, + + // Communication / video + {"zoom.us", "Zoom"}, + {"zoomgov", "Zoom"}, + {"webex", "Webex"}, + {"meet.google", "Google Meet"}, + + // News / misc + {"wikipedia", "Wikipedia"}, + {"wikimedia", "Wikipedia"}, + + // Mozilla / privacy + {"mozilla", "Mozilla"}, + {"firefox", "Firefox"}, + {"duckduckgo", "DuckDuckGo"}, + {"protonmail", "Proton"}, + {"proton.me", "Proton"}, + }); + + const std::string lower = toLowerCopy(hostname); + for (const auto& pat : catalog) { + if (lower.find(pat.needle) != std::string::npos) return std::string(pat.label); + } + return {}; + } + +} // namespace core diff --git a/src/core/QuicTracker.hpp b/src/core/QuicTracker.hpp index 49e5b14..f48ef5d 100644 --- a/src/core/QuicTracker.hpp +++ b/src/core/QuicTracker.hpp @@ -1,54 +1,54 @@ -#pragma once - -#include "core/QuicParser.hpp" - -#include -#include -#include -#include -#include - -namespace core { - - // Reassembles a QUIC ClientHello that spans several Initial packets. - // - // QUIC's minimum datagram is 1200 bytes, and a ClientHello carrying a - // post-quantum key share exceeds that — so the CRYPTO stream arrives split - // across two or more Initials. Each is independently decryptable (the keys - // come from the Destination Connection ID, which the client repeats), but - // the TLS message only parses once the fragments are stitched together. - // - // Keyed by DCID, which is stable across the client's first flight. - class QuicTracker { - public: - // Feeds one UDP payload that looked like a QUIC long header. Returns the - // SNI on the packet that completes the ClientHello. - std::optional feed(const uint8_t* payload, size_t length, - int64_t timestampMicroseconds); - - void clear(); - size_t trackedConnectionCount() const { return m_connections.size(); } - - private: - static constexpr size_t kMaxStreamBytes = 64 * 1024; - static constexpr int64_t kConnectionTimeoutMicroseconds = 10'000'000; - static constexpr size_t kMaxTrackedConnections = 1'024; - - struct Connection { - std::vector stream; // TLS handshake bytes, indexed by offset - std::vector present; // which of those bytes have arrived - int64_t lastSeen = 0; - }; - - struct ConnectionIdHash { - size_t operator()(const std::vector& id) const noexcept; - }; - - // Number of bytes available contiguously from offset 0. - static size_t contiguousPrefix(const Connection& connection); - void expireStale(int64_t now); - - std::unordered_map, Connection, ConnectionIdHash> m_connections; - }; - -} // namespace core +#pragma once + +#include "core/QuicParser.hpp" + +#include +#include +#include +#include +#include + +namespace core { + + // Reassembles a QUIC ClientHello that spans several Initial packets. + // + // QUIC's minimum datagram is 1200 bytes, and a ClientHello carrying a + // post-quantum key share exceeds that — so the CRYPTO stream arrives split + // across two or more Initials. Each is independently decryptable (the keys + // come from the Destination Connection ID, which the client repeats), but + // the TLS message only parses once the fragments are stitched together. + // + // Keyed by DCID, which is stable across the client's first flight. + class QuicTracker { + public: + // Feeds one UDP payload that looked like a QUIC long header. Returns the + // SNI on the packet that completes the ClientHello. + std::optional feed(const uint8_t* payload, size_t length, + int64_t timestampMicroseconds); + + void clear(); + size_t trackedConnectionCount() const { return m_connections.size(); } + + private: + static constexpr size_t kMaxStreamBytes = size_t{64} * 1024; + static constexpr int64_t kConnectionTimeoutMicroseconds = 10'000'000; + static constexpr size_t kMaxTrackedConnections = 1'024; + + struct Connection { + std::vector stream; // TLS handshake bytes, indexed by offset + std::vector present; // which of those bytes have arrived + int64_t lastSeen = 0; + }; + + struct ConnectionIdHash { + size_t operator()(const std::vector& id) const noexcept; + }; + + // Number of bytes available contiguously from offset 0. + static size_t contiguousPrefix(const Connection& connection); + void expireStale(int64_t now); + + std::unordered_map, Connection, ConnectionIdHash> m_connections; + }; + +} // namespace core diff --git a/src/core/TlsReassembler.hpp b/src/core/TlsReassembler.hpp index e062f36..e2607fc 100644 --- a/src/core/TlsReassembler.hpp +++ b/src/core/TlsReassembler.hpp @@ -1,66 +1,66 @@ -#pragma once - -#include "core/ParsedPacket.hpp" - -#include -#include -#include -#include -#include - -namespace core { - - // Buffers TLS ClientHello bytes across TCP segments. - // - // A ClientHello used to fit comfortably in one segment. It no longer does: - // post-quantum key shares (X25519MLKEM768, now the default in Chrome and - // Firefox) push the message past a typical 1460-byte MSS, so a parser that - // only ever looks at a single segment silently stops finding SNI on a - // growing share of real traffic. - // - // Scope is deliberately narrow — we only track the client's first flight, - // in order, until the record is complete. Out-of-order segments, oversized - // records, and non-handshake streams are dropped rather than buffered. - class TlsReassembler { - public: - // Feeds one TCP segment's payload. Returns the SNI on the segment that - // completes the ClientHello, and std::nullopt otherwise. - std::optional feed(const ParsedPacket& packet, - const uint8_t* payload, size_t length); - - void clear(); - size_t trackedStreamCount() const { return m_streams.size(); } - - private: - // Beyond this a "ClientHello" is either malicious or not a ClientHello. - static constexpr size_t kMaxBufferedBytes = 64 * 1024; - // Stop tracking a stream that has gone quiet, so a long capture does not - // accumulate half-finished handshakes forever. - static constexpr int64_t kStreamTimeoutMicroseconds = 30'000'000; - static constexpr size_t kMaxTrackedStreams = 2'048; - - struct StreamKey { - std::string srcIP; - std::string dstIP; - uint16_t srcPort = 0; - uint16_t dstPort = 0; - bool operator==(const StreamKey& other) const = default; - }; - - struct StreamKeyHash { - size_t operator()(const StreamKey& key) const noexcept; - }; - - struct Stream { - std::vector buffer; - uint32_t nextSeq = 0; // TCP sequence number expected next - size_t needed = 0; // total record bytes required, 0 until known - int64_t lastSeen = 0; - }; - - void expireStale(int64_t now); - - std::unordered_map m_streams; - }; - -} // namespace core +#pragma once + +#include "core/ParsedPacket.hpp" + +#include +#include +#include +#include +#include + +namespace core { + + // Buffers TLS ClientHello bytes across TCP segments. + // + // A ClientHello used to fit comfortably in one segment. It no longer does: + // post-quantum key shares (X25519MLKEM768, now the default in Chrome and + // Firefox) push the message past a typical 1460-byte MSS, so a parser that + // only ever looks at a single segment silently stops finding SNI on a + // growing share of real traffic. + // + // Scope is deliberately narrow — we only track the client's first flight, + // in order, until the record is complete. Out-of-order segments, oversized + // records, and non-handshake streams are dropped rather than buffered. + class TlsReassembler { + public: + // Feeds one TCP segment's payload. Returns the SNI on the segment that + // completes the ClientHello, and std::nullopt otherwise. + std::optional feed(const ParsedPacket& packet, + const uint8_t* payload, size_t length); + + void clear(); + size_t trackedStreamCount() const { return m_streams.size(); } + + private: + // Beyond this a "ClientHello" is either malicious or not a ClientHello. + static constexpr size_t kMaxBufferedBytes = size_t{64} * 1024; + // Stop tracking a stream that has gone quiet, so a long capture does not + // accumulate half-finished handshakes forever. + static constexpr int64_t kStreamTimeoutMicroseconds = 30'000'000; + static constexpr size_t kMaxTrackedStreams = 2'048; + + struct StreamKey { + std::string srcIP; + std::string dstIP; + uint16_t srcPort = 0; + uint16_t dstPort = 0; + bool operator==(const StreamKey& other) const = default; + }; + + struct StreamKeyHash { + size_t operator()(const StreamKey& key) const noexcept; + }; + + struct Stream { + std::vector buffer; + uint32_t nextSeq = 0; // TCP sequence number expected next + size_t needed = 0; // total record bytes required, 0 until known + int64_t lastSeen = 0; + }; + + void expireStale(int64_t now); + + std::unordered_map m_streams; + }; + +} // namespace core diff --git a/src/main.cpp b/src/main.cpp index 407eed4..31db1fd 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,76 +1,96 @@ -#include -#include -#include "capture/CaptureEngine.hpp" -#include "core/PacketQueue.hpp" -#include "ui/GuiLayer.hpp" - -int main() { - std::cout << "Starting NetProbe..." << std::endl; - - // 1. Initialize Core Components - core::PacketQueue queue; - capture::CaptureEngine engine(queue); - - // 2. Select Network Adapter - auto devices = engine.getAvailableDevices(); - if (devices.empty()) { - std::cerr << "No live network devices found. Offline PCAP loading is still available." << std::endl; - } - - // Convert capture::DeviceInfo to ui::GuiLayer::DeviceInfo - std::vector uiDevices; - for (const auto& d : devices) { - uiDevices.push_back({d.name, d.description}); - } - - // 3. Initialize and Run UI - ui::GuiLayer gui(queue); - - // Pass devices to UI - gui.setDevices(uiDevices); - - if (!gui.init()) { - std::cerr << "Failed to initialize GUI." << std::endl; - return 1; - } - - // Handle Device Selection only after the UI is ready to consume packets. - gui.onDeviceSelected = [&](std::string deviceName) { - std::cout << "Switching to device: " << deviceName << std::endl; - engine.startCapture(deviceName); - }; - - gui.onCaptureStopRequested = [&]() { - std::cout << "Stopping live capture." << std::endl; - engine.stopCapture(); - }; - - gui.onPcapFileSelected = [&](std::string path) { - std::cout << "Loading capture file: " << path << std::endl; - if (!engine.openFile(path)) { - std::cerr << "Failed to load capture file." << std::endl; - } - }; - - gui.onBpfFilterRequested = [&](const std::string& filter, std::string& error) { - return engine.setFilter(filter, error); - }; - - gui.onPcapSaveRequested = [&](const std::string& path, std::string& error) { - return engine.exportSession(path, error); - }; - - // Auto-start on the first device when live capture is available. - if (!devices.empty()) { - std::cout << "Auto-selecting device: " << devices[0].description << std::endl; - engine.startCapture(devices[0].name); - gui.setCaptureActive(true); - } - - // This blocks until window is closed - gui.run(); - - // 5. Shutdown (RAII handles engine/thread stop) - std::cout << "NetProbe shutting down." << std::endl; - return 0; -} +#include +#include +#include +#include +#include "capture/CaptureEngine.hpp" +#include "core/PacketQueue.hpp" +#include "ui/GuiLayer.hpp" + +static int run() { + std::cout << "Starting NetProbe..." << '\n'; + + // 1. Initialize Core Components + core::PacketQueue queue; + capture::CaptureEngine engine(queue); + + // 2. Select Network Adapter + auto devices = engine.getAvailableDevices(); + if (devices.empty()) { + std::cerr << "No live network devices found. Offline PCAP loading is still available." << '\n'; + } + + // Convert capture::DeviceInfo to ui::GuiLayer::DeviceInfo + std::vector uiDevices; + uiDevices.reserve(devices.size()); + for (const auto& d : devices) { + uiDevices.push_back({d.name, d.description}); + } + + // 3. Initialize and Run UI + ui::GuiLayer gui(queue); + + // Pass devices to UI + gui.setDevices(uiDevices); + + if (!gui.init()) { + std::cerr << "Failed to initialize GUI." << '\n'; + return 1; + } + + // Handle Device Selection only after the UI is ready to consume packets. + gui.onDeviceSelected = [&](const std::string& deviceName) { + std::cout << "Switching to device: " << deviceName << '\n'; + engine.startCapture(deviceName); + }; + + gui.onCaptureStopRequested = [&]() { + std::cout << "Stopping live capture." << '\n'; + engine.stopCapture(); + }; + + gui.onPcapFileSelected = [&](const std::string& path) { + std::cout << "Loading capture file: " << path << '\n'; + if (!engine.openFile(path)) { + std::cerr << "Failed to load capture file." << '\n'; + } + }; + + gui.onBpfFilterRequested = [&](const std::string& filter, std::string& error) { + return engine.setFilter(filter, error); + }; + + gui.onPcapSaveRequested = [&](const std::string& path, std::string& error) { + return engine.exportSession(path, error); + }; + + // Auto-start on the first device when live capture is available. + if (!devices.empty()) { + std::cout << "Auto-selecting device: " << devices[0].description << '\n'; + engine.startCapture(devices[0].name); + gui.setCaptureActive(true); + } + + // This blocks until window is closed + gui.run(); + + // 5. Shutdown (RAII handles engine/thread stop) + std::cout << "NetProbe shutting down." << '\n'; + return 0; +} + +int main() { + // An exception escaping main terminates with no diagnostic at all, which + // for a windowed application means the window simply vanishes. Report it + // on a path that cannot itself throw, so the user has something to go on. + try { + return run(); + } catch (const std::exception& error) { + std::fputs("NetProbe: unexpected failure: ", stderr); + std::fputs(error.what(), stderr); + std::fputs("\n", stderr); + return 1; + } catch (...) { + std::fputs("NetProbe: unexpected failure\n", stderr); + return 1; + } +} diff --git a/src/ui/GuiLayer.cpp b/src/ui/GuiLayer.cpp index 95f2e97..9dfe73a 100644 --- a/src/ui/GuiLayer.cpp +++ b/src/ui/GuiLayer.cpp @@ -1,936 +1,952 @@ -#include "ui/GuiLayer.hpp" -#include "ui/GuiTheme.hpp" -#include "ui/EmbeddedIcon.hpp" -#include "imgui.h" -#include "imgui_internal.h" -#include "imgui_impl_glfw.h" -#include "imgui_impl_opengl3.h" -#include "implot.h" -#include "nfd.h" -#include -#include -#include -#include -#include -#include -#include - -namespace ui { - - GuiLayer::GuiLayer(core::PacketQueue& queue) : m_queue(queue) {} - - GuiLayer::~GuiLayer() { - captureWindowGeometry(); - saveSettings(); - if (m_window) { - ImGui_ImplOpenGL3_Shutdown(); - ImGui_ImplGlfw_Shutdown(); - ImPlot::DestroyContext(); - ImGui::DestroyContext(); - glfwDestroyWindow(m_window); - glfwTerminate(); - } - if (m_nfdInitialized) { - NFD_Quit(); - } - } - - bool GuiLayer::init() { - if (!glfwInit()) return false; - - // Settings are needed before window creation: they carry the saved - // window geometry and maximized state from the previous run. - loadSettings(); - - const char* glsl_version = "#version 130"; - glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 3); - glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 0); - glfwWindowHint(GLFW_VISIBLE, GLFW_FALSE); - if (m_savedWindowMaximized) glfwWindowHint(GLFW_MAXIMIZED, GLFW_TRUE); - - // A saved size is already in physical pixels; the default size is - // authored for 100% scale and multiplied by the primary monitor's - // content scale so first launch isn't tiny on high-DPI displays. - int width = 1440, height = 900; - if (m_savedWindowW > 0 && m_savedWindowH > 0) { - width = std::clamp(m_savedWindowW, 800, 16384); - height = std::clamp(m_savedWindowH, 500, 16384); - } else if (GLFWmonitor* primary = glfwGetPrimaryMonitor()) { - float scaleX = 1.0f, scaleY = 1.0f; - glfwGetMonitorContentScale(primary, &scaleX, &scaleY); - width = static_cast(width * scaleX); - height = static_cast(height * scaleY); - } - - m_window = glfwCreateWindow(width, height, "NetProbe - Network Analyzer", NULL, NULL); - if (!m_window) { - glfwTerminate(); - return false; - } - - // Restore the previous window position, but only when its center still - // falls on a connected monitor — displays come and go between runs. - if (m_hasSavedWindowPos && !m_savedWindowMaximized) { - int monitorCount = 0; - GLFWmonitor** monitors = glfwGetMonitors(&monitorCount); - const int centerX = m_savedWindowX + width / 2; - const int centerY = m_savedWindowY + height / 2; - for (int i = 0; i < monitorCount; ++i) { - int mx, my, mw, mh; - glfwGetMonitorWorkarea(monitors[i], &mx, &my, &mw, &mh); - if (centerX >= mx && centerX < mx + mw && centerY >= my && centerY < my + mh) { - glfwSetWindowPos(m_window, m_savedWindowX, m_savedWindowY); - break; - } - } - } - - // Title-bar and taskbar icon — the embedded .ico covers Explorer / - // shortcuts, but GLFW windows need an explicit set call to override - // the default OS window icon while the app is running. - GLFWimage icons[3]; - icons[0] = {kEmbeddedIcon16Size, kEmbeddedIcon16Size, - const_cast(kEmbeddedIcon16Pixels)}; - icons[1] = {kEmbeddedIcon32Size, kEmbeddedIcon32Size, - const_cast(kEmbeddedIcon32Pixels)}; - icons[2] = {kEmbeddedIcon48Size, kEmbeddedIcon48Size, - const_cast(kEmbeddedIcon48Pixels)}; - glfwSetWindowIcon(m_window, 3, icons); - glfwShowWindow(m_window); - - glfwMakeContextCurrent(m_window); - glfwSwapInterval(1); // Enable vsync - glfwSetWindowUserPointer(m_window, this); - - // Per-monitor DPI: pick up the content scale of whichever monitor the - // window actually landed on, and re-theme when it moves to another. - float contentScale = 1.0f; - glfwGetWindowContentScale(m_window, &contentScale, nullptr); - if (contentScale > 0.0f) m_contentScale = contentScale; - glfwSetWindowContentScaleCallback(m_window, - [](GLFWwindow* window, float xScale, float) { - auto* gui = static_cast(glfwGetWindowUserPointer(window)); - if (gui && xScale > 0.0f && std::abs(xScale - gui->m_contentScale) > 0.01f) { - gui->m_contentScale = xScale; - gui->applyTheme(); - } - }); - glfwSetDropCallback(m_window, [](GLFWwindow* window, int pathCount, const char** paths) { - auto* gui = static_cast(glfwGetWindowUserPointer(window)); - if (gui && pathCount > 0 && paths[0]) { - gui->openPcapFile(paths[0]); - } - }); - - m_nfdInitialized = NFD_Init() == NFD_OKAY; - if (!m_nfdInitialized) { - std::cerr << "Native file dialog is unavailable: " << NFD_GetError() << std::endl; - } - - IMGUI_CHECKVERSION(); - ImGui::CreateContext(); - ImPlot::CreateContext(); - - ImGuiIO& io = ImGui::GetIO(); - io.ConfigFlags |= ImGuiConfigFlags_NavEnableKeyboard; - io.ConfigFlags |= ImGuiConfigFlags_DockingEnable; - - // Use a dedicated ini so a previously-saved imgui.ini with the legacy layout - // doesn't override the new default dockspace on the first run. - io.IniFilename = "netprobe-layout.ini"; - - loadFonts(); - applyTheme(); - - ImGui_ImplGlfw_InitForOpenGL(m_window, true); - ImGui_ImplOpenGL3_Init(glsl_version); - - return true; - } - - void GuiLayer::loadFonts() { - ImGuiIO& io = ImGui::GetIO(); - ImFontConfig cfg; - cfg.OversampleH = 3; - cfg.OversampleV = 1; - cfg.PixelSnapH = false; - - const std::filesystem::path regular = "C:/Windows/Fonts/segoeui.ttf"; - const std::filesystem::path semibold = "C:/Windows/Fonts/seguisb.ttf"; - const std::filesystem::path mono = "C:/Windows/Fonts/consola.ttf"; - - if (std::filesystem::exists(regular)) { - m_fontDefault = io.Fonts->AddFontFromFileTTF(regular.string().c_str(), 15.0f, &cfg); - m_fontSmall = io.Fonts->AddFontFromFileTTF(regular.string().c_str(), 12.0f, &cfg); - const auto& semiPath = std::filesystem::exists(semibold) ? semibold : regular; - m_fontHeadline = io.Fonts->AddFontFromFileTTF(semiPath.string().c_str(), 28.0f, &cfg); - m_fontBrand = io.Fonts->AddFontFromFileTTF(semiPath.string().c_str(), 16.0f, &cfg); - m_fontMono = std::filesystem::exists(mono) - ? io.Fonts->AddFontFromFileTTF(mono.string().c_str(), 13.0f, &cfg) - : m_fontDefault; - } else { - // Fall back to the bundled bitmap font if Segoe UI is unavailable. - m_fontDefault = io.Fonts->AddFontDefault(); - m_fontSmall = m_fontDefault; - m_fontHeadline = m_fontDefault; - m_fontBrand = m_fontDefault; - m_fontMono = m_fontDefault; - } - } - - void GuiLayer::loadSettings() { - std::ifstream file("netprobe-settings.ini"); - if (!file) return; - std::string line; - while (std::getline(file, line)) { - const auto eq = line.find('='); - if (eq == std::string::npos) continue; - const std::string key = line.substr(0, eq); - const std::string value = line.substr(eq + 1); - if (key == "theme") { - m_darkTheme = value != "light"; - } else if (key == "scale") { - try { - m_uiScale = std::clamp(std::stof(value), 0.75f, 1.5f); - } catch (...) { - m_uiScale = 1.0f; - } - } else if (key == "window") { - int x = 0, y = 0, w = 0, h = 0, maximized = 0; - if (std::sscanf(value.c_str(), "%d %d %d %d %d", &x, &y, &w, &h, &maximized) == 5) { - m_savedWindowX = x; - m_savedWindowY = y; - m_savedWindowW = w; - m_savedWindowH = h; - m_savedWindowMaximized = maximized != 0; - m_hasSavedWindowPos = true; - } - } - } - } - - void GuiLayer::saveSettings() const { - std::ofstream file("netprobe-settings.ini", std::ios::trunc); - if (!file) return; - file << "theme=" << (m_darkTheme ? "dark" : "light") << '\n'; - file << "scale=" << m_uiScale << '\n'; - if (m_savedWindowW > 0 && m_savedWindowH > 0) { - file << "window=" << m_savedWindowX << ' ' << m_savedWindowY << ' ' - << m_savedWindowW << ' ' << m_savedWindowH << ' ' - << (m_savedWindowMaximized ? 1 : 0) << '\n'; - } - } - - void GuiLayer::captureWindowGeometry() { - if (!m_window) return; - m_savedWindowMaximized = glfwGetWindowAttrib(m_window, GLFW_MAXIMIZED) == GLFW_TRUE; - // While maximized (or minimized) the current rect is not the one worth - // restoring — keep the last normal geometry already recorded. - if (m_savedWindowMaximized - || glfwGetWindowAttrib(m_window, GLFW_ICONIFIED) == GLFW_TRUE) { - return; - } - glfwGetWindowPos(m_window, &m_savedWindowX, &m_savedWindowY); - glfwGetWindowSize(m_window, &m_savedWindowW, &m_savedWindowH); - m_hasSavedWindowPos = true; - } - - void GuiLayer::setStatus(std::string message, bool isError) { - m_captureStatus = std::move(message); - m_captureStatusIsError = isError; - m_captureStatusTime = glfwGetTime(); - } - - void GuiLayer::applyTheme() { - applyPalette(m_darkTheme); - - ImGuiStyle& style = ImGui::GetStyle(); - // Rebuild from defaults so repeated calls (theme toggle, monitor DPI - // change) never compound the ScaleAllSizes multiplication below. - style = ImGuiStyle(); - style.FontScaleMain = m_uiScale * m_contentScale; - - style.WindowPadding = ImVec2(18, 16); - style.FramePadding = ImVec2(12, 7); - style.CellPadding = ImVec2(10, 8); - style.ItemSpacing = ImVec2(10, 10); - style.ItemInnerSpacing = ImVec2(8, 6); - style.IndentSpacing = 22.0f; - style.ScrollbarSize = 10.0f; - style.GrabMinSize = 10.0f; - - style.WindowBorderSize = 1.0f; - style.ChildBorderSize = 1.0f; - style.FrameBorderSize = 0.0f; - style.PopupBorderSize = 1.0f; - style.TabBorderSize = 0.0f; - style.SeparatorTextBorderSize = 1.0f; - - style.WindowRounding = 10.0f; - style.ChildRounding = 8.0f; - style.FrameRounding = 7.0f; - style.PopupRounding = 8.0f; - style.GrabRounding = 7.0f; - style.TabRounding = 6.0f; - style.ScrollbarRounding = 6.0f; - - ImVec4* c = style.Colors; - c[ImGuiCol_Text] = kText1; - c[ImGuiCol_TextDisabled] = kText3; - c[ImGuiCol_WindowBg] = kBgBase; - c[ImGuiCol_ChildBg] = kBgBase; - c[ImGuiCol_PopupBg] = kBgElevated; - c[ImGuiCol_Border] = kBorderSoft; - c[ImGuiCol_BorderShadow] = ImVec4(0, 0, 0, 0); - c[ImGuiCol_FrameBg] = kBgInput; - c[ImGuiCol_FrameBgHovered] = kBgHover; - c[ImGuiCol_FrameBgActive] = kBgActive; - c[ImGuiCol_TitleBg] = kBgBase; - c[ImGuiCol_TitleBgActive] = kBgBase; - c[ImGuiCol_TitleBgCollapsed] = kBgBase; - c[ImGuiCol_MenuBarBg] = kBgBase; - c[ImGuiCol_ScrollbarBg] = ImVec4(0, 0, 0, 0); - c[ImGuiCol_ScrollbarGrab] = kBgActive; - c[ImGuiCol_ScrollbarGrabHovered] = kBorder; - c[ImGuiCol_ScrollbarGrabActive] = kAccentSoft; - c[ImGuiCol_CheckMark] = kAccent; - c[ImGuiCol_SliderGrab] = kAccent; - c[ImGuiCol_SliderGrabActive] = kAccent; - c[ImGuiCol_Button] = kBgInput; - c[ImGuiCol_ButtonHovered] = kBgHover; - c[ImGuiCol_ButtonActive] = kBgActive; - c[ImGuiCol_Header] = kAccentFaint; - c[ImGuiCol_HeaderHovered] = kAccentSoft; - c[ImGuiCol_HeaderActive] = kAccentSoft; - c[ImGuiCol_Separator] = kBorderSoft; - c[ImGuiCol_SeparatorHovered] = kBorder; - c[ImGuiCol_SeparatorActive] = kAccent; - c[ImGuiCol_ResizeGrip] = ImVec4(0, 0, 0, 0); - c[ImGuiCol_ResizeGripHovered] = kAccentSoft; - c[ImGuiCol_ResizeGripActive] = kAccent; - c[ImGuiCol_Tab] = kBgBase; - c[ImGuiCol_TabHovered] = kBgElevated; - c[ImGuiCol_TabActive] = kBgSurface; - c[ImGuiCol_TabUnfocused] = kBgBase; - c[ImGuiCol_TabUnfocusedActive] = kBgSurface; - c[ImGuiCol_DockingPreview] = kAccentSoft; - c[ImGuiCol_DockingEmptyBg] = kBgBase; - c[ImGuiCol_PlotLines] = kAccent; - c[ImGuiCol_PlotLinesHovered] = kAccent; - c[ImGuiCol_PlotHistogram] = kAccent; - c[ImGuiCol_PlotHistogramHovered] = kAccent; - c[ImGuiCol_TableHeaderBg] = kBgBase; - c[ImGuiCol_TableBorderStrong] = kBorderSoft; - c[ImGuiCol_TableBorderLight] = kBorderSoft; - c[ImGuiCol_TableRowBg] = ImVec4(0, 0, 0, 0); - c[ImGuiCol_TableRowBgAlt] = kIsDarkTheme - ? ImVec4(1, 1, 1, 0.015f) - : ImVec4(0, 0, 0, 0.020f); - c[ImGuiCol_TextSelectedBg] = kAccentSoft; - c[ImGuiCol_NavHighlight] = kAccent; - - // All style metrics above are authored at 100% scale; stretch them to - // the monitor's DPI so padding and spacing match the scaled fonts. - style.ScaleAllSizes(m_contentScale); - - // ImPlot — minimal, removes most chrome. - ImPlotStyle& ps = ImPlot::GetStyle(); - ps.PlotPadding = ImVec2(8, 8); - ps.LabelPadding = ImVec2(4, 4); - ps.LegendPadding = ImVec2(8, 6); - ps.PlotBorderSize = 0.0f; - ps.MajorTickLen = ImVec2(0, 0); - ps.MinorTickLen = ImVec2(0, 0); - ps.MajorTickSize = ImVec2(0, 0); - ps.MinorTickSize = ImVec2(0, 0); - ImVec4* pc = ps.Colors; - pc[ImPlotCol_FrameBg] = ImVec4(0, 0, 0, 0); - pc[ImPlotCol_PlotBg] = ImVec4(0, 0, 0, 0); - pc[ImPlotCol_PlotBorder] = ImVec4(0, 0, 0, 0); - pc[ImPlotCol_AxisBg] = ImVec4(0, 0, 0, 0); - pc[ImPlotCol_AxisGrid] = ImVec4(kBorder.x, kBorder.y, kBorder.z, 0.60f); - pc[ImPlotCol_AxisText] = kText3; - pc[ImPlotCol_TitleText] = kText1; - pc[ImPlotCol_LegendBg] = kBgElevated; - pc[ImPlotCol_LegendBorder] = kBorderSoft; - pc[ImPlotCol_LegendText] = kText1; - } - - void GuiLayer::run() { - while (!glfwWindowShouldClose(m_window)) { - // While minimized there is nothing to draw: keep draining the - // packet queue at a low rate so counters and flows stay current, - // but skip rendering entirely. - if (glfwGetWindowAttrib(m_window, GLFW_ICONIFIED) == GLFW_TRUE) { - glfwWaitEventsTimeout(0.1); - processQueue(); - continue; - } - - // Frame pacing: run at full (vsync) rate only while live traffic - // is streaming or the user recently interacted. Otherwise sleep on - // the event queue — any input wakes the loop immediately, so the - // UI stays responsive while an idle window stops burning CPU/GPU. - const bool streaming = m_captureActive && !m_capturePaused; - const bool recentInput = glfwGetTime() - m_lastInputTime < 0.75; - if (streaming || recentInput) { - glfwPollEvents(); - } else { - glfwWaitEventsTimeout(1.0 / 15.0); - } - processQueue(); - - // Track the last normal (non-maximized) window rect so it can be - // persisted on exit. - captureWindowGeometry(); - - ImGui_ImplOpenGL3_NewFrame(); - ImGui_ImplGlfw_NewFrame(); - ImGui::NewFrame(); - - const ImGuiIO& io = ImGui::GetIO(); - if (io.MouseDelta.x != 0.0f || io.MouseDelta.y != 0.0f - || io.MouseWheel != 0.0f || io.MouseWheelH != 0.0f - || ImGui::IsAnyMouseDown() || io.WantTextInput) { - m_lastInputTime = glfwGetTime(); - } - - renderUI(); - - ImGui::Render(); - int display_w, display_h; - glfwGetFramebufferSize(m_window, &display_w, &display_h); - glViewport(0, 0, display_w, display_h); - glClearColor(kBgBase.x, kBgBase.y, kBgBase.z, 1.00f); - glClear(GL_COLOR_BUFFER_BIT); - ImGui_ImplOpenGL3_RenderDrawData(ImGui::GetDrawData()); - - glfwSwapBuffers(m_window); - } - } - - void GuiLayer::setPaused(bool paused) { - if (m_capturePaused == paused) return; - m_capturePaused = paused; - if (!paused) { - // Packets that piled up while paused are stale — the engine - // retains them for PCAP export, but the live view should resume - // with fresh traffic instead of replaying the backlog. - m_queue.clear(); - m_bytesThisSec = 0; - m_tcpBytesThisSec = 0; - m_udpBytesThisSec = 0; - m_lastUpdateTime = glfwGetTime(); - } - } - - void GuiLayer::linearizeBuffer(const ScrollingBuffer& buffer, - std::vector& time, std::vector& data) { - time.clear(); - data.clear(); - const size_t sampleCount = buffer.Data.size(); - if (sampleCount == 0) return; - const size_t start = sampleCount == static_cast(buffer.MaxSize) - ? static_cast(buffer.Offset) - : 0; - time.reserve(sampleCount); - data.reserve(sampleCount); - for (size_t i = 0; i < sampleCount; ++i) { - const size_t index = (start + i) % sampleCount; - time.push_back(buffer.Time[index]); - data.push_back(buffer.Data[index]); - } - } - - void GuiLayer::processQueue() { - if (m_capturePaused) return; - - // Process up to 1000 packets per frame to maintain responsiveness - constexpr size_t maxPacketHistory = 10'000; - int count = 0; - double currentTime = glfwGetTime(); - - while (count < 1000) { - auto packetOpt = m_queue.try_pop(); - if (!packetOpt) break; - - // Decode, recover cross-packet SNI, harvest DNS answers, attribute - // the owning process, and fold the packet into its flow. - auto analyzed = m_session.feed(*packetOpt); - - // Keep the visual history bounded while retaining all-time counters separately. - if (m_packetHistory.size() >= maxPacketHistory) { - m_packetHistory.pop_front(); - if (m_selectedPacketIndex >= 0) --m_selectedPacketIndex; - } - m_packetHistory.push_back({*packetOpt, std::move(analyzed.parsed), - std::move(analyzed.process)}); - ++m_packetHistoryVersion; - const core::ParsedPacket& stored = m_packetHistory.back().parsed; - - ++m_totalPackets; - m_bytesThisSec += stored.length; - if (stored.protocol == "TCP") { - m_tcpBytesThisSec += stored.length; - } else if (stored.protocol == "UDP") { - m_udpBytesThisSec += stored.length; - } - m_protocolCounts[stored.protocol]++; - m_protocolBytes[stored.protocol] += stored.length; - if (!stored.service.empty()) { - m_appCounts[stored.service]++; - } - - count++; - } - - // Update bandwidth chart every 0.5s. - if (currentTime - m_lastUpdateTime >= 0.5) { - const auto toMbps = [](uint64_t bytes) { - return static_cast(bytes) / (1024.0f * 1024.0f) / 0.5f; - }; - const float mbps = toMbps(m_bytesThisSec); - m_bandwidthData.AddPoint((float)currentTime, mbps); - m_tcpBandwidth.AddPoint((float)currentTime, toMbps(m_tcpBytesThisSec)); - m_udpBandwidth.AddPoint((float)currentTime, toMbps(m_udpBytesThisSec)); - m_currentMbps = mbps; - if (mbps > m_peakMbps) m_peakMbps = mbps; - m_bytesThisSec = 0; - m_tcpBytesThisSec = 0; - m_udpBytesThisSec = 0; - m_lastUpdateTime = currentTime; - } - - // Refresh KPI cache (cheap derived values). - m_topService.clear(); - m_topServiceCount = 0; - for (const auto& [name, c] : m_appCounts) { - if (c > m_topServiceCount) { m_topService = name; m_topServiceCount = c; } - } - } - - void GuiLayer::handleShortcuts() { - ImGuiIO& io = ImGui::GetIO(); - if (ImGui::IsKeyChordPressed(ImGuiMod_Ctrl | ImGuiKey_O)) { - openPcapDialog(); - } - // Text-editing keys only act as shortcuts when no input field owns them. - if (!io.WantTextInput) { - if (ImGui::IsKeyPressed(ImGuiKey_Space, false)) { - setPaused(!m_capturePaused); - } - if (ImGui::IsKeyPressed(ImGuiKey_Slash, false)) { - m_focusDisplayFilter = true; - } - } - } - - void GuiLayer::renderUI() { - handleShortcuts(); - - // Host window that owns the menu bar, control bar, and dockspace. - const ImGuiViewport* viewport = ImGui::GetMainViewport(); - ImGui::SetNextWindowPos(viewport->WorkPos); - ImGui::SetNextWindowSize(viewport->WorkSize); - ImGui::SetNextWindowViewport(viewport->ID); - - const ImGuiWindowFlags hostFlags = - ImGuiWindowFlags_NoTitleBar | ImGuiWindowFlags_NoCollapse | - ImGuiWindowFlags_NoResize | ImGuiWindowFlags_NoMove | - ImGuiWindowFlags_NoBringToFrontOnFocus | ImGuiWindowFlags_NoNavFocus | - ImGuiWindowFlags_NoDocking | ImGuiWindowFlags_MenuBar; - - ImGui::PushStyleVar(ImGuiStyleVar_WindowRounding, 0.0f); - ImGui::PushStyleVar(ImGuiStyleVar_WindowBorderSize, 0.0f); - ImGui::PushStyleVar(ImGuiStyleVar_WindowPadding, ImVec2(0, 0)); - ImGui::Begin("##NetProbeHost", nullptr, hostFlags); - ImGui::PopStyleVar(3); - - renderTopBar(); - renderControlBar(); - renderEncryptedDnsNotice(); - - const ImGuiID dockspaceId = ImGui::GetID("NetProbeDockspace"); - if (!m_layoutBuilt) { - m_layoutBuilt = true; - ImGuiDockNode* root = ImGui::DockBuilderGetNode(dockspaceId); - const bool needsLayout = !root || root->IsLeafNode(); - if (needsLayout) { - ImGui::DockBuilderRemoveNode(dockspaceId); - ImGui::DockBuilderAddNode(dockspaceId, ImGuiDockNodeFlags_DockSpace); - ImGui::DockBuilderSetNodeSize(dockspaceId, ImGui::GetContentRegionAvail()); - - ImGuiID topId, bottomId; - ImGui::DockBuilderSplitNode(dockspaceId, ImGuiDir_Up, 0.50f, &topId, &bottomId); - - ImGuiID flowsId, rightId; - ImGui::DockBuilderSplitNode(bottomId, ImGuiDir_Left, 0.45f, &flowsId, &rightId); - - ImGuiID packetsId, detailId; - ImGui::DockBuilderSplitNode(rightId, ImGuiDir_Up, 0.55f, &packetsId, &detailId); - - ImGui::DockBuilderDockWindow("Dashboard", topId); - ImGui::DockBuilderDockWindow("Statistics", topId); - ImGui::DockBuilderDockWindow("Flows", flowsId); - ImGui::DockBuilderDockWindow("Live Packets", packetsId); - ImGui::DockBuilderDockWindow("Packet Detail", detailId); - ImGui::DockBuilderFinish(dockspaceId); - - // Keep the Dashboard tab in front of Statistics on first run. - if (ImGuiDockNode* topNode = ImGui::DockBuilderGetNode(topId)) { - topNode->SelectedTabId = ImHashStr("Dashboard"); - } - } - } - ImGui::DockSpace(dockspaceId, ImVec2(0, 0), ImGuiDockNodeFlags_None); - - ImGui::End(); - - renderCharts(); - renderStatsView(); - renderFlowsTable(); - renderPacketTable(); - renderPacketDetail(); - } - - void GuiLayer::setDevices(const std::vector& devices) { - m_devices = devices; - } - - void GuiLayer::clearCaptureView() { - m_packetHistory.clear(); - ++m_packetHistoryVersion; - m_filteredPacketIndexes.clear(); - m_filterCacheHistoryVersion = UINT64_MAX; - m_flowsCache.clear(); - m_flowGeoCache.clear(); - m_lastFlowsRefresh = -1.0; - m_maxFlowBytes = 0; - m_activeFlowCount = 0; - m_selectedPacketIndex = -1; - m_appCounts.clear(); - m_protocolCounts.clear(); - m_protocolBytes.clear(); - m_session.clear(); - m_packetFlowFilter.reset(); - m_flowRateHistory.Erase(); - m_flowRateKey.reset(); - m_bandwidthData.Erase(); - m_tcpBandwidth.Erase(); - m_udpBandwidth.Erase(); - m_bytesThisSec = 0; - m_tcpBytesThisSec = 0; - m_udpBytesThisSec = 0; - m_totalPackets = 0; - m_lastUpdateTime = glfwGetTime(); - } - - void GuiLayer::openPcapFile(const std::string& path) { - if (path.empty()) return; - clearCaptureView(); - setPaused(false); - m_captureActive = false; - if (onPcapFileSelected) { - onPcapFileSelected(path); - } - } - - void GuiLayer::openPcapDialog() { - if (!m_nfdInitialized) return; - const nfdu8filteritem_t filters[] = { - {"Packet captures", "pcap,pcapng,cap"}, - }; - nfdu8char_t* selectedPath = nullptr; - const nfdresult_t result = NFD_OpenDialogU8(&selectedPath, filters, 1, nullptr); - if (result == NFD_OKAY) { - openPcapFile(selectedPath); - NFD_FreePathU8(selectedPath); - } else if (result == NFD_ERROR) { - std::cerr << "Unable to open file dialog: " << NFD_GetError() << std::endl; - } - } - - void GuiLayer::renderTopBar() { - if (!ImGui::BeginMenuBar()) return; - - // Brand - if (m_fontBrand) ImGui::PushFont(m_fontBrand); - ImGui::TextColored(kText1, "NetProbe"); - if (m_fontBrand) ImGui::PopFont(); - - // Capture state indicator: pulsing green when live traffic is being - // consumed, amber when paused, muted when idle. - ImGui::SameLine(0.0f, 14.0f); - { - const bool paused = m_capturePaused; - const bool live = !paused && m_currentMbps > 0.0001f; - const float pulse = live - ? 0.55f + 0.45f * (0.5f + 0.5f * std::sin(static_cast(glfwGetTime()) * 3.0f)) - : 1.0f; - const ImVec4 dotColor = paused - ? kWarning - : live - ? ImVec4(kSuccess.x, kSuccess.y, kSuccess.z, pulse) - : ImVec4(kText3.x, kText3.y, kText3.z, 1.0f); - const float r = 4.0f; - const ImVec2 cursor = ImGui::GetCursorScreenPos(); - ImGui::GetWindowDrawList()->AddCircleFilled( - ImVec2(cursor.x + r, cursor.y + ImGui::GetTextLineHeight() * 0.5f), - r, ImGui::GetColorU32(dotColor)); - ImGui::Dummy(ImVec2(r * 2.0f + 6.0f, ImGui::GetTextLineHeight())); - ImGui::SameLine(); - const char* label = paused ? "Paused" : live ? "Live" : "Idle"; - ImGui::TextColored(paused || live ? kText2 : kText3, "%s", label); - } - - ImGui::SameLine(0.0f, 18.0f); - if (ImGui::BeginMenu("File")) { - if (ImGui::MenuItem("Open PCAP...", "Ctrl+O", false, m_nfdInitialized)) { - openPcapDialog(); - } - if (ImGui::MenuItem("Save PCAP...", nullptr, false, m_nfdInitialized)) { - const nfdu8filteritem_t filters[] = { - {"Packet captures", "pcap"}, - }; - nfdu8char_t* selectedPath = nullptr; - const nfdresult_t result = NFD_SaveDialogU8( - &selectedPath, filters, 1, nullptr, "netprobe-session.pcap"); - if (result == NFD_OKAY) { - std::string error; - if (onPcapSaveRequested && onPcapSaveRequested(selectedPath, error)) { - setStatus("PCAP session saved.", false); - } else { - setStatus(error.empty() ? "Unable to save the PCAP session." : error, true); - } - NFD_FreePathU8(selectedPath); - } else if (result == NFD_ERROR) { - setStatus(NFD_GetError() ? NFD_GetError() : "Unable to open the save dialog.", true); - } - } - if (ImGui::MenuItem("Export Flows CSV...", nullptr, false, m_nfdInitialized)) { - const nfdu8filteritem_t filters[] = { - {"CSV", "csv"}, - }; - nfdu8char_t* selectedPath = nullptr; - const nfdresult_t result = NFD_SaveDialogU8( - &selectedPath, filters, 1, nullptr, "netprobe-flows.csv"); - if (result == NFD_OKAY) { - std::string error; - if (exportFlowsCsv(selectedPath, error)) { - setStatus("Flows exported to CSV.", false); - } else { - setStatus(error.empty() ? "Unable to export flows." : error, true); - } - NFD_FreePathU8(selectedPath); - } else if (result == NFD_ERROR) { - setStatus(NFD_GetError() ? NFD_GetError() : "Unable to open the save dialog.", true); - } - } - ImGui::Separator(); - if (ImGui::MenuItem("Exit", "Alt+F4")) glfwSetWindowShouldClose(m_window, true); - ImGui::EndMenu(); - } - if (ImGui::BeginMenu("View")) { - if (ImGui::MenuItem("Dark theme", nullptr, m_darkTheme) && !m_darkTheme) { - m_darkTheme = true; - applyTheme(); - saveSettings(); - } - if (ImGui::MenuItem("Light theme", nullptr, !m_darkTheme) && m_darkTheme) { - m_darkTheme = false; - applyTheme(); - saveSettings(); - } - ImGui::Separator(); - if (ImGui::BeginMenu("UI scale")) { - constexpr float scales[] = {0.9f, 1.0f, 1.1f, 1.25f}; - const char* labels[] = {"90%", "100%", "110%", "125%"}; - for (int i = 0; i < 4; ++i) { - const bool selected = std::abs(m_uiScale - scales[i]) < 0.01f; - if (ImGui::MenuItem(labels[i], nullptr, selected) && !selected) { - m_uiScale = scales[i]; - ImGui::GetStyle().FontScaleMain = m_uiScale; - saveSettings(); - } - } - ImGui::EndMenu(); - } - ImGui::Separator(); - if (ImGui::MenuItem("Reset layout")) { - m_layoutBuilt = false; - } - ImGui::EndMenu(); - } - - ImGui::EndMenuBar(); - } - - void GuiLayer::renderControlBar() { - ImGui::PushStyleColor(ImGuiCol_ChildBg, kBgBase); - ImGui::PushStyleVar(ImGuiStyleVar_WindowPadding, ImVec2(18, 10)); - ImGui::PushStyleVar(ImGuiStyleVar_ChildBorderSize, 0.0f); - const float barHeight = ImGui::GetFrameHeightWithSpacing() + 6.0f; - - ImGui::BeginChild("##controlBar", ImVec2(0, barHeight), false, ImGuiWindowFlags_NoScrollbar); - - // Adapter selector - ImGui::TextColored(kText3, "ADAPTER"); - ImGui::SameLine(); - ImGui::SetNextItemWidth(scaled(300.0f)); - if (m_devices.empty()) { - ImGui::BeginDisabled(); - ImGui::BeginCombo("##deviceCombo", "No adapters detected"); - ImGui::EndCombo(); - ImGui::EndDisabled(); - } else if (ImGui::BeginCombo("##deviceCombo", m_devices[m_selectedDeviceIndex].description.c_str())) { - for (int n = 0; n < static_cast(m_devices.size()); n++) { - const bool is_selected = (m_selectedDeviceIndex == n); - if (ImGui::Selectable(m_devices[n].description.c_str(), is_selected)) { - m_selectedDeviceIndex = n; - clearCaptureView(); - setPaused(false); - if (onDeviceSelected) { - onDeviceSelected(m_devices[n].name); - m_captureActive = true; - setStatus("Capture started.", false); - } - } - if (is_selected) ImGui::SetItemDefaultFocus(); - } - ImGui::EndCombo(); - } - - // Start / Stop / Pause capture controls. - ImGui::SameLine(0.0f, 10.0f); - if (m_captureActive) { - ImGui::PushStyleColor(ImGuiCol_Text, kDanger); - if (ImGui::Button("Stop")) { - if (onCaptureStopRequested) onCaptureStopRequested(); - m_captureActive = false; - setStatus("Capture stopped.", false); - } - ImGui::PopStyleColor(); - } else { - ImGui::BeginDisabled(m_devices.empty()); - ImGui::PushStyleColor(ImGuiCol_Text, kSuccess); - if (ImGui::Button("Start")) { - clearCaptureView(); - setPaused(false); - if (onDeviceSelected) { - onDeviceSelected(m_devices[m_selectedDeviceIndex].name); - m_captureActive = true; - setStatus("Capture started.", false); - } - } - ImGui::PopStyleColor(); - ImGui::EndDisabled(); - } - ImGui::SetItemTooltip(m_captureActive - ? "Stop the live capture (retained packets stay available for export)" - : "Start capturing on the selected adapter"); - - ImGui::SameLine(0.0f, 6.0f); - if (ImGui::Button(m_capturePaused ? "Resume" : "Pause")) { - setPaused(!m_capturePaused); - } - ImGui::SetItemTooltip("Freeze the live view without stopping capture (Space)"); - - ImGui::SameLine(0.0f, 24.0f); - ImGui::TextColored(kText3, "FILTER"); - ImGui::SameLine(); - ImGui::SetNextItemWidth(scaled(240.0f)); - if (ImGui::InputTextWithHint("##bpfFilter", "tcp port 443", - m_bpfFilter, sizeof(m_bpfFilter), ImGuiInputTextFlags_EnterReturnsTrue)) { - std::string error; - if (onBpfFilterRequested && onBpfFilterRequested(m_bpfFilter, error)) { - setStatus(m_bpfFilter[0] == '\0' ? "BPF filter cleared." : "BPF filter applied.", false); - } else { - setStatus(error.empty() ? "Unable to apply BPF filter." : error, true); - } - } - ImGui::SetItemTooltip("Capture filter (BPF syntax) — press Enter to apply.\nOnly affects what is captured, not what is shown."); - - // Status messages expire after a few seconds instead of lingering - // until the next action replaces them. - if (!m_captureStatus.empty() && glfwGetTime() - m_captureStatusTime > 6.0) { - m_captureStatus.clear(); - } - if (!m_captureStatus.empty()) { - ImGui::SameLine(0.0f, 18.0f); - ImGui::TextColored(m_captureStatusIsError ? kDanger : kSuccess, "%s", m_captureStatus.c_str()); - } - - // Right-aligned overrun badge so users see when the UI is dropping. - if (const size_t dropped = m_queue.droppedPackets(); dropped > 0) { - const std::string badge = std::format("{} dropped", formatCount(dropped)); - const float w = ImGui::CalcTextSize(badge.c_str()).x; - ImGui::SameLine(ImGui::GetContentRegionMax().x - w - 18.0f); - ImGui::TextColored(kWarning, "%s", badge.c_str()); - ImGui::SetItemTooltip("Packets evicted from the UI queue because capture outpaced the display.\nThe oldest packets are dropped first; counters keep running."); - } - - ImGui::EndChild(); - ImGui::PopStyleVar(2); - ImGui::PopStyleColor(); - - // Subtle separator line below the control bar. - const ImVec2 lineP = ImGui::GetCursorScreenPos(); - ImGui::GetWindowDrawList()->AddLine( - lineP, ImVec2(lineP.x + ImGui::GetContentRegionAvail().x, lineP.y), - ImGui::GetColorU32(kBorderSoft)); - } - - bool GuiLayer::encryptedDnsDominates() const { - // A handful of DoH packets alongside healthy plaintext DNS is normal. - // The notice is for the case where name resolution has genuinely moved - // off the wire and the hostname columns are going to stay empty. - // - // The bar is low deliberately: a DoH connection is identified by its - // ClientHello SNI, and browsers reuse one connection for every lookup, - // so a handful of sightings can already mean all resolution is hidden. - constexpr uint64_t minimumEvidence = 3; - return m_session.encryptedDnsPackets() >= minimumEvidence - && m_session.encryptedDnsPackets() > m_session.plaintextDnsResponses() * 2; - } - - void GuiLayer::renderEncryptedDnsNotice() { - // ECH alone is not banner-worthy — most Cloudflare-fronted domains - // advertise it, so it would show on nearly every capture. It is reported - // as a standing fact in the Statistics view instead. - const bool show = !m_dnsNoticeDismissed && encryptedDnsDominates(); - if (!show) return; - - ImGui::PushStyleColor(ImGuiCol_ChildBg, - ImVec4(kWarning.x, kWarning.y, kWarning.z, kIsDarkTheme ? 0.10f : 0.16f)); - ImGui::PushStyleColor(ImGuiCol_Border, - ImVec4(kWarning.x, kWarning.y, kWarning.z, 0.45f)); - ImGui::PushStyleVar(ImGuiStyleVar_ChildRounding, 8.0f); - ImGui::PushStyleVar(ImGuiStyleVar_ChildBorderSize, 1.0f); - ImGui::PushStyleVar(ImGuiStyleVar_WindowPadding, ImVec2(14, 8)); - - const float height = ImGui::GetFrameHeight() + 20.0f; - ImGui::SetCursorPosX(ImGui::GetCursorPosX() + 18.0f); - ImGui::BeginChild("##dnsNotice", - ImVec2(ImGui::GetContentRegionAvail().x - 18.0f, height), true, - ImGuiWindowFlags_NoScrollbar); - - drawDot(kWarning, 4.0f); - ImGui::SameLine(); - ImGui::TextColored(kText1, "Encrypted DNS in use."); - ImGui::SameLine(); - ImGui::TextColored(kText2, m_session.echAdvertised() - ? "Name resolution is hidden, and Encrypted Client Hello was advertised, so some hostnames will show only an IP." - : "Name resolution is not visible on the wire, so hostnames come only from TLS/QUIC SNI and may be incomplete."); - - ImGui::SameLine(ImGui::GetContentRegionMax().x - 70.0f); - if (ImGui::SmallButton("Dismiss")) { - m_dnsNoticeDismissed = true; - } - - ImGui::EndChild(); - ImGui::PopStyleVar(3); - ImGui::PopStyleColor(2); - ImGui::Dummy(ImVec2(0, 4)); - } - -} // namespace ui +#include "ui/GuiLayer.hpp" +#include "ui/GuiTheme.hpp" +#include "ui/EmbeddedIcon.hpp" +#include "imgui.h" +#include "imgui_internal.h" +#include "imgui_impl_glfw.h" +#include "imgui_impl_opengl3.h" +#include "implot.h" +#include "nfd.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ui { + + GuiLayer::GuiLayer(core::PacketQueue& queue) : m_queue(queue) {} + + GuiLayer::~GuiLayer() { + // Persisting preferences touches the filesystem, and an exception + // escaping a destructor terminates the process. Losing the saved + // window geometry is an acceptable outcome; crashing on exit is not. + try { + captureWindowGeometry(); + saveSettings(); + } catch (...) { + // Reported with fputs rather than iostreams: this runs during + // shutdown, where anything that can throw defeats the point. + std::fputs("NetProbe: could not save window geometry or preferences.\n", stderr); + } + + if (m_window) { + ImGui_ImplOpenGL3_Shutdown(); + ImGui_ImplGlfw_Shutdown(); + ImPlot::DestroyContext(); + ImGui::DestroyContext(); + glfwDestroyWindow(m_window); + glfwTerminate(); + } + if (m_nfdInitialized) { + NFD_Quit(); + } + } + + bool GuiLayer::init() { + if (!glfwInit()) return false; + + // Settings are needed before window creation: they carry the saved + // window geometry and maximized state from the previous run. + loadSettings(); + + const char* glsl_version = "#version 130"; + glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 3); + glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 0); + glfwWindowHint(GLFW_VISIBLE, GLFW_FALSE); + if (m_savedWindowMaximized) glfwWindowHint(GLFW_MAXIMIZED, GLFW_TRUE); + + // A saved size is already in physical pixels; the default size is + // authored for 100% scale and multiplied by the primary monitor's + // content scale so first launch isn't tiny on high-DPI displays. + int width = 1440, height = 900; + if (m_savedWindowW > 0 && m_savedWindowH > 0) { + width = std::clamp(m_savedWindowW, 800, 16384); + height = std::clamp(m_savedWindowH, 500, 16384); + } else if (GLFWmonitor* primary = glfwGetPrimaryMonitor()) { + float scaleX = 1.0f, scaleY = 1.0f; + glfwGetMonitorContentScale(primary, &scaleX, &scaleY); + width = static_cast(width * scaleX); + height = static_cast(height * scaleY); + } + + m_window = glfwCreateWindow(width, height, "NetProbe - Network Analyzer", nullptr, nullptr); + if (!m_window) { + glfwTerminate(); + return false; + } + + // Restore the previous window position, but only when its center still + // falls on a connected monitor — displays come and go between runs. + if (m_hasSavedWindowPos && !m_savedWindowMaximized) { + int monitorCount = 0; + GLFWmonitor** monitors = glfwGetMonitors(&monitorCount); + const int centerX = m_savedWindowX + width / 2; + const int centerY = m_savedWindowY + height / 2; + for (int i = 0; i < monitorCount; ++i) { + int mx, my, mw, mh; + glfwGetMonitorWorkarea(monitors[i], &mx, &my, &mw, &mh); + if (centerX >= mx && centerX < mx + mw && centerY >= my && centerY < my + mh) { + glfwSetWindowPos(m_window, m_savedWindowX, m_savedWindowY); + break; + } + } + } + + // Title-bar and taskbar icon — the embedded .ico covers Explorer / + // shortcuts, but GLFW windows need an explicit set call to override + // the default OS window icon while the app is running. + GLFWimage icons[3]; + icons[0] = {kEmbeddedIcon16Size, kEmbeddedIcon16Size, + const_cast(kEmbeddedIcon16Pixels)}; + icons[1] = {kEmbeddedIcon32Size, kEmbeddedIcon32Size, + const_cast(kEmbeddedIcon32Pixels)}; + icons[2] = {kEmbeddedIcon48Size, kEmbeddedIcon48Size, + const_cast(kEmbeddedIcon48Pixels)}; + glfwSetWindowIcon(m_window, 3, icons); + glfwShowWindow(m_window); + + glfwMakeContextCurrent(m_window); + glfwSwapInterval(1); // Enable vsync + glfwSetWindowUserPointer(m_window, this); + + // Per-monitor DPI: pick up the content scale of whichever monitor the + // window actually landed on, and re-theme when it moves to another. + float contentScale = 1.0f; + glfwGetWindowContentScale(m_window, &contentScale, nullptr); + if (contentScale > 0.0f) m_contentScale = contentScale; + glfwSetWindowContentScaleCallback(m_window, + [](GLFWwindow* window, float xScale, float) { + auto* gui = static_cast(glfwGetWindowUserPointer(window)); + if (gui && xScale > 0.0f && std::abs(xScale - gui->m_contentScale) > 0.01f) { + gui->m_contentScale = xScale; + gui->applyTheme(); + } + }); + glfwSetDropCallback(m_window, [](GLFWwindow* window, int pathCount, const char** paths) { + auto* gui = static_cast(glfwGetWindowUserPointer(window)); + if (gui && pathCount > 0 && paths[0]) { + gui->openPcapFile(paths[0]); + } + }); + + m_nfdInitialized = NFD_Init() == NFD_OKAY; + if (!m_nfdInitialized) { + std::cerr << "Native file dialog is unavailable: " << NFD_GetError() << '\n'; + } + + IMGUI_CHECKVERSION(); + ImGui::CreateContext(); + ImPlot::CreateContext(); + + ImGuiIO& io = ImGui::GetIO(); + io.ConfigFlags |= ImGuiConfigFlags_NavEnableKeyboard; + io.ConfigFlags |= ImGuiConfigFlags_DockingEnable; + + // Use a dedicated ini so a previously-saved imgui.ini with the legacy layout + // doesn't override the new default dockspace on the first run. + io.IniFilename = "netprobe-layout.ini"; + + loadFonts(); + applyTheme(); + + ImGui_ImplGlfw_InitForOpenGL(m_window, true); + ImGui_ImplOpenGL3_Init(glsl_version); + + return true; + } + + void GuiLayer::loadFonts() { + ImGuiIO& io = ImGui::GetIO(); + ImFontConfig cfg; + cfg.OversampleH = 3; + cfg.OversampleV = 1; + cfg.PixelSnapH = false; + + const std::filesystem::path regular = "C:/Windows/Fonts/segoeui.ttf"; + const std::filesystem::path semibold = "C:/Windows/Fonts/seguisb.ttf"; + const std::filesystem::path mono = "C:/Windows/Fonts/consola.ttf"; + + if (std::filesystem::exists(regular)) { + m_fontDefault = io.Fonts->AddFontFromFileTTF(regular.string().c_str(), 15.0f, &cfg); + m_fontSmall = io.Fonts->AddFontFromFileTTF(regular.string().c_str(), 12.0f, &cfg); + const auto& semiPath = std::filesystem::exists(semibold) ? semibold : regular; + m_fontHeadline = io.Fonts->AddFontFromFileTTF(semiPath.string().c_str(), 28.0f, &cfg); + m_fontBrand = io.Fonts->AddFontFromFileTTF(semiPath.string().c_str(), 16.0f, &cfg); + m_fontMono = std::filesystem::exists(mono) + ? io.Fonts->AddFontFromFileTTF(mono.string().c_str(), 13.0f, &cfg) + : m_fontDefault; + } else { + // Fall back to the bundled bitmap font if Segoe UI is unavailable. + m_fontDefault = io.Fonts->AddFontDefault(); + m_fontSmall = m_fontDefault; + m_fontHeadline = m_fontDefault; + m_fontBrand = m_fontDefault; + m_fontMono = m_fontDefault; + } + } + + void GuiLayer::loadSettings() { + std::ifstream file("netprobe-settings.ini"); + if (!file) return; + std::string line; + while (std::getline(file, line)) { + const auto eq = line.find('='); + if (eq == std::string::npos) continue; + const std::string key = line.substr(0, eq); + const std::string value = line.substr(eq + 1); + if (key == "theme") { + m_darkTheme = value != "light"; + } else if (key == "scale") { + try { + m_uiScale = std::clamp(std::stof(value), 0.75f, 1.5f); + } catch (...) { + m_uiScale = 1.0f; + } + } else if (key == "window") { + int x = 0, y = 0, w = 0, h = 0, maximized = 0; + // Extracted rather than scanf'd: type-safe, and it avoids the + // MSVC deprecation of sscanf without reaching for the + // Windows-only sscanf_s. + std::istringstream fields(value); + if (fields >> x >> y >> w >> h >> maximized) { + m_savedWindowX = x; + m_savedWindowY = y; + m_savedWindowW = w; + m_savedWindowH = h; + m_savedWindowMaximized = maximized != 0; + m_hasSavedWindowPos = true; + } + } + } + } + + void GuiLayer::saveSettings() const { + std::ofstream file("netprobe-settings.ini", std::ios::trunc); + if (!file) return; + file << "theme=" << (m_darkTheme ? "dark" : "light") << '\n'; + file << "scale=" << m_uiScale << '\n'; + if (m_savedWindowW > 0 && m_savedWindowH > 0) { + file << "window=" << m_savedWindowX << ' ' << m_savedWindowY << ' ' + << m_savedWindowW << ' ' << m_savedWindowH << ' ' + << (m_savedWindowMaximized ? 1 : 0) << '\n'; + } + } + + void GuiLayer::captureWindowGeometry() { + if (!m_window) return; + m_savedWindowMaximized = glfwGetWindowAttrib(m_window, GLFW_MAXIMIZED) == GLFW_TRUE; + // While maximized (or minimized) the current rect is not the one worth + // restoring — keep the last normal geometry already recorded. + if (m_savedWindowMaximized + || glfwGetWindowAttrib(m_window, GLFW_ICONIFIED) == GLFW_TRUE) { + return; + } + glfwGetWindowPos(m_window, &m_savedWindowX, &m_savedWindowY); + glfwGetWindowSize(m_window, &m_savedWindowW, &m_savedWindowH); + m_hasSavedWindowPos = true; + } + + void GuiLayer::setStatus(std::string message, bool isError) { + m_captureStatus = std::move(message); + m_captureStatusIsError = isError; + m_captureStatusTime = glfwGetTime(); + } + + void GuiLayer::applyTheme() { + applyPalette(m_darkTheme); + + ImGuiStyle& style = ImGui::GetStyle(); + // Rebuild from defaults so repeated calls (theme toggle, monitor DPI + // change) never compound the ScaleAllSizes multiplication below. + style = ImGuiStyle(); + style.FontScaleMain = m_uiScale * m_contentScale; + + style.WindowPadding = ImVec2(18, 16); + style.FramePadding = ImVec2(12, 7); + style.CellPadding = ImVec2(10, 8); + style.ItemSpacing = ImVec2(10, 10); + style.ItemInnerSpacing = ImVec2(8, 6); + style.IndentSpacing = 22.0f; + style.ScrollbarSize = 10.0f; + style.GrabMinSize = 10.0f; + + style.WindowBorderSize = 1.0f; + style.ChildBorderSize = 1.0f; + style.FrameBorderSize = 0.0f; + style.PopupBorderSize = 1.0f; + style.TabBorderSize = 0.0f; + style.SeparatorTextBorderSize = 1.0f; + + style.WindowRounding = 10.0f; + style.ChildRounding = 8.0f; + style.FrameRounding = 7.0f; + style.PopupRounding = 8.0f; + style.GrabRounding = 7.0f; + style.TabRounding = 6.0f; + style.ScrollbarRounding = 6.0f; + + ImVec4* c = style.Colors; + c[ImGuiCol_Text] = kText1; + c[ImGuiCol_TextDisabled] = kText3; + c[ImGuiCol_WindowBg] = kBgBase; + c[ImGuiCol_ChildBg] = kBgBase; + c[ImGuiCol_PopupBg] = kBgElevated; + c[ImGuiCol_Border] = kBorderSoft; + c[ImGuiCol_BorderShadow] = ImVec4(0, 0, 0, 0); + c[ImGuiCol_FrameBg] = kBgInput; + c[ImGuiCol_FrameBgHovered] = kBgHover; + c[ImGuiCol_FrameBgActive] = kBgActive; + c[ImGuiCol_TitleBg] = kBgBase; + c[ImGuiCol_TitleBgActive] = kBgBase; + c[ImGuiCol_TitleBgCollapsed] = kBgBase; + c[ImGuiCol_MenuBarBg] = kBgBase; + c[ImGuiCol_ScrollbarBg] = ImVec4(0, 0, 0, 0); + c[ImGuiCol_ScrollbarGrab] = kBgActive; + c[ImGuiCol_ScrollbarGrabHovered] = kBorder; + c[ImGuiCol_ScrollbarGrabActive] = kAccentSoft; + c[ImGuiCol_CheckMark] = kAccent; + c[ImGuiCol_SliderGrab] = kAccent; + c[ImGuiCol_SliderGrabActive] = kAccent; + c[ImGuiCol_Button] = kBgInput; + c[ImGuiCol_ButtonHovered] = kBgHover; + c[ImGuiCol_ButtonActive] = kBgActive; + c[ImGuiCol_Header] = kAccentFaint; + c[ImGuiCol_HeaderHovered] = kAccentSoft; + c[ImGuiCol_HeaderActive] = kAccentSoft; + c[ImGuiCol_Separator] = kBorderSoft; + c[ImGuiCol_SeparatorHovered] = kBorder; + c[ImGuiCol_SeparatorActive] = kAccent; + c[ImGuiCol_ResizeGrip] = ImVec4(0, 0, 0, 0); + c[ImGuiCol_ResizeGripHovered] = kAccentSoft; + c[ImGuiCol_ResizeGripActive] = kAccent; + c[ImGuiCol_Tab] = kBgBase; + c[ImGuiCol_TabHovered] = kBgElevated; + c[ImGuiCol_TabActive] = kBgSurface; + c[ImGuiCol_TabUnfocused] = kBgBase; + c[ImGuiCol_TabUnfocusedActive] = kBgSurface; + c[ImGuiCol_DockingPreview] = kAccentSoft; + c[ImGuiCol_DockingEmptyBg] = kBgBase; + c[ImGuiCol_PlotLines] = kAccent; + c[ImGuiCol_PlotLinesHovered] = kAccent; + c[ImGuiCol_PlotHistogram] = kAccent; + c[ImGuiCol_PlotHistogramHovered] = kAccent; + c[ImGuiCol_TableHeaderBg] = kBgBase; + c[ImGuiCol_TableBorderStrong] = kBorderSoft; + c[ImGuiCol_TableBorderLight] = kBorderSoft; + c[ImGuiCol_TableRowBg] = ImVec4(0, 0, 0, 0); + c[ImGuiCol_TableRowBgAlt] = kIsDarkTheme + ? ImVec4(1, 1, 1, 0.015f) + : ImVec4(0, 0, 0, 0.020f); + c[ImGuiCol_TextSelectedBg] = kAccentSoft; + c[ImGuiCol_NavHighlight] = kAccent; + + // All style metrics above are authored at 100% scale; stretch them to + // the monitor's DPI so padding and spacing match the scaled fonts. + style.ScaleAllSizes(m_contentScale); + + // ImPlot — minimal, removes most chrome. + ImPlotStyle& ps = ImPlot::GetStyle(); + ps.PlotPadding = ImVec2(8, 8); + ps.LabelPadding = ImVec2(4, 4); + ps.LegendPadding = ImVec2(8, 6); + ps.PlotBorderSize = 0.0f; + ps.MajorTickLen = ImVec2(0, 0); + ps.MinorTickLen = ImVec2(0, 0); + ps.MajorTickSize = ImVec2(0, 0); + ps.MinorTickSize = ImVec2(0, 0); + ImVec4* pc = ps.Colors; + pc[ImPlotCol_FrameBg] = ImVec4(0, 0, 0, 0); + pc[ImPlotCol_PlotBg] = ImVec4(0, 0, 0, 0); + pc[ImPlotCol_PlotBorder] = ImVec4(0, 0, 0, 0); + pc[ImPlotCol_AxisBg] = ImVec4(0, 0, 0, 0); + pc[ImPlotCol_AxisGrid] = ImVec4(kBorder.x, kBorder.y, kBorder.z, 0.60f); + pc[ImPlotCol_AxisText] = kText3; + pc[ImPlotCol_TitleText] = kText1; + pc[ImPlotCol_LegendBg] = kBgElevated; + pc[ImPlotCol_LegendBorder] = kBorderSoft; + pc[ImPlotCol_LegendText] = kText1; + } + + void GuiLayer::run() { + while (!glfwWindowShouldClose(m_window)) { + // While minimized there is nothing to draw: keep draining the + // packet queue at a low rate so counters and flows stay current, + // but skip rendering entirely. + if (glfwGetWindowAttrib(m_window, GLFW_ICONIFIED) == GLFW_TRUE) { + glfwWaitEventsTimeout(0.1); + processQueue(); + continue; + } + + // Frame pacing: run at full (vsync) rate only while live traffic + // is streaming or the user recently interacted. Otherwise sleep on + // the event queue — any input wakes the loop immediately, so the + // UI stays responsive while an idle window stops burning CPU/GPU. + const bool streaming = m_captureActive && !m_capturePaused; + const bool recentInput = glfwGetTime() - m_lastInputTime < 0.75; + if (streaming || recentInput) { + glfwPollEvents(); + } else { + glfwWaitEventsTimeout(1.0 / 15.0); + } + processQueue(); + + // Track the last normal (non-maximized) window rect so it can be + // persisted on exit. + captureWindowGeometry(); + + ImGui_ImplOpenGL3_NewFrame(); + ImGui_ImplGlfw_NewFrame(); + ImGui::NewFrame(); + + const ImGuiIO& io = ImGui::GetIO(); + if (io.MouseDelta.x != 0.0f || io.MouseDelta.y != 0.0f + || io.MouseWheel != 0.0f || io.MouseWheelH != 0.0f + || ImGui::IsAnyMouseDown() || io.WantTextInput) { + m_lastInputTime = glfwGetTime(); + } + + renderUI(); + + ImGui::Render(); + int display_w, display_h; + glfwGetFramebufferSize(m_window, &display_w, &display_h); + glViewport(0, 0, display_w, display_h); + glClearColor(kBgBase.x, kBgBase.y, kBgBase.z, 1.00f); + glClear(GL_COLOR_BUFFER_BIT); + ImGui_ImplOpenGL3_RenderDrawData(ImGui::GetDrawData()); + + glfwSwapBuffers(m_window); + } + } + + void GuiLayer::setPaused(bool paused) { + if (m_capturePaused == paused) return; + m_capturePaused = paused; + if (!paused) { + // Packets that piled up while paused are stale — the engine + // retains them for PCAP export, but the live view should resume + // with fresh traffic instead of replaying the backlog. + m_queue.clear(); + m_bytesThisSec = 0; + m_tcpBytesThisSec = 0; + m_udpBytesThisSec = 0; + m_lastUpdateTime = glfwGetTime(); + } + } + + void GuiLayer::linearizeBuffer(const ScrollingBuffer& buffer, + std::vector& time, std::vector& data) { + time.clear(); + data.clear(); + const size_t sampleCount = buffer.Data.size(); + if (sampleCount == 0) return; + const size_t start = sampleCount == static_cast(buffer.MaxSize) + ? static_cast(buffer.Offset) + : 0; + time.reserve(sampleCount); + data.reserve(sampleCount); + for (size_t i = 0; i < sampleCount; ++i) { + const size_t index = (start + i) % sampleCount; + time.push_back(buffer.Time[index]); + data.push_back(buffer.Data[index]); + } + } + + void GuiLayer::processQueue() { + if (m_capturePaused) return; + + // Process up to 1000 packets per frame to maintain responsiveness + constexpr size_t maxPacketHistory = 10'000; + int count = 0; + double currentTime = glfwGetTime(); + + while (count < 1000) { + auto packetOpt = m_queue.try_pop(); + if (!packetOpt) break; + + // Decode, recover cross-packet SNI, harvest DNS answers, attribute + // the owning process, and fold the packet into its flow. + auto analyzed = m_session.feed(*packetOpt); + + // Keep the visual history bounded while retaining all-time counters separately. + if (m_packetHistory.size() >= maxPacketHistory) { + m_packetHistory.pop_front(); + if (m_selectedPacketIndex >= 0) --m_selectedPacketIndex; + } + m_packetHistory.push_back({*packetOpt, std::move(analyzed.parsed), + std::move(analyzed.process)}); + ++m_packetHistoryVersion; + const core::ParsedPacket& stored = m_packetHistory.back().parsed; + + ++m_totalPackets; + m_bytesThisSec += stored.length; + if (stored.protocol == "TCP") { + m_tcpBytesThisSec += stored.length; + } else if (stored.protocol == "UDP") { + m_udpBytesThisSec += stored.length; + } + m_protocolCounts[stored.protocol]++; + m_protocolBytes[stored.protocol] += stored.length; + if (!stored.service.empty()) { + m_appCounts[stored.service]++; + } + + count++; + } + + // Update bandwidth chart every 0.5s. + if (currentTime - m_lastUpdateTime >= 0.5) { + const auto toMbps = [](uint64_t bytes) { + return static_cast(bytes) / (1024.0f * 1024.0f) / 0.5f; + }; + const float mbps = toMbps(m_bytesThisSec); + m_bandwidthData.AddPoint((float)currentTime, mbps); + m_tcpBandwidth.AddPoint((float)currentTime, toMbps(m_tcpBytesThisSec)); + m_udpBandwidth.AddPoint((float)currentTime, toMbps(m_udpBytesThisSec)); + m_currentMbps = mbps; + if (mbps > m_peakMbps) m_peakMbps = mbps; + m_bytesThisSec = 0; + m_tcpBytesThisSec = 0; + m_udpBytesThisSec = 0; + m_lastUpdateTime = currentTime; + } + + // Refresh KPI cache (cheap derived values). + m_topService.clear(); + m_topServiceCount = 0; + for (const auto& [name, c] : m_appCounts) { + if (c > m_topServiceCount) { m_topService = name; m_topServiceCount = c; } + } + } + + void GuiLayer::handleShortcuts() { + ImGuiIO& io = ImGui::GetIO(); + if (ImGui::IsKeyChordPressed(ImGuiMod_Ctrl | ImGuiKey_O)) { + openPcapDialog(); + } + // Text-editing keys only act as shortcuts when no input field owns them. + if (!io.WantTextInput) { + if (ImGui::IsKeyPressed(ImGuiKey_Space, false)) { + setPaused(!m_capturePaused); + } + if (ImGui::IsKeyPressed(ImGuiKey_Slash, false)) { + m_focusDisplayFilter = true; + } + } + } + + void GuiLayer::renderUI() { + handleShortcuts(); + + // Host window that owns the menu bar, control bar, and dockspace. + const ImGuiViewport* viewport = ImGui::GetMainViewport(); + ImGui::SetNextWindowPos(viewport->WorkPos); + ImGui::SetNextWindowSize(viewport->WorkSize); + ImGui::SetNextWindowViewport(viewport->ID); + + const ImGuiWindowFlags hostFlags = + ImGuiWindowFlags_NoTitleBar | ImGuiWindowFlags_NoCollapse | + ImGuiWindowFlags_NoResize | ImGuiWindowFlags_NoMove | + ImGuiWindowFlags_NoBringToFrontOnFocus | ImGuiWindowFlags_NoNavFocus | + ImGuiWindowFlags_NoDocking | ImGuiWindowFlags_MenuBar; + + ImGui::PushStyleVar(ImGuiStyleVar_WindowRounding, 0.0f); + ImGui::PushStyleVar(ImGuiStyleVar_WindowBorderSize, 0.0f); + ImGui::PushStyleVar(ImGuiStyleVar_WindowPadding, ImVec2(0, 0)); + ImGui::Begin("##NetProbeHost", nullptr, hostFlags); + ImGui::PopStyleVar(3); + + renderTopBar(); + renderControlBar(); + renderEncryptedDnsNotice(); + + const ImGuiID dockspaceId = ImGui::GetID("NetProbeDockspace"); + if (!m_layoutBuilt) { + m_layoutBuilt = true; + ImGuiDockNode* root = ImGui::DockBuilderGetNode(dockspaceId); + const bool needsLayout = !root || root->IsLeafNode(); + if (needsLayout) { + ImGui::DockBuilderRemoveNode(dockspaceId); + ImGui::DockBuilderAddNode(dockspaceId, ImGuiDockNodeFlags_DockSpace); + ImGui::DockBuilderSetNodeSize(dockspaceId, ImGui::GetContentRegionAvail()); + + ImGuiID topId, bottomId; + ImGui::DockBuilderSplitNode(dockspaceId, ImGuiDir_Up, 0.50f, &topId, &bottomId); + + ImGuiID flowsId, rightId; + ImGui::DockBuilderSplitNode(bottomId, ImGuiDir_Left, 0.45f, &flowsId, &rightId); + + ImGuiID packetsId, detailId; + ImGui::DockBuilderSplitNode(rightId, ImGuiDir_Up, 0.55f, &packetsId, &detailId); + + ImGui::DockBuilderDockWindow("Dashboard", topId); + ImGui::DockBuilderDockWindow("Statistics", topId); + ImGui::DockBuilderDockWindow("Flows", flowsId); + ImGui::DockBuilderDockWindow("Live Packets", packetsId); + ImGui::DockBuilderDockWindow("Packet Detail", detailId); + ImGui::DockBuilderFinish(dockspaceId); + + // Keep the Dashboard tab in front of Statistics on first run. + if (ImGuiDockNode* topNode = ImGui::DockBuilderGetNode(topId)) { + topNode->SelectedTabId = ImHashStr("Dashboard"); + } + } + } + ImGui::DockSpace(dockspaceId, ImVec2(0, 0), ImGuiDockNodeFlags_None); + + ImGui::End(); + + renderCharts(); + renderStatsView(); + renderFlowsTable(); + renderPacketTable(); + renderPacketDetail(); + } + + void GuiLayer::setDevices(const std::vector& devices) { + m_devices = devices; + } + + void GuiLayer::clearCaptureView() { + m_packetHistory.clear(); + ++m_packetHistoryVersion; + m_filteredPacketIndexes.clear(); + m_filterCacheHistoryVersion = UINT64_MAX; + m_flowsCache.clear(); + m_flowGeoCache.clear(); + m_lastFlowsRefresh = -1.0; + m_maxFlowBytes = 0; + m_activeFlowCount = 0; + m_selectedPacketIndex = -1; + m_appCounts.clear(); + m_protocolCounts.clear(); + m_protocolBytes.clear(); + m_session.clear(); + m_packetFlowFilter.reset(); + m_flowRateHistory.Erase(); + m_flowRateKey.reset(); + m_bandwidthData.Erase(); + m_tcpBandwidth.Erase(); + m_udpBandwidth.Erase(); + m_bytesThisSec = 0; + m_tcpBytesThisSec = 0; + m_udpBytesThisSec = 0; + m_totalPackets = 0; + m_lastUpdateTime = glfwGetTime(); + } + + void GuiLayer::openPcapFile(const std::string& path) { + if (path.empty()) return; + clearCaptureView(); + setPaused(false); + m_captureActive = false; + if (onPcapFileSelected) { + onPcapFileSelected(path); + } + } + + void GuiLayer::openPcapDialog() { + if (!m_nfdInitialized) return; + const nfdu8filteritem_t filters[] = { + {"Packet captures", "pcap,pcapng,cap"}, + }; + nfdu8char_t* selectedPath = nullptr; + const nfdresult_t result = NFD_OpenDialogU8(&selectedPath, filters, 1, nullptr); + if (result == NFD_OKAY) { + openPcapFile(selectedPath); + NFD_FreePathU8(selectedPath); + } else if (result == NFD_ERROR) { + std::cerr << "Unable to open file dialog: " << NFD_GetError() << '\n'; + } + } + + void GuiLayer::renderTopBar() { + if (!ImGui::BeginMenuBar()) return; + + // Brand + if (m_fontBrand) ImGui::PushFont(m_fontBrand); + ImGui::TextColored(kText1, "NetProbe"); + if (m_fontBrand) ImGui::PopFont(); + + // Capture state indicator: pulsing green when live traffic is being + // consumed, amber when paused, muted when idle. + ImGui::SameLine(0.0f, 14.0f); + { + const bool paused = m_capturePaused; + const bool live = !paused && m_currentMbps > 0.0001f; + const float pulse = live + ? 0.55f + 0.45f * (0.5f + 0.5f * std::sin(static_cast(glfwGetTime()) * 3.0f)) + : 1.0f; + const ImVec4 dotColor = paused + ? kWarning + : live + ? ImVec4(kSuccess.x, kSuccess.y, kSuccess.z, pulse) + : ImVec4(kText3.x, kText3.y, kText3.z, 1.0f); + const float r = 4.0f; + const ImVec2 cursor = ImGui::GetCursorScreenPos(); + ImGui::GetWindowDrawList()->AddCircleFilled( + ImVec2(cursor.x + r, cursor.y + ImGui::GetTextLineHeight() * 0.5f), + r, ImGui::GetColorU32(dotColor)); + ImGui::Dummy(ImVec2(r * 2.0f + 6.0f, ImGui::GetTextLineHeight())); + ImGui::SameLine(); + const char* label = paused ? "Paused" : live ? "Live" : "Idle"; + ImGui::TextColored(paused || live ? kText2 : kText3, "%s", label); + } + + ImGui::SameLine(0.0f, 18.0f); + if (ImGui::BeginMenu("File")) { + if (ImGui::MenuItem("Open PCAP...", "Ctrl+O", false, m_nfdInitialized)) { + openPcapDialog(); + } + if (ImGui::MenuItem("Save PCAP...", nullptr, false, m_nfdInitialized)) { + const nfdu8filteritem_t filters[] = { + {"Packet captures", "pcap"}, + }; + nfdu8char_t* selectedPath = nullptr; + const nfdresult_t result = NFD_SaveDialogU8( + &selectedPath, filters, 1, nullptr, "netprobe-session.pcap"); + if (result == NFD_OKAY) { + std::string error; + if (onPcapSaveRequested && onPcapSaveRequested(selectedPath, error)) { + setStatus("PCAP session saved.", false); + } else { + setStatus(error.empty() ? "Unable to save the PCAP session." : error, true); + } + NFD_FreePathU8(selectedPath); + } else if (result == NFD_ERROR) { + setStatus(NFD_GetError() ? NFD_GetError() : "Unable to open the save dialog.", true); + } + } + if (ImGui::MenuItem("Export Flows CSV...", nullptr, false, m_nfdInitialized)) { + const nfdu8filteritem_t filters[] = { + {"CSV", "csv"}, + }; + nfdu8char_t* selectedPath = nullptr; + const nfdresult_t result = NFD_SaveDialogU8( + &selectedPath, filters, 1, nullptr, "netprobe-flows.csv"); + if (result == NFD_OKAY) { + std::string error; + if (exportFlowsCsv(selectedPath, error)) { + setStatus("Flows exported to CSV.", false); + } else { + setStatus(error.empty() ? "Unable to export flows." : error, true); + } + NFD_FreePathU8(selectedPath); + } else if (result == NFD_ERROR) { + setStatus(NFD_GetError() ? NFD_GetError() : "Unable to open the save dialog.", true); + } + } + ImGui::Separator(); + if (ImGui::MenuItem("Exit", "Alt+F4")) glfwSetWindowShouldClose(m_window, true); + ImGui::EndMenu(); + } + if (ImGui::BeginMenu("View")) { + if (ImGui::MenuItem("Dark theme", nullptr, m_darkTheme) && !m_darkTheme) { + m_darkTheme = true; + applyTheme(); + saveSettings(); + } + if (ImGui::MenuItem("Light theme", nullptr, !m_darkTheme) && m_darkTheme) { + m_darkTheme = false; + applyTheme(); + saveSettings(); + } + ImGui::Separator(); + if (ImGui::BeginMenu("UI scale")) { + constexpr float scales[] = {0.9f, 1.0f, 1.1f, 1.25f}; + const char* labels[] = {"90%", "100%", "110%", "125%"}; + for (int i = 0; i < 4; ++i) { + const bool selected = std::abs(m_uiScale - scales[i]) < 0.01f; + if (ImGui::MenuItem(labels[i], nullptr, selected) && !selected) { + m_uiScale = scales[i]; + ImGui::GetStyle().FontScaleMain = m_uiScale; + saveSettings(); + } + } + ImGui::EndMenu(); + } + ImGui::Separator(); + if (ImGui::MenuItem("Reset layout")) { + m_layoutBuilt = false; + } + ImGui::EndMenu(); + } + + ImGui::EndMenuBar(); + } + + void GuiLayer::renderControlBar() { + ImGui::PushStyleColor(ImGuiCol_ChildBg, kBgBase); + ImGui::PushStyleVar(ImGuiStyleVar_WindowPadding, ImVec2(18, 10)); + ImGui::PushStyleVar(ImGuiStyleVar_ChildBorderSize, 0.0f); + const float barHeight = ImGui::GetFrameHeightWithSpacing() + 6.0f; + + ImGui::BeginChild("##controlBar", ImVec2(0, barHeight), false, ImGuiWindowFlags_NoScrollbar); + + // Adapter selector + ImGui::TextColored(kText3, "ADAPTER"); + ImGui::SameLine(); + ImGui::SetNextItemWidth(scaled(300.0f)); + if (m_devices.empty()) { + ImGui::BeginDisabled(); + ImGui::BeginCombo("##deviceCombo", "No adapters detected"); + ImGui::EndCombo(); + ImGui::EndDisabled(); + } else if (ImGui::BeginCombo("##deviceCombo", m_devices[m_selectedDeviceIndex].description.c_str())) { + for (int n = 0; n < static_cast(m_devices.size()); n++) { + const bool is_selected = (m_selectedDeviceIndex == n); + if (ImGui::Selectable(m_devices[n].description.c_str(), is_selected)) { + m_selectedDeviceIndex = n; + clearCaptureView(); + setPaused(false); + if (onDeviceSelected) { + onDeviceSelected(m_devices[n].name); + m_captureActive = true; + setStatus("Capture started.", false); + } + } + if (is_selected) ImGui::SetItemDefaultFocus(); + } + ImGui::EndCombo(); + } + + // Start / Stop / Pause capture controls. + ImGui::SameLine(0.0f, 10.0f); + if (m_captureActive) { + ImGui::PushStyleColor(ImGuiCol_Text, kDanger); + if (ImGui::Button("Stop")) { + if (onCaptureStopRequested) onCaptureStopRequested(); + m_captureActive = false; + setStatus("Capture stopped.", false); + } + ImGui::PopStyleColor(); + } else { + ImGui::BeginDisabled(m_devices.empty()); + ImGui::PushStyleColor(ImGuiCol_Text, kSuccess); + if (ImGui::Button("Start")) { + clearCaptureView(); + setPaused(false); + if (onDeviceSelected) { + onDeviceSelected(m_devices[m_selectedDeviceIndex].name); + m_captureActive = true; + setStatus("Capture started.", false); + } + } + ImGui::PopStyleColor(); + ImGui::EndDisabled(); + } + ImGui::SetItemTooltip(m_captureActive + ? "Stop the live capture (retained packets stay available for export)" + : "Start capturing on the selected adapter"); + + ImGui::SameLine(0.0f, 6.0f); + if (ImGui::Button(m_capturePaused ? "Resume" : "Pause")) { + setPaused(!m_capturePaused); + } + ImGui::SetItemTooltip("Freeze the live view without stopping capture (Space)"); + + ImGui::SameLine(0.0f, 24.0f); + ImGui::TextColored(kText3, "FILTER"); + ImGui::SameLine(); + ImGui::SetNextItemWidth(scaled(240.0f)); + if (ImGui::InputTextWithHint("##bpfFilter", "tcp port 443", + m_bpfFilter, sizeof(m_bpfFilter), ImGuiInputTextFlags_EnterReturnsTrue)) { + std::string error; + if (onBpfFilterRequested && onBpfFilterRequested(m_bpfFilter, error)) { + setStatus(m_bpfFilter[0] == '\0' ? "BPF filter cleared." : "BPF filter applied.", false); + } else { + setStatus(error.empty() ? "Unable to apply BPF filter." : error, true); + } + } + ImGui::SetItemTooltip("Capture filter (BPF syntax) — press Enter to apply.\nOnly affects what is captured, not what is shown."); + + // Status messages expire after a few seconds instead of lingering + // until the next action replaces them. + if (!m_captureStatus.empty() && glfwGetTime() - m_captureStatusTime > 6.0) { + m_captureStatus.clear(); + } + if (!m_captureStatus.empty()) { + ImGui::SameLine(0.0f, 18.0f); + ImGui::TextColored(m_captureStatusIsError ? kDanger : kSuccess, "%s", m_captureStatus.c_str()); + } + + // Right-aligned overrun badge so users see when the UI is dropping. + if (const size_t dropped = m_queue.droppedPackets(); dropped > 0) { + const std::string badge = std::format("{} dropped", formatCount(dropped)); + const float w = ImGui::CalcTextSize(badge.c_str()).x; + ImGui::SameLine(ImGui::GetContentRegionMax().x - w - 18.0f); + ImGui::TextColored(kWarning, "%s", badge.c_str()); + ImGui::SetItemTooltip("Packets evicted from the UI queue because capture outpaced the display.\nThe oldest packets are dropped first; counters keep running."); + } + + ImGui::EndChild(); + ImGui::PopStyleVar(2); + ImGui::PopStyleColor(); + + // Subtle separator line below the control bar. + const ImVec2 lineP = ImGui::GetCursorScreenPos(); + ImGui::GetWindowDrawList()->AddLine( + lineP, ImVec2(lineP.x + ImGui::GetContentRegionAvail().x, lineP.y), + ImGui::GetColorU32(kBorderSoft)); + } + + bool GuiLayer::encryptedDnsDominates() const { + // A handful of DoH packets alongside healthy plaintext DNS is normal. + // The notice is for the case where name resolution has genuinely moved + // off the wire and the hostname columns are going to stay empty. + // + // The bar is low deliberately: a DoH connection is identified by its + // ClientHello SNI, and browsers reuse one connection for every lookup, + // so a handful of sightings can already mean all resolution is hidden. + constexpr uint64_t minimumEvidence = 3; + return m_session.encryptedDnsPackets() >= minimumEvidence + && m_session.encryptedDnsPackets() > m_session.plaintextDnsResponses() * 2; + } + + void GuiLayer::renderEncryptedDnsNotice() { + // ECH alone is not banner-worthy — most Cloudflare-fronted domains + // advertise it, so it would show on nearly every capture. It is reported + // as a standing fact in the Statistics view instead. + const bool show = !m_dnsNoticeDismissed && encryptedDnsDominates(); + if (!show) return; + + ImGui::PushStyleColor(ImGuiCol_ChildBg, + ImVec4(kWarning.x, kWarning.y, kWarning.z, kIsDarkTheme ? 0.10f : 0.16f)); + ImGui::PushStyleColor(ImGuiCol_Border, + ImVec4(kWarning.x, kWarning.y, kWarning.z, 0.45f)); + ImGui::PushStyleVar(ImGuiStyleVar_ChildRounding, 8.0f); + ImGui::PushStyleVar(ImGuiStyleVar_ChildBorderSize, 1.0f); + ImGui::PushStyleVar(ImGuiStyleVar_WindowPadding, ImVec2(14, 8)); + + const float height = ImGui::GetFrameHeight() + 20.0f; + ImGui::SetCursorPosX(ImGui::GetCursorPosX() + 18.0f); + ImGui::BeginChild("##dnsNotice", + ImVec2(ImGui::GetContentRegionAvail().x - 18.0f, height), true, + ImGuiWindowFlags_NoScrollbar); + + drawDot(kWarning, 4.0f); + ImGui::SameLine(); + ImGui::TextColored(kText1, "Encrypted DNS in use."); + ImGui::SameLine(); + ImGui::TextColored(kText2, m_session.echAdvertised() + ? "Name resolution is hidden, and Encrypted Client Hello was advertised, so some hostnames will show only an IP." + : "Name resolution is not visible on the wire, so hostnames come only from TLS/QUIC SNI and may be incomplete."); + + ImGui::SameLine(ImGui::GetContentRegionMax().x - 70.0f); + if (ImGui::SmallButton("Dismiss")) { + m_dnsNoticeDismissed = true; + } + + ImGui::EndChild(); + ImGui::PopStyleVar(3); + ImGui::PopStyleColor(2); + ImGui::Dummy(ImVec2(0, 4)); + } + +} // namespace ui diff --git a/src/ui/GuiLayer.hpp b/src/ui/GuiLayer.hpp index c6c3189..f0354c2 100644 --- a/src/ui/GuiLayer.hpp +++ b/src/ui/GuiLayer.hpp @@ -77,13 +77,13 @@ namespace ui { void setCaptureActive(bool active) { m_captureActive = active; } // Callback to start capture on new device - std::function onDeviceSelected; + std::function onDeviceSelected; // Callback to stop the running live capture. std::function onCaptureStopRequested; // Callback to load an offline capture file. - std::function onPcapFileSelected; + std::function onPcapFileSelected; // Callbacks for live BPF filtering and exporting the retained raw session. std::function onBpfFilterRequested; From d262df8fe63a46b510ca8c36287605cca6508af7 Mon Sep 17 00:00:00 2001 From: YehiaGewily <79144005+YehiaGewily@users.noreply.github.com> Date: Sun, 19 Jul 2026 12:26:10 +0300 Subject: [PATCH 09/12] fix: address audit findings against the literal plan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Corrections after an audit found several places where the implementation diverged from the plan without justification. - JSON now exports initial_rtt_us as the plan specified, not initial_rtt_ms. Integer microseconds is the better contract anyway: the JSON is for machines, and it avoids handing consumers a rounded float to convert back. CSV keeps milliseconds for spreadsheets. - The flows column is "Retrans" showing retransmissions only, as specified, rather than a combined "Loss" figure. Merging retransmissions with reordering produced a number that cannot be acted on: the two have different causes — loss versus multipath or queueing. Out-of-order counts remain in the tooltip and the detail pane. - Adds the TCP option fixtures the plan asked for and the first pass missed: option length zero (the classic infinite-loop bait), an option overrunning the header, a data offset claiming absent option bytes, and a well-formed option area as the control. - Malformed fixtures now pin their expected degraded outcome instead of only asserting generic sanity. This immediately earned its keep by catching four wrong assumptions of mine — the parser reports "Truncated" rather than "Unknown" for sub-Ethernet frames, distinguishes a zero-length capture from a truncated one, and correctly clamps a lying IPv4 total-length field to the real buffer end, which means recovering the SNI behind it is right rather than invented. Every mismatch was my expectation being wrong; no parser defect was found. - README architecture section no longer claims the UI owns the reassemblers and name cache; that moved to AnalysisSession. Also makes offline replay interruptible. Ctrl+C during a long replay previously did nothing until the whole file had been read, because replayFile never checked for cancellation. SIGBREAK is handled too, since Windows delivers Ctrl+Break that way. Verified by interrupting a real live capture: 13,772 packets on a Wi-Fi adapter, Ctrl+Break, clean exit 0, valid JSON with 39 flows. That capture also validates the loss statistics against real traffic — 4 of 29 TCP flows showed retransmissions, and the busiest showed 152 retransmitted and 133 out of order in 6,492 packets, the near-equal counts being the expected signature of reordering producing one classification of each. 89/89 tests pass; clang-tidy clean; fuzz corpus now 44 seeds. --- README.md | 7 +- src/capture/CaptureEngine.cpp | 8 +- src/capture/CaptureEngine.hpp | 7 +- src/cli/main.cpp | 8 +- src/core/FlowExporter.cpp | 7 +- src/ui/FlowsView.cpp | 53 +- tests/PacketBuilders.hpp | 885 +++++++++++++++++++--------------- tests/flow_exporter_test.cpp | 2 +- tests/malformed_test.cpp | 22 + 9 files changed, 570 insertions(+), 429 deletions(-) diff --git a/README.md b/README.md index c4280b5..bb38449 100644 --- a/README.md +++ b/README.md @@ -193,13 +193,16 @@ flowchart LR - **Protocol stack (`src/core/`)**: - `LinkType` maps libpcap `DLT_*` values to supported encapsulations, so cooked, loopback, raw-IP, and Ethernet-family captures are decoded intentionally. - `ProtocolParser` is the stateless fast path: link layer -> IPv4/IPv6 -> tunnel descent -> transport -> application hints and service classification. - - `TlsReassembler` buffers split TCP ClientHellos with size and expiry caps; it is owned by the UI layer because it is stream state. + - `TlsReassembler` buffers split TCP ClientHellos with size and expiry caps. - `QuicParser` decrypts QUIC v1/v2 Initial packets with mbedTLS and extracts TLS ClientHello data from CRYPTO frames. - `QuicTracker` stitches CRYPTO fragments across multiple Initial packets, keyed by connection id. - `DNSParser` extracts DNS/mDNS/LLMNR answers, feeds `HostnameCache`, and flags advertised ECH configs. - `FlowAggregator` collapses both directions of a conversation into one canonical key, tracks rates and byte totals, and measures initial TCP RTT from SYN/SYN-ACK timing. - `GeoIPResolver` and `ProcessResolver` add country/ASN and process ownership where platform data is available. -- **GUI layer (`src/ui/`)**: `GuiLayer` owns the Dear ImGui dockspace, capture controls, settings, queue drain, stateful reassemblers, DNS name cache, flow aggregator, and bounded packet history. `Dashboard`, `FlowsView`, `PacketView`, `PacketDetail`, and `StatsView` render those derived models. + - `AnalysisSession` drives the whole per-packet pipeline — decode, cross-packet SNI recovery, DNS harvesting, hostname attribution, process lookup, flow aggregation — and owns the stateful reassemblers, the name cache, and the flow table. The GUI and the headless CLI both run this same object, which is what keeps them from drifting apart. + - `FlowExporter` serializes a flow snapshot to CSV or JSON. +- **GUI layer (`src/ui/`)**: `GuiLayer` owns the Dear ImGui dockspace, capture controls, settings, queue drain, and bounded packet history, layered over an `AnalysisSession`. `Dashboard`, `FlowsView`, `PacketView`, `PacketDetail`, and `StatsView` render those derived models. +- **Headless CLI (`src/cli/`)**: the same `CaptureEngine` and `AnalysisSession` with no window, exporting flows for scripts and pipelines. ## Quality diff --git a/src/capture/CaptureEngine.cpp b/src/capture/CaptureEngine.cpp index 85de355..d370fca 100644 --- a/src/capture/CaptureEngine.cpp +++ b/src/capture/CaptureEngine.cpp @@ -84,7 +84,8 @@ namespace capture { } bool CaptureEngine::replayFile(const std::string& path, - const std::function& onPacket, std::string& error) { + const std::function& onPacket, std::string& error, + const std::function& shouldStop) { stopCapture(); m_queue.clear(); clearSession(); @@ -100,6 +101,11 @@ namespace capture { } while (true) { + if (shouldStop && shouldStop()) { + m_backend->close(); + return true; + } + core::PacketData packet; std::string readError; switch (m_backend->nextPacket(packet, readError)) { diff --git a/src/capture/CaptureEngine.hpp b/src/capture/CaptureEngine.hpp index 588d182..8421d15 100644 --- a/src/capture/CaptureEngine.hpp +++ b/src/capture/CaptureEngine.hpp @@ -42,9 +42,14 @@ namespace capture { // beginning of the file before anything could consume it — silently // wrong totals, which is unacceptable when the output is an exported // flow table. + // `shouldStop`, when supplied, is polled between packets so a long + // replay can be interrupted. Stopping early still reports success: + // the caller knows how many packets it consumed, and a partial + // analysis of a huge capture is a legitimate thing to ask for. bool replayFile(const std::string& path, const std::function& onPacket, - std::string& error); + std::string& error, + const std::function& shouldStop = {}); bool setFilter(const std::string& filter, std::string& error); bool exportSession(const std::string& path, std::string& error) const; void stopCapture(); diff --git a/src/cli/main.cpp b/src/cli/main.cpp index d6ac347..a31b077 100644 --- a/src/cli/main.cpp +++ b/src/cli/main.cpp @@ -272,7 +272,8 @@ namespace { [&](const core::PacketData& packet) { session.feed(packet); ++packetsAnalyzed; - }, error); + }, error, + [] { return g_stopRequested.load(std::memory_order_acquire); }); if (!ok) { std::cerr << "netprobe-cli: unable to read '" << options.readPath << "': " @@ -400,6 +401,11 @@ static int run(int argc, char** argv) { #ifdef SIGTERM std::signal(SIGTERM, handleInterrupt); #endif +#ifdef SIGBREAK + // Windows delivers Ctrl+Break as SIGBREAK; it should stop as cleanly as + // Ctrl+C rather than killing the process before the output is written. + std::signal(SIGBREAK, handleInterrupt); +#endif // Replaying a file means the sockets that owned those packets are long // gone, so the socket-table walk is pure overhead with nothing to find. diff --git a/src/core/FlowExporter.cpp b/src/core/FlowExporter.cpp index ef5656a..2f86d60 100644 --- a/src/core/FlowExporter.cpp +++ b/src/core/FlowExporter.cpp @@ -208,8 +208,11 @@ namespace core { document += std::format("{}", durationSecondsOf(flow)); document += ",\n \"rate_bytes_per_sec\": "; document += std::format("{:.1f}", flow.rateBytesPerSecond); - document += ",\n \"initial_rtt_ms\": "; - document += std::format("{:.2f}", rttMillisecondsOf(flow)); + // Microseconds, as an integer: the JSON is for machines, and the + // raw measurement avoids handing consumers a rounded float to + // convert back. The CSV keeps milliseconds for spreadsheets. + document += ",\n \"initial_rtt_us\": "; + document += std::format("{}", flow.initialRttMicroseconds); document += ",\n \"retransmissions_up\": "; document += std::format("{}", flow.retransmissionsUp); document += ",\n \"retransmissions_down\": "; diff --git a/src/ui/FlowsView.cpp b/src/ui/FlowsView.cpp index 23e36bf..209a8e3 100644 --- a/src/ui/FlowsView.cpp +++ b/src/ui/FlowsView.cpp @@ -21,17 +21,25 @@ namespace ui { using core::organizationLabel; - uint64_t deliveryProblemsOf(const core::Flow& flow) { - return flow.retransmissionsUp + flow.retransmissionsDown - + flow.outOfOrderUp + flow.outOfOrderDown; + uint64_t retransmissionsOf(const core::Flow& flow) { + return flow.retransmissionsUp + flow.retransmissionsDown; } - // Retransmits and reorderings as a share of the flow's packets. A rate - // reads better than a raw count here: 20 retransmits out of 50 packets - // is a broken connection, out of 500,000 it is noise. - double deliveryProblemRateOf(const core::Flow& flow) { + uint64_t outOfOrderOf(const core::Flow& flow) { + return flow.outOfOrderUp + flow.outOfOrderDown; + } + + // Retransmissions as a share of the flow's packets. A rate reads + // better than a raw count: 20 retransmits out of 50 packets is a + // broken connection, out of 500,000 it is noise. + // + // Reordering is deliberately kept out of this number. The two have + // different causes — loss versus multipath or queueing — and merging + // them produces a figure that cannot be acted on. Out-of-order counts + // live in the tooltip and the detail pane. + double retransmissionRateOf(const core::Flow& flow) { if (flow.packets == 0) return 0.0; - return 100.0 * static_cast(deliveryProblemsOf(flow)) + return 100.0 * static_cast(retransmissionsOf(flow)) / static_cast(flow.packets); } @@ -122,9 +130,9 @@ namespace ui { break; } case 9: { - const double leftLoss = deliveryProblemRateOf(left); - const double rightLoss = deliveryProblemRateOf(right); - comparison = leftLoss < rightLoss ? -1 : leftLoss > rightLoss ? 1 : 0; + const double leftRate = retransmissionRateOf(left); + const double rightRate = retransmissionRateOf(right); + comparison = leftRate < rightRate ? -1 : leftRate > rightRate ? 1 : 0; break; } case 10: comparison = (left.lastSeen - left.firstSeen) < (right.lastSeen - right.firstSeen) ? -1 @@ -297,7 +305,7 @@ namespace ui { ImGui::TableSetupColumn("Bytes", ImGuiTableColumnFlags_WidthFixed, scaled(90.0f)); ImGui::TableSetupColumn("Rate", ImGuiTableColumnFlags_WidthFixed | ImGuiTableColumnFlags_DefaultSort | ImGuiTableColumnFlags_PreferSortDescending, scaled(90.0f)); ImGui::TableSetupColumn("RTT", ImGuiTableColumnFlags_WidthFixed, scaled(85.0f)); - ImGui::TableSetupColumn("Loss", ImGuiTableColumnFlags_WidthFixed | ImGuiTableColumnFlags_PreferSortDescending, scaled(70.0f)); + ImGui::TableSetupColumn("Retrans", ImGuiTableColumnFlags_WidthFixed | ImGuiTableColumnFlags_PreferSortDescending, scaled(78.0f)); ImGui::TableSetupColumn("Dur", ImGuiTableColumnFlags_WidthFixed, scaled(70.0f)); ImGui::TableSetupColumn("App", ImGuiTableColumnFlags_WidthFixed, scaled(130.0f)); ImGui::TableHeadersRow(); @@ -400,18 +408,19 @@ namespace ui { // Only TCP carries the sequence numbers this is derived from. ImGui::TextDisabled("--"); } else { - const uint64_t problems = deliveryProblemsOf(flow); - const double lossRate = deliveryProblemRateOf(flow); - const ImVec4& lossColor = lossRate < 1.0 ? kSuccess - : lossRate < 3.0 ? kWarning : kDanger; - ImGui::TextColored(lossColor, "%.1f%%", lossRate); - if (problems > 0 && ImGui::IsItemHovered(ImGuiHoveredFlags_ForTooltip)) { + const uint64_t retransmissions = retransmissionsOf(flow); + const double rate = retransmissionRateOf(flow); + // A couple of percent is where a connection stops + // feeling fast, so that is where the colour turns. + const ImVec4& rateColor = rate < 2.0 ? kSuccess + : rate < 5.0 ? kWarning : kDanger; + ImGui::TextColored(rateColor, "%.1f%%", rate); + if ((retransmissions > 0 || outOfOrderOf(flow) > 0) + && ImGui::IsItemHovered(ImGuiHoveredFlags_ForTooltip)) { ImGui::SetTooltip( "%llu retransmitted, %llu out of order, of %llu packets", - static_cast( - flow.retransmissionsUp + flow.retransmissionsDown), - static_cast( - flow.outOfOrderUp + flow.outOfOrderDown), + static_cast(retransmissions), + static_cast(outOfOrderOf(flow)), static_cast(flow.packets)); } } diff --git a/tests/PacketBuilders.hpp b/tests/PacketBuilders.hpp index 224e144..c161c3e 100644 --- a/tests/PacketBuilders.hpp +++ b/tests/PacketBuilders.hpp @@ -1,399 +1,486 @@ -#pragma once - -#include -#include -#include - -// Wire-format constructors for synthetic Ethernet/IPv4/DNS/TLS packets. -// Shared between the unit tests (which compare parsed output against expected -// values) and the fuzz seed generator (which writes them out as a starting -// corpus for libFuzzer). -namespace test { - - inline void appendU16(std::vector& bytes, uint16_t value) { - bytes.push_back(static_cast(value >> 8)); - bytes.push_back(static_cast(value & 0xFF)); - } - - inline void appendU32LE(std::vector& bytes, uint32_t value) { - bytes.push_back(static_cast(value & 0xFF)); - bytes.push_back(static_cast((value >> 8) & 0xFF)); - bytes.push_back(static_cast((value >> 16) & 0xFF)); - bytes.push_back(static_cast((value >> 24) & 0xFF)); - } - - inline void appendU32BE(std::vector& bytes, uint32_t value) { - bytes.push_back(static_cast((value >> 24) & 0xFF)); - bytes.push_back(static_cast((value >> 16) & 0xFF)); - bytes.push_back(static_cast((value >> 8) & 0xFF)); - bytes.push_back(static_cast(value & 0xFF)); - } - - inline void appendDnsName(std::vector& bytes, const std::string& name) { - size_t labelStart = 0; - while (labelStart < name.size()) { - const size_t labelEnd = name.find('.', labelStart); - const size_t labelLength = (labelEnd == std::string::npos ? name.size() : labelEnd) - labelStart; - bytes.push_back(static_cast(labelLength)); - bytes.insert(bytes.end(), name.begin() + static_cast(labelStart), - name.begin() + static_cast(labelStart + labelLength)); - if (labelEnd == std::string::npos) break; - labelStart = labelEnd + 1; - } - bytes.push_back(0x00); - } - - inline void appendEthernetHeader(std::vector& bytes, uint16_t etherType) { - bytes.insert(bytes.end(), {0x00, 0x11, 0x22, 0x33, 0x44, 0x55}); - bytes.insert(bytes.end(), {0x66, 0x77, 0x88, 0x99, 0xAA, 0xBB}); - appendU16(bytes, etherType); - } - - inline void appendIPv4Header(std::vector& bytes, uint16_t totalLength, uint8_t protocol, - const std::vector& source, const std::vector& destination) { - bytes.insert(bytes.end(), {0x45, 0x00}); - appendU16(bytes, totalLength); - bytes.insert(bytes.end(), {0x00, 0x01, 0x40, 0x00, 64, protocol, 0x00, 0x00}); - bytes.insert(bytes.end(), source.begin(), source.end()); - bytes.insert(bytes.end(), destination.begin(), destination.end()); - } - - // Minimal 20-byte TCP header. `flags` uses the wire bit layout - // (0x02 SYN, 0x10 ACK, 0x18 PSH|ACK). - inline void appendTcpHeader(std::vector& bytes, uint16_t sourcePort, - uint16_t destinationPort, uint32_t sequence, uint8_t flags) { - appendU16(bytes, sourcePort); - appendU16(bytes, destinationPort); - appendU32BE(bytes, sequence); - appendU32BE(bytes, 0); // acknowledgement number - bytes.push_back(0x50); // data offset = 5 words, no options - bytes.push_back(flags); - appendU16(bytes, 0xFFFF); // window - appendU16(bytes, 0x0000); // checksum - appendU16(bytes, 0x0000); // urgent pointer - } - - inline void appendUdpHeader(std::vector& bytes, uint16_t sourcePort, - uint16_t destinationPort, uint16_t payloadLength) { - appendU16(bytes, sourcePort); - appendU16(bytes, destinationPort); - appendU16(bytes, static_cast(8 + payloadLength)); - appendU16(bytes, 0x0000); // checksum - } - - // 40-byte fixed IPv6 header. - inline void appendIPv6Header(std::vector& bytes, uint16_t payloadLength, - uint8_t nextHeader, const std::vector& source, const std::vector& destination) { - bytes.insert(bytes.end(), {0x60, 0x00, 0x00, 0x00}); - appendU16(bytes, payloadLength); - bytes.push_back(nextHeader); - bytes.push_back(64); // hop limit - bytes.insert(bytes.end(), source.begin(), source.end()); - bytes.insert(bytes.end(), destination.begin(), destination.end()); - } - - inline std::vector makeHttpRequest(const std::string& host, const std::string& path = "/index.html") { - const std::string text = "GET " + path + " HTTP/1.1\r\n" - "User-Agent: netprobe-test\r\n" - "Host: " + host + "\r\n" - "Accept: */*\r\n\r\n"; - return std::vector(text.begin(), text.end()); - } - - inline std::vector makeTlsClientHello(const std::string& hostname) { - std::vector body = {0x03, 0x03}; - body.insert(body.end(), 32, 0x00); // ClientHello random - body.push_back(0x00); // Empty session ID - appendU16(body, 2); - appendU16(body, 0x1301); - body.insert(body.end(), {0x01, 0x00}); // Null compression - - std::vector extensions; - appendU16(extensions, 0x0000); // Server Name Indication - appendU16(extensions, static_cast(5 + hostname.size())); - appendU16(extensions, static_cast(3 + hostname.size())); - extensions.push_back(0x00); // host_name - appendU16(extensions, static_cast(hostname.size())); - extensions.insert(extensions.end(), hostname.begin(), hostname.end()); - appendU16(body, static_cast(extensions.size())); - body.insert(body.end(), extensions.begin(), extensions.end()); - - std::vector record = {0x16, 0x03, 0x01}; - appendU16(record, static_cast(4 + body.size())); - record.push_back(0x01); // ClientHello handshake - record.push_back(static_cast(body.size() >> 16)); - record.push_back(static_cast(body.size() >> 8)); - record.push_back(static_cast(body.size())); - record.insert(record.end(), body.begin(), body.end()); - return record; - } - - // --------------------------------------------------------------------- - // Adversarial inputs. - // - // Wire-format constructors that lie about their own structure: headers - // claiming lengths the buffer does not contain, chains that run off the - // end, counts that exceed the data present. These are the shapes that - // crash capture tools in production, and they are shared with the fuzz - // seed corpus so libFuzzer starts from them rather than rediscovering - // them byte by byte. - // --------------------------------------------------------------------- - namespace malformed { - - struct Case { - std::string name; - std::vector bytes; - }; - - // Offsets into an Ethernet + IPv4 frame. - inline constexpr size_t kEthernetSize = 14; - inline constexpr size_t kIPv4Offset = kEthernetSize; - inline constexpr size_t kIPv4VersionIhl = kIPv4Offset; - inline constexpr size_t kIPv4TotalLength = kIPv4Offset + 2; - inline constexpr size_t kTcpOffset = kIPv4Offset + 20; - inline constexpr size_t kTcpDataOffset = kTcpOffset + 12; - - inline std::vector validTcpFrame(const std::vector& payload) { - std::vector packet; - appendEthernetHeader(packet, 0x0800); - appendIPv4Header(packet, static_cast(20 + 20 + payload.size()), 6, - {192, 168, 1, 10}, {93, 184, 216, 34}); - appendTcpHeader(packet, 50000, 443, 1000, 0x18); - packet.insert(packet.end(), payload.begin(), payload.end()); - return packet; - } - - inline std::vector validUdpFrame(const std::vector& payload) { - std::vector packet; - appendEthernetHeader(packet, 0x0800); - appendIPv4Header(packet, static_cast(20 + 8 + payload.size()), 17, - {8, 8, 8, 8}, {192, 168, 1, 10}); - appendUdpHeader(packet, 53, 53000, static_cast(payload.size())); - packet.insert(packet.end(), payload.begin(), payload.end()); - return packet; - } - - inline std::vector truncatedTo(std::vector bytes, size_t size) { - if (bytes.size() > size) bytes.resize(size); - return bytes; - } - - // An IPv6 frame whose next-header chain is supplied verbatim, so a - // caller can build chains that overrun or run long. - inline std::vector ipv6WithChain(uint8_t firstNextHeader, - const std::vector& chain) { - std::vector packet; - appendEthernetHeader(packet, 0x86DD); - appendIPv6Header(packet, static_cast(chain.size()), firstNextHeader, - {0x20, 0x01, 0x0D, 0xB8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}, - {0x20, 0x01, 0x0D, 0xB8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}); - packet.insert(packet.end(), chain.begin(), chain.end()); - return packet; - } - - inline std::vector allCases() { - std::vector cases; - const auto add = [&cases](std::string name, std::vector bytes) { - cases.push_back({std::move(name), std::move(bytes)}); - }; - - const auto tlsFrame = validTcpFrame(makeTlsClientHello("truncated.example.com")); - const auto httpFrame = validTcpFrame(makeHttpRequest("truncated.example.com")); - - // --- Truncation at every layer boundary -------------------------- - add("empty", {}); - add("one-byte", {0x00}); - for (const size_t size : {6u, 13u, 14u, 15u, 20u, 33u, 34u, 40u, 45u, 53u}) { - add("tls-frame-truncated-to-" + std::to_string(size), - truncatedTo(tlsFrame, size)); - } - add("http-frame-truncated-mid-payload", truncatedTo(httpFrame, 60)); - - // --- IPv4 headers that lie -------------------------------------- - { - auto packet = tlsFrame; - packet[kIPv4VersionIhl] = 0x40; // IHL = 0, below the 20-byte minimum - add("ipv4-ihl-zero", packet); - } - { - auto packet = tlsFrame; - packet[kIPv4VersionIhl] = 0x44; // IHL = 4 words = 16 bytes - add("ipv4-ihl-below-minimum", packet); - } - { - auto packet = tlsFrame; - packet[kIPv4VersionIhl] = 0x4F; // IHL = 15 words = 60 bytes of options - add("ipv4-ihl-claims-options-past-header", packet); - } - { - auto packet = tlsFrame; - packet[kIPv4TotalLength] = 0xFF; // total length far beyond the buffer - packet[kIPv4TotalLength + 1] = 0xFF; - add("ipv4-total-length-exceeds-buffer", packet); - } - { - auto packet = tlsFrame; - packet[kIPv4TotalLength] = 0x00; // total length below the header size - packet[kIPv4TotalLength + 1] = 0x08; - add("ipv4-total-length-below-header", packet); - } - { - auto packet = tlsFrame; - packet[kIPv4VersionIhl] = 0x65; // version 6 in an IPv4 ethertype frame - add("ipv4-version-mismatch", packet); - } - - // --- TCP headers that lie --------------------------------------- - { - auto packet = tlsFrame; - packet[kTcpDataOffset] = 0xF0; // data offset = 15 words = 60 bytes - add("tcp-data-offset-past-payload", packet); - } - { - auto packet = tlsFrame; - packet[kTcpDataOffset] = 0x00; // data offset = 0, below the 20-byte minimum - add("tcp-data-offset-zero", packet); - } - - // --- UDP headers that lie --------------------------------------- - { - auto packet = validUdpFrame({0xDE, 0xAD, 0xBE, 0xEF}); - packet[kIPv4Offset + 20 + 4] = 0xFF; // UDP length beyond the buffer - packet[kIPv4Offset + 20 + 5] = 0xFF; - add("udp-length-exceeds-buffer", packet); - } - { - auto packet = validUdpFrame({0xDE, 0xAD, 0xBE, 0xEF}); - packet[kIPv4Offset + 20 + 4] = 0x00; // UDP length below its own 8-byte header - packet[kIPv4Offset + 20 + 5] = 0x02; - add("udp-length-below-header", packet); - } - - // --- IPv6 extension header chains ------------------------------- - { - // Hop-by-hop -> routing -> fragment -> TCP, all well formed. - std::vector chain = {43, 0}; // hop-by-hop: next=routing, len=0 (8 bytes) - chain.resize(8, 0x00); - chain.insert(chain.end(), {44, 0}); // routing: next=fragment - chain.resize(16, 0x00); - chain.insert(chain.end(), {6, 0}); // fragment: next=TCP - chain.resize(24, 0x00); - add("ipv6-valid-extension-chain", ipv6WithChain(0, chain)); - } - { - // Hop-by-hop claiming 2048 bytes of options that are not there. - const std::vector chain = {6, 0xFF, 0, 0, 0, 0, 0, 0}; - add("ipv6-extension-length-past-packet", ipv6WithChain(0, chain)); - } - { - // A long but individually valid chain: the walk must terminate - // because every step advances at least 8 bytes. - std::vector chain; - constexpr int kLinks = 64; - for (int link = 0; link < kLinks; ++link) { - chain.push_back(link + 1 == kLinks ? uint8_t{6} : uint8_t{0}); - chain.push_back(0); // length 0 => this header occupies 8 bytes - chain.resize(chain.size() + 6, 0x00); - } - add("ipv6-long-extension-chain", ipv6WithChain(0, chain)); - } - { - // Destination-options header pointing at itself as the next - // header, repeatedly. Terminates only because pos advances. - std::vector chain; - for (int link = 0; link < 16; ++link) { - chain.push_back(60); // next header = destination options again - chain.push_back(0); - chain.resize(chain.size() + 6, 0x00); - } - add("ipv6-self-referential-extension-chain", ipv6WithChain(60, chain)); - } - - // --- DNS payloads that lie -------------------------------------- - { - std::vector dns; - appendU16(dns, 0x1234); - appendU16(dns, 0x8180); - appendU16(dns, 1); - appendU16(dns, 20); // claims 20 answers - appendU16(dns, 0); - appendU16(dns, 0); - appendDnsName(dns, "www.example.com"); - appendU16(dns, 1); - appendU16(dns, 1); - add("dns-answer-count-exceeds-payload", validUdpFrame(dns)); - } - { - // A compression pointer that points back at itself. - std::vector dns; - appendU16(dns, 0x1234); - appendU16(dns, 0x8180); - appendU16(dns, 1); - appendU16(dns, 1); - appendU16(dns, 0); - appendU16(dns, 0); - dns.insert(dns.end(), {0xC0, 0x0C}); // pointer to offset 12 == itself - appendU16(dns, 1); - appendU16(dns, 1); - add("dns-self-referential-compression-pointer", validUdpFrame(dns)); - } - { - // A label whose length runs past the end of the payload. - std::vector dns; - appendU16(dns, 0x1234); - appendU16(dns, 0x8180); - appendU16(dns, 1); - appendU16(dns, 0); - appendU16(dns, 0); - appendU16(dns, 0); - dns.push_back(0x3F); // 63-byte label with 3 bytes following - dns.insert(dns.end(), {'a', 'b', 'c'}); - add("dns-label-runs-past-payload", validUdpFrame(dns)); - } - { - std::vector dns; - appendU16(dns, 0x1234); - appendU16(dns, 0x8180); - add("dns-truncated-mid-header", validUdpFrame(dns)); - } - - // --- TLS records that lie --------------------------------------- - { - // Record header announcing 64 KB with almost nothing behind it. - std::vector tls = {0x16, 0x03, 0x01, 0xFF, 0xFF, 0x01, 0x00, 0xFF, 0xFB}; - add("tls-record-length-exceeds-payload", validTcpFrame(tls)); - } - { - auto tls = makeTlsClientHello("half.example.com"); - tls.resize(tls.size() / 2); - add("tls-client-hello-truncated", validTcpFrame(tls)); - } - - // --- QUIC long headers that lie --------------------------------- - { - // Long header, version 1, destination CID length 255. - std::vector quic = {0xC0, 0x00, 0x00, 0x00, 0x01, 0xFF}; - quic.resize(32, 0x41); - add("quic-connection-id-length-exceeds-packet", validUdpFrame(quic)); - } - { - // Truncated part way through the version field. - const std::vector quic = {0xC0, 0x00, 0x00}; - add("quic-truncated-mid-version", validUdpFrame(quic)); - } - { - // Token length varint claiming an implausible size. - std::vector quic = {0xC0, 0x00, 0x00, 0x00, 0x01, 0x08}; - quic.resize(14, 0x42); // 8-byte destination CID - quic.push_back(0x00); // source CID length 0 - quic.push_back(0xFF); // token length varint, 8-byte form, huge - quic.resize(quic.size() + 3, 0xFF); - add("quic-token-length-exceeds-packet", validUdpFrame(quic)); - } - - return cases; - } - - } // namespace malformed - -} // namespace test +#pragma once + +#include +#include +#include + +// Wire-format constructors for synthetic Ethernet/IPv4/DNS/TLS packets. +// Shared between the unit tests (which compare parsed output against expected +// values) and the fuzz seed generator (which writes them out as a starting +// corpus for libFuzzer). +namespace test { + + inline void appendU16(std::vector& bytes, uint16_t value) { + bytes.push_back(static_cast(value >> 8)); + bytes.push_back(static_cast(value & 0xFF)); + } + + inline void appendU32LE(std::vector& bytes, uint32_t value) { + bytes.push_back(static_cast(value & 0xFF)); + bytes.push_back(static_cast((value >> 8) & 0xFF)); + bytes.push_back(static_cast((value >> 16) & 0xFF)); + bytes.push_back(static_cast((value >> 24) & 0xFF)); + } + + inline void appendU32BE(std::vector& bytes, uint32_t value) { + bytes.push_back(static_cast((value >> 24) & 0xFF)); + bytes.push_back(static_cast((value >> 16) & 0xFF)); + bytes.push_back(static_cast((value >> 8) & 0xFF)); + bytes.push_back(static_cast(value & 0xFF)); + } + + inline void appendDnsName(std::vector& bytes, const std::string& name) { + size_t labelStart = 0; + while (labelStart < name.size()) { + const size_t labelEnd = name.find('.', labelStart); + const size_t labelLength = (labelEnd == std::string::npos ? name.size() : labelEnd) - labelStart; + bytes.push_back(static_cast(labelLength)); + bytes.insert(bytes.end(), name.begin() + static_cast(labelStart), + name.begin() + static_cast(labelStart + labelLength)); + if (labelEnd == std::string::npos) break; + labelStart = labelEnd + 1; + } + bytes.push_back(0x00); + } + + inline void appendEthernetHeader(std::vector& bytes, uint16_t etherType) { + bytes.insert(bytes.end(), {0x00, 0x11, 0x22, 0x33, 0x44, 0x55}); + bytes.insert(bytes.end(), {0x66, 0x77, 0x88, 0x99, 0xAA, 0xBB}); + appendU16(bytes, etherType); + } + + inline void appendIPv4Header(std::vector& bytes, uint16_t totalLength, uint8_t protocol, + const std::vector& source, const std::vector& destination) { + bytes.insert(bytes.end(), {0x45, 0x00}); + appendU16(bytes, totalLength); + bytes.insert(bytes.end(), {0x00, 0x01, 0x40, 0x00, 64, protocol, 0x00, 0x00}); + bytes.insert(bytes.end(), source.begin(), source.end()); + bytes.insert(bytes.end(), destination.begin(), destination.end()); + } + + // Minimal 20-byte TCP header. `flags` uses the wire bit layout + // (0x02 SYN, 0x10 ACK, 0x18 PSH|ACK). + inline void appendTcpHeader(std::vector& bytes, uint16_t sourcePort, + uint16_t destinationPort, uint32_t sequence, uint8_t flags) { + appendU16(bytes, sourcePort); + appendU16(bytes, destinationPort); + appendU32BE(bytes, sequence); + appendU32BE(bytes, 0); // acknowledgement number + bytes.push_back(0x50); // data offset = 5 words, no options + bytes.push_back(flags); + appendU16(bytes, 0xFFFF); // window + appendU16(bytes, 0x0000); // checksum + appendU16(bytes, 0x0000); // urgent pointer + } + + inline void appendUdpHeader(std::vector& bytes, uint16_t sourcePort, + uint16_t destinationPort, uint16_t payloadLength) { + appendU16(bytes, sourcePort); + appendU16(bytes, destinationPort); + appendU16(bytes, static_cast(8 + payloadLength)); + appendU16(bytes, 0x0000); // checksum + } + + // 40-byte fixed IPv6 header. + inline void appendIPv6Header(std::vector& bytes, uint16_t payloadLength, + uint8_t nextHeader, const std::vector& source, const std::vector& destination) { + bytes.insert(bytes.end(), {0x60, 0x00, 0x00, 0x00}); + appendU16(bytes, payloadLength); + bytes.push_back(nextHeader); + bytes.push_back(64); // hop limit + bytes.insert(bytes.end(), source.begin(), source.end()); + bytes.insert(bytes.end(), destination.begin(), destination.end()); + } + + inline std::vector makeHttpRequest(const std::string& host, const std::string& path = "/index.html") { + const std::string text = "GET " + path + " HTTP/1.1\r\n" + "User-Agent: netprobe-test\r\n" + "Host: " + host + "\r\n" + "Accept: */*\r\n\r\n"; + return std::vector(text.begin(), text.end()); + } + + inline std::vector makeTlsClientHello(const std::string& hostname) { + std::vector body = {0x03, 0x03}; + body.insert(body.end(), 32, 0x00); // ClientHello random + body.push_back(0x00); // Empty session ID + appendU16(body, 2); + appendU16(body, 0x1301); + body.insert(body.end(), {0x01, 0x00}); // Null compression + + std::vector extensions; + appendU16(extensions, 0x0000); // Server Name Indication + appendU16(extensions, static_cast(5 + hostname.size())); + appendU16(extensions, static_cast(3 + hostname.size())); + extensions.push_back(0x00); // host_name + appendU16(extensions, static_cast(hostname.size())); + extensions.insert(extensions.end(), hostname.begin(), hostname.end()); + appendU16(body, static_cast(extensions.size())); + body.insert(body.end(), extensions.begin(), extensions.end()); + + std::vector record = {0x16, 0x03, 0x01}; + appendU16(record, static_cast(4 + body.size())); + record.push_back(0x01); // ClientHello handshake + record.push_back(static_cast(body.size() >> 16)); + record.push_back(static_cast(body.size() >> 8)); + record.push_back(static_cast(body.size())); + record.insert(record.end(), body.begin(), body.end()); + return record; + } + + // --------------------------------------------------------------------- + // Adversarial inputs. + // + // Wire-format constructors that lie about their own structure: headers + // claiming lengths the buffer does not contain, chains that run off the + // end, counts that exceed the data present. These are the shapes that + // crash capture tools in production, and they are shared with the fuzz + // seed corpus so libFuzzer starts from them rather than rediscovering + // them byte by byte. + // --------------------------------------------------------------------- + namespace malformed { + + struct Case { + std::string name; + std::vector bytes; + + // The degraded outcome this input must produce. Pinning it is the + // point: a parser that walks off the end of a packet and returns a + // plausible-looking result is worse than one that gives up, and + // only an expected value catches the difference. + // + // Empty expectedProtocol means "unconstrained" — used where more + // than one answer is defensible. + std::string expectedProtocol; + // True when the transport payload cannot be located, so the + // payload window must stay empty. + bool expectNoPayload = false; + // Most fixtures destroy the framing badly enough that no server + // name can survive. A few corrupt exactly one length field while + // leaving a well-formed ClientHello intact and reachable — there, + // recovering the SNI is correct behaviour, not invention. + bool expectNoSni = true; + }; + + // Offsets into an Ethernet + IPv4 frame. + inline constexpr size_t kEthernetSize = 14; + inline constexpr size_t kIPv4Offset = kEthernetSize; + inline constexpr size_t kIPv4VersionIhl = kIPv4Offset; + inline constexpr size_t kIPv4TotalLength = kIPv4Offset + 2; + inline constexpr size_t kTcpOffset = kIPv4Offset + 20; + inline constexpr size_t kTcpDataOffset = kTcpOffset + 12; + + inline std::vector validTcpFrame(const std::vector& payload) { + std::vector packet; + appendEthernetHeader(packet, 0x0800); + appendIPv4Header(packet, static_cast(20 + 20 + payload.size()), 6, + {192, 168, 1, 10}, {93, 184, 216, 34}); + appendTcpHeader(packet, 50000, 443, 1000, 0x18); + packet.insert(packet.end(), payload.begin(), payload.end()); + return packet; + } + + inline std::vector validUdpFrame(const std::vector& payload) { + std::vector packet; + appendEthernetHeader(packet, 0x0800); + appendIPv4Header(packet, static_cast(20 + 8 + payload.size()), 17, + {8, 8, 8, 8}, {192, 168, 1, 10}); + appendUdpHeader(packet, 53, 53000, static_cast(payload.size())); + packet.insert(packet.end(), payload.begin(), payload.end()); + return packet; + } + + inline std::vector truncatedTo(std::vector bytes, size_t size) { + if (bytes.size() > size) bytes.resize(size); + return bytes; + } + + // An IPv6 frame whose next-header chain is supplied verbatim, so a + // caller can build chains that overrun or run long. + inline std::vector ipv6WithChain(uint8_t firstNextHeader, + const std::vector& chain) { + std::vector packet; + appendEthernetHeader(packet, 0x86DD); + appendIPv6Header(packet, static_cast(chain.size()), firstNextHeader, + {0x20, 0x01, 0x0D, 0xB8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}, + {0x20, 0x01, 0x0D, 0xB8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}); + packet.insert(packet.end(), chain.begin(), chain.end()); + return packet; + } + + // A TCP frame whose header claims `dataOffsetWords` 32-bit words and + // carries `options` between the fixed header and the payload. + inline std::vector tcpFrameWithOptions(uint8_t dataOffsetWords, + const std::vector& options, const std::vector& payload) { + std::vector packet; + appendEthernetHeader(packet, 0x0800); + appendIPv4Header(packet, + static_cast(20 + 20 + options.size() + payload.size()), 6, + {192, 168, 1, 10}, {93, 184, 216, 34}); + appendTcpHeader(packet, 50000, 443, 1000, 0x18); + // appendTcpHeader writes a fixed data offset of 5 words; override + // it so the header can claim the option bytes that follow. + packet[kTcpDataOffset] = static_cast(dataOffsetWords << 4); + packet.insert(packet.end(), options.begin(), options.end()); + packet.insert(packet.end(), payload.begin(), payload.end()); + return packet; + } + + inline std::vector allCases() { + std::vector cases; + const auto add = [&cases](std::string name, std::vector bytes, + std::string expectedProtocol = {}, bool expectNoPayload = false, + bool expectNoSni = true) { + cases.push_back({std::move(name), std::move(bytes), + std::move(expectedProtocol), expectNoPayload, expectNoSni}); + }; + + const auto tlsFrame = validTcpFrame(makeTlsClientHello("truncated.example.com")); + const auto httpFrame = validTcpFrame(makeHttpRequest("truncated.example.com")); + + // --- Truncation at every layer boundary -------------------------- + // A zero-length capture is distinguishable from a truncated one: + // there are no bytes to have been cut short. + add("empty", {}, "Unknown", /*expectNoPayload=*/true); + add("one-byte", {0x00}, "Truncated", /*expectNoPayload=*/true); + // Below 34 bytes there is not a complete Ethernet + IPv4 header, + // so no transport can be named and no payload located. + for (const size_t size : {6u, 13u}) { + add("tls-frame-truncated-to-" + std::to_string(size), + truncatedTo(tlsFrame, size), "Truncated", /*expectNoPayload=*/true); + } + // A complete Ethernet header but an incomplete IPv4 one: no + // transport is nameable and no payload can be located. + for (const size_t size : {14u, 15u, 20u, 33u}) { + add("tls-frame-truncated-to-" + std::to_string(size), + truncatedTo(tlsFrame, size), {}, /*expectNoPayload=*/true); + } + // From here the headers are present but the payload is cut short; + // more than one answer is defensible, so only sanity is asserted. + for (const size_t size : {34u, 40u, 45u, 53u}) { + add("tls-frame-truncated-to-" + std::to_string(size), + truncatedTo(tlsFrame, size)); + } + add("http-frame-truncated-mid-payload", truncatedTo(httpFrame, 60)); + + // --- IPv4 headers that lie -------------------------------------- + { + auto packet = tlsFrame; + packet[kIPv4VersionIhl] = 0x40; // IHL = 0, below the 20-byte minimum + add("ipv4-ihl-zero", packet, "Unknown", /*expectNoPayload=*/true); + } + { + auto packet = tlsFrame; + packet[kIPv4VersionIhl] = 0x44; // IHL = 4 words = 16 bytes + add("ipv4-ihl-below-minimum", packet, "Unknown", /*expectNoPayload=*/true); + } + { + auto packet = tlsFrame; + packet[kIPv4VersionIhl] = 0x4F; // IHL = 15 words = 60 bytes of options + add("ipv4-ihl-claims-options-past-header", packet); + } + { + auto packet = tlsFrame; + packet[kIPv4TotalLength] = 0xFF; // total length far beyond the buffer + packet[kIPv4TotalLength + 1] = 0xFF; + add("ipv4-total-length-exceeds-buffer", packet, "TCP", /*expectNoPayload=*/false, + /*expectNoSni=*/false); + } + { + auto packet = tlsFrame; + packet[kIPv4TotalLength] = 0x00; // total length below the header size + packet[kIPv4TotalLength + 1] = 0x08; + add("ipv4-total-length-below-header", packet, "Unknown", /*expectNoPayload=*/true); + } + { + auto packet = tlsFrame; + packet[kIPv4VersionIhl] = 0x65; // version 6 in an IPv4 ethertype frame + add("ipv4-version-mismatch", packet, "Unknown", /*expectNoPayload=*/true); + } + + // --- TCP headers that lie --------------------------------------- + { + auto packet = tlsFrame; + packet[kTcpDataOffset] = 0xF0; // data offset = 15 words = 60 bytes + add("tcp-data-offset-past-payload", packet, "TCP"); + } + { + auto packet = tlsFrame; + packet[kTcpDataOffset] = 0x00; // data offset = 0, below the 20-byte minimum + add("tcp-data-offset-zero", packet, "TCP", /*expectNoPayload=*/true); + } + + // --- TCP options that lie --------------------------------------- + { + // Option kind 2 (MSS) declaring length 0. A parser that walks + // the option list by adding the declared length never advances + // past this one: the classic infinite-loop bait. + const std::vector options = {0x02, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; + add("tcp-option-length-zero", + tcpFrameWithOptions(8, options, {0xDE, 0xAD, 0xBE, 0xEF})); + } + { + // Option kind 2 declaring 200 bytes inside a 12-byte option + // area, so the option runs past the end of the header. + const std::vector options = {0x02, 0xC8, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; + add("tcp-option-length-overruns-header", + tcpFrameWithOptions(8, options, {0xDE, 0xAD, 0xBE, 0xEF})); + } + { + // A well-formed option area, as the control for the two above: + // NOP, NOP, then a valid 10-byte timestamp option. + std::vector options = {0x01, 0x01, 0x08, 0x0A}; + options.resize(12, 0x00); + add("tcp-valid-options", + tcpFrameWithOptions(8, options, {0xDE, 0xAD, 0xBE, 0xEF}), "TCP"); + } + { + // Data offset claims option bytes the frame does not contain. + add("tcp-options-claimed-but-absent", + tcpFrameWithOptions(15, {}, {}), "TCP", /*expectNoPayload=*/true); + } + + // --- UDP headers that lie --------------------------------------- + { + auto packet = validUdpFrame({0xDE, 0xAD, 0xBE, 0xEF}); + packet[kIPv4Offset + 20 + 4] = 0xFF; // UDP length beyond the buffer + packet[kIPv4Offset + 20 + 5] = 0xFF; + add("udp-length-exceeds-buffer", packet); + } + { + auto packet = validUdpFrame({0xDE, 0xAD, 0xBE, 0xEF}); + packet[kIPv4Offset + 20 + 4] = 0x00; // UDP length below its own 8-byte header + packet[kIPv4Offset + 20 + 5] = 0x02; + add("udp-length-below-header", packet); + } + + // --- IPv6 extension header chains ------------------------------- + { + // Hop-by-hop -> routing -> fragment -> TCP, all well formed. + std::vector chain = {43, 0}; // hop-by-hop: next=routing, len=0 (8 bytes) + chain.resize(8, 0x00); + chain.insert(chain.end(), {44, 0}); // routing: next=fragment + chain.resize(16, 0x00); + chain.insert(chain.end(), {6, 0}); // fragment: next=TCP + chain.resize(24, 0x00); + add("ipv6-valid-extension-chain", ipv6WithChain(0, chain)); + } + { + // Hop-by-hop claiming 2048 bytes of options that are not there. + const std::vector chain = {6, 0xFF, 0, 0, 0, 0, 0, 0}; + add("ipv6-extension-length-past-packet", ipv6WithChain(0, chain)); + } + { + // A long but individually valid chain: the walk must terminate + // because every step advances at least 8 bytes. + std::vector chain; + constexpr int kLinks = 64; + for (int link = 0; link < kLinks; ++link) { + chain.push_back(link + 1 == kLinks ? uint8_t{6} : uint8_t{0}); + chain.push_back(0); // length 0 => this header occupies 8 bytes + chain.resize(chain.size() + 6, 0x00); + } + add("ipv6-long-extension-chain", ipv6WithChain(0, chain)); + } + { + // Destination-options header pointing at itself as the next + // header, repeatedly. Terminates only because pos advances. + std::vector chain; + for (int link = 0; link < 16; ++link) { + chain.push_back(60); // next header = destination options again + chain.push_back(0); + chain.resize(chain.size() + 6, 0x00); + } + add("ipv6-self-referential-extension-chain", ipv6WithChain(60, chain)); + } + + // --- DNS payloads that lie -------------------------------------- + { + std::vector dns; + appendU16(dns, 0x1234); + appendU16(dns, 0x8180); + appendU16(dns, 1); + appendU16(dns, 20); // claims 20 answers + appendU16(dns, 0); + appendU16(dns, 0); + appendDnsName(dns, "www.example.com"); + appendU16(dns, 1); + appendU16(dns, 1); + add("dns-answer-count-exceeds-payload", validUdpFrame(dns)); + } + { + // A compression pointer that points back at itself. + std::vector dns; + appendU16(dns, 0x1234); + appendU16(dns, 0x8180); + appendU16(dns, 1); + appendU16(dns, 1); + appendU16(dns, 0); + appendU16(dns, 0); + dns.insert(dns.end(), {0xC0, 0x0C}); // pointer to offset 12 == itself + appendU16(dns, 1); + appendU16(dns, 1); + add("dns-self-referential-compression-pointer", validUdpFrame(dns)); + } + { + // A label whose length runs past the end of the payload. + std::vector dns; + appendU16(dns, 0x1234); + appendU16(dns, 0x8180); + appendU16(dns, 1); + appendU16(dns, 0); + appendU16(dns, 0); + appendU16(dns, 0); + dns.push_back(0x3F); // 63-byte label with 3 bytes following + dns.insert(dns.end(), {'a', 'b', 'c'}); + add("dns-label-runs-past-payload", validUdpFrame(dns)); + } + { + std::vector dns; + appendU16(dns, 0x1234); + appendU16(dns, 0x8180); + add("dns-truncated-mid-header", validUdpFrame(dns)); + } + + // --- TLS records that lie --------------------------------------- + { + // Record header announcing 64 KB with almost nothing behind it. + std::vector tls = {0x16, 0x03, 0x01, 0xFF, 0xFF, 0x01, 0x00, 0xFF, 0xFB}; + add("tls-record-length-exceeds-payload", validTcpFrame(tls)); + } + { + auto tls = makeTlsClientHello("half.example.com"); + tls.resize(tls.size() / 2); + add("tls-client-hello-truncated", validTcpFrame(tls)); + } + + // --- QUIC long headers that lie --------------------------------- + { + // Long header, version 1, destination CID length 255. + std::vector quic = {0xC0, 0x00, 0x00, 0x00, 0x01, 0xFF}; + quic.resize(32, 0x41); + add("quic-connection-id-length-exceeds-packet", validUdpFrame(quic)); + } + { + // Truncated part way through the version field. + const std::vector quic = {0xC0, 0x00, 0x00}; + add("quic-truncated-mid-version", validUdpFrame(quic)); + } + { + // Token length varint claiming an implausible size. + std::vector quic = {0xC0, 0x00, 0x00, 0x00, 0x01, 0x08}; + quic.resize(14, 0x42); // 8-byte destination CID + quic.push_back(0x00); // source CID length 0 + quic.push_back(0xFF); // token length varint, 8-byte form, huge + quic.resize(quic.size() + 3, 0xFF); + add("quic-token-length-exceeds-packet", validUdpFrame(quic)); + } + + return cases; + } + + } // namespace malformed + +} // namespace test diff --git a/tests/flow_exporter_test.cpp b/tests/flow_exporter_test.cpp index 362f51f..1afd5e1 100644 --- a/tests/flow_exporter_test.cpp +++ b/tests/flow_exporter_test.cpp @@ -92,7 +92,7 @@ TEST(FlowExporterTest, JsonContainsExpectedFieldsAndValues) { EXPECT_NE(document.find("\"hostname\": \"example.com\""), std::string::npos); EXPECT_NE(document.find("\"bytes_down\": 8192"), std::string::npos); EXPECT_NE(document.find("\"packets\": 12"), std::string::npos); - EXPECT_NE(document.find("\"initial_rtt_ms\": 24.50"), std::string::npos); + EXPECT_NE(document.find("\"initial_rtt_us\": 24500"), std::string::npos); EXPECT_NE(document.find("\"duration_sec\": 5"), std::string::npos); EXPECT_NE(document.find("\"encrypted_tunnel\": false"), std::string::npos); EXPECT_NE(document.find("\"retransmissions_up\": 3"), std::string::npos); diff --git a/tests/malformed_test.cpp b/tests/malformed_test.cpp index e3bfb43..3c5f95c 100644 --- a/tests/malformed_test.cpp +++ b/tests/malformed_test.cpp @@ -42,6 +42,27 @@ namespace { // Everything a malformed packet is allowed to produce. The point is not // that parsing succeeds, but that failure is expressed as absent or // clearly-marked fields rather than as garbage that reads as real. + // Asserts the specific degraded result a fixture declares. This is the + // half that catches a parser quietly starting to invent answers; the + // generic sanity checks below only catch it going off the rails. + void expectPinnedOutcome(const core::ParsedPacket& parsed, + const malformed::Case& testCase) { + if (!testCase.expectedProtocol.empty()) { + EXPECT_EQ(parsed.protocol, testCase.expectedProtocol) + << testCase.name << ": parsed as a different transport than expected"; + } + if (testCase.expectNoPayload) { + EXPECT_EQ(parsed.payloadLength, 0u) + << testCase.name << ": located a payload that is not there"; + } + if (testCase.expectNoSni) { + // An SNI here would be fabricated from bytes that do not encode + // one — and it would be shown to the user as fact. + EXPECT_TRUE(parsed.sni.empty()) + << testCase.name << ": invented an SNI (" << parsed.sni << ")"; + } + } + void expectDegradedButSane(const core::ParsedPacket& parsed, const std::vector& bytes, const std::string& name) { // A payload window must never point outside the captured bytes. @@ -72,6 +93,7 @@ TEST(MalformedPacketTest, ProtocolParserSurvivesEveryAdversarialCase) { const auto raw = toPacketData(testCase.bytes); const auto parsed = core::ProtocolParser::parse(raw); expectDegradedButSane(parsed, testCase.bytes, testCase.name); + expectPinnedOutcome(parsed, testCase); } } From ba4054e2b0a239aad9899c830c93f99905aabac7 Mon Sep 17 00:00:00 2001 From: YehiaGewily <79144005+YehiaGewily@users.noreply.github.com> Date: Sun, 19 Jul 2026 12:28:19 +0300 Subject: [PATCH 10/12] docs: add QUIC Initial decryption deep-dive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 6's technical writing deliverable: a walk through the full path from a long-header UDP packet to a recovered server name — HKDF-Extract and Expand-Label, header protection, AES-128-GCM, CRYPTO reassembly, and the ClientHello parse — using this repository's implementation as the worked example. Written around the things that actually cost time rather than a summary of the RFCs: the "tls13 " label prefix, QUIC v2's different salt AND labels AND Initial type bits (a v1-only parser fails silently on v2), the chicken-and-egg that header protection exists to solve, the nonce being iv XOR packet number rather than the iv, and a failed GCM tag being the normal answer for a packet that was never QUIC rather than an error. The section on CRYPTO reassembly is the one that justifies the post: post-quantum key shares have pushed real ClientHellos past a single Initial, so a parser that reads only the first CRYPTO frame does not break loudly — it quietly stops finding SNI on a growing share of exactly the traffic you most want to see. Draft, not published. Needs a human read-through before it goes anywhere. --- docs/blog/decrypting-quic-initial-packets.md | 291 +++++++++++++++++++ 1 file changed, 291 insertions(+) create mode 100644 docs/blog/decrypting-quic-initial-packets.md diff --git a/docs/blog/decrypting-quic-initial-packets.md b/docs/blog/decrypting-quic-initial-packets.md new file mode 100644 index 0000000..f01a5a3 --- /dev/null +++ b/docs/blog/decrypting-quic-initial-packets.md @@ -0,0 +1,291 @@ +# Decrypting QUIC Initial packets by hand + +*How a passive analyzer recovers the server name from an encrypted QUIC handshake: HKDF, header protection, AES-GCM, and CRYPTO reassembly.* + +--- + +## Why this is worth explaining + +If you run a packet analyzer on a modern network, a growing share of your traffic is QUIC, and QUIC looks like noise. TLS over TCP at least hands you a plaintext ClientHello: the SNI is sitting there in the clear, and any tool can read it. QUIC encrypts its handshake from the very first packet. Point Wireshark at a QUIC flow without keys and you get UDP payloads of undifferentiated bytes. + +Except it isn't really undifferentiated, because QUIC's *Initial* packets are encrypted with keys derived from a value that is printed in the clear in the packet header. This is not a flaw — the encryption exists to stop middleboxes ossifying the protocol, not to hide anything from an observer — but it means a passive analyzer *can* decrypt Initial packets and read the ClientHello inside. + +Very few people write this up accessibly. The RFCs describe it precisely and unhelpfully; most blog posts stop at "QUIC encrypts its handshake." This post walks the entire path in the order the code executes, using [NetProbe](https://github.com/YehiaGewily/netprobe-cpp)'s implementation in [`src/core/QuicParser.cpp`](../../src/core/QuicParser.cpp) as the worked example. + +The five stages: + +1. Parse enough of the long header to find the Destination Connection ID +2. Derive Initial secrets from that DCID (HKDF-Extract, then HKDF-Expand-Label) +3. Remove header protection to recover the packet number +4. Decrypt the payload with AES-128-GCM +5. Reassemble CRYPTO frames — possibly across several packets — and parse the ClientHello + +Stage 5 is the one most implementations get wrong, and it is the one that matters most in 2026. + +--- + +## Stage 0: the long header + +A QUIC Initial packet uses the *long header* form. The first byte tells you which: + +``` + 0 1 2 3 4 5 6 7 ++-+-+-+-+-+-+-+-+ +|1|1|T T|R R|P P| ++-+-+-+-+-+-+-+-+ +``` + +Bit 7 set means long header; bit 6 is the "fixed bit", always 1. NetProbe's cheap pre-filter checks exactly those two bits before doing any work: + +```cpp +bool QuicParser::looksLikeLongHeader(const uint8_t* data, size_t size) { + // Long header (bit 7) + fixed bit (bit 6). + ... +} +``` + +This matters for performance. Every UDP packet on the wire gets this test; only the ones that pass go anywhere near the crypto. + +After the first byte comes a 32-bit version, then the connection IDs: + +``` +first byte (1) | version (4) | DCID len (1) | DCID (0-20) | SCID len (1) | SCID (0-20) | token len (varint) | token | length (varint) | packet number (1-4, protected) | payload +``` + +Two versions matter today: + +| Version | Value | Spec | +| --- | --- | --- | +| QUIC v1 | `0x00000001` | RFC 9000 | +| QUIC v2 | `0x6b3343cf` | RFC 9369 | + +**The v2 trap.** QUIC v2 is not a cosmetic revision of v1. It uses a *different Initial salt* and *different key-derivation labels*. A v1-only parser does not error on v2 — it derives the wrong keys, AES-GCM authentication fails, and the packet is silently discarded as "not decryptable." You get no SNI and no indication why. NetProbe carries both parameter sets: + +```cpp +struct VersionParams { + const uint8_t* salt; + const char* keyLabel; + const char* ivLabel; + const char* hpLabel; + uint8_t initialTypeBits; // value of (firstByte & 0x30) for an Initial +}; + +// v1: {kInitialSaltV1, "quic key", "quic iv", "quic hp", 0x00} +// v2: {kInitialSaltV2, "quicv2 key", "quicv2 iv", "quicv2 hp", 0x10} +``` + +Note the last field. The two-bit packet *type* encoding also changed between versions: an Initial is `0b00` in v1 and `0b01` in v2. Hardcode the v1 value and you will reject every v2 Initial before you even get to the keys. + +--- + +## Stage 1: deriving keys from a public value + +Here is the part that surprises people. The Initial keys come from the **Destination Connection ID**, which is transmitted in plaintext in the header of the very packet you are trying to decrypt. + +RFC 9001 §5.2 defines: + +``` +initial_secret = HKDF-Extract(initial_salt, client_dst_connection_id) +``` + +The salt is a fixed 20-byte constant published in the RFC. For v1 it begins `0x38 0x76 0x2c 0xf7 …`. So the "secret" is derived from a public constant and a public field. Anyone observing the packet can compute it. + +That is deliberate. The purpose of Initial encryption is to make the handshake opaque to middleboxes so they cannot come to depend on its internals and freeze the protocol in place — not to provide confidentiality against an observer. Real confidentiality begins with the Handshake keys, which come from the ephemeral key exchange inside the ClientHello. Those you cannot derive passively, and NetProbe does not try. + +From the initial secret, one more expansion gives the client's secret: + +``` +client_initial_secret = HKDF-Expand-Label(initial_secret, "client in", "", 32) +``` + +Then three more give the actual key material: + +``` +key = HKDF-Expand-Label(client_initial_secret, "quic key", "", 16) // AES-128 +iv = HKDF-Expand-Label(client_initial_secret, "quic iv", "", 12) // GCM nonce base +hp = HKDF-Expand-Label(client_initial_secret, "quic hp", "", 16) // header protection +``` + +In code, that whole chain is: + +```cpp +bool deriveClientInitialKeys(const VersionParams& params, + const uint8_t* dcid, size_t dcidLen, InitialKeys& out) { + uint8_t initialSecret[32]; + if (!hkdfExtract(params.salt, 20, dcid, dcidLen, initialSecret)) return false; + + uint8_t clientSecret[32]; + if (!hkdfExpandLabel(initialSecret, 32, "client in", 9, clientSecret, 32)) return false; + + if (!hkdfExpandLabel(clientSecret, 32, params.keyLabel, ..., out.key, 16)) return false; + if (!hkdfExpandLabel(clientSecret, 32, params.ivLabel, ..., out.iv, 12)) return false; + if (!hkdfExpandLabel(clientSecret, 32, params.hpLabel, ..., out.hp, 16)) return false; + return true; +} +``` + +### HKDF-Expand-Label is not HKDF-Expand + +This trips people up. `HKDF-Expand-Label` is a TLS 1.3 construction (RFC 8446 §7.1) that builds a structured `info` parameter before calling ordinary HKDF-Expand: + +``` +struct { + uint16 length; + opaque label<7..255>; // "tls13 " || label + opaque context<0..255>; +} HkdfLabel; +``` + +The literal ASCII prefix `"tls13 "` — with the trailing space — is prepended to every label. QUIC uses the TLS 1.3 KDF unmodified, so `"quic key"` is really `"tls13 quic key"` on the wire: + +```cpp +static constexpr char kPrefix[] = "tls13 "; +... +info[p++] = static_cast(outLen >> 8); +info[p++] = static_cast(outLen); +info[p++] = static_cast(fullLabelLen); +std::memcpy(info + p, kPrefix, kPrefixLen); p += kPrefixLen; +std::memcpy(info + p, label, labelLen); p += labelLen; +info[p++] = 0x00; // empty context +``` + +Forget the prefix, or the two-byte big-endian output length, or the trailing empty-context byte, and you get 16 perfectly valid-looking bytes that decrypt nothing. There is no diagnostic. This is the single most common place to lose an afternoon. + +--- + +## Stage 2: the chicken-and-egg of header protection + +You now have keys. You cannot yet decrypt, because you do not know where the ciphertext starts. + +The packet number is 1–4 bytes, and *how many* is encoded in the low two bits of the first byte — which is itself encrypted. Worse, the packet number is an input to the AEAD nonce. So: + +- To find the packet number length, you must decrypt the first byte. +- To decrypt the payload, you need the packet number. + +QUIC resolves this with **header protection**: a separate cipher pass that masks only the header bits, keyed by a sample of the ciphertext taken from a position that does not depend on the packet number length. + +The trick is that the sample is taken at a *fixed* offset — 4 bytes past where the packet number starts, as though the packet number were always the maximum 4 bytes: + +```cpp +// The HP sample is 16 bytes at offset (pnOffset + 4), assuming the +// maximum 4-byte packet number. +``` + +Encrypt that 16-byte sample with AES-128-ECB under the `hp` key. The output is a 5-byte mask: + +```cpp +mbedtls_aes_setkey_enc(&ctx, keys.hp, 128); +mbedtls_aes_crypt_ecb(&ctx, MBEDTLS_AES_ENCRYPT, sample, mask); + +// Long header: the low nibble (bits 0..3) is masked. +inOutFirstByte ^= (mask[0] & 0x0F); +outPnLen = static_cast(inOutFirstByte & 0x03) + 1; +for (size_t i = 0; i < outPnLen; ++i) { + pnBytes[i] ^= mask[i + 1]; +} +``` + +`mask[0]` unmasks the first byte — the low **nibble** for a long header, the low 5 bits for a short header. Once the first byte is clear, its low two bits give the packet number length, and `mask[1..4]` unmask that many packet-number bytes. + +Note this is AES-ECB, the mode everyone is told never to use. It is correct here precisely because it is a single block of a value that is never reused: one deterministic permutation of a unique sample. Using it as a general-purpose mode is what's dangerous. + +--- + +## Stage 3: AES-128-GCM, and the nonce that is not the IV + +Now the payload. QUIC uses AEAD_AES_128_GCM for Initial packets, with two details that differ from how people usually reach for GCM. + +**The nonce is the IV XOR the packet number**, big-endian and right-aligned into 12 bytes: + +```cpp +uint8_t nonce[12]; +std::memcpy(nonce, keys.iv, 12); +for (int i = 0; i < 8; ++i) { + nonce[12 - 1 - i] ^= static_cast((packetNumber >> (i * 8)) & 0xFFu); +} +``` + +The derived `iv` is not used directly — it is a per-connection baseline that the packet number perturbs, guaranteeing a unique nonce per packet without transmitting one. + +**The associated data is the entire header, unprotected.** Everything from the first byte through the end of the packet number, *after* header protection has been removed, is authenticated but not encrypted. Get the AAD boundaries wrong by a single byte and GCM authentication fails with no hint as to why. + +The 16-byte authentication tag occupies the last 16 bytes of the payload: + +```cpp +mbedtls_gcm_auth_decrypt(&ctx, ciphertextLen, + nonce, sizeof(nonce), + aad, aadLen, + tag, 16, + ciphertext, outPlaintext.data()); +``` + +This authentication check is doing real work for an analyzer, not just crypto hygiene. Because Initial keys are derivable by anyone, a passive parser will happily attempt decryption on any UDP packet whose first two bits look like a long header. The GCM tag is what distinguishes an actual QUIC Initial from a coincidence. **A failed tag is not an error to report — it is the normal answer for a packet that was never QUIC.** + +--- + +## Stage 4: CRYPTO frames, and the part everyone gets wrong + +Decryption gives you a QUIC frame sequence, not a ClientHello. Walk it, collecting CRYPTO frames (type `0x06`) and skipping the frames that legitimately appear in an Initial: + +- `0x00` PADDING — usually a *lot* of it, since Initials are padded to at least 1200 bytes +- `0x01` PING +- `0x02`/`0x03` ACK +- `0x06` CRYPTO — carries `offset` and `length` varints, then the TLS bytes + +Each CRYPTO frame carries an **offset**. That is the whole story: the ClientHello is a byte stream that may be split across multiple frames, in multiple packets, arriving in any order. + +For years you could ignore this. A ClientHello fit comfortably in one Initial packet, so a parser that grabbed the first CRYPTO frame and parsed it worked essentially always. + +**That stopped being true.** Post-quantum key exchange — X25519MLKEM768, now the default in Chrome and Firefox — adds roughly a kilobyte to the ClientHello. A post-quantum key share does not fit alongside everything else in a single Initial. Real ClientHellos are now routinely split across two or more Initial packets. + +So a parser that reads only the first CRYPTO frame does not fail loudly. It silently stops finding SNI on a steadily growing share of real traffic — exactly the browsers and traffic you most want visibility into. This is the highest-value correctness detail in the entire pipeline, and it is the one most likely to be skipped. + +NetProbe handles it in [`QuicTracker`](../../src/core/QuicTracker.cpp), which buffers fragments per connection, keyed by the Destination Connection ID, and reassembles by offset until the TLS record is complete. Because it holds state across packets from the network, it is bounded on every axis: + +```cpp +static constexpr size_t kMaxStreamBytes = size_t{64} * 1024; +``` + +with a cap on tracked connections and a timeout that evicts handshakes that never complete. An attacker who can send packets should not be able to make an analyzer allocate without limit — a reassembler that trusts offsets is a memory-exhaustion primitive wearing a parser costume. + +The frame walk applies the same suspicion: + +```cpp +constexpr uint64_t kSaneCryptoBound = 1u << 24; +``` + +A CRYPTO frame claiming a 16 MB length is not a large handshake; it is an attack or a corrupt packet, and either way the right move is to stop. + +--- + +## Stage 5: reading the SNI + +With a contiguous ClientHello you are back in ordinary TLS 1.3 territory: skip `legacy_version` (2 bytes) and `random` (32), then the variable-length session ID, cipher suites, and compression methods, to reach the extension list. Find extension type `0x0000`, server_name, and read the first `host_name` entry. + +Every one of those length fields comes from the network. Each one gets bounds-checked against the buffer, and any inconsistency aborts the parse rather than clamping and continuing — a parser that "recovers" from a malformed length is a parser that eventually reports a hostname assembled from adjacent memory. + +--- + +## What you end up with + +For a passive observer with no keys and no cooperation from either endpoint: + +- The server name a QUIC client is connecting to, on any UDP port +- For both QUIC v1 and v2 +- Including handshakes split across multiple Initial packets + +And what you explicitly do *not* get: anything after the Initial. Handshake and 1-RTT packets are protected by keys derived from the ephemeral exchange, and no amount of passive observation recovers them. That boundary is worth stating plainly, because the ability to decrypt Initials is sometimes mistaken for a break in QUIC's confidentiality. It is not. It is a deliberate design point, and the visibility it gives you ends exactly where real encryption begins. + +There is also a live counter-pressure worth knowing about: **Encrypted Client Hello**. When ECH is in use the true server name is encrypted inside an outer ClientHello, and none of the above recovers it. NetProbe detects advertised ECH configs in DNS HTTPS records and reports that name resolution has genuinely moved out of view, rather than showing a bare IP and letting you assume the tool failed. + +--- + +## Testing something you cannot eyeball + +Crypto code fails silently. A wrong label, a nonce assembled in the wrong byte order, an AAD off by one — every one of these produces a clean "authentication failed" and no clue. Print-debugging is nearly useless because every intermediate value is uniform-looking bytes. + +The only approach that holds up is deriving known-good vectors from the RFC and asserting on each stage independently, so a failure localizes to one transformation. NetProbe's suite covers the full path end-to-end, including a two-packet split ClientHello, and the parsers are continuously fuzzed under ASan/UBSan with a corpus seeded by hand-built adversarial packets: connection ID lengths beyond the packet, token lengths claiming more than exists, truncated varints. That last category matters here more than almost anywhere else in a network tool, because this code path takes attacker-controlled bytes and does pointer arithmetic on lengths derived from them. + +--- + +*The implementation described here is in [`src/core/QuicParser.cpp`](../../src/core/QuicParser.cpp) and [`src/core/QuicTracker.cpp`](../../src/core/QuicTracker.cpp). Corrections welcome — particularly from anyone who has fought the v2 label change.* From 8c447fa4afd2f32cb3f1a4c91b3faed4c053dbbb Mon Sep 17 00:00:00 2001 From: YehiaGewily <79144005+YehiaGewily@users.noreply.github.com> Date: Sun, 19 Jul 2026 14:01:24 +0300 Subject: [PATCH 11/12] fix: correct clang-tidy header filter and its verification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The clang-tidy gate was reported clean and was not. Running it over all of src/ exits 1: Mbed TLS, GLFW, and NFD headers produce macro-parentheses, reserved-identifier, and use-nullptr findings which WarningsAsErrors turns into failures. QuicParser.cpp, Dashboard.cpp, FlowsView.cpp, and GuiLayer.cpp all fail — precisely the files that include those headers. Two compounding mistakes: The filter. HeaderFilterRegex was '.*[/\](src|include)[/\].*', which also matches _deps/mbedtls-src/include/mbedtls/aes.h — that path contains both "src" and "include". Combined with WarningsAsErrors, every dependency header became a build failure. Fixed by naming the first-party module directories explicitly: src/(capture|cli|core|ui)/. There are no headers directly in src/ and the repository's include/ directory is empty, so this covers everything of ours and cannot match a dependency — verified no _deps path shadows those four names. The verification. I checked this by grepping stdout for first-party paths rather than by looking at the exit code, so failures in dependency headers were invisible to the check. The deliberate-defect test that appeared to prove the gate worked ran on FlowExporter.cpp, which happens to include no third-party header, and so passed for the wrong reason. Re-verified properly: exit code 0 across every file in src/, and the gate still fails on a defect injected into a first-party .cpp and, separately, into a first-party .hpp — in QuicParser, which does include Mbed TLS. That proves dependency headers are excluded while ours are still analysed. The CI job now prints the clang-tidy version and names the failing files, and both .clang-tidy and CONTRIBUTING.md state that this check is judged by exit code, not by reading the log. Also completes the parts of Phase 6 that do not need a human: - The QUIC deep-dive gains an ASCII diagram for every stage — the pipeline overview, the HKDF key schedule, the HkdfLabel byte layout, the header-protection sample offset and mask usage, the AEAD nonce and AAD layout, and CRYPTO reassembly across two Initials — plus line-specific source links throughout and a source index table mapping each stage to its function and line. - CONTRIBUTING.md: per-platform build instructions, how to run each check CI runs, and what review looks for in a tool that parses hostile input (never trust a wire length, degrade visibly rather than plausibly, bound anything holding cross-packet state), plus four good first issues. 89/89 tests pass; clang-tidy exits 0. --- .clang-tidy | 22 +- .github/workflows/ci.yml | 17 +- CONTRIBUTING.md | 101 +++ README.md | 3 +- docs/blog/decrypting-quic-initial-packets.md | 737 +++++++++++-------- 5 files changed, 580 insertions(+), 300 deletions(-) create mode 100644 CONTRIBUTING.md diff --git a/.clang-tidy b/.clang-tidy index 871b113..a41cdde 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -28,11 +28,27 @@ Checks: > # The tree is clean under this configuration, so anything new is a # regression and should fail the build rather than scroll past in a log. +# +# This makes HeaderFilterRegex load-bearing rather than cosmetic: any header +# the filter admits can fail the build, which is why the filter below names +# first-party directories explicitly instead of pattern-matching on "src" or +# "include". It also means a clang-tidy version bump can surface new findings +# — that is intended, and the fix is to address them, not to widen the filter. WarningsAsErrors: '*' -# Only first-party headers. Without this, findings from the FetchContent'd -# ImGui, Mbed TLS, and libmaxminddb headers drown out our own. -HeaderFilterRegex: '.*[/\\](src|include)[/\\].*' +# Only first-party headers. +# +# This must name the source subdirectories explicitly. The obvious pattern — +# matching any path containing "src" or "include" — also matches the +# FetchContent trees, because a dependency checked out to +# _deps/mbedtls-src/include/mbedtls/aes.h contains BOTH. Combined with +# WarningsAsErrors below, that turned every macro-parentheses finding in Mbed +# TLS, GLFW, and NFD into a build failure. +# +# There are no headers directly in src/, and the repository's include/ +# directory is empty, so enumerating the four module directories covers all +# first-party headers and cannot match a dependency. +HeaderFilterRegex: '.*[/\\]src[/\\](capture|cli|core|ui)[/\\]' # clang-tidy must not rewrite formatting; that belongs to the author. FormatStyle: none diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a5df39..c410b85 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -173,17 +173,26 @@ jobs: # FetchContent and is not ours to fix; .clang-tidy's HeaderFilterRegex # keeps their headers out of the report as well. # Driving clang-tidy directly rather than through run-clang-tidy, which - # is packaged inconsistently across distributions. .clang-tidy sets - # WarningsAsErrors, so any finding makes the invocation fail. + # is packaged inconsistently across distributions. + # + # The pass/fail signal is the EXIT CODE, not the output. .clang-tidy + # sets WarningsAsErrors, so a finding fails the invocation; grepping the + # log for first-party paths instead would miss exactly the failures that + # matter, since a finding in a dependency header still fails the run. - name: Run clang-tidy run: | + clang-tidy --version status=0 + failed="" while IFS= read -r file; do echo "--- $file" - clang-tidy -p build-tidy --quiet "$file" || status=1 + if ! clang-tidy -p build-tidy --quiet "$file"; then + status=1 + failed="$failed $file" + fi done < <(find src -name '*.cpp' | sort) if [ "$status" -ne 0 ]; then - echo "::error::clang-tidy reported findings (see the log above)" + echo "::error::clang-tidy reported findings in:$failed" fi exit $status shell: bash diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..3fa28ff --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,101 @@ +# Contributing to NetProbe + +Thanks for taking a look. NetProbe is a packet capture and analysis tool, which means most of its code reads bytes that came off a hostile network. That shapes what review looks for, so this document is mostly about that rather than about style. + +## Building + +You need CMake 3.20+, a C++20 compiler, and libpcap (or the Npcap SDK on Windows). All other dependencies are fetched and pinned automatically by CMake. + +**Windows** — Visual Studio 2022 with the C++ Desktop workload, the [Npcap driver](https://npcap.com/#download), and the Npcap SDK extracted to `C:\Npcap-SDK`: + +```powershell +.\build_project.bat +``` + +**Linux** — `sudo apt install build-essential cmake libpcap-dev libgl1-mesa-dev libgtk-3-dev pkg-config` + +**macOS** — Xcode Command Line Tools and CMake; libpcap and OpenGL ship with the OS. + +```bash +cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=ON +cmake --build build --parallel +ctest --test-dir build --output-on-failure +``` + +The build also produces `data/sample.pcap`, a small synthetic capture you can open without any capture privileges — useful for exercising the whole pipeline while developing. + +## Running the checks CI runs + +Before opening a pull request, run whichever of these your platform supports. CI runs all of them. + +**Tests** (all platforms): + +```bash +ctest --test-dir build --output-on-failure +``` + +**Sanitizers** (Clang or GCC): + +```bash +cmake -S . -B build-asan -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_SANITIZERS=ON -DBUILD_TESTING=ON +cmake --build build-asan --target NetProbeTests --parallel +ctest --test-dir build-asan --output-on-failure +``` + +This is the most valuable single check for parser work. A test can pass while reading past the end of a buffer; under ASan it cannot. + +**Static analysis** (needs `clang-tidy` and a compile database, so configure with Ninja — the Visual Studio generator does not emit one): + +```bash +cmake -S . -B build-tidy -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF +find src -name '*.cpp' -print0 | xargs -0 -n1 clang-tidy -p build-tidy --quiet +``` + +`.clang-tidy` sets `WarningsAsErrors`, so **judge this by the exit code, not by reading the output**. A finding anywhere — including in a header the filter admits — fails the run. + +**Fuzzing** (Clang only): + +```bash +cmake -S . -B build-fuzz -DCMAKE_BUILD_TYPE=Release -DBUILD_FUZZERS=ON -DBUILD_TESTING=OFF +cmake --build build-fuzz --target NetProbeFuzzerSeeds NetProbeParserFuzzer --parallel +./build-fuzz/NetProbeFuzzerSeeds build-fuzz/corpus +./build-fuzz/NetProbeParserFuzzer build-fuzz/corpus -max_total_time=60 +``` + +## What review looks for + +Ordinary code review, plus three things specific to this project: + +**Never trust a length that came off the wire.** Every offset and length in a packet is attacker-controlled. Check it against the actual buffer before using it, and prefer bailing out to clamping and continuing — a parser that "recovers" from an inconsistent length is a parser that eventually reports a hostname assembled from adjacent memory. + +**Degrade visibly, not plausibly.** When a packet cannot be parsed, the right outcome is an empty or clearly-marked field. The dangerous failure is not a crash; it is returning something that looks like a real answer, because it gets shown to the user as fact. Tests in `tests/malformed_test.cpp` pin the expected degraded result for each malformed fixture for exactly this reason. + +**Bound anything that holds state across packets.** The TLS and QUIC reassemblers buffer data keyed by values from the network. Every such structure needs a size cap, a count cap, and a timeout, or it is a memory-exhaustion primitive. + +## Adding tests + +- Unit and integration tests go in `tests/`, using GoogleTest. `tests/PacketBuilders.hpp` has wire-format constructors for synthetic frames; extend it rather than hand-rolling bytes in a test. +- Malformed-input fixtures go in `test::malformed::allCases()` in the same header. They are shared with the fuzz seed generator, so one addition improves both the regression suite and the fuzzing corpus. +- If you fix a bug the fuzzer found, check the reproducing input in as a named fixture. That loop is what keeps fixes from regressing. + +## Commit and pull request conventions + +- Conventional-commit prefixes: `feat:`, `fix:`, `test:`, `docs:`, `build:`, `ci:`, `refactor:`, `perf:`, `chore:`. +- Explain **why** in the commit body, not just what. The diff already shows what changed; what it cannot show is the reasoning or the alternative you rejected. +- One logical change per commit. Keep a refactor and a behaviour change in separate commits. +- Say what you verified and how. "Tests pass" is less useful than naming the case you checked. + +## Good first issues + +If you are looking for somewhere to start, these are small, self-contained, and need no architectural context: + +- **Add a malformed-packet fixture.** Pick a protocol field NetProbe parses, construct a frame that lies about it, add it to `test::malformed::allCases()` with its expected degraded outcome, and confirm the parser handles it. Each one is a few lines and strengthens both the test suite and the fuzz corpus. +- **Extend service identification.** `ProtocolParser::identifyService` maps hostnames to friendly service names. Adding well-known services is a self-contained change with an obvious test. +- **Add a DNS record type.** `DNSParser` handles A, AAAA, CNAME, PTR, SRV, TXT, and HTTPS/SVCB. Others (MX, NS, CAA) follow the same shape. +- **CLI usability.** The headless CLI is deliberately minimal; small additions like a `--quiet` flag or a summary line format are easy to scope. + +Larger items worth discussing in an issue first: additional link-layer types, IPv6 extension-header coverage, and anything touching the QUIC decryption path. + +## Reporting bugs + +A capture file that reproduces the problem is worth more than any description. If the traffic is sensitive, a synthetic `PacketBuilders.hpp`-style reproduction is just as good. Include your OS, how NetProbe was built, and whether it was running elevated. diff --git a/README.md b/README.md index bb38449..5dae094 100644 --- a/README.md +++ b/README.md @@ -215,8 +215,7 @@ flowchart LR ## Contributing - -Contributions are welcome! Please feel free to submit a Pull Request. +Contributions are welcome. See [CONTRIBUTING.md](CONTRIBUTING.md) for build instructions on each platform, how to run the same checks CI runs (tests, sanitizers, clang-tidy, fuzzing), and what review looks for in a tool that parses hostile input. It also lists some good first issues. ## License diff --git a/docs/blog/decrypting-quic-initial-packets.md b/docs/blog/decrypting-quic-initial-packets.md index f01a5a3..5aeaab4 100644 --- a/docs/blog/decrypting-quic-initial-packets.md +++ b/docs/blog/decrypting-quic-initial-packets.md @@ -1,291 +1,446 @@ -# Decrypting QUIC Initial packets by hand - -*How a passive analyzer recovers the server name from an encrypted QUIC handshake: HKDF, header protection, AES-GCM, and CRYPTO reassembly.* - ---- - -## Why this is worth explaining - -If you run a packet analyzer on a modern network, a growing share of your traffic is QUIC, and QUIC looks like noise. TLS over TCP at least hands you a plaintext ClientHello: the SNI is sitting there in the clear, and any tool can read it. QUIC encrypts its handshake from the very first packet. Point Wireshark at a QUIC flow without keys and you get UDP payloads of undifferentiated bytes. - -Except it isn't really undifferentiated, because QUIC's *Initial* packets are encrypted with keys derived from a value that is printed in the clear in the packet header. This is not a flaw — the encryption exists to stop middleboxes ossifying the protocol, not to hide anything from an observer — but it means a passive analyzer *can* decrypt Initial packets and read the ClientHello inside. - -Very few people write this up accessibly. The RFCs describe it precisely and unhelpfully; most blog posts stop at "QUIC encrypts its handshake." This post walks the entire path in the order the code executes, using [NetProbe](https://github.com/YehiaGewily/netprobe-cpp)'s implementation in [`src/core/QuicParser.cpp`](../../src/core/QuicParser.cpp) as the worked example. - -The five stages: - -1. Parse enough of the long header to find the Destination Connection ID -2. Derive Initial secrets from that DCID (HKDF-Extract, then HKDF-Expand-Label) -3. Remove header protection to recover the packet number -4. Decrypt the payload with AES-128-GCM -5. Reassemble CRYPTO frames — possibly across several packets — and parse the ClientHello - -Stage 5 is the one most implementations get wrong, and it is the one that matters most in 2026. - ---- - -## Stage 0: the long header - -A QUIC Initial packet uses the *long header* form. The first byte tells you which: - -``` - 0 1 2 3 4 5 6 7 -+-+-+-+-+-+-+-+-+ -|1|1|T T|R R|P P| -+-+-+-+-+-+-+-+-+ -``` - -Bit 7 set means long header; bit 6 is the "fixed bit", always 1. NetProbe's cheap pre-filter checks exactly those two bits before doing any work: - -```cpp -bool QuicParser::looksLikeLongHeader(const uint8_t* data, size_t size) { - // Long header (bit 7) + fixed bit (bit 6). - ... -} -``` - -This matters for performance. Every UDP packet on the wire gets this test; only the ones that pass go anywhere near the crypto. - -After the first byte comes a 32-bit version, then the connection IDs: - -``` -first byte (1) | version (4) | DCID len (1) | DCID (0-20) | SCID len (1) | SCID (0-20) | token len (varint) | token | length (varint) | packet number (1-4, protected) | payload -``` - -Two versions matter today: - -| Version | Value | Spec | -| --- | --- | --- | -| QUIC v1 | `0x00000001` | RFC 9000 | -| QUIC v2 | `0x6b3343cf` | RFC 9369 | - -**The v2 trap.** QUIC v2 is not a cosmetic revision of v1. It uses a *different Initial salt* and *different key-derivation labels*. A v1-only parser does not error on v2 — it derives the wrong keys, AES-GCM authentication fails, and the packet is silently discarded as "not decryptable." You get no SNI and no indication why. NetProbe carries both parameter sets: - -```cpp -struct VersionParams { - const uint8_t* salt; - const char* keyLabel; - const char* ivLabel; - const char* hpLabel; - uint8_t initialTypeBits; // value of (firstByte & 0x30) for an Initial -}; - -// v1: {kInitialSaltV1, "quic key", "quic iv", "quic hp", 0x00} -// v2: {kInitialSaltV2, "quicv2 key", "quicv2 iv", "quicv2 hp", 0x10} -``` - -Note the last field. The two-bit packet *type* encoding also changed between versions: an Initial is `0b00` in v1 and `0b01` in v2. Hardcode the v1 value and you will reject every v2 Initial before you even get to the keys. - ---- - -## Stage 1: deriving keys from a public value - -Here is the part that surprises people. The Initial keys come from the **Destination Connection ID**, which is transmitted in plaintext in the header of the very packet you are trying to decrypt. - -RFC 9001 §5.2 defines: - -``` -initial_secret = HKDF-Extract(initial_salt, client_dst_connection_id) -``` - -The salt is a fixed 20-byte constant published in the RFC. For v1 it begins `0x38 0x76 0x2c 0xf7 …`. So the "secret" is derived from a public constant and a public field. Anyone observing the packet can compute it. - -That is deliberate. The purpose of Initial encryption is to make the handshake opaque to middleboxes so they cannot come to depend on its internals and freeze the protocol in place — not to provide confidentiality against an observer. Real confidentiality begins with the Handshake keys, which come from the ephemeral key exchange inside the ClientHello. Those you cannot derive passively, and NetProbe does not try. - -From the initial secret, one more expansion gives the client's secret: - -``` -client_initial_secret = HKDF-Expand-Label(initial_secret, "client in", "", 32) -``` - -Then three more give the actual key material: - -``` -key = HKDF-Expand-Label(client_initial_secret, "quic key", "", 16) // AES-128 -iv = HKDF-Expand-Label(client_initial_secret, "quic iv", "", 12) // GCM nonce base -hp = HKDF-Expand-Label(client_initial_secret, "quic hp", "", 16) // header protection -``` - -In code, that whole chain is: - -```cpp -bool deriveClientInitialKeys(const VersionParams& params, - const uint8_t* dcid, size_t dcidLen, InitialKeys& out) { - uint8_t initialSecret[32]; - if (!hkdfExtract(params.salt, 20, dcid, dcidLen, initialSecret)) return false; - - uint8_t clientSecret[32]; - if (!hkdfExpandLabel(initialSecret, 32, "client in", 9, clientSecret, 32)) return false; - - if (!hkdfExpandLabel(clientSecret, 32, params.keyLabel, ..., out.key, 16)) return false; - if (!hkdfExpandLabel(clientSecret, 32, params.ivLabel, ..., out.iv, 12)) return false; - if (!hkdfExpandLabel(clientSecret, 32, params.hpLabel, ..., out.hp, 16)) return false; - return true; -} -``` - -### HKDF-Expand-Label is not HKDF-Expand - -This trips people up. `HKDF-Expand-Label` is a TLS 1.3 construction (RFC 8446 §7.1) that builds a structured `info` parameter before calling ordinary HKDF-Expand: - -``` -struct { - uint16 length; - opaque label<7..255>; // "tls13 " || label - opaque context<0..255>; -} HkdfLabel; -``` - -The literal ASCII prefix `"tls13 "` — with the trailing space — is prepended to every label. QUIC uses the TLS 1.3 KDF unmodified, so `"quic key"` is really `"tls13 quic key"` on the wire: - -```cpp -static constexpr char kPrefix[] = "tls13 "; -... -info[p++] = static_cast(outLen >> 8); -info[p++] = static_cast(outLen); -info[p++] = static_cast(fullLabelLen); -std::memcpy(info + p, kPrefix, kPrefixLen); p += kPrefixLen; -std::memcpy(info + p, label, labelLen); p += labelLen; -info[p++] = 0x00; // empty context -``` - -Forget the prefix, or the two-byte big-endian output length, or the trailing empty-context byte, and you get 16 perfectly valid-looking bytes that decrypt nothing. There is no diagnostic. This is the single most common place to lose an afternoon. - ---- - -## Stage 2: the chicken-and-egg of header protection - -You now have keys. You cannot yet decrypt, because you do not know where the ciphertext starts. - -The packet number is 1–4 bytes, and *how many* is encoded in the low two bits of the first byte — which is itself encrypted. Worse, the packet number is an input to the AEAD nonce. So: - -- To find the packet number length, you must decrypt the first byte. -- To decrypt the payload, you need the packet number. - -QUIC resolves this with **header protection**: a separate cipher pass that masks only the header bits, keyed by a sample of the ciphertext taken from a position that does not depend on the packet number length. - -The trick is that the sample is taken at a *fixed* offset — 4 bytes past where the packet number starts, as though the packet number were always the maximum 4 bytes: - -```cpp -// The HP sample is 16 bytes at offset (pnOffset + 4), assuming the -// maximum 4-byte packet number. -``` - -Encrypt that 16-byte sample with AES-128-ECB under the `hp` key. The output is a 5-byte mask: - -```cpp -mbedtls_aes_setkey_enc(&ctx, keys.hp, 128); -mbedtls_aes_crypt_ecb(&ctx, MBEDTLS_AES_ENCRYPT, sample, mask); - -// Long header: the low nibble (bits 0..3) is masked. -inOutFirstByte ^= (mask[0] & 0x0F); -outPnLen = static_cast(inOutFirstByte & 0x03) + 1; -for (size_t i = 0; i < outPnLen; ++i) { - pnBytes[i] ^= mask[i + 1]; -} -``` - -`mask[0]` unmasks the first byte — the low **nibble** for a long header, the low 5 bits for a short header. Once the first byte is clear, its low two bits give the packet number length, and `mask[1..4]` unmask that many packet-number bytes. - -Note this is AES-ECB, the mode everyone is told never to use. It is correct here precisely because it is a single block of a value that is never reused: one deterministic permutation of a unique sample. Using it as a general-purpose mode is what's dangerous. - ---- - -## Stage 3: AES-128-GCM, and the nonce that is not the IV - -Now the payload. QUIC uses AEAD_AES_128_GCM for Initial packets, with two details that differ from how people usually reach for GCM. - -**The nonce is the IV XOR the packet number**, big-endian and right-aligned into 12 bytes: - -```cpp -uint8_t nonce[12]; -std::memcpy(nonce, keys.iv, 12); -for (int i = 0; i < 8; ++i) { - nonce[12 - 1 - i] ^= static_cast((packetNumber >> (i * 8)) & 0xFFu); -} -``` - -The derived `iv` is not used directly — it is a per-connection baseline that the packet number perturbs, guaranteeing a unique nonce per packet without transmitting one. - -**The associated data is the entire header, unprotected.** Everything from the first byte through the end of the packet number, *after* header protection has been removed, is authenticated but not encrypted. Get the AAD boundaries wrong by a single byte and GCM authentication fails with no hint as to why. - -The 16-byte authentication tag occupies the last 16 bytes of the payload: - -```cpp -mbedtls_gcm_auth_decrypt(&ctx, ciphertextLen, - nonce, sizeof(nonce), - aad, aadLen, - tag, 16, - ciphertext, outPlaintext.data()); -``` - -This authentication check is doing real work for an analyzer, not just crypto hygiene. Because Initial keys are derivable by anyone, a passive parser will happily attempt decryption on any UDP packet whose first two bits look like a long header. The GCM tag is what distinguishes an actual QUIC Initial from a coincidence. **A failed tag is not an error to report — it is the normal answer for a packet that was never QUIC.** - ---- - -## Stage 4: CRYPTO frames, and the part everyone gets wrong - -Decryption gives you a QUIC frame sequence, not a ClientHello. Walk it, collecting CRYPTO frames (type `0x06`) and skipping the frames that legitimately appear in an Initial: - -- `0x00` PADDING — usually a *lot* of it, since Initials are padded to at least 1200 bytes -- `0x01` PING -- `0x02`/`0x03` ACK -- `0x06` CRYPTO — carries `offset` and `length` varints, then the TLS bytes - -Each CRYPTO frame carries an **offset**. That is the whole story: the ClientHello is a byte stream that may be split across multiple frames, in multiple packets, arriving in any order. - -For years you could ignore this. A ClientHello fit comfortably in one Initial packet, so a parser that grabbed the first CRYPTO frame and parsed it worked essentially always. - -**That stopped being true.** Post-quantum key exchange — X25519MLKEM768, now the default in Chrome and Firefox — adds roughly a kilobyte to the ClientHello. A post-quantum key share does not fit alongside everything else in a single Initial. Real ClientHellos are now routinely split across two or more Initial packets. - -So a parser that reads only the first CRYPTO frame does not fail loudly. It silently stops finding SNI on a steadily growing share of real traffic — exactly the browsers and traffic you most want visibility into. This is the highest-value correctness detail in the entire pipeline, and it is the one most likely to be skipped. - -NetProbe handles it in [`QuicTracker`](../../src/core/QuicTracker.cpp), which buffers fragments per connection, keyed by the Destination Connection ID, and reassembles by offset until the TLS record is complete. Because it holds state across packets from the network, it is bounded on every axis: - -```cpp -static constexpr size_t kMaxStreamBytes = size_t{64} * 1024; -``` - -with a cap on tracked connections and a timeout that evicts handshakes that never complete. An attacker who can send packets should not be able to make an analyzer allocate without limit — a reassembler that trusts offsets is a memory-exhaustion primitive wearing a parser costume. - -The frame walk applies the same suspicion: - -```cpp -constexpr uint64_t kSaneCryptoBound = 1u << 24; -``` - -A CRYPTO frame claiming a 16 MB length is not a large handshake; it is an attack or a corrupt packet, and either way the right move is to stop. - ---- - -## Stage 5: reading the SNI - -With a contiguous ClientHello you are back in ordinary TLS 1.3 territory: skip `legacy_version` (2 bytes) and `random` (32), then the variable-length session ID, cipher suites, and compression methods, to reach the extension list. Find extension type `0x0000`, server_name, and read the first `host_name` entry. - -Every one of those length fields comes from the network. Each one gets bounds-checked against the buffer, and any inconsistency aborts the parse rather than clamping and continuing — a parser that "recovers" from a malformed length is a parser that eventually reports a hostname assembled from adjacent memory. - ---- - -## What you end up with - -For a passive observer with no keys and no cooperation from either endpoint: - -- The server name a QUIC client is connecting to, on any UDP port -- For both QUIC v1 and v2 -- Including handshakes split across multiple Initial packets - -And what you explicitly do *not* get: anything after the Initial. Handshake and 1-RTT packets are protected by keys derived from the ephemeral exchange, and no amount of passive observation recovers them. That boundary is worth stating plainly, because the ability to decrypt Initials is sometimes mistaken for a break in QUIC's confidentiality. It is not. It is a deliberate design point, and the visibility it gives you ends exactly where real encryption begins. - -There is also a live counter-pressure worth knowing about: **Encrypted Client Hello**. When ECH is in use the true server name is encrypted inside an outer ClientHello, and none of the above recovers it. NetProbe detects advertised ECH configs in DNS HTTPS records and reports that name resolution has genuinely moved out of view, rather than showing a bare IP and letting you assume the tool failed. - ---- - -## Testing something you cannot eyeball - -Crypto code fails silently. A wrong label, a nonce assembled in the wrong byte order, an AAD off by one — every one of these produces a clean "authentication failed" and no clue. Print-debugging is nearly useless because every intermediate value is uniform-looking bytes. - -The only approach that holds up is deriving known-good vectors from the RFC and asserting on each stage independently, so a failure localizes to one transformation. NetProbe's suite covers the full path end-to-end, including a two-packet split ClientHello, and the parsers are continuously fuzzed under ASan/UBSan with a corpus seeded by hand-built adversarial packets: connection ID lengths beyond the packet, token lengths claiming more than exists, truncated varints. That last category matters here more than almost anywhere else in a network tool, because this code path takes attacker-controlled bytes and does pointer arithmetic on lengths derived from them. - ---- - -*The implementation described here is in [`src/core/QuicParser.cpp`](../../src/core/QuicParser.cpp) and [`src/core/QuicTracker.cpp`](../../src/core/QuicTracker.cpp). Corrections welcome — particularly from anyone who has fought the v2 label change.* +# Decrypting QUIC Initial packets by hand + +*How a passive analyzer recovers the server name from an encrypted QUIC handshake: HKDF, header protection, AES-GCM, and CRYPTO reassembly.* + +--- + +## Why this is worth explaining + +If you run a packet analyzer on a modern network, a growing share of your traffic is QUIC, and QUIC looks like noise. TLS over TCP at least hands you a plaintext ClientHello: the SNI is sitting there in the clear, and any tool can read it. QUIC encrypts its handshake from the very first packet. Point Wireshark at a QUIC flow without keys and you get UDP payloads of undifferentiated bytes. + +Except it isn't really undifferentiated, because QUIC's *Initial* packets are encrypted with keys derived from a value that is printed in the clear in the packet header. This is not a flaw — the encryption exists to stop middleboxes ossifying the protocol, not to hide anything from an observer — but it means a passive analyzer *can* decrypt Initial packets and read the ClientHello inside. + +Very few people write this up accessibly. The RFCs describe it precisely and unhelpfully; most blog posts stop at "QUIC encrypts its handshake." This post walks the entire path in the order the code executes, using [NetProbe](https://github.com/YehiaGewily/netprobe-cpp)'s implementation in [`src/core/QuicParser.cpp`](../../src/core/QuicParser.cpp) as the worked example. + +The five stages: + +``` + UDP payload + | + [0] long header parse ....... find version + Destination Connection ID + | QuicParser.cpp:322 (parseInitial) + v + [1] key derivation .......... HKDF-Extract(salt, DCID) -> initial_secret + | HKDF-Expand-Label -> key / iv / hp + | QuicParser.cpp:117 (deriveClientInitialKeys) + v + [2] header protection ....... AES-128-ECB over a ciphertext sample + | -> unmask first byte + packet number + | QuicParser.cpp:136 (removeHeaderProtection) + v + [3] payload decryption ...... AES-128-GCM, nonce = iv XOR packet number + | AAD = the unprotected header + | QuicParser.cpp:163 (decryptPayload) + v + [4] frame walk .............. collect CRYPTO frames by offset + | QuicParser.cpp:194 (collectCryptoFrames) + v + [5] reassembly + parse ...... stitch fragments across packets, read SNI + QuicTracker.cpp / QuicParser.cpp:258 +``` + +1. Parse enough of the long header to find the Destination Connection ID +2. Derive Initial secrets from that DCID (HKDF-Extract, then HKDF-Expand-Label) +3. Remove header protection to recover the packet number +4. Decrypt the payload with AES-128-GCM +5. Reassemble CRYPTO frames — possibly across several packets — and parse the ClientHello + +Stage 5 is the one most implementations get wrong, and it is the one that matters most in 2026. + +--- + +## Stage 0: the long header + +A QUIC Initial packet uses the *long header* form. The first byte tells you which: + +``` + 0 1 2 3 4 5 6 7 ++-+-+-+-+-+-+-+-+ +|1|1|T T|R R|P P| ++-+-+-+-+-+-+-+-+ +``` + +Bit 7 set means long header; bit 6 is the "fixed bit", always 1. NetProbe's cheap pre-filter checks exactly those two bits before doing any work: + +```cpp +// QuicParser.cpp:249 +bool QuicParser::looksLikeLongHeader(const uint8_t* data, size_t size) { + // Long header (bit 7) + fixed bit (bit 6). + ... +} +``` + +[`QuicParser.cpp:249`](../../src/core/QuicParser.cpp#L249) + +This matters for performance. Every UDP packet on the wire gets this test; only the ones that pass go anywhere near the crypto. + +After the first byte comes a 32-bit version, then the connection IDs: + +``` +first byte (1) | version (4) | DCID len (1) | DCID (0-20) | SCID len (1) | SCID (0-20) | token len (varint) | token | length (varint) | packet number (1-4, protected) | payload +``` + +Two versions matter today: + +| Version | Value | Spec | +| --- | --- | --- | +| QUIC v1 | `0x00000001` | RFC 9000 | +| QUIC v2 | `0x6b3343cf` | RFC 9369 | + +**The v2 trap.** QUIC v2 is not a cosmetic revision of v1. It uses a *different Initial salt* and *different key-derivation labels*. A v1-only parser does not error on v2 — it derives the wrong keys, AES-GCM authentication fails, and the packet is silently discarded as "not decryptable." You get no SNI and no indication why. NetProbe carries both parameter sets: + +```cpp +// QuicParser.cpp:35, selected at QuicParser.cpp:43 +struct VersionParams { + const uint8_t* salt; + const char* keyLabel; + const char* ivLabel; + const char* hpLabel; + uint8_t initialTypeBits; // value of (firstByte & 0x30) for an Initial +}; + +// v1: {kInitialSaltV1, "quic key", "quic iv", "quic hp", 0x00} +// v2: {kInitialSaltV2, "quicv2 key", "quicv2 iv", "quicv2 hp", 0x10} +``` + +Salts at [`QuicParser.cpp:20`](../../src/core/QuicParser.cpp#L20) (v1) and [`QuicParser.cpp:28`](../../src/core/QuicParser.cpp#L28) (v2); selection at [`QuicParser.cpp:43`](../../src/core/QuicParser.cpp#L43). + +Note the last field. The two-bit packet *type* encoding also changed between versions: an Initial is `0b00` in v1 and `0b01` in v2. Hardcode the v1 value and you will reject every v2 Initial before you even get to the keys. + +--- + +## Stage 1: deriving keys from a public value + +Here is the part that surprises people. The Initial keys come from the **Destination Connection ID**, which is transmitted in plaintext in the header of the very packet you are trying to decrypt. + +RFC 9001 §5.2 defines: + +``` +initial_secret = HKDF-Extract(initial_salt, client_dst_connection_id) +``` + +The salt is a fixed 20-byte constant published in the RFC. For v1 it begins `0x38 0x76 0x2c 0xf7 …`. So the "secret" is derived from a public constant and a public field. Anyone observing the packet can compute it. + +That is deliberate. The purpose of Initial encryption is to make the handshake opaque to middleboxes so they cannot come to depend on its internals and freeze the protocol in place — not to provide confidentiality against an observer. Real confidentiality begins with the Handshake keys, which come from the ephemeral key exchange inside the ClientHello. Those you cannot derive passively, and NetProbe does not try. + +From the initial secret, one more expansion gives the client's secret: + +``` +client_initial_secret = HKDF-Expand-Label(initial_secret, "client in", "", 32) +``` + +Then three more give the actual key material: + +``` +key = HKDF-Expand-Label(client_initial_secret, "quic key", "", 16) // AES-128 +iv = HKDF-Expand-Label(client_initial_secret, "quic iv", "", 12) // GCM nonce base +hp = HKDF-Expand-Label(client_initial_secret, "quic hp", "", 16) // header protection +``` + +In code, that whole chain is: + +``` + initial_salt (RFC constant) DCID (from the packet, plaintext) + | | + +----------- HKDF-Extract ---------+ + | + initial_secret (32) + | + HKDF-Expand-Label "client in" + | + client_initial_secret (32) + | + +---------------------+---------------------+ + | | | + "quic key" "quic iv" "quic hp" + | | | + key (16) iv (12) hp (16) + AES-128-GCM nonce baseline header protection +``` + +```cpp +// QuicParser.cpp:117 +bool deriveClientInitialKeys(const VersionParams& params, + const uint8_t* dcid, size_t dcidLen, InitialKeys& out) { + uint8_t initialSecret[32]; + if (!hkdfExtract(params.salt, 20, dcid, dcidLen, initialSecret)) return false; + + uint8_t clientSecret[32]; + if (!hkdfExpandLabel(initialSecret, 32, "client in", 9, clientSecret, 32)) return false; + + if (!hkdfExpandLabel(clientSecret, 32, params.keyLabel, ..., out.key, 16)) return false; + if (!hkdfExpandLabel(clientSecret, 32, params.ivLabel, ..., out.iv, 12)) return false; + if (!hkdfExpandLabel(clientSecret, 32, params.hpLabel, ..., out.hp, 16)) return false; + return true; +} +``` + +[`QuicParser.cpp:117`](../../src/core/QuicParser.cpp#L117) + +### HKDF-Expand-Label is not HKDF-Expand + +This trips people up. `HKDF-Expand-Label` is a TLS 1.3 construction (RFC 8446 §7.1) that builds a structured `info` parameter before calling ordinary HKDF-Expand: + +``` +struct { + uint16 length; + opaque label<7..255>; // "tls13 " || label + opaque context<0..255>; +} HkdfLabel; +``` + +The literal ASCII prefix `"tls13 "` — with the trailing space — is prepended to every label. QUIC uses the TLS 1.3 KDF unmodified, so `"quic key"` is really `"tls13 quic key"` on the wire: + +``` + HkdfLabel info bytes: + + +--------+--------+--------+---------------------------+--------+ + | len hi | len lo | lblLen | "tls13 " || label | 0x00 | + +--------+--------+--------+---------------------------+--------+ + output length length prefix is literal ASCII, empty + (big endian) of the including the trailing context + label space +``` + +```cpp +// QuicParser.cpp:95 +static constexpr char kPrefix[] = "tls13 "; +... +info[p++] = static_cast(outLen >> 8); +info[p++] = static_cast(outLen); +info[p++] = static_cast(fullLabelLen); +std::memcpy(info + p, kPrefix, kPrefixLen); p += kPrefixLen; +std::memcpy(info + p, label, labelLen); p += labelLen; +info[p++] = 0x00; // empty context +``` + +[`QuicParser.cpp:95`](../../src/core/QuicParser.cpp#L95) + +Forget the prefix, or the two-byte big-endian output length, or the trailing empty-context byte, and you get 16 perfectly valid-looking bytes that decrypt nothing. There is no diagnostic. This is the single most common place to lose an afternoon. + +--- + +## Stage 2: the chicken-and-egg of header protection + +You now have keys. You cannot yet decrypt, because you do not know where the ciphertext starts. + +The packet number is 1–4 bytes, and *how many* is encoded in the low two bits of the first byte — which is itself encrypted. Worse, the packet number is an input to the AEAD nonce. So: + +- To find the packet number length, you must decrypt the first byte. +- To decrypt the payload, you need the packet number. + +QUIC resolves this with **header protection**: a separate cipher pass that masks only the header bits, keyed by a sample of the ciphertext taken from a position that does not depend on the packet number length. + +The trick is that the sample is taken at a *fixed* offset — 4 bytes past where the packet number starts, as though the packet number were always the maximum 4 bytes: + +``` + ... | packet number (1-4 bytes, protected) | payload ... + ^ + pnOffset + |<------- 4 bytes ------->|<--- 16-byte HP sample --->| + + AES-128-ECB(hp_key, sample) -> mask[16], of which 5 bytes are used: + + mask[0] -> XOR into first byte (low nibble, long header) + the unmasked low 2 bits give the packet number length N + mask[1..N] -> XOR into the N packet-number bytes +``` + + +```cpp +// The HP sample is 16 bytes at offset (pnOffset + 4), assuming the +// maximum 4-byte packet number. +``` + +Encrypt that 16-byte sample with AES-128-ECB under the `hp` key. The output is a 5-byte mask: + +```cpp +// QuicParser.cpp:136 +mbedtls_aes_setkey_enc(&ctx, keys.hp, 128); +mbedtls_aes_crypt_ecb(&ctx, MBEDTLS_AES_ENCRYPT, sample, mask); + +// Long header: the low nibble (bits 0..3) is masked. +inOutFirstByte ^= (mask[0] & 0x0F); +outPnLen = static_cast(inOutFirstByte & 0x03) + 1; +for (size_t i = 0; i < outPnLen; ++i) { + pnBytes[i] ^= mask[i + 1]; +} +``` + +[`QuicParser.cpp:136`](../../src/core/QuicParser.cpp#L136) + +`mask[0]` unmasks the first byte — the low **nibble** for a long header, the low 5 bits for a short header. Once the first byte is clear, its low two bits give the packet number length, and `mask[1..4]` unmask that many packet-number bytes. + +Note this is AES-ECB, the mode everyone is told never to use. It is correct here precisely because it is a single block of a value that is never reused: one deterministic permutation of a unique sample. Using it as a general-purpose mode is what's dangerous. + +--- + +## Stage 3: AES-128-GCM, and the nonce that is not the IV + +Now the payload. QUIC uses AEAD_AES_128_GCM for Initial packets, with two details that differ from how people usually reach for GCM. + +**The nonce is the IV XOR the packet number**, big-endian and right-aligned into 12 bytes: + +``` + iv (12 bytes, from HKDF) b0 b1 b2 b3 b4 b5 b6 b7 b8 b9 b10 b11 + packet number (8 bytes) XOR right-aligned -> b4..b11 + ------------------------------ + nonce (12 bytes) b0 b1 b2 b3 (b4^pn0) ... (b11^pn7) + + AEAD layout of the packet: + + +--------------------------------+-------------------+------------+ + | header, header-protection | ciphertext | GCM tag | + | REMOVED (first byte .. pn) | | (16 bytes) | + +--------------------------------+-------------------+------------+ + |<---------- AAD --------------->|<-- decrypted -->| +``` + + +```cpp +// QuicParser.cpp:163 +uint8_t nonce[12]; +std::memcpy(nonce, keys.iv, 12); +for (int i = 0; i < 8; ++i) { + nonce[12 - 1 - i] ^= static_cast((packetNumber >> (i * 8)) & 0xFFu); +} +``` + +The derived `iv` is not used directly — it is a per-connection baseline that the packet number perturbs, guaranteeing a unique nonce per packet without transmitting one. + +**The associated data is the entire header, unprotected.** Everything from the first byte through the end of the packet number, *after* header protection has been removed, is authenticated but not encrypted. Get the AAD boundaries wrong by a single byte and GCM authentication fails with no hint as to why. + +The 16-byte authentication tag occupies the last 16 bytes of the payload: + +```cpp +mbedtls_gcm_auth_decrypt(&ctx, ciphertextLen, + nonce, sizeof(nonce), + aad, aadLen, + tag, 16, + ciphertext, outPlaintext.data()); +``` + +[`QuicParser.cpp:163`](../../src/core/QuicParser.cpp#L163) + +This authentication check is doing real work for an analyzer, not just crypto hygiene. Because Initial keys are derivable by anyone, a passive parser will happily attempt decryption on any UDP packet whose first two bits look like a long header. The GCM tag is what distinguishes an actual QUIC Initial from a coincidence. **A failed tag is not an error to report — it is the normal answer for a packet that was never QUIC.** + +--- + +## Stage 4: CRYPTO frames, and the part everyone gets wrong + +Decryption gives you a QUIC frame sequence, not a ClientHello. Walk it, collecting CRYPTO frames (type `0x06`) and skipping the frames that legitimately appear in an Initial: + +- `0x00` PADDING — usually a *lot* of it, since Initials are padded to at least 1200 bytes +- `0x01` PING +- `0x02`/`0x03` ACK +- `0x06` CRYPTO — carries `offset` and `length` varints, then the TLS bytes + +The frame walk is [`QuicParser.cpp:194`](../../src/core/QuicParser.cpp#L194). + +Each CRYPTO frame carries an **offset**. That is the whole story: the ClientHello is a byte stream that may be split across multiple frames, in multiple packets, arriving in any order. + +``` + Initial packet #1 (decrypted) Initial packet #2 (decrypted) + +----------------------------+ +----------------------------+ + | PADDING ... | | CRYPTO off=1100 len=800 | + | CRYPTO off=0 len=1100 | | PADDING ... | + +----------------------------+ +----------------------------+ + | | + +------------------+-------------------+ + v + reassembly buffer, keyed by DCID, ordered by offset + +--------------------------------------------+ + | 0 .......................... 1900 (bytes) | + +--------------------------------------------+ + | + v + one contiguous TLS ClientHello +``` + + +For years you could ignore this. A ClientHello fit comfortably in one Initial packet, so a parser that grabbed the first CRYPTO frame and parsed it worked essentially always. + +**That stopped being true.** Post-quantum key exchange — X25519MLKEM768, now the default in Chrome and Firefox — adds roughly a kilobyte to the ClientHello. A post-quantum key share does not fit alongside everything else in a single Initial. Real ClientHellos are now routinely split across two or more Initial packets. + +So a parser that reads only the first CRYPTO frame does not fail loudly. It silently stops finding SNI on a steadily growing share of real traffic — exactly the browsers and traffic you most want visibility into. This is the highest-value correctness detail in the entire pipeline, and it is the one most likely to be skipped. + +NetProbe handles it in [`QuicTracker`](../../src/core/QuicTracker.cpp), which buffers fragments per connection, keyed by the Destination Connection ID, and reassembles by offset until the TLS record is complete. Because it holds state across packets from the network, it is bounded on every axis: + +```cpp +// QuicTracker.hpp:33-35 +static constexpr size_t kMaxStreamBytes = size_t{64} * 1024; +static constexpr int64_t kConnectionTimeoutMicroseconds = 10'000'000; +static constexpr size_t kMaxTrackedConnections = 1'024; +``` + +[`QuicTracker.hpp:33`](../../src/core/QuicTracker.hpp#L33) + +A cap on tracked connections and a timeout evict handshakes that never complete. An attacker who can send packets should not be able to make an analyzer allocate without limit — a reassembler that trusts offsets is a memory-exhaustion primitive wearing a parser costume. + +The frame walk applies the same suspicion: + +```cpp +// QuicParser.cpp:196 +constexpr uint64_t kSaneCryptoBound = 1u << 24; +``` + +[`QuicParser.cpp:196`](../../src/core/QuicParser.cpp#L196) + +A CRYPTO frame claiming a 16 MB length is not a large handshake; it is an attack or a corrupt packet, and either way the right move is to stop. + +--- + +## Stage 5: reading the SNI + +The ClientHello parse is [`QuicParser.cpp:258`](../../src/core/QuicParser.cpp#L258). + +With a contiguous ClientHello you are back in ordinary TLS 1.3 territory: skip `legacy_version` (2 bytes) and `random` (32), then the variable-length session ID, cipher suites, and compression methods, to reach the extension list. Find extension type `0x0000`, server_name, and read the first `host_name` entry. + +Every one of those length fields comes from the network. Each one gets bounds-checked against the buffer, and any inconsistency aborts the parse rather than clamping and continuing — a parser that "recovers" from a malformed length is a parser that eventually reports a hostname assembled from adjacent memory. + +--- + +## What you end up with + +For a passive observer with no keys and no cooperation from either endpoint: + +- The server name a QUIC client is connecting to, on any UDP port +- For both QUIC v1 and v2 +- Including handshakes split across multiple Initial packets + +And what you explicitly do *not* get: anything after the Initial. Handshake and 1-RTT packets are protected by keys derived from the ephemeral exchange, and no amount of passive observation recovers them. That boundary is worth stating plainly, because the ability to decrypt Initials is sometimes mistaken for a break in QUIC's confidentiality. It is not. It is a deliberate design point, and the visibility it gives you ends exactly where real encryption begins. + +There is also a live counter-pressure worth knowing about: **Encrypted Client Hello**. When ECH is in use the true server name is encrypted inside an outer ClientHello, and none of the above recovers it. NetProbe detects advertised ECH configs in DNS HTTPS records and reports that name resolution has genuinely moved out of view, rather than showing a bare IP and letting you assume the tool failed. + +--- + +## Testing something you cannot eyeball + +Crypto code fails silently. A wrong label, a nonce assembled in the wrong byte order, an AAD off by one — every one of these produces a clean "authentication failed" and no clue. Print-debugging is nearly useless because every intermediate value is uniform-looking bytes. + +The only approach that holds up is deriving known-good vectors from the RFC and asserting on each stage independently, so a failure localizes to one transformation. NetProbe's suite covers the full path end-to-end, including a two-packet split ClientHello, and the parsers are continuously fuzzed under ASan/UBSan with a corpus seeded by hand-built adversarial packets: connection ID lengths beyond the packet, token lengths claiming more than exists, truncated varints. That last category matters here more than almost anywhere else in a network tool, because this code path takes attacker-controlled bytes and does pointer arithmetic on lengths derived from them. + +--- + +## Source index + +Every stage above, in the order it executes: + +| Stage | Function | Location | +| --- | --- | --- | +| Pre-filter | `looksLikeLongHeader` | [QuicParser.cpp:249](../../src/core/QuicParser.cpp#L249) | +| 0 — header parse | `parseInitial` | [QuicParser.cpp:322](../../src/core/QuicParser.cpp#L322) | +| 0 — version params | `versionParams` | [QuicParser.cpp:43](../../src/core/QuicParser.cpp#L43) | +| 0 — v1 salt | `kInitialSaltV1` | [QuicParser.cpp:20](../../src/core/QuicParser.cpp#L20) | +| 0 — v2 salt | `kInitialSaltV2` | [QuicParser.cpp:28](../../src/core/QuicParser.cpp#L28) | +| 1 — expand-label | `hkdfExpandLabel` | [QuicParser.cpp:95](../../src/core/QuicParser.cpp#L95) | +| 1 — key schedule | `deriveClientInitialKeys` | [QuicParser.cpp:117](../../src/core/QuicParser.cpp#L117) | +| 2 — header protection | `removeHeaderProtection` | [QuicParser.cpp:136](../../src/core/QuicParser.cpp#L136) | +| 3 — AEAD | `decryptPayload` | [QuicParser.cpp:163](../../src/core/QuicParser.cpp#L163) | +| 4 — frame walk | `collectCryptoFrames` | [QuicParser.cpp:194](../../src/core/QuicParser.cpp#L194) | +| 4 — sanity bound | `kSaneCryptoBound` | [QuicParser.cpp:196](../../src/core/QuicParser.cpp#L196) | +| 5 — reassembly | `QuicTracker::feed` | [QuicTracker.hpp:26](../../src/core/QuicTracker.hpp#L26) | +| 5 — reassembly bounds | `kMaxStreamBytes` et al. | [QuicTracker.hpp:33](../../src/core/QuicTracker.hpp#L33) | +| 5 — ClientHello parse | `sniFromClientHello` | [QuicParser.cpp:258](../../src/core/QuicParser.cpp#L258) | + +*Line numbers refer to the commit this post was written against; the functions are stable even if the lines drift. Corrections welcome — particularly from anyone who has fought the v2 label change.* From f916109c0c4b56de35c561bb87e22a97e398aaa1 Mon Sep 17 00:00:00 2001 From: YehiaGewily <79144005+YehiaGewily@users.noreply.github.com> Date: Sun, 19 Jul 2026 14:54:50 +0300 Subject: [PATCH 12/12] docs: correct stale counts and add the Stage 5 wire-format diagram MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Audit follow-ups, all documentation. README claimed 35 malformed fixtures; there are 40 (39 of which seed the fuzz corpus — the empty-capture case has no bytes to write, taking the corpus to 44 entries). The description also predated the TCP option fixtures and the pinned degraded outcomes, so it now mentions both. The plan's status header said 10 commits, carried placeholders for commits that had already landed, and listed CONTRIBUTING.md as both done and not started. Corrected, and the outstanding list now separates "CONTRIBUTING.md written" from "good first issue labels created on GitHub" — the four candidates exist only in the file; the repository still shows zero labelled issues, and creating them is an outward-facing action. The blog claimed a diagram at every stage and Stage 5 had none. Added the ClientHello field walk: the fixed-size prefix, the three variable-length skips, the extension list, and the nested server_name structure down to host_name. The nesting is the point — host_name_len sits inside the server_name entry, inside ext_len, inside extensions_len, inside the reassembled buffer, so a bounds check at only the innermost level passes for a packet that lied at the outermost one. Verified: build clean, 89/89 tests, clang-tidy exit 0, every one of the blog's source anchors still within its file. --- README.md | 2 +- docs/blog/decrypting-quic-initial-packets.md | 35 ++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5dae094..bcf7c28 100644 --- a/README.md +++ b/README.md @@ -207,7 +207,7 @@ flowchart LR ## Quality - **89 unit and integration tests** (GoogleTest) covering protocol parsing, link-type handling, tunnel descent, TLS/QUIC ClientHello reassembly, HTTP header extraction, DNS record coverage, canonical flow keying, RTT measurement, queue concurrency, GeoIP lookup, QUIC Initial decryption end-to-end, and both classic-PCAP + PCAPNG fixtures — including an end-to-end test that drives a crafted capture file through the same pipeline the UI runs, and end-to-end tests that run the real `netprobe-cli` binary over a capture file and validate the exported JSON and CSV. -- **Adversarial input suite**: 35 named malformed-packet fixtures — headers claiming lengths the buffer does not contain, IPv4 IHL and total-length lies, IPv6 extension-header chains that overrun or self-reference, DNS compression-pointer loops and inflated record counts, truncated TLS records, QUIC connection-ID and token lengths beyond the packet — plus an exhaustive check that every prefix of a valid frame parses without crashing. The same fixtures seed the fuzz corpus. +- **Adversarial input suite**: 40 named malformed-packet fixtures — headers claiming lengths the buffer does not contain, IPv4 IHL and total-length lies, IPv6 extension-header chains that overrun or self-reference, DNS compression-pointer loops and inflated record counts, truncated TLS records, QUIC connection-ID and token lengths beyond the packet — plus TCP option lengths of zero and options overrunning their header, and an exhaustive check that every prefix of a valid frame parses without crashing. Each fixture pins the degraded result it must produce, so a parser that starts inventing plausible answers fails the build rather than passing quietly. The same fixtures seed the fuzz corpus, taking it to 44 entries. - **libFuzzer harness** on `ProtocolParser::parse`, `DNSParser::parseResponse`, and the stateful `TlsReassembler` / `QuicTracker`, across every supported link type, with a deterministic seed corpus. CI fuzzes every push for 60 seconds on Ubuntu + Clang and uploads any crash inputs as build artifacts. - **AddressSanitizer + UndefinedBehaviorSanitizer** CI job runs the full test suite under ASan/UBSan on every push. - **clang-tidy static analysis** in CI over every first-party source, with `WarningsAsErrors` enabled: the tree is clean, so a new finding fails the build rather than scrolling past in a log. Checks are curated for signal — those that fire constantly on idiomatic code are disabled in `.clang-tidy` with the reasoning recorded there, rather than silenced case by case. diff --git a/docs/blog/decrypting-quic-initial-packets.md b/docs/blog/decrypting-quic-initial-packets.md index 5aeaab4..faa3cab 100644 --- a/docs/blog/decrypting-quic-initial-packets.md +++ b/docs/blog/decrypting-quic-initial-packets.md @@ -396,6 +396,41 @@ The ClientHello parse is [`QuicParser.cpp:258`](../../src/core/QuicParser.cpp#L2 With a contiguous ClientHello you are back in ordinary TLS 1.3 territory: skip `legacy_version` (2 bytes) and `random` (32), then the variable-length session ID, cipher suites, and compression methods, to reach the extension list. Find extension type `0x0000`, server_name, and read the first `host_name` entry. +Every hop below is a length read from the network, and every one is a place to walk off the end of the buffer: + +``` + ClientHello body + +--------------------------------+ + | legacy_version 2 | skip + | random 32 | skip + | session_id_len 1 ----+-> session_id (variable) skip + | cipher_suites_len 2 ----+-> cipher_suites (variable) skip + | compression_len 1 ----+-> compression (variable) skip + | extensions_len 2 ----+-> extension list + +--------------------------------+ + | + v + extension list (walk until extensions_len is consumed) + +----------------+----------------+---------------------------+ + | ext_type 2 | ext_len 2 | ext_data (ext_len) | + +----------------+----------------+---------------------------+ + | + | ext_type == 0x0000 (server_name) + v + +--------------------------------+ + | server_name_list_len 2 | + | name_type 1 | 0x00 = host_name + | host_name_len 2 | + | host_name (len) | <-- the answer + +--------------------------------+ +``` + +The nesting is what makes this hazardous: `host_name_len` must fit inside the +server_name entry, which must fit inside `ext_len`, which must fit inside +`extensions_len`, which must fit inside the reassembled buffer. A check at +only the innermost level passes for a packet that has already lied at the +outermost one. + Every one of those length fields comes from the network. Each one gets bounds-checked against the buffer, and any inconsistency aborts the parse rather than clamping and continuing — a parser that "recovers" from a malformed length is a parser that eventually reports a hostname assembled from adjacent memory. ---