From f500fa6f1d62d7605154991f3501371bd6fccaea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tor=20Arne=20Vestb=C3=B8?= Date: Tue, 28 Apr 2026 02:32:18 +0200 Subject: [PATCH 1/7] Prepare for network modes that do not provide pre-allocated IP addresses - Attachment.ipv4Address and .ipv4Gateway made optional; bridge attachments carry no pre-allocated IP (assigned via DHCP at runtime) - Update all call sites and tests for optional IP fields Co-authored-by: Curd Becker --- Sources/APIServer/ContainerDNSHandler.swift | 6 ++- .../Builder/BuilderStatus.swift | 2 +- .../Container/ContainerList.swift | 38 +++++++++++++++++++ .../ManagedContainer+ListDisplayable.swift | 2 +- .../Network/Attachment.swift | 30 +++++++-------- .../Server/MachinesService.swift | 4 +- .../Server/DefaultNetworkService.swift | 4 +- .../Server/IsolatedInterfaceStrategy.swift | 8 +++- .../Server/NonisolatedInterfaceStrategy.swift | 5 ++- .../RuntimeLinux/Server/RuntimeService.swift | 8 ++-- .../Subcommands/Networks/TestCLINetwork.swift | 4 +- .../Subcommands/Run/TestCLIRunCommand.swift | 4 +- .../AttachmentTest.swift | 34 +++++++++++++++++ 13 files changed, 115 insertions(+), 34 deletions(-) create mode 100644 Tests/ContainerResourceTests/AttachmentTest.swift diff --git a/Sources/APIServer/ContainerDNSHandler.swift b/Sources/APIServer/ContainerDNSHandler.swift index 78a207467..dd0996283 100644 --- a/Sources/APIServer/ContainerDNSHandler.swift +++ b/Sources/APIServer/ContainerDNSHandler.swift @@ -76,10 +76,12 @@ struct ContainerDNSHandler: DNSHandler { } private func answerHost(question: Question) async throws -> ResourceRecord? { - guard let ipAllocation = try await networkService.lookup(hostname: question.name) else { + guard let ipAllocation = try await networkService.lookup(hostname: question.name), + let ipv4Address = ipAllocation.ipv4Address + else { return nil } - let ipv4 = ipAllocation.ipv4Address.address.description + let ipv4 = ipv4Address.address.description guard let ip = try? IPv4Address(ipv4) else { throw DNSResolverError.serverError("failed to parse IP address: \(ipv4)") } diff --git a/Sources/ContainerCommands/Builder/BuilderStatus.swift b/Sources/ContainerCommands/Builder/BuilderStatus.swift index a7b102f93..54698cbe6 100644 --- a/Sources/ContainerCommands/Builder/BuilderStatus.swift +++ b/Sources/ContainerCommands/Builder/BuilderStatus.swift @@ -81,7 +81,7 @@ private struct PrintableBuilder: ListDisplayable { snapshot.id, snapshot.configuration.image.reference, snapshot.status.rawValue, - snapshot.networks.map { $0.ipv4Address.description }.joined(separator: ","), + snapshot.networks.map { $0.ipv4Address?.description ?? "" }.joined(separator: ","), "\(snapshot.configuration.resources.cpus)", "\(snapshot.configuration.resources.memoryInBytes / (1024 * 1024)) MB", ] diff --git a/Sources/ContainerCommands/Container/ContainerList.swift b/Sources/ContainerCommands/Container/ContainerList.swift index a61cedc0e..720d9491d 100644 --- a/Sources/ContainerCommands/Container/ContainerList.swift +++ b/Sources/ContainerCommands/Container/ContainerList.swift @@ -52,3 +52,41 @@ extension Application { } } } + +extension PrintableContainer: ListDisplayable { + static var tableHeader: [String] { + ["ID", "IMAGE", "OS", "ARCH", "STATE", "IP", "CPUS", "MEMORY", "STARTED"] + } + + var tableRow: [String] { + [ + self.configuration.id, + self.configuration.image.reference, + self.configuration.platform.os, + self.configuration.platform.architecture, + self.status.rawValue, + self.networks.map { $0.ipv4Address?.description ?? "" }.joined(separator: ","), + "\(self.configuration.resources.cpus)", + "\(self.configuration.resources.memoryInBytes / (1024 * 1024)) MB", + self.startedDate?.ISO8601Format() ?? "", + ] + } + + var quietValue: String { + self.configuration.id + } +} + +struct PrintableContainer: Codable, Sendable { + let status: RuntimeStatus + let configuration: ContainerConfiguration + let networks: [Attachment] + let startedDate: Date? + + init(_ container: ContainerSnapshot) { + self.status = container.status + self.configuration = container.configuration + self.networks = container.networks + self.startedDate = container.startedDate + } +} diff --git a/Sources/ContainerCommands/Container/ManagedContainer+ListDisplayable.swift b/Sources/ContainerCommands/Container/ManagedContainer+ListDisplayable.swift index f70791ce6..59b5751f6 100644 --- a/Sources/ContainerCommands/Container/ManagedContainer+ListDisplayable.swift +++ b/Sources/ContainerCommands/Container/ManagedContainer+ListDisplayable.swift @@ -29,7 +29,7 @@ extension ManagedContainer: ListDisplayable { configuration.platform.os, configuration.platform.architecture, status.state.rawValue, - status.networks.map { $0.ipv4Address.description }.joined(separator: ","), + status.networks.map { $0.ipv4Address?.description ?? "" }.joined(separator: ","), "\(configuration.resources.cpus)", "\(configuration.resources.memoryInBytes / (1024 * 1024)) MB", status.startedDate?.ISO8601Format() ?? "", diff --git a/Sources/ContainerResource/Network/Attachment.swift b/Sources/ContainerResource/Network/Attachment.swift index bbadad759..bce860544 100644 --- a/Sources/ContainerResource/Network/Attachment.swift +++ b/Sources/ContainerResource/Network/Attachment.swift @@ -23,9 +23,11 @@ public struct Attachment: Codable, Sendable { /// The hostname associated with the attachment. public let hostname: String /// The CIDR address describing the interface IPv4 address, with the prefix length of the subnet. - public let ipv4Address: CIDRv4 + /// Nil for bridge-mode attachments where the address is assigned by DHCP at runtime. + public let ipv4Address: CIDRv4? /// The IPv4 gateway address. - public let ipv4Gateway: IPv4Address + /// Nil for bridge-mode attachments where the gateway is discovered via DHCP at runtime. + public let ipv4Gateway: IPv4Address? /// The CIDR address describing the interface IPv6 address, with the prefix length of the subnet. /// The address is nil if the IPv6 subnet could not be determined at network creation time. public let ipv6Address: CIDRv6? @@ -37,8 +39,8 @@ public struct Attachment: Codable, Sendable { public init( network: String, hostname: String, - ipv4Address: CIDRv4, - ipv4Gateway: IPv4Address, + ipv4Address: CIDRv4?, + ipv4Gateway: IPv4Address?, ipv6Address: CIDRv6?, macAddress: MACAddress?, mtu: UInt32? = nil @@ -72,16 +74,12 @@ public struct Attachment: Codable, Sendable { network = try container.decode(String.self, forKey: .network) hostname = try container.decode(String.self, forKey: .hostname) - if let address = try? container.decode(CIDRv4.self, forKey: .ipv4Address) { - ipv4Address = address - } else { - ipv4Address = try container.decode(CIDRv4.self, forKey: .address) - } - if let gateway = try? container.decode(IPv4Address.self, forKey: .ipv4Gateway) { - ipv4Gateway = gateway - } else { - ipv4Gateway = try container.decode(IPv4Address.self, forKey: .gateway) - } + ipv4Address = + try container.decodeIfPresent(CIDRv4.self, forKey: .ipv4Address) + ?? container.decodeIfPresent(CIDRv4.self, forKey: .address) + ipv4Gateway = + try container.decodeIfPresent(IPv4Address.self, forKey: .ipv4Gateway) + ?? container.decodeIfPresent(IPv4Address.self, forKey: .gateway) ipv6Address = try container.decodeIfPresent(CIDRv6.self, forKey: .ipv6Address) macAddress = try container.decodeIfPresent(MACAddress.self, forKey: .macAddress) mtu = try container.decodeIfPresent(UInt32.self, forKey: .mtu) @@ -93,8 +91,8 @@ public struct Attachment: Codable, Sendable { try container.encode(network, forKey: .network) try container.encode(hostname, forKey: .hostname) - try container.encode(ipv4Address, forKey: .ipv4Address) - try container.encode(ipv4Gateway, forKey: .ipv4Gateway) + try container.encodeIfPresent(ipv4Address, forKey: .ipv4Address) + try container.encodeIfPresent(ipv4Gateway, forKey: .ipv4Gateway) try container.encodeIfPresent(ipv6Address, forKey: .ipv6Address) try container.encodeIfPresent(macAddress, forKey: .macAddress) try container.encodeIfPresent(mtu, forKey: .mtu) diff --git a/Sources/Services/MachineAPIService/Server/MachinesService.swift b/Sources/Services/MachineAPIService/Server/MachinesService.swift index a80cc7698..764d3b6c7 100644 --- a/Sources/Services/MachineAPIService/Server/MachinesService.swift +++ b/Sources/Services/MachineAPIService/Server/MachinesService.swift @@ -162,7 +162,7 @@ public actor MachinesService { self.log.warning("failed to fetch container addresses: \(error)") } let addressMap = (containers ?? []).reduce(into: [String: String]()) { result, c in - if let addr = c.networks.first?.ipv4Address.address.description { + if let addr = c.networks.first?.ipv4Address?.address.description { result[c.id] = addr } } @@ -536,7 +536,7 @@ public actor MachinesService { if snapshot.status == .running, let cid = snapshot.containerId { do { let container = try await self.client.get(id: cid) - snapshot.ipAddress = container.networks.first?.ipv4Address.address.description + snapshot.ipAddress = container.networks.first?.ipv4Address?.address.description } catch { self.log.warning("failed to fetch container address for \(cid): \(error)") } diff --git a/Sources/Services/Network/Server/DefaultNetworkService.swift b/Sources/Services/Network/Server/DefaultNetworkService.swift index 29f3fbe82..efbfc883b 100644 --- a/Sources/Services/Network/Server/DefaultNetworkService.swift +++ b/Sources/Services/Network/Server/DefaultNetworkService.swift @@ -83,8 +83,8 @@ public actor DefaultNetworkService: NetworkService { "allocated attachment", metadata: [ "hostname": "\(hostname)", - "ipv4Address": "\(attachment.ipv4Address)", - "ipv4Gateway": "\(attachment.ipv4Gateway)", + "ipv4Address": "\(attachment.ipv4Address?.description ?? "none")", + "ipv4Gateway": "\(attachment.ipv4Gateway?.description ?? "none")", "ipv6Address": "\(attachment.ipv6Address?.description ?? "unavailable")", "macAddress": "\(attachment.macAddress?.description ?? "unspecified")", ]) diff --git a/Sources/Services/RuntimeLinux/Server/IsolatedInterfaceStrategy.swift b/Sources/Services/RuntimeLinux/Server/IsolatedInterfaceStrategy.swift index d180fac5c..ac975b48e 100644 --- a/Sources/Services/RuntimeLinux/Server/IsolatedInterfaceStrategy.swift +++ b/Sources/Services/RuntimeLinux/Server/IsolatedInterfaceStrategy.swift @@ -18,6 +18,7 @@ import ContainerResource import ContainerRuntimeClient import ContainerXPC import Containerization +import ContainerizationError /// Isolated container network interface strategy. This strategy prohibits /// container to container networking, but it is the only approach that @@ -25,10 +26,13 @@ import Containerization public struct IsolatedInterfaceStrategy: InterfaceStrategy { public init() {} - public func toInterface(attachment: Attachment, interfaceIndex: Int, additionalData: XPCMessage?) -> Interface { + public func toInterface(attachment: Attachment, interfaceIndex: Int, additionalData: XPCMessage?) throws -> Interface { + guard let ipv4Address = attachment.ipv4Address else { + throw ContainerizationError(.invalidState, message: "NAT attachment missing IPv4 address") + } let ipv4Gateway = interfaceIndex == 0 ? attachment.ipv4Gateway : nil return NATInterface( - ipv4Address: attachment.ipv4Address, + ipv4Address: ipv4Address, ipv4Gateway: ipv4Gateway, macAddress: attachment.macAddress, // https://github.com/apple/containerization/pull/38 diff --git a/Sources/Services/RuntimeLinux/Server/NonisolatedInterfaceStrategy.swift b/Sources/Services/RuntimeLinux/Server/NonisolatedInterfaceStrategy.swift index 38c1b6764..ea1602d65 100644 --- a/Sources/Services/RuntimeLinux/Server/NonisolatedInterfaceStrategy.swift +++ b/Sources/Services/RuntimeLinux/Server/NonisolatedInterfaceStrategy.swift @@ -42,10 +42,13 @@ public struct NonisolatedInterfaceStrategy: InterfaceStrategy { throw ContainerizationError(.invalidState, message: "cannot deserialize custom network reference, status \(status)") } + guard let ipv4Address = attachment.ipv4Address else { + throw ContainerizationError(.invalidState, message: "NAT attachment missing IPv4 address") + } log.info("creating NATNetworkInterface with network reference") let ipv4Gateway = interfaceIndex == 0 ? attachment.ipv4Gateway : nil return NATNetworkInterface( - ipv4Address: attachment.ipv4Address, + ipv4Address: ipv4Address, ipv4Gateway: ipv4Gateway, reference: networkRef, macAddress: attachment.macAddress, diff --git a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift index e51ad14b0..e208567ce 100644 --- a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift +++ b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift @@ -261,8 +261,7 @@ public actor RuntimeService { // NOTE: We can support a user providing new entries eventually, but for now craft // a default /etc/hosts. var hostsEntries = [Hosts.Entry.localHostIPV4()] - if !interfaces.isEmpty { - let primaryIfaceAddr = interfaces[0].ipv4Address + if !interfaces.isEmpty, let primaryIfaceAddr = interfaces[0].ipv4Address { hostsEntries.append( Hosts.Entry( ipAddress: primaryIfaceAddr.address.description, @@ -891,7 +890,10 @@ public actor RuntimeService { let containerIPAddress: String switch publishedPort.hostAddress { case .v4(_): - containerIPAddress = attachment.ipv4Address.address.description + guard let ipv4Address = attachment.ipv4Address else { + throw ContainerizationError(.invalidState, message: "cannot configure IPv4 port forwarding for container with unknown IPv4 address") + } + containerIPAddress = ipv4Address.address.description case .v6(_): guard let ipv6Address = attachment.ipv6Address else { throw ContainerizationError(.invalidState, message: "cannot configure IPv6 port forwarding for container with unknown IPv6 address") diff --git a/Tests/CLITests/Subcommands/Networks/TestCLINetwork.swift b/Tests/CLITests/Subcommands/Networks/TestCLINetwork.swift index 9de319717..00c8d6820 100644 --- a/Tests/CLITests/Subcommands/Networks/TestCLINetwork.swift +++ b/Tests/CLITests/Subcommands/Networks/TestCLINetwork.swift @@ -72,7 +72,7 @@ class TestCLINetwork: CLITest { let container = try inspectContainer(name) #expect(container.networks.count > 0) - let cidrAddress = container.networks[0].ipv4Address + let cidrAddress = try #require(container.networks[0].ipv4Address) let url = "http://\(cidrAddress.address):\(port)" var request = HTTPClientRequest(url: url) request.method = .GET @@ -245,7 +245,7 @@ class TestCLINetwork: CLITest { let container = try inspectContainer(name) #expect(container.networks.count > 0) let curlImage = "docker.io/curlimages/curl:8.6.0" - let cidrAddress = container.networks[0].ipv4Address + let cidrAddress = try #require(container.networks[0].ipv4Address) let url = "http://\(cidrAddress.address):\(port)" let (_, _, _, succeed) = try run(arguments: [ "run", diff --git a/Tests/CLITests/Subcommands/Run/TestCLIRunCommand.swift b/Tests/CLITests/Subcommands/Run/TestCLIRunCommand.swift index 4d7ec61bd..2b9e12f3b 100644 --- a/Tests/CLITests/Subcommands/Run/TestCLIRunCommand.swift +++ b/Tests/CLITests/Subcommands/Run/TestCLIRunCommand.swift @@ -610,7 +610,7 @@ class TestCLIRunCommand3: CLITest { .map { $0.joined(separator: " ") } let inspectOutput = try inspectContainer(name) - let ip = inspectOutput.networks[0].ipv4Address.address + let ip = try #require(inspectOutput.networks[0].ipv4Address).address let expectedNameserver = IPv4Address((ip.value & Prefix(length: 24)!.prefixMask32) + 1).description let defaultDomain = try getDefaultDomain() let expectedLines: [String] = [ @@ -672,7 +672,7 @@ class TestCLIRunCommand3: CLITest { } let inspectOutput = try inspectContainer(name) - let ip = inspectOutput.networks[0].ipv4Address.address + let ip = try #require(inspectOutput.networks[0].ipv4Address).address let output = try doExec(name: name, cmd: ["cat", "/etc/hosts"]) let lines = output.split(separator: "\n") diff --git a/Tests/ContainerResourceTests/AttachmentTest.swift b/Tests/ContainerResourceTests/AttachmentTest.swift new file mode 100644 index 000000000..60b7e50ef --- /dev/null +++ b/Tests/ContainerResourceTests/AttachmentTest.swift @@ -0,0 +1,34 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Testing + +@testable import ContainerResource + +struct AttachmentTest { + @Test func testAttachmentNilIPFields() { + let attachment = Attachment( + network: "my-net", + hostname: "host1", + ipv4Address: nil, + ipv4Gateway: nil, + ipv6Address: nil, + macAddress: nil + ) + #expect(attachment.ipv4Address == nil) + #expect(attachment.ipv4Gateway == nil) + } +} From b91fc5beb81457e9d216ea3c538643cd1cb4f962 Mon Sep 17 00:00:00 2001 From: Curd Becker Date: Mon, 1 Jun 2026 01:53:01 +0200 Subject: [PATCH 2/7] Add missing getter to retrieve XPCDictionary from an XPCMessage The setter for adding an XPCDictionary to an XPCMessage is present, but the getter to retrieve it again is missing. This might be connected to the fact that xpc_endpoint_t is a type alias to xpc_object_t, so they appear to use an identical setter implementation (unless there is some implicit argument overloading that I am not aware of), but they apparently they cannot use the same getter method, since there is a dedicated `xpc_dictionary_get_dictionary` method to retrieve the dictionary again. --- Sources/ContainerXPC/XPCMessage.swift | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Sources/ContainerXPC/XPCMessage.swift b/Sources/ContainerXPC/XPCMessage.swift index 3c6a3dca8..d5f3ba385 100644 --- a/Sources/ContainerXPC/XPCMessage.swift +++ b/Sources/ContainerXPC/XPCMessage.swift @@ -275,6 +275,12 @@ extension XPCMessage { } } + public func xpcDictionary(key: String) -> xpc_object_t? { + lock.withLock { + xpc_dictionary_get_dictionary(self.object, key) + } + } + public func endpoint(key: String) -> xpc_endpoint_t? { lock.withLock { xpc_dictionary_get_value(self.object, key) From 8d0397d3a17790269541f05b6a92d510e254d6af Mon Sep 17 00:00:00 2001 From: Curd Becker Date: Mon, 1 Jun 2026 01:58:02 +0200 Subject: [PATCH 3/7] Pass network configuration options along to vmnet helper Right now, the configuration for a network is only saved in the `NetworksService` after the network has been successfully started. However, that constitutes a dependency issue when the vmnet helper implementation depends on options in order to be able to create and start the network in the first place. This commit fixes this issue by passing the options (excluding the variant) explicitly along as a serialized json dictionary. This is not that elegant, but works well for now. To me it looks also that this area is still undergoing quite some refactoring, so this might be a workaround until a more elegant permanent solution is implace. --- .../NetworkVmnet/NetworkVmnetHelper+Start.swift | 13 ++++++++++++- .../Server/Networks/NetworksService.swift | 11 +++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift b/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift index 7d67f1f32..ec32ea383 100644 --- a/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift +++ b/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift @@ -67,10 +67,19 @@ extension NetworkVmnetHelper { return .reserved }() + @Option(name: .customLong("option-json"), help: "UTF8-encoded JSON string that contains the configuration options for this network") + var stringifiedOptions: String = "{}" + var logRoot = LogRoot.path func run() async throws { let commandName = NetworkVmnetHelper._commandName + guard let encodedOptions = stringifiedOptions.data(using: .utf8), + let options = try? JSONSerialization.jsonObject(with: encodedOptions) as? [String: String] + else { + throw ContainerizationError(.invalidArgument, message: "failed to decode network configuration options from JSON string") + } + let logPath = logRoot.map { $0.appending("\(commandName)-\(id).log") } let log = ServiceLogger.bootstrap(category: "NetworkVmnetHelper", metadata: ["id": "\(id)"], debug: debug, logPath: logPath) log.info("starting helper", metadata: ["name": "\(commandName)"]) @@ -89,7 +98,9 @@ extension NetworkVmnetHelper { ipv4Subnet: ipv4Subnet, ipv6Subnet: ipv6Subnet, plugin: NetworkVmnetHelper._commandName, - options: ["variant": self.variant.rawValue] + options: options.merging( + ["variant": self.variant.rawValue], + uniquingKeysWith: { _, new in new }) ) let network = try Self.createNetwork( configuration: configuration, diff --git a/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift b/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift index 6be54acb4..5727e31e9 100644 --- a/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift +++ b/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift @@ -403,6 +403,17 @@ public actor NetworksService { args += ["--variant", variant] } + // TODO: variant could possibly stay inside the options and does not need to be a dedicated commandline argument? + let options = configuration.options.filter({ key, _ in key != "variant" }) + if !options.isEmpty { + guard let encodedOptions = try? JSONSerialization.data(withJSONObject: options), + let stringifiedOptions = String(data: encodedOptions, encoding: .utf8) + else { + throw ContainerizationError(.internalError, message: "failed to encode network configuration options as json string") + } + args += ["--option-json", stringifiedOptions] + } + let entityPath = try store.entityPath(configuration.id) try pluginLoader.registerWithLaunchd( plugin: networkPlugin, From e582270b70e454369e12ca934873a114d13feab5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tor=20Arne=20Vestb=C3=B8?= Date: Tue, 28 Apr 2026 16:47:38 +0200 Subject: [PATCH 4/7] Add kernel IP_PNP cmdline args for bridge networking For each bridge attachment (ipv4Address == nil) at interface index n, append ip=:::::eth:dhcp to the kernel cmdline before booting the VM. The kernel's built-in DHCP client (CONFIG_IP_PNP_DHCP=y) acquires the lease before userspace starts, so no DHCP binary is needed in the rootfs. We also pass the hostname to the DHCP server. VZVirtualMachineManager creation is moved after attachment allocation and interface creation, so interface indices are known when the cmdline is finalized. Co-authored-by: Curd Becker --- .../RuntimeLinux/Server/RuntimeService.swift | 39 ++++++++++++------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift index e208567ce..d57d9a631 100644 --- a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift +++ b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift @@ -162,12 +162,6 @@ public actor RuntimeService { var kernel = try bundle.kernel kernel.commandLine.kernelArgs.append("oops=panic") kernel.commandLine.kernelArgs.append("lsm=lockdown,capability,landlock,yama,apparmor") - let vmm = VZVirtualMachineManager( - kernel: kernel, - initialFilesystem: bundle.initialFilesystem.asMount, - rosetta: config.rosetta, - logger: self.log - ) let networkBootstrapInfos = try message.networkBootstrapInfos() @@ -196,6 +190,12 @@ public actor RuntimeService { mtu: mtu ) } + + // enable DHCP if the attachment has not been assigned an explicit IP address + if attachment.ipv4Address == nil { + kernel.commandLine.kernelArgs.append("ip=::::\(attachment.hostname):eth\(index):dhcp") + } + guard let iStrategy = self.interfaceStrategies[NetworkInterfaceKey(plugin: info.plugin, variant: info.options["variant"])] else { throw ContainerizationError( .internalError, @@ -214,17 +214,23 @@ public actor RuntimeService { throw error } + let vmm = VZVirtualMachineManager( + kernel: kernel, + initialFilesystem: bundle.initialFilesystem.asMount, + rosetta: config.rosetta, + logger: self.log + ) + // Dynamically configure the DNS nameserver from a network if no explicit configuration + // For bridge networks (unspecified gateway), nameservers and domain come from DHCP (/proc/net/pnp). if let dns = config.dns, dns.nameservers.isEmpty { let defaultNameservers = self.getDefaultNameservers(from: attachments) - if !defaultNameservers.isEmpty { - config.dns = ContainerConfiguration.DNSConfiguration( - nameservers: defaultNameservers, - domain: dns.domain, - searchDomains: dns.searchDomains, - options: dns.options - ) - } + config.dns = ContainerConfiguration.DNSConfiguration( + nameservers: defaultNameservers.isEmpty ? dns.nameservers : defaultNameservers, + domain: defaultNameservers.isEmpty ? nil : dns.domain, + searchDomains: dns.searchDomains, + options: dns.options + ) } let stdio = message.stdio() @@ -1058,7 +1064,10 @@ public actor RuntimeService { private nonisolated func getDefaultNameservers(from attachments: [Attachment]) -> [String] { for attachment in attachments { - return [attachment.ipv4Gateway.description] + guard let ipv4Gateway: IPv4Address = attachment.ipv4Gateway else { + continue + } + return [ipv4Gateway.description] } return [] } From 64e0c82e9eb8a5d7508fdae8a30d72a25d7f08ba Mon Sep 17 00:00:00 2001 From: Curd Becker Date: Mon, 1 Jun 2026 14:01:47 +0200 Subject: [PATCH 5/7] Expose NetworkVariant as global enum in ContainerNetworkClient This was previously just a private enum inside the NetworkVmnetHelper entrypoint, but it makes sense to make this actually globally accessible ... at least for now. I imagine this is also going to be likely the focus of the ongoing refactoring. --- .../NetworkVmnetHelper+Start.swift | 10 +++------- .../Network/Client/NetworkVariant.swift | 20 +++++++++++++++++++ 2 files changed, 23 insertions(+), 7 deletions(-) create mode 100644 Sources/Services/Network/Client/NetworkVariant.swift diff --git a/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift b/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift index ec32ea383..026288766 100644 --- a/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift +++ b/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift @@ -27,12 +27,8 @@ import ContainerizationExtras import Foundation import Logging -enum Variant: String, ExpressibleByArgument { - case reserved - case allocationOnly -} - extension NetworkMode: ExpressibleByArgument {} +extension NetworkVariant: ExpressibleByArgument {} extension NetworkVmnetHelper { struct Start: AsyncParsableCommand { @@ -60,7 +56,7 @@ extension NetworkVmnetHelper { var ipv6Subnet: String? @Option(name: .long, help: "Variant of the network helper to use.") - var variant: Variant = { + var variant: NetworkVariant = { guard #available(macOS 26, *) else { return .allocationOnly } @@ -133,7 +129,7 @@ extension NetworkVmnetHelper { } } - private static func createNetwork(configuration: NetworkConfiguration, variant: Variant, log: Logger) throws -> Network { + private static func createNetwork(configuration: NetworkConfiguration, variant: NetworkVariant, log: Logger) throws -> Network { switch variant { case .allocationOnly: return try AllocationOnlyVmnetNetwork(configuration: configuration, log: log) diff --git a/Sources/Services/Network/Client/NetworkVariant.swift b/Sources/Services/Network/Client/NetworkVariant.swift new file mode 100644 index 000000000..99a30b7b3 --- /dev/null +++ b/Sources/Services/Network/Client/NetworkVariant.swift @@ -0,0 +1,20 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +public enum NetworkVariant: String, Sendable { + case reserved + case allocationOnly +} From b33db8bcfdd5da17866daab65b66d8679fb2b782 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tor=20Arne=20Vestb=C3=B8?= Date: Tue, 28 Apr 2026 17:06:52 +0200 Subject: [PATCH 6/7] Add bridged vmnet network plugin variant including BridgedInterfaceStrategy Implements the vmnet plugin side of bridge networking: a new BridgedVmnetNetwork actor that uses placeholder subnets (no IP pool) and a .bridged/.bridgedViaHelper NetworkVariant for the plugin helper. The BridgeInterfaceStrategy communicates XPC additional data payload forwarded by the BridgedVmnetNetwork to the concrete network interface. BridgeInterfaceStrategy maps: - .bridged network attachments to a BridgedNetworkInterface for the given host interface. - .bridgedViaHelper network attachments to a FileHandleNetworkInterface for the FileHandle endpoint where the plugin helper will provide bridging services to the runtime. Co-authored-by: Curd Becker --- .../NetworkVmnetHelper+Start.swift | 2 + .../RuntimeLinuxHelper+Start.swift | 2 + .../Network/Client/BridgeNetworkKeys.swift | 23 +++ .../Network/Client/NetworkVariant.swift | 2 + .../Server/BridgedVmnetNetwork.swift | 155 ++++++++++++++++++ .../Server/BridgedInterfaceStrategy.swift | 56 +++++++ .../RuntimeLinux/Server/RuntimeService.swift | 4 +- 7 files changed, 242 insertions(+), 2 deletions(-) create mode 100644 Sources/Services/Network/Client/BridgeNetworkKeys.swift create mode 100644 Sources/Services/NetworkVmnet/Server/BridgedVmnetNetwork.swift create mode 100644 Sources/Services/RuntimeLinux/Server/BridgedInterfaceStrategy.swift diff --git a/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift b/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift index 026288766..a2e826644 100644 --- a/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift +++ b/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift @@ -141,6 +141,8 @@ extension NetworkVmnetHelper { ) } return try ReservedVmnetNetwork(configuration: configuration, log: log) + case .bridged, .bridgedViaHelper: + return try BridgedVmnetNetwork(configuration: configuration, log: log) } } } diff --git a/Sources/Plugins/RuntimeLinux/RuntimeLinuxHelper+Start.swift b/Sources/Plugins/RuntimeLinux/RuntimeLinuxHelper+Start.swift index 3c7938b8e..796bf2aff 100644 --- a/Sources/Plugins/RuntimeLinux/RuntimeLinuxHelper+Start.swift +++ b/Sources/Plugins/RuntimeLinux/RuntimeLinuxHelper+Start.swift @@ -68,6 +68,8 @@ extension RuntimeLinuxHelper { NetworkInterfaceKey(plugin: "container-network-vmnet", variant: "allocationOnly"): IsolatedInterfaceStrategy() ] if #available(macOS 26, *) { + interfaceStrategies[NetworkInterfaceKey(plugin: "container-network-vmnet", variant: "bridged")] = BridgedInterfaceStrategy(log: log) + interfaceStrategies[NetworkInterfaceKey(plugin: "container-network-vmnet", variant: "bridgedViaHelper")] = BridgedInterfaceStrategy(log: log) interfaceStrategies[NetworkInterfaceKey(plugin: "container-network-vmnet", variant: "reserved")] = NonisolatedInterfaceStrategy(log: log) } diff --git a/Sources/Services/Network/Client/BridgeNetworkKeys.swift b/Sources/Services/Network/Client/BridgeNetworkKeys.swift new file mode 100644 index 000000000..fd600f9bd --- /dev/null +++ b/Sources/Services/Network/Client/BridgeNetworkKeys.swift @@ -0,0 +1,23 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +public enum BridgeNetworkKeys: String { + case hostInterface + case enableTso + case enableChecksumOffload + case bufferedPacketCount + case sandboxEndpoint +} diff --git a/Sources/Services/Network/Client/NetworkVariant.swift b/Sources/Services/Network/Client/NetworkVariant.swift index 99a30b7b3..a557081a1 100644 --- a/Sources/Services/Network/Client/NetworkVariant.swift +++ b/Sources/Services/Network/Client/NetworkVariant.swift @@ -17,4 +17,6 @@ public enum NetworkVariant: String, Sendable { case reserved case allocationOnly + case bridged + case bridgedViaHelper } diff --git a/Sources/Services/NetworkVmnet/Server/BridgedVmnetNetwork.swift b/Sources/Services/NetworkVmnet/Server/BridgedVmnetNetwork.swift new file mode 100644 index 000000000..8ac016eea --- /dev/null +++ b/Sources/Services/NetworkVmnet/Server/BridgedVmnetNetwork.swift @@ -0,0 +1,155 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerNetworkClient +import ContainerNetworkServer +import ContainerResource +import ContainerXPC +import Containerization +import ContainerizationError +import ContainerizationExtras +import Foundation +import Logging +import Synchronization +import Virtualization +import XPC + +public final class BridgedVmnetNetwork: ContainerNetworkServer.Network { + // FIXME: NetworkPluginStatus requires non-optional ipv4Subnet/ipv4Gateway; use placeholder + // values until the type is refactored to make them optional. + private static let placeholderSubnet = try! CIDRv4("0.0.0.0/0") + private static let placeholderGateway = IPv4Address(0) + private static let placeholderSubnetv6 = try! CIDRv6("::/0") + + private let log: Logger + private let configuration: NetworkConfiguration + private let hostInterface: String + private let statusMutex: Mutex + private let enableTso: Bool? + private let enableChecksumOffload: Bool? + private let bufferedPacketCount: Int? + + public init(configuration: NetworkConfiguration, log: Logger) throws { + // TODO: ignore the network type for now - having both network type and plugin variant is a bit awkward... + // I imagine this will get resolved later on + // guard configuration.mode == .bridge else { + // throw ContainerizationError(.unsupported, message: "invalid network mode \(configuration.mode)") + // } + guard let variantStr = configuration.options["variant"], + let variant = NetworkVariant(rawValue: variantStr), + variant == .bridged || variant == .bridgedViaHelper + else { + throw ContainerizationError( + .unsupported, + message: "invalid network variant \(configuration.options["variant"] ?? "unspecified")") + } + + guard let hostInterface = configuration.options["hostInterface"] else { + throw ContainerizationError(.invalidArgument, message: "hostInterface must be given as a plugin option") + } + let available = VZBridgedNetworkInterface.networkInterfaces.map { $0.identifier } + guard available.contains(hostInterface) else { + let list = available.isEmpty ? "none available" : available.joined(separator: ", ") + throw ContainerizationError(.invalidArgument, message: "no host interface '\(hostInterface)'; available: \(list)") + } + self.hostInterface = hostInterface + + self.configuration = configuration + self.log = log + self.statusMutex = Mutex(nil) + + if variant == .bridged { + self.enableTso = nil + self.enableChecksumOffload = nil + self.bufferedPacketCount = nil + } else { + let rawEnableTso = configuration.options[BridgeNetworkKeys.enableTso.rawValue] + if rawEnableTso != nil { + guard let enableTso = Bool(rawEnableTso!) else { + throw ContainerizationError(.invalidArgument, message: "enableTso must be a boolean") + } + self.enableTso = enableTso + } else { + self.enableTso = nil + } + + let rawEnableChecksumOffload: String? = configuration.options[BridgeNetworkKeys.enableChecksumOffload.rawValue] + if rawEnableChecksumOffload != nil { + guard let enableChecksumOffload = Bool(rawEnableChecksumOffload!) else { + throw ContainerizationError(.invalidArgument, message: "enableChecksumOffload must be a boolean") + } + self.enableChecksumOffload = enableChecksumOffload + } else { + self.enableChecksumOffload = nil + } + + let rawBufferedPacketCount = configuration.options[BridgeNetworkKeys.bufferedPacketCount.rawValue] + if rawBufferedPacketCount != nil { + guard let bufferedPacketCount = Int(rawBufferedPacketCount!), bufferedPacketCount > 0 else { + throw ContainerizationError(.invalidArgument, message: "bufferedPacketCount must be a positive integer") + } + self.bufferedPacketCount = bufferedPacketCount + } else { + self.bufferedPacketCount = nil + } + } + } + + public nonisolated var id: String { configuration.id } + + public var status: NetworkStatus? { + self.statusMutex.withLock { $0 } + } + + public nonisolated func withAdditionalData(_ handler: (XPCMessage?) throws -> Void) throws { + let bridgeConfigMsg = XPCMessage(object: xpc_dictionary_create_empty()) + bridgeConfigMsg.set(key: BridgeNetworkKeys.hostInterface.rawValue, value: hostInterface) + if self.enableTso != nil { + bridgeConfigMsg.set(key: BridgeNetworkKeys.enableTso.rawValue, value: self.enableTso!) + } + if self.enableChecksumOffload != nil { + bridgeConfigMsg.set(key: BridgeNetworkKeys.enableChecksumOffload.rawValue, value: self.enableChecksumOffload!) + } + if self.bufferedPacketCount != nil { + bridgeConfigMsg.set(key: BridgeNetworkKeys.bufferedPacketCount.rawValue, value: UInt64(self.bufferedPacketCount!)) + } + + let msg: XPCMessage = XPCMessage(object: xpc_dictionary_create_empty()) + msg.set(key: NetworkKeys.additionalData.rawValue, value: bridgeConfigMsg.underlying) + try handler(msg) + } + + public func start() async throws { + try statusMutex.withLock { status in + + guard status == nil else { + throw ContainerizationError(.invalidArgument, message: "cannot start network \(configuration.id): already started") + } + + status = NetworkStatus( + ipv4Subnet: Self.placeholderSubnet, ipv4Gateway: Self.placeholderGateway, ipv6Subnet: Self.placeholderSubnetv6 + ) + + log.info( + "started bridged network", + metadata: [ + "id": "\(configuration.id)", + "hostInterface": "\(hostInterface)", + ] + ) + } + } +} diff --git a/Sources/Services/RuntimeLinux/Server/BridgedInterfaceStrategy.swift b/Sources/Services/RuntimeLinux/Server/BridgedInterfaceStrategy.swift new file mode 100644 index 000000000..29ff55f1b --- /dev/null +++ b/Sources/Services/RuntimeLinux/Server/BridgedInterfaceStrategy.swift @@ -0,0 +1,56 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerNetworkClient +import ContainerResource +import ContainerRuntimeClient +import ContainerXPC +import Containerization +import ContainerizationError +import Logging + +/// Interface strategy for containers that use macOS's custom network feature. +@available(macOS 26, *) +public struct BridgedInterfaceStrategy: InterfaceStrategy { + private let log: Logger + + public init(log: Logger) { + self.log = log + } + + public func toInterface(attachment: Attachment, interfaceIndex: Int, additionalData: XPCMessage?) throws -> Interface { + guard let bridgeDictionary = additionalData?.xpcDictionary(key: NetworkKeys.additionalData.rawValue) else { + throw ContainerizationError(.internalError, message: "did not receive bridge dictionary in interface additional message") + } + let bridgeData = XPCMessage(object: bridgeDictionary) + + guard let ifaceName = bridgeData.string(key: BridgeNetworkKeys.hostInterface.rawValue) + else { + throw ContainerizationError(.invalidState, message: "bridge network missing host interface name") + } + guard let containerBridgeEndpoint = bridgeData.fileHandle(key: BridgeNetworkKeys.sandboxEndpoint.rawValue) else { + return BridgedNetworkInterface( + hostInterfaceName: ifaceName, + macAddress: attachment.macAddress + ) + } + + return FileHandleNetworkInterface( + fileHandle: containerBridgeEndpoint, + macAddress: attachment.macAddress + ) + } +} diff --git a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift index d57d9a631..eb46d22a4 100644 --- a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift +++ b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift @@ -267,10 +267,10 @@ public actor RuntimeService { // NOTE: We can support a user providing new entries eventually, but for now craft // a default /etc/hosts. var hostsEntries = [Hosts.Entry.localHostIPV4()] - if !interfaces.isEmpty, let primaryIfaceAddr = interfaces[0].ipv4Address { + if !interfaces.isEmpty, let ipv4Address = interfaces[0].ipv4Address, !ipv4Address.address.isUnspecified { hostsEntries.append( Hosts.Entry( - ipAddress: primaryIfaceAddr.address.description, + ipAddress: ipv4Address.address.description, hostnames: [czConfig.hostname ?? id], )) } From 9f6fef9935fd7e56995597abea539987e1db5075 Mon Sep 17 00:00:00 2001 From: Curd Becker Date: Mon, 1 Jun 2026 01:52:43 +0200 Subject: [PATCH 7/7] Add a BridgeNetworkService for socket-based bridge forwarding The BridgeNetworkService is heavily inspired by nirs/vmnet-helper and draws on the same concept. This allow bridging to a real host network interface without having a binary signed with the com.apple.vm.networking entitlement, e.g. during container development. Therefore, the service allocates a standalone vmnet interface and a dedicate UNIX datagram socket pair. One end of the pair remains local to the service as the network attachment of the runtime, the other end is sent to the runtime where it is assigned in the underlying VM configuration as `VZFileHandleNetworkDeviceAttachment`. This setup allows the service to perform manual bridging between the vmnnet network interface attached to the host interface and the runtime using event handlers on both sides. While this is obviously certainly less performant and generally inferior to having the same capabilities integrated directly in the Virtualization Framework, it provides a very similar setting and is actually possible without root privileges or any special entitlements. --- .../NetworkVmnetHelper+Start.swift | 11 +- .../Network/Server/BridgeNetworkService.swift | 488 ++++++++++++++++++ 2 files changed, 498 insertions(+), 1 deletion(-) create mode 100644 Sources/Services/Network/Server/BridgeNetworkService.swift diff --git a/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift b/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift index a2e826644..a3d89619f 100644 --- a/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift +++ b/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift @@ -104,7 +104,7 @@ extension NetworkVmnetHelper { log: log ) try await network.start() - let service = try await DefaultNetworkService(network: network, log: log) + let service: NetworkService = try await Self.createNetworkService(network: network, variant: variant, log: log) let harness = NetworkHarness(service: service) let xpc = XPCServer( identifier: serviceIdentifier, @@ -129,6 +129,15 @@ extension NetworkVmnetHelper { } } + private static func createNetworkService(network: Network, variant: NetworkVariant, log: Logger) async throws -> NetworkService { + switch variant { + case .bridged, .bridgedViaHelper: + return try await BridgeNetworkService(network: network, variant: variant, log: log) + default: + return try await DefaultNetworkService(network: network, log: log) + } + } + private static func createNetwork(configuration: NetworkConfiguration, variant: NetworkVariant, log: Logger) throws -> Network { switch variant { case .allocationOnly: diff --git a/Sources/Services/Network/Server/BridgeNetworkService.swift b/Sources/Services/Network/Server/BridgeNetworkService.swift new file mode 100644 index 000000000..ebf6b629c --- /dev/null +++ b/Sources/Services/Network/Server/BridgeNetworkService.swift @@ -0,0 +1,488 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerNetworkClient +import ContainerResource +import ContainerXPC +import ContainerizationError +import ContainerizationExtras +import Foundation +import Logging +import vmnet + +internal final class Bridge { + + internal final class PacketBuffer { + public let buffers: UnsafeMutableRawPointer + public let iovs: UnsafeMutablePointer + public let packets: UnsafeMutablePointer + + public let maxPacketSize: Int + public let maxPktCount: Int + + public init(maxPktCount: Int, maxPktSize: Int) { + self.buffers = .allocate(byteCount: maxPktCount * maxPktSize, alignment: 16) + self.iovs = .allocate(capacity: maxPktCount) + self.packets = .allocate(capacity: maxPktCount) + for i in 0..sandbox + private let hostPacketBuffer: PacketBuffer + // sandbox->host + private let sandboxPacketBuffer: PacketBuffer + + public init( + identifier: String, + hostInterface: String, + logger: Logger, + enableTso: Bool, + enableChecksumOffload: Bool, + bufferedPacketCount: Int + ) throws { + var fds: [Int32] = [-1, -1] + let rc = fds.withUnsafeMutableBufferPointer { + socketpair(AF_UNIX, SOCK_DGRAM, 0, $0.baseAddress) + } + guard rc == 0 else { + throw ContainerizationError(.internalError, message: "unable to create socket pairs for UNIX bridge endpoints") + } + + let networkEndpoint = FileHandle(fileDescriptor: fds[0], closeOnDealloc: true) + let sandboxEndpoint = FileHandle(fileDescriptor: fds[1], closeOnDealloc: true) + + // Make our endpoint side non blocking, so the read source can drain it in a loop. + let flags = fcntl(networkEndpoint.fileDescriptor, F_GETFL) + _ = fcntl(networkEndpoint.fileDescriptor, F_SETFL, flags | O_NONBLOCK) + + // vmnet interface descriptor. + let desc = xpc_dictionary_create(nil, nil, 0) + xpc_dictionary_set_uint64( + desc, vmnet_operation_mode_key, + UInt64(vmnet.operating_modes_t.VMNET_BRIDGED_MODE.rawValue) + ) + xpc_dictionary_set_string(desc, vmnet_shared_interface_name_key, hostInterface) + xpc_dictionary_set_bool(desc, vmnet_enable_tso_key, enableTso) + xpc_dictionary_set_bool( + desc, vmnet_enable_checksum_offload_key, + enableChecksumOffload) + + // Unfortunately, apparently not understood currently - otherwise this would maybe provide us the opportunity to + // set a custom MAC address + // xpc_dictionary_set_bool(desc, vmnet_allocate_mac_address_key, false) + + // start interface synchronously — vmnet posts the completion on `queue`. + let queue = DispatchQueue(label: identifier, qos: .userInitiated) + let sema = DispatchSemaphore(value: 0) + var startStatus: vmnet_return_t = .VMNET_FAILURE + var maxPacketSize = 0 + var mtu: UInt32 = 0 + var mac = "" + + let iface = vmnet_start_interface(desc, queue) { status, param in + startStatus = status + if status == .VMNET_SUCCESS, let param = param { + maxPacketSize = Int(xpc_dictionary_get_uint64(param, vmnet_max_packet_size_key)) + mtu = UInt32(xpc_dictionary_get_uint64(param, vmnet_mtu_key)) + if let cstr = xpc_dictionary_get_string(param, vmnet_mac_address_key) { + mac = String(cString: cstr) + } + } + sema.signal() + } + sema.wait() + + guard startStatus == .VMNET_SUCCESS, let iface = iface else { + throw ContainerizationError(.internalError, message: "unable to start vmnet bridge interface") + } + guard let macAddress = try? MACAddress(mac) else { + throw ContainerizationError(.internalError, message: "vmnet returned an invalid mac address for vmnet bridge interface") + } + + self.log = logger + + self.identifier = identifier + self.networkEndpoint = networkEndpoint + self.sandboxEndpoint = sandboxEndpoint + self.macAddress = macAddress + self.mtu = mtu + self.interfaceRef = iface + self.queue = queue + + self.hostPacketBuffer = PacketBuffer(maxPktCount: bufferedPacketCount, maxPktSize: maxPacketSize) + self.sandboxPacketBuffer = PacketBuffer(maxPktCount: 1, maxPktSize: maxPacketSize) + } + + public func start() throws { + guard self.readSource == nil, !stopped else { + throw ContainerizationError(.invalidState, message: "bridge was already started") + } + + // TODO: despite Swift's claim that there are no function calls inside the event handlers that could throw exceptions, + // this is unfortunately simply not true. this is possibly connected due to some Objective-C legacy behavior + // of the vmnet_interface API. + // further investigation in catching these exceptions and making them visible might prove useful, since + // otherwise the network service can silently crash without any trace in the logs. the sandbox service will + // also keep running and will only notice that its interface has lost its carrier (quite fitting). only the + // console logging does provide then a crash dump including stack trace featuring some message like this: + // + // *** Terminating app due to uncaught exception 'NSFileHandleOperationException', reason: '*** -[NSConcreteFileHandle fileDescriptor]: Bad file descriptor' + + // host -> vm: vmnet fires an event when packets are buffered. + let st = vmnet_interface_set_event_callback( + interfaceRef, .VMNET_INTERFACE_PACKETS_AVAILABLE, queue + ) { [weak self] _, _ in + self?.relayFromHostToSandbox() + } + guard st == .VMNET_SUCCESS else { + throw ContainerizationError(.internalError, message: "vmnet_interface_set_event_callback failed") + } + + // vm -> host: dispatch source on the socket. + let src = DispatchSource.makeReadSource(fileDescriptor: networkEndpoint.fileDescriptor, queue: queue) + src.setEventHandler { [weak self] in self?.relayFromSandboxToHost() } + src.resume() + self.readSource = src + } + + public func stop() throws { + guard !stopped else { + throw ContainerizationError(.invalidState, message: "bridge was already stopped") + } + stopped = true + + guard let readSource else { + throw ContainerizationError(.internalError, message: "read source for network endpoint has not been created") + } + readSource.cancel() + + let sema = DispatchSemaphore(value: 0) + let st = vmnet_stop_interface(interfaceRef, queue, { _ in sema.signal() }) + sema.wait() + + guard st == .VMNET_SUCCESS else { + throw ContainerizationError(.internalError, message: "failed to stop vmnet bridge interface") + } + } + + // bridge input and output helpers + private func readFromHost() -> Int32 { + hostPacketBuffer.reset() + var count = Int32(hostPacketBuffer.maxPktCount) + let st = vmnet_read(interfaceRef, hostPacketBuffer.packets, &count) + guard st == .VMNET_SUCCESS, count > 0 else { + return -1 + } + return count + } + + private func writeToSandbox(count: Int) -> Int { + var written = 0 + for i in 0.. 0 else { + log.warning("received invalid number of packets from host: \(pktsIn)") + return + } + let pktsOut = writeToSandbox(count: pktsIn) + guard pktsOut == pktsIn else { + log.warning("could not forward all packets to sandbox: \(pktsIn) != \(pktsOut)") + return + } + } + + private func relayFromSandboxToHost() { + // vmnet-helper uses Apple-private system calls sendmsg_x/recmsg_x to forward multiple packets + // in a single system call on the bridge... that sounds intuitively as it will likely achieve quite a + // performance improvement, but it also might break without further notice + // + // in doubt: this is just a fallback for the likely even more optimized internal VirtualizationFramework + // bridging functionality + let pktSize = read( + networkEndpoint.fileDescriptor, sandboxPacketBuffer.buffers, + sandboxPacketBuffer.maxPacketSize) + guard pktSize > 0 else { + log.warning("failed to read from sandbox socket: \(pktSize) - EOF?") + return + } + + sandboxPacketBuffer.iovs.pointee.iov_len = pktSize + sandboxPacketBuffer.packets.pointee.vm_pkt_size = pktSize + sandboxPacketBuffer.packets.pointee.vm_flags = 0 + + var pktCount: Int32 = 1 + guard vmnet_write(interfaceRef, sandboxPacketBuffer.packets, &pktCount) == .VMNET_SUCCESS, pktCount == 1 else { + log.warning("failed to forward packet from sandbox to host") + return + } + } + +} + +public actor BridgeNetworkService: NetworkService { + + private let network: any Network + private let variant: NetworkVariant + private let log: Logger + private var allocationsBySession: [XPCServerSession: [(hostname: String, bridge: Bridge?)]] + private var macAddresses: [String: MACAddress] + + /// Set up a network service for the specified network. + public init( + network: any Network, + variant: NetworkVariant, + log: Logger + ) async throws { + guard await network.status != nil else { + throw ContainerizationError(.invalidState, message: "network \(network.id) must be running") + } + + self.network = network + self.variant = variant + self.log = log + self.allocationsBySession = [:] + self.macAddresses = [:] + } + + @Sendable + public func status() async throws -> NetworkStatus { + guard let status = await network.status else { + throw ContainerizationError(.invalidState, message: "network \(network.id) is not running") + } + return status + } + + @Sendable + public func allocate( + hostname: String, + macAddress: MACAddress?, + session: XPCServerSession + ) async throws -> (attachment: Attachment, additionalData: XPCMessage?) { + log.debug("enter", metadata: ["func": "\(#function)"]) + defer { log.debug("exit", metadata: ["func": "\(#function)"]) } + + guard await network.status != nil else { + throw ContainerizationError(.invalidState, message: "network \(network.id) must be running") + } + + // retrieve configuration from the network + // this is a bit awkward compared to the regular purpose of the network in managing allocations, since this is all + // handled externally by the network infrastructure on the other side of the bridge. therefore the network is for us + // simply a quite overblown struct/adapter that passes information along that are still unknown during creation time. + var additionalData: XPCMessage? + try network.withAdditionalData { + additionalData = $0 + } + + guard let bridgeDictionary = additionalData?.xpcDictionary(key: NetworkKeys.additionalData.rawValue) else { + throw ContainerizationError(.internalError, message: "did not receive bridge dictionary in network additional message") + } + let bridgeMessage = XPCMessage(object: bridgeDictionary) + + guard let hostInterface = bridgeMessage.string(key: BridgeNetworkKeys.hostInterface.rawValue) + else { + throw ContainerizationError(.invalidState, message: "bridge network is not assigned to a host interface") + } + + var attachment: Attachment + var bridge: Bridge? + if variant == .bridged { + let macAddress = macAddress ?? MACAddress((UInt64.random(in: 0...UInt64.max) & 0x0cff_ffff_ffff) | 0xf200_0000_0000) + attachment = Attachment( + network: network.id, + hostname: hostname, + ipv4Address: nil, + ipv4Gateway: nil, + ipv6Address: nil, + macAddress: macAddress, + mtu: nil, + ) + bridge = nil + } else { + let enableTso = + bridgeMessage.dataNoCopy(key: BridgeNetworkKeys.enableTso.rawValue) != nil ? bridgeMessage.bool(key: BridgeNetworkKeys.enableTso.rawValue) : nil + let enableChecksumOffload = + bridgeMessage.dataNoCopy(key: BridgeNetworkKeys.enableChecksumOffload.rawValue) != nil + ? bridgeMessage.bool(key: BridgeNetworkKeys.enableChecksumOffload.rawValue) : nil + let batchSize = + bridgeMessage.dataNoCopy(key: BridgeNetworkKeys.bufferedPacketCount.rawValue) != nil + ? Int(bridgeMessage.uint64(key: BridgeNetworkKeys.bufferedPacketCount.rawValue)) : nil + + bridge = try! Bridge( + identifier: "vmnet-bridge-\(network.id)-\(hostname)", hostInterface: hostInterface, logger: log, enableTso: enableTso ?? false, + enableChecksumOffload: enableChecksumOffload ?? false, bufferedPacketCount: batchSize ?? 64) + try bridge!.start() + + attachment = Attachment( + network: network.id, + hostname: hostname, + ipv4Address: nil, + ipv4Gateway: nil, + ipv6Address: nil, + macAddress: bridge!.macAddress, + mtu: bridge!.mtu + ) + + // add the sandbox-side fd endpoint to the bridge message + bridgeMessage.set(key: BridgeNetworkKeys.sandboxEndpoint.rawValue, value: bridge!.sandboxEndpoint) + + // close the sandbox endpoint's fd, since we do not need it anymore and it has been just dup-ed + // when we attached it to the XPC message above + try bridge!.sandboxEndpoint.close() + } + + log.info( + "allocated attachment", + metadata: [ + "hostname": "\(hostname)", + "hostInterface": "\(hostInterface)", + "macAddress": "\(attachment.macAddress!)", + "bridge": "\(bridge?.identifier ?? "vf-based")", + ]) + + if allocationsBySession[session] == nil { + allocationsBySession[session] = [] + await session.onDisconnect { [weak self] in + await self?.releaseSession(session) + } + } + allocationsBySession[session]!.append((hostname: hostname, bridge: bridge)) + macAddresses[hostname] = attachment.macAddress + + return (attachment: attachment, additionalData: additionalData) + } + + private func releaseSession(_ session: XPCServerSession) async { + guard let allocations = allocationsBySession.removeValue(forKey: session) else { + return + } + for allocation: (hostname: String, bridge: Bridge?) in allocations { + macAddresses[allocation.hostname] = nil + guard let bridge = allocation.bridge else { + continue + } + do { + try bridge.stop() + } catch let error as ContainerizationError { + log.error( + "failed to stop bridge for attachment", + metadata: [ + "error": Logger.MetadataValue.string(error.message) + ]) + } catch { + assert(false, "should never happen") + } + } + log.info("released session", metadata: ["allocations": "\(allocations.count)"]) + } + + @Sendable + public func lookup(hostname: String) async throws -> Attachment? { + log.debug("enter", metadata: ["func": "\(#function)"]) + defer { log.debug("exit", metadata: ["func": "\(#function)"]) } + + guard await network.status != nil else { + throw ContainerizationError(.invalidState, message: "network \(network.id) must be running") + } + guard let macAddress = macAddresses[hostname] else { + log.warning("MAC address for hostname \(hostname) is not a valid attachment") + return nil + } + + let attachment = Attachment( + network: network.id, + hostname: hostname, + ipv4Address: nil, + ipv4Gateway: nil, + ipv6Address: nil, + macAddress: macAddress + ) + log.info( + "lookup attachment", + metadata: [ + "hostname": "\(hostname)", + "macAddress": "\(macAddress)", + ]) + + return attachment + } +}