From 14e431fc8fd1d9ded96ccd553ba02f9a1d66eda7 Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Mon, 27 Nov 2023 01:58:36 +0100 Subject: [PATCH 01/49] q-dev: keep one DeviceInfo API the same as in client --- qubes/api/__init__.py | 4 +- qubes/api/admin.py | 31 +- qubes/api/misc.py | 16 +- qubes/app.py | 8 +- qubes/devices.py | 484 ++++++++++++++++++++++++----- qubes/ext/__init__.py | 4 +- qubes/ext/block.py | 92 +++++- qubes/ext/pci.py | 32 +- qubes/tests/devices.py | 81 ++++- qubes/tests/devices_block.py | 30 +- qubes/tests/integ/devices_block.py | 3 +- qubes/vm/__init__.py | 4 +- 12 files changed, 657 insertions(+), 132 deletions(-) diff --git a/qubes/api/__init__.py b/qubes/api/__init__.py index 700eca39b..ca548d3b4 100644 --- a/qubes/api/__init__.py +++ b/qubes/api/__init__.py @@ -127,8 +127,8 @@ def __init__(self, app, src, method_name, dest, arg, send_event=None): #: destination qube self.dest = self.app.domains[vm] except KeyError: - # normally this should filtered out by qrexec policy, but there are - # two cases it might not be: + # normally this should be filtered out by qrexec policy, but there + # are two cases it might not be: # 1. The call comes from dom0, which bypasses qrexec policy # 2. Domain was removed between checking the policy and here # we inform the client accordingly diff --git a/qubes/api/admin.py b/qubes/api/admin.py index cfcc113a6..bc0b5f484 100644 --- a/qubes/api/admin.py +++ b/qubes/api/admin.py @@ -1200,26 +1200,31 @@ async def vm_device_available(self, endpoint): devices = self.dest.devices[devclass].available() if self.arg: devices = [dev for dev in devices if dev.ident == self.arg] - # no duplicated devices, but device may not exists, in which case + # no duplicated devices, but device may not exist, in which case # the list is empty self.enforce(len(devices) <= 1) devices = self.fire_event_for_filter(devices, devclass=devclass) + # dev_info = {dev.ident: dev.serialize() for dev in devices} dev_info = {} for dev in devices: - non_default_attrs = set(attr for attr in dir(dev) if - not attr.startswith('_')).difference(( + # TODO: + if hasattr(dev, "serialize"): + properties_txt = dev.serialize().decode() + else: + non_default_attrs = set(attr for attr in dir(dev) if + not attr.startswith('_')).difference(( 'backend_domain', 'ident', 'frontend_domain', 'description', 'options', 'regex')) - properties_txt = ' '.join( - '{}={!s}'.format(prop, value) for prop, value - in itertools.chain( - ((key, getattr(dev, key)) for key in non_default_attrs), - # keep description as the last one, according to API - # specification - (('description', dev.description),) - )) - self.enforce('\n' not in properties_txt) + properties_txt = ' '.join( + '{}={!s}'.format(prop, value) for prop, value + in itertools.chain( + ((key, getattr(dev, key)) for key in non_default_attrs), + # keep description as the last one, according to API + # specification + (('description', dev.description),) + )) + self.enforce('\n' not in properties_txt) dev_info[dev.ident] = properties_txt return ''.join('{} {}\n'.format(ident, dev_info[ident]) @@ -1237,7 +1242,7 @@ async def vm_device_list(self, endpoint): device_assignments = [dev for dev in device_assignments if (str(dev.backend_domain), dev.ident) == (select_backend, select_ident)] - # no duplicated devices, but device may not exists, in which case + # no duplicated devices, but device may not exist, in which case # the list is empty self.enforce(len(device_assignments) <= 1) device_assignments = self.fire_event_for_filter(device_assignments, diff --git a/qubes/api/misc.py b/qubes/api/misc.py index 85591796c..ebc2ff955 100644 --- a/qubes/api/misc.py +++ b/qubes/api/misc.py @@ -18,8 +18,8 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, see . -''' Interface for methods not being part of Admin API, but still handled by -qubesd. ''' +""" Interface for methods not being part of Admin API, but still handled by +qubesd. """ import string from datetime import datetime @@ -34,7 +34,7 @@ class QubesMiscAPI(qubes.api.AbstractQubesAPI): @qubes.api.method('qubes.FeaturesRequest', no_payload=True) async def qubes_features_request(self): - ''' qubes.FeaturesRequest handler + """ qubes.FeaturesRequest handler VM (mostly templates) can request some features from dom0 for itself. Then dom0 (qubesd extension) may respect this request or ignore it. @@ -44,7 +44,7 @@ async def qubes_features_request(self): dispatch 'features-request' event, which may be handled by appropriate extensions. Requests not explicitly handled by some extension are ignored. - ''' + """ self.enforce(self.dest.name == 'dom0') self.enforce(not self.arg) @@ -66,9 +66,9 @@ async def qubes_features_request(self): @qubes.api.method('qubes.NotifyTools', no_payload=True) async def qubes_notify_tools(self): - ''' + """ Legacy version of qubes.FeaturesRequest, used by Qubes Windows Tools - ''' + """ self.enforce(self.dest.name == 'dom0') self.enforce(not self.arg) @@ -92,12 +92,12 @@ async def qubes_notify_tools(self): @qubes.api.method('qubes.NotifyUpdates') async def qubes_notify_updates(self, untrusted_payload): - ''' + """ Receive VM notification about updates availability Payload contains a single integer - either 0 (no updates) or some positive value (some updates). - ''' + """ untrusted_update_count = untrusted_payload.strip() self.enforce(untrusted_update_count.isdigit()) diff --git a/qubes/app.py b/qubes/app.py index f85b4848e..d8bd77a7a 100644 --- a/qubes/app.py +++ b/qubes/app.py @@ -1052,7 +1052,7 @@ def load(self, lock=False): vm.fire_event('domain-load') # get a file timestamp (before closing it - still holding the lock!), - # to detect whether anyone else have modified it in the meantime + # to detect whether anyone else has modified it in the meantime self.__load_timestamp = os.path.getmtime(self._store) if not lock: @@ -1509,7 +1509,7 @@ def on_domain_pre_deleted(self, event, vm): assignment.ident for assignment in assignments) raise qubes.exc.QubesVMInUseError( vm, - 'VM has devices attached persistently to other VMs: ' + + 'VM has devices attached persistently to other VMs: ' + # TODO desc) @qubes.events.handler('domain-delete') @@ -1564,7 +1564,7 @@ def on_property_set_default_netvm(self, event, name, newvalue, if hasattr(vm, 'provides_network') and not vm.provides_network and \ hasattr(vm, 'netvm') and vm.property_is_default('netvm'): # fire property-reset:netvm as it is responsible for resetting - # netvm to it's default value + # netvm to its default value vm.fire_event('property-reset:netvm', name='netvm', oldvalue=oldvalue) @@ -1576,6 +1576,6 @@ def on_property_set_default_dispvm(self, event, name, newvalue, if hasattr(vm, 'default_dispvm') and \ vm.property_is_default('default_dispvm'): # fire property-reset:default_dispvm as it is responsible for - # resetting dispvm to it's default value + # resetting dispvm to its default value vm.fire_event('property-reset:default_dispvm', name='default_dispvm', oldvalue=oldvalue) diff --git a/qubes/devices.py b/qubes/devices.py index a277aeeb2..de72a8da5 100644 --- a/qubes/devices.py +++ b/qubes/devices.py @@ -19,11 +19,11 @@ # License along with this library; if not, see . # -'''API for various types of devices. +"""API for various types of devices. Main concept is that some domain may expose (potentially multiple) devices, which can be attached to other domains. -Devices can be of different buses (like 'pci', 'usb', etc). Each device +Devices can be of different buses (like 'pci', 'usb', etc.). Each device bus is implemented by an extension. Devices are identified by pair of (backend domain, `ident`), where `ident` is @@ -54,50 +54,33 @@ Extension may use QubesDB watch API (QubesVM.watch_qdb_path(path), then handle `domain-qdb-change:path`) to detect changes and fire `device-list-change:class` event. -''' -from typing import Optional +""" +import itertools +import base64 +import sys +from enum import Enum +from typing import Optional, List, Type import qubes.utils +from qubes.api import PermissionDenied + class DeviceNotAttached(qubes.exc.QubesException, KeyError): - '''Trying to detach not attached device''' + """Trying to detach not attached device""" -class DeviceAlreadyAttached(qubes.exc.QubesException, KeyError): - '''Trying to attach already attached device''' -class DeviceInfo: - ''' Holds all information about a device ''' - # pylint: disable=too-few-public-methods - def __init__(self, backend_domain, ident, description=None, - frontend_domain=None): - #: domain providing this device - self.backend_domain = backend_domain - #: device identifier (unique for given domain and device type) - self.ident = ident - # allow redefining those as dynamic properties in subclasses - try: - #: human readable description/name of the device - self.description = description - except AttributeError: - pass - try: - #: (running) domain to which device is currently attached - self.frontend_domain = frontend_domain - except AttributeError: - pass +class DeviceAlreadyAttached(qubes.exc.QubesException, KeyError): + """Trying to attach already attached device""" - if hasattr(self, 'regex'): - # pylint: disable=no-member - dev_match = self.regex.match(ident) - if not dev_match: - raise ValueError('Invalid device identifier: {!r}'.format( - ident)) - for group in self.regex.groupindex: - setattr(self, group, dev_match.group(group)) +class Device: + def __init__(self, backend_domain, ident, devclass=None): + self.__backend_domain = backend_domain + self.__ident = ident + self.__bus = devclass def __hash__(self): - return hash((self.backend_domain, self.ident)) + return hash((str(self.backend_domain), self.ident)) def __eq__(self, other): return ( @@ -106,58 +89,408 @@ def __eq__(self, other): ) def __lt__(self, other): - if isinstance(other, DeviceInfo): + if isinstance(other, Device): return (self.backend_domain, self.ident) < \ (other.backend_domain, other.ident) return NotImplemented + def __repr__(self): + return "[%s]:%s" % (self.backend_domain, self.ident) + def __str__(self): return '{!s}:{!s}'.format(self.backend_domain, self.ident) + @property + def ident(self) -> str: + """ + Immutable device identifier. -class DeviceAssignment: # pylint: disable=too-few-public-methods - ''' Maps a device to a frontend_domain. ''' + Unique for given domain and device type. + """ + return self.__ident - def __init__(self, backend_domain, ident, options=None, persistent=False, - bus=None): - self.backend_domain = backend_domain - self.ident = ident - self.options = options or {} - self.persistent = persistent - self.bus = bus + @property + def backend_domain(self) -> 'qubesadmin.vm.QubesVM': + """ Which domain provides this device. (immutable)""" + return self.__backend_domain - def __repr__(self): - return "[%s]:%s" % (self.backend_domain, self.ident) + @property + def devclass(self) -> Optional[str]: + """ Immutable* Device class such like: 'usb', 'pci' etc. + + * see `@devclass.setter` + """ + return self.__bus + + @devclass.setter + def devclass(self, devclass: str): + """ Once a value is set, it should not be overridden. + + However, if it has not been set, i.e., the value is `None`, + we can override it.""" + if self.__bus != None: + raise TypeError("Attribute devclass is immutable") + self.__bus = devclass + + +class DeviceInterface(Enum): + # USB interfaces: + # https://www.usb.org/defined-class-codes#anchor_BaseClass03h + Other = "******" + USB_Audio = "01****" + USB_CDC = "02****" # Communications Device Class + USB_HID = "03****" + USB_HID_Keyboard = "03**01" + USB_HID_Mouse = "03**02" + # USB_Physical = "05****" + # USB_Still_Imaging = "06****" # Camera + USB_Printer = "07****" + USB_Mass_Storage = "08****" + USB_Hub = "09****" + USB_CDC_Data = "0a****" + USB_Smart_Card = "0b****" + # USB_Content_Security = "0d****" + USB_Video = "0e****" # Video Camera + # USB_Personal_Healthcare = "0f****" + USB_Audio_Video = "10****" + # USB_Billboard = "11****" + # USB_C_Bridge = "12****" + # and more... + + @staticmethod + def from_str(interface_encoding: str) -> 'DeviceInterface': + result = DeviceInterface.Other + best_score = 0 + + for interface in DeviceInterface: + pattern = interface.value + score = 0 + for t, p in zip(interface_encoding, pattern): + if t == p: + score += 1 + elif p != "*": + score = -1 # inconsistent with pattern + break + + if score > best_score: + best_score = score + result = interface - def __hash__(self): - # it's important to use the same hash as DeviceInfo - return hash((self.backend_domain, self.ident)) + return result - def __eq__(self, other): - if not isinstance(self, other.__class__): - return NotImplemented - return self.backend_domain == other.backend_domain \ - and self.ident == other.ident +class DeviceInfo(Device): + """ Holds all information about a device """ + + # pylint: disable=too-few-public-methods + def __init__( + self, + backend_domain: 'qubes.vm.qubesvm.QubesVM', # TODO + ident: str, + devclass: Optional[str] = None, + vendor: Optional[str] = None, + product: Optional[str] = None, + manufacturer: Optional[str] = None, + name: Optional[str] = None, + serial: Optional[str] = None, + interfaces: Optional[List[DeviceInterface]] = None, + parent: Optional[Device] = None, + **kwargs + ): + super().__init__(backend_domain, ident, devclass) + + self._vendor = vendor + self._product = product + self._manufacturer = manufacturer + self._name = name + self._serial = serial + self._interfaces = interfaces + self._parent = parent + + self.data = kwargs + + @property + def vendor(self) -> str: + """ + Device vendor name from local database. + + Could be empty string or "unknown". + + Override this method to return proper name from `/usr/share/hwdata/*`. + """ + if not self._vendor: + return "unknown" + return self._vendor + + @property + def product(self) -> str: + """ + Device name from local database. + + Could be empty string or "unknown". + + Override this method to return proper name from `/usr/share/hwdata/*`. + """ + if not self._product: + return "unknown" + return self._product + + @property + def manufacturer(self) -> str: + """ + The name of the manufacturer of the device introduced by device itself. + + Could be empty string or "unknown". + + Override this method to return proper name directly from device itself. + """ + if not self._manufacturer: + return "unknown" + return self._manufacturer + + @property + def name(self) -> str: + """ + The name of the device it introduced itself with. + + Could be empty string or "unknown". + + Override this method to return proper name directly from device itself. + """ + if not self._name: + return "unknown" + return self._name + + @property + def serial(self) -> str: + """ + The serial number of the device it introduced itself with. + + Could be empty string or "unknown". + + Override this method to return proper name directly from device itself. + """ + if not self._serial: + return "unknown" + return self._serial + + @property + def description(self) -> str: + """ + Short human-readable description. + + For unknown device returns `unknown device (unknown vendor)`. + For unknown USB device returns `unknown usb device (unknown vendor)`. + For unknown USB device with known serial number returns + ` (unknown vendor)`. + """ + if self.product and self.product != "unknown": + prod = self.product + elif self.name and self.name != "unknown": + prod = self.name + elif self.serial and self.serial != "unknown": + prod = self.serial + elif self.parent_device is not None: + return f"partition of {self.parent_device}" + else: + prod = f"unknown {self.devclass if self.devclass else ''} device" + + if self.vendor and self.vendor != "unknown": + vendor = self.vendor + elif self.manufacturer and self.manufacturer != "unknown": + vendor = self.manufacturer + else: + vendor = "unknown vendor" + + return f"{prod} ({vendor})" + + @property + def interfaces(self) -> List[DeviceInterface]: + """ + Non-empty list of device interfaces. + + Every device should have at least one interface. + """ + if not self._interfaces: + return [DeviceInterface.Other] + return self._interfaces + + @property + def parent_device(self) -> Optional['DeviceInfo']: + """ + The parent device if any. + + If the device is part of another device (e.g. it's a single + partition of an usb stick), the parent device id should be here. + """ + if self._parent is None: + return None + return self.backend_domain.devices.get( + self._parent.devclass, {}).get(self._parent.ident, None) + + @property + def subdevices(self) -> List['DeviceInfo']: + """ + The list of children devices if any. + + If the device has subdevices (e.g. partitions of an usb stick), + the subdevices id should be here. + """ + return [dev for dev in self.backend_domain.devices[self.devclass] + if dev.parent_device.ident == self.ident] + + # @property + # def port_id(self) -> str: + # """ + # Which port the device is connected to. + # """ + # return self.ident # TODO: ??? + + @property + def attachments(self) -> List['DeviceAssignment']: + """ + Device attachments + """ + return [] # TODO + + def serialize(self) -> bytes: + """ + Serialize object to be transmitted via Qubes API. + """ + # 'backend_domain', 'interfaces', 'data', 'parent_device' + # are not string, so they need special treatment + default_attrs = { + 'ident', 'devclass', 'vendor', 'product', 'manufacturer', 'name', + 'serial'} + properties = b' '.join( + base64.b64encode(f'{prop}={value!s}'.encode('ascii')) + for prop, value in ( + (key, getattr(self, key)) for key in default_attrs) + ) + + backend_domain_name = self.backend_domain.name + backend_domain_prop = (b'backend_domain=' + + backend_domain_name.encode('ascii')) + properties += b' ' + base64.b64encode(backend_domain_prop) + + interfaces = ''.join(ifc.value for ifc in self.interfaces) + interfaces_prop = b'interfaces=' + str(interfaces).encode('ascii') + properties += b' ' + base64.b64encode(interfaces_prop) + + if self.parent_device is not None: + parent_prop = b'parent=' + self.parent_device.ident.encode('ascii') + properties += b' ' + base64.b64encode(parent_prop) + + data = b' '.join( + base64.b64encode(f'_{prop}={value!s}'.encode('ascii')) + for prop, value in ((key, self.data[key]) for key in self.data) + ) + if data: + properties += b' ' + data + + return properties + + @classmethod + def deserialize( + cls, + serialization: bytes, + expected_backend_domain: 'qubes.vm.qubesvm.QubesVM', + expected_devclass: Optional[str] = None, + ) -> 'DeviceInfo': + try: + result = DeviceInfo._deserialize( + cls, serialization, expected_backend_domain, expected_devclass) + except Exception as exc: + print(exc, file=sys.stderr) # TODO + ident = serialization.split(b' ')[0].decode( + 'ascii', errors='ignore') + result = UnknownDevice( + backend_domain=expected_backend_domain, + ident=ident, + devclass=expected_devclass, + ) + return result + + @staticmethod + def _deserialize( + cls: Type, + serialization: bytes, + expected_backend_domain: 'qubes.vm.qubesvm.QubesVM', + expected_devclass: Optional[str] = None, + ) -> 'DeviceInfo': + properties_str = [ + base64.b64decode(line).decode('ascii', errors='ignore') + for line in serialization.split(b' ')[1:]] + + properties = dict() + for line in properties_str: + key, _, param = line.partition("=") + if key.startswith("_"): + properties[key[1:]] = param + else: + properties[key] = param + + if properties['backend_domain'] != expected_backend_domain.name: + raise ValueError("TODO") # TODO + properties['backend_domain'] = expected_backend_domain + # if expected_devclass and properties['devclass'] != expected_devclass: + # raise ValueError("TODO") # TODO + + interfaces = properties['interfaces'] + interfaces = [ + DeviceInterface.from_str(interfaces[i:i + 6]) + for i in range(0, len(interfaces), 6)] + properties['interfaces'] = interfaces + + if 'parent' in properties: + properties['parent'] = Device( + backend_domain=expected_backend_domain, + ident=properties['parent'] + ) + + return cls(**properties) + + @property + def frontend_domain(self): + return self.data.get("frontend_domain", None) + + +class UnknownDevice(DeviceInfo): + # pylint: disable=too-few-public-methods + """Unknown device - for example exposed by domain not running currently""" + + def __init__(self, backend_domain, devclass, ident, **kwargs): + super().__init__(backend_domain, ident, devclass=devclass, **kwargs) + + +class DeviceAssignment(Device): # pylint: disable=too-few-public-methods + """ Maps a device to a frontend_domain. """ + + def __init__( + self, backend_domain, ident, options=None, persistent=False, bus=None + ): + super().__init__(backend_domain, ident, bus) # TODO + self.options = options or {} + self.persistent = persistent # TODO def clone(self): - '''Clone object instance''' + """Clone object instance""" return self.__class__( self.backend_domain, self.ident, self.options, self.persistent, - self.bus, + self.devclass, ) @property - def device(self): - '''Get DeviceInfo object corresponding to this DeviceAssignment''' - return self.backend_domain.devices[self.bus][self.ident] + def device(self) -> DeviceInfo: + """Get DeviceInfo object corresponding to this DeviceAssignment""" + return self.backend_domain.devices[self.devclass][self.ident] class DeviceCollection: - '''Bag for devices. + """Bag for devices. Used as default value for :py:meth:`DeviceManager.__missing__` factory. @@ -166,6 +499,10 @@ class DeviceCollection: This class emits following events on VM object: + .. event:: device-added: (device) + + Fired when new device is discovered to a VM. + .. event:: device-attach: (device, options) Fired when device is attached to a VM. @@ -218,12 +555,12 @@ class DeviceCollection: of this event should return list of devices actually attached to a domain, regardless of its settings. - ''' + """ def __init__(self, vm, bus): self._vm = vm self._bus = bus - self._set = PersistentCollection() + self._set = PersistentCollection() # TODO self.devclass = qubes.utils.get_entry_point_one( 'qubes.devices', self._bus) @@ -234,13 +571,13 @@ async def attach(self, device_assignment: DeviceAssignment): :param DeviceInfo device: device object ''' - if device_assignment.bus is None: - device_assignment.bus = self._bus - elif device_assignment.bus != self._bus: + if device_assignment.devclass is None: + device_assignment.devclass = self._bus + elif device_assignment.devclass != self._bus: raise ValueError( 'Trying to attach DeviceAssignment of a different device class') - if not device_assignment.persistent and self._vm.is_halted(): + if not device_assignment.persistent and self._vm.is_halted(): # TODO raise qubes.exc.QubesVMNotRunningError(self._vm, "VM not running, can only attach device with persistent flag") device = device_assignment.device @@ -264,7 +601,7 @@ def load_persistent(self, device_assignment: DeviceAssignment): ''' assert not self._vm.events_enabled assert device_assignment.persistent - device_assignment.bus = self._bus + device_assignment.devclass = self._bus self._set.add(device_assignment) def update_persistent(self, device: DeviceInfo, persistent: bool): @@ -294,10 +631,10 @@ async def detach(self, device_assignment: DeviceAssignment): :param DeviceInfo device: device object ''' - if device_assignment.bus is None: - device_assignment.bus = self._bus + if device_assignment.devclass is None: + device_assignment.devclass = self._bus else: - assert device_assignment.bus == self._bus, \ + assert device_assignment.devclass == self._bus, \ "Trying to attach DeviceAssignment of a different device class" if device_assignment in self._set and not self._vm.is_halted(): @@ -420,8 +757,7 @@ def __init__(self, backend_domain, ident, description=None, frontend_domain=None): if description is None: description = "Unknown device" - super().__init__(backend_domain, ident, description, - frontend_domain) + super().__init__(backend_domain, ident, description, frontend_domain) class PersistentCollection: diff --git a/qubes/ext/__init__.py b/qubes/ext/__init__.py index 0530a3442..4a1949815 100644 --- a/qubes/ext/__init__.py +++ b/qubes/ext/__init__.py @@ -30,8 +30,8 @@ class Extension: - '''Base class for all extensions - ''' # pylint: disable=too-few-public-methods + '''Base class for all extensions''' + # pylint: disable=too-few-public-methods def __new__(cls): if '_instance' not in cls.__dict__: diff --git a/qubes/ext/block.py b/qubes/ext/block.py index 4d53c2c13..f19b2bb38 100644 --- a/qubes/ext/block.py +++ b/qubes/ext/block.py @@ -18,9 +18,12 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, see . -''' Qubes block devices extensions ''' +""" Qubes block devices extensions """ +import collections import re import string +from typing import Optional + import lxml.etree import qubes.devices @@ -44,15 +47,15 @@ class BlockDevice(qubes.devices.DeviceInfo): def __init__(self, backend_domain, ident): - super().__init__(backend_domain=backend_domain, - ident=ident) - self._description = None + super().__init__( + backend_domain=backend_domain, ident=ident, devclass="block") + self._mode = None self._size = None @property def description(self): - '''Human readable device description''' + """Human readable device description""" if self._description is None: if not self.backend_domain.is_running(): return self.ident @@ -69,7 +72,7 @@ def description(self): @property def mode(self): - '''Device mode, either 'w' for read-write, or 'r' for read-only''' + """Device mode, either 'w' for read-write, or 'r' for read-only""" if self._mode is None: if not self.backend_domain.is_running(): return 'w' @@ -87,7 +90,7 @@ def mode(self): @property def size(self): - '''Device size in bytes''' + """Device size in bytes""" if self._size is None: if not self.backend_domain.is_running(): return None @@ -105,22 +108,91 @@ def size(self): @property def device_node(self): - '''Device node in backend domain''' + """Device node in backend domain""" return '/dev/' + self.ident.replace('_', '/') + @property + def parent_device(self) -> Optional[qubes.devices.DeviceInfo]: + """ + The parent device if any. + + If the device is part of another device (e.g. it's a single + partition of an usb stick), the parent device id should be here. + """ + if self._parent is None: + if not self.backend_domain.is_running(): + return None + untrusted_parent: bytes = self.backend_domain.untrusted_qdb.read( + f'/qubes-block-devices/{self.ident}/parent') + if untrusted_parent is None: + return None + else: + parent_ident = self._sanitize(untrusted_parent) + self._parent = qubes.devices.Device( + self.backend_domain, parent_ident) + return self.backend_domain.devices.get( + self._parent.devclass, {}).get( + self._parent.ident, qubes.devices.UnknownDevice( + backend_domain=self._parent.backend_domain, + ident=self._parent.ident + ) + ) + + @staticmethod + def _sanitize( + untrusted_parent: bytes, + safe_chars: str = + string.ascii_letters + string.digits + string.punctuation + ) -> str: + untrusted_device_desc = untrusted_parent.decode( + 'ascii', errors='ignore') + return ''.join( + c if c in set(safe_chars) else '_' for c in untrusted_device_desc + ) + class BlockDeviceExtension(qubes.ext.Extension): + def __init__(self): + super().__init__() + self.devices_cache = collections.defaultdict(dict) + @qubes.ext.handler('domain-init', 'domain-load') def on_domain_init_load(self, vm, event): - '''Initialize watching for changes''' + """Initialize watching for changes""" # pylint: disable=unused-argument vm.watch_qdb_path('/qubes-block-devices') + if event == 'domain-load': + # avoid building a cache on domain-init, as it isn't fully set yet, + # and definitely isn't running yet + current_devices = { + dev.ident: dev.frontend_domain + for dev in self.on_device_list_block(vm, None) + } + self.devices_cache[vm.name] = current_devices + else: + self.devices_cache[vm.name] = {} @qubes.ext.handler('domain-qdb-change:/qubes-block-devices') def on_qdb_change(self, vm, event, path): - '''A change in QubesDB means a change in device list''' + """A change in QubesDB means a change in device list.""" # pylint: disable=unused-argument vm.fire_event('device-list-change:block') + current_devices = dict((dev.ident, dev.frontend_domain) + for dev in self.on_device_list_block(vm, None)) + + devices_cache_for_vm = self.devices_cache[vm.name] + for dev_id, connected_to in current_devices.items(): + if dev_id not in devices_cache_for_vm: + device = BlockDevice(vm, dev_id) + vm.fire_event('device-added:block', device=device) + for dev_id, connected_to in devices_cache_for_vm.items(): + if dev_id not in current_devices: + device = BlockDevice(vm, dev_id) + vm.fire_event('device-removed:block', device=device) + + # TODO: if not removed and not added + # TODO attach/detach + self.devices_cache[vm.name] = current_devices def device_get(self, vm, ident): '''Read information about device from QubesDB diff --git a/qubes/ext/pci.py b/qubes/ext/pci.py index 4b1cf32db..09f8839da 100644 --- a/qubes/ext/pci.py +++ b/qubes/ext/pci.py @@ -18,7 +18,7 @@ # License along with this library; if not, see . # -''' Qubes PCI Extensions ''' +""" Qubes PCI Extensions """ import functools import os @@ -35,7 +35,7 @@ pci_classes = None -#: emit warning on unspported device only once +#: emit warning on unsupported device only once unsupported_devices_warned = set() @@ -44,14 +44,14 @@ class UnsupportedDevice(Exception): def load_pci_classes(): - ''' List of known device classes, subclasses and programming interfaces. ''' + """ List of known device classes, subclasses and programming interfaces. """ # Syntax: # C class class_name # subclass subclass_name <-- single tab # prog-if prog-if_name <-- two tabs result = {} with open('/usr/share/hwdata/pci.ids', - encoding='utf-8', errors='ignore') as pciids: + encoding='utf-8', errors='ignore') as pciids: class_id = None subclass_id = None for line in pciids.readlines(): @@ -150,7 +150,18 @@ def __init__(self, backend_domain, ident, libvirt_name=None): raise UnsupportedDevice(libvirt_name) ident = '{bus}_{device}.{function}'.format(**dev_match.groupdict()) - super().__init__(backend_domain, ident, None) + super().__init__( + backend_domain=backend_domain, ident=ident, devclass="pci") + + if hasattr(self, 'regex'): + # pylint: disable=no-member + dev_match = self.regex.match(ident) + if not dev_match: + raise ValueError('Invalid device identifier: {!r}'.format( + ident)) + + for group in self.regex.groupindex: + setattr(self, group, dev_match.group(group)) # lazy loading self._description = None @@ -172,12 +183,11 @@ def description(self): hostdev_details.XMLDesc())) return self._description - @property - def frontend_domain(self): - # TODO: cache this - all_attached = attached_devices(self.backend_domain.app) - return all_attached.get(self.ident, None) - + # @property + # def frontend_domain(self): # TODO: possibly could be removed + # # TODO: cache this + # all_attached = attached_devices(self.backend_domain.app) + # return all_attached.get(self.ident, None) class PCIDeviceExtension(qubes.ext.Extension): diff --git a/qubes/tests/devices.py b/qubes/tests/devices.py index 50bb6f131..562ea22ab 100644 --- a/qubes/tests/devices.py +++ b/qubes/tests/devices.py @@ -21,6 +21,7 @@ # import qubes.devices +from qubes.devices import DeviceInfo, DeviceInterface import qubes.tests @@ -60,7 +61,7 @@ def __str__(self): @qubes.events.handler('device-list-attached:testclass') def dev_testclass_list_attached(self, event, persistent = False): for vm in self.app.domains: - if vm.device.frontend_domain == self: + if vm.device.data.get('test_frontend_domain', None) == self: yield (vm.device, {}) @qubes.events.handler('device-list:testclass') @@ -73,6 +74,10 @@ def is_halted(self): def is_running(self): return self.running + class log: + @staticmethod + def exception(message): + pass class TC_00_DeviceCollection(qubes.tests.QubesTestCase): @@ -140,7 +145,7 @@ def test_014_list_attached_non_persistent(self): self.emitter.running = True self.loop.run_until_complete(self.collection.attach(self.assignment)) # device-attach event not implemented, so manipulate object manually - self.device.frontend_domain = self.emitter + self.device.data['test_frontend_domain'] = self.emitter self.assertEqual({self.device}, set(self.collection.attached())) self.assertEqual(set([]), @@ -158,7 +163,7 @@ def test_020_update_persistent_to_false(self): self.assertEqual(set([]), set(self.collection.persistent())) self.loop.run_until_complete(self.collection.attach(self.assignment)) # device-attach event not implemented, so manipulate object manually - self.device.frontend_domain = self.emitter + self.device.data['test_frontend_domain'] = self.emitter self.assertEqual({self.device}, set(self.collection.persistent())) self.assertEqual({self.device}, set(self.collection.attached())) self.assertEqual({self.device}, set(self.collection.persistent())) @@ -173,7 +178,7 @@ def test_021_update_persistent_to_true(self): self.assertEqual(set([]), set(self.collection.persistent())) self.loop.run_until_complete(self.collection.attach(self.assignment)) # device-attach event not implemented, so manipulate object manually - self.device.frontend_domain = self.emitter + self.device.data['test_frontend_domain'] = self.emitter self.assertEqual(set(), set(self.collection.persistent())) self.assertEqual({self.device}, set(self.collection.attached())) self.assertEqual(set(), set(self.collection.persistent())) @@ -220,3 +225,71 @@ def test_001_missing(self): self.manager['testclass'].attach(assignment)) self.assertEventFired(self.emitter, 'device-attach:testclass') + +class TC_02_DeviceInfo(qubes.tests.QubesTestCase): + def setUp(self): + super().setUp() + self.app = TestApp() + self.vm = TestVM(self.app, 'vm') + + def test_001_init(self): + pass # TODO + + def test_010_serialize(self): + device = DeviceInfo( + backend_domain=self.vm, + ident="1-1.1.1", + devclass="bus", + vendor="ITL", + product="Qubes", + manufacturer="", + name="Some untrusted garbage", + serial=None, + interfaces=[DeviceInterface.Other, DeviceInterface.USB_HID], + # additional_info="", # TODO + # date="06.12.23", # TODO + ) + actual = sorted(device.serialize().split(b' ')) + expected = [ + b'YmFja2VuZF9kb21haW49dm0=', b'ZGV2Y2xhc3M9YnVz', + b'aW50ZXJmYWNlcz0qKioqKiowMyoqKio=', b'aWRlbnQ9MS0xLjEuMQ==', + b'bWFudWZhY3R1cmVyPXVua25vd24=', + b'bmFtZT1Tb21lIHVudHJ1c3RlZCBnYXJiYWdl', + b'c2VyaWFsPXVua25vd24=', b'cHJvZHVjdD1RdWJlcw==', + b'dmVuZG9yPUlUTA=='] + self.assertEqual(actual, expected) + + def test_020_deserialize(self): + expected = [ + b'YmFja2VuZF9kb21haW49dm0=', b'ZGV2Y2xhc3M9YnVz', + b'aW50ZXJmYWNlcz0qKioqKiowMyoqKio=', b'aWRlbnQ9MS0xLjEuMQ==', + b'bWFudWZhY3R1cmVyPXVua25vd24=', + b'bmFtZT1Tb21lIHVudHJ1c3RlZCBnYXJiYWdl', + b'c2VyaWFsPXVua25vd24=', b'cHJvZHVjdD1RdWJlcw==', + b'dmVuZG9yPUlUTA=='] + actual = DeviceInfo.deserialize(b' '.join(expected), self.vm) + expected = DeviceInfo( + backend_domain=self.vm, + ident="1-1.1.1", + devclass="bus", + vendor="ITL", + product="Qubes", + manufacturer="", + name="Some untrusted garbage", + serial=None, + interfaces=[DeviceInterface.Other, DeviceInterface.USB_HID], + # additional_info="", # TODO + # date="06.12.23", # TODO + ) + + self.assertEqual(actual.backend_domain, expected.backend_domain) + self.assertEqual(actual.ident, expected.ident) + self.assertEqual(actual.devclass, expected.devclass) + self.assertEqual(actual.vendor, expected.vendor) + self.assertEqual(actual.product, expected.product) + self.assertEqual(actual.manufacturer, expected.manufacturer) + self.assertEqual(actual.name, expected.name) + self.assertEqual(actual.serial, expected.serial) + self.assertEqual(actual.interfaces, expected.interfaces) + # self.assertEqual(actual.data, expected.data) # TODO + diff --git a/qubes/tests/devices_block.py b/qubes/tests/devices_block.py index 07daf211c..aa0f83526 100644 --- a/qubes/tests/devices_block.py +++ b/qubes/tests/devices_block.py @@ -159,7 +159,8 @@ def test_000_device_get(self): self.assertEqual(device_info._description, 'Test device') self.assertEqual(device_info.size, 1024000) self.assertEqual(device_info.mode, 'w') - self.assertEqual(device_info.frontend_domain, None) + self.assertEqual( + device_info.data.get('test_frontend_domain', None), None) self.assertEqual(device_info.device_node, '/dev/sda') def test_001_device_get_other_node(self): @@ -177,7 +178,8 @@ def test_001_device_get_other_node(self): self.assertEqual(device_info._description, 'Test device') self.assertEqual(device_info.size, 1024000) self.assertEqual(device_info.mode, 'w') - self.assertEqual(device_info.frontend_domain, None) + self.assertEqual( + device_info.data.get('test_frontend_domain', None), None) self.assertEqual(device_info.device_node, '/dev/mapper/dmroot') def test_002_device_get_invalid_desc(self): @@ -583,3 +585,27 @@ def test_051_detach_not_attached(self): dev = qubes.ext.block.BlockDevice(back_vm, 'sda') self.ext.on_device_pre_detached_block(vm, '', dev) self.assertFalse(vm.libvirt_domain.detachDevice.called) + + def test_060_devices_added(self): + vm = TestVM({ + '/qubes-block-devices/sda': b'', + '/qubes-block-devices/sda/desc': b'Test device', + '/qubes-block-devices/sda/size': b'1024000', + '/qubes-block-devices/sda/mode': b'w', + '/qubes-block-devices/sdb': b'', + '/qubes-block-devices/sdb/desc': b'Test device2', + '/qubes-block-devices/sdb/size': b'2048000', + '/qubes-block-devices/sdb/mode': b'r', + }) + devices = sorted(list(self.ext.on_qdb_change(vm, ''))) + self.assertEqual(len(devices), 2) + self.assertEqual(devices[0].backend_domain, vm) + self.assertEqual(devices[0].ident, 'sda') + self.assertEqual(devices[0].description, 'Test device') + self.assertEqual(devices[0].size, 1024000) + self.assertEqual(devices[0].mode, 'w') + self.assertEqual(devices[1].backend_domain, vm) + self.assertEqual(devices[1].ident, 'sdb') + self.assertEqual(devices[1].description, 'Test device2') + self.assertEqual(devices[1].size, 2048000) + self.assertEqual(devices[1].mode, 'r') diff --git a/qubes/tests/integ/devices_block.py b/qubes/tests/integ/devices_block.py index 3897881a2..5eea5aff8 100644 --- a/qubes/tests/integ/devices_block.py +++ b/qubes/tests/integ/devices_block.py @@ -346,7 +346,8 @@ def test_000_attach_reattach(self): self.loop.run_until_complete( self.frontend.run_for_stdio('! ls /dev/xvdi')) - self.assertIsNone(self.device.frontend_domain) + self.assertIsNone( + self.device.data.get('test_frontend_domain', None)) with self.subTest('reattach'): self.loop.run_until_complete( diff --git a/qubes/vm/__init__.py b/qubes/vm/__init__.py index b3be66e92..9c2b09f3a 100644 --- a/qubes/vm/__init__.py +++ b/qubes/vm/__init__.py @@ -279,7 +279,8 @@ def load_extras(self): self.app.domains[node.get('backend-domain')], node.get('id'), options, - persistent=True + persistent=True, + # TODO: required=node.get('required') ) self.devices[devclass].load_persistent(device_assignment) except KeyError: @@ -333,6 +334,7 @@ def __xml__(self): node = lxml.etree.Element('device') node.set('backend-domain', device.backend_domain.name) node.set('id', device.ident) + # TODO: node.set('required', device.required) for key, val in device.options.items(): option_node = lxml.etree.Element('option') option_node.set('name', key) From 69adb221f6f9483f9c1d1d6dd29ea56899f1a1e6 Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Fri, 29 Dec 2023 15:43:13 +0100 Subject: [PATCH 02/49] q-dev: simplify parent_device --- qubes/devices.py | 9 +++------ qubes/ext/block.py | 10 ++-------- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/qubes/devices.py b/qubes/devices.py index de72a8da5..4faad4bb3 100644 --- a/qubes/devices.py +++ b/qubes/devices.py @@ -291,7 +291,7 @@ def description(self) -> str: elif self.serial and self.serial != "unknown": prod = self.serial elif self.parent_device is not None: - return f"partition of {self.parent_device}" + return f"sub-device of {self.parent_device}" else: prod = f"unknown {self.devclass if self.devclass else ''} device" @@ -316,17 +316,14 @@ def interfaces(self) -> List[DeviceInterface]: return self._interfaces @property - def parent_device(self) -> Optional['DeviceInfo']: + def parent_device(self) -> Optional[Device]: """ The parent device if any. If the device is part of another device (e.g. it's a single partition of an usb stick), the parent device id should be here. """ - if self._parent is None: - return None - return self.backend_domain.devices.get( - self._parent.devclass, {}).get(self._parent.ident, None) + return self._parent @property def subdevices(self) -> List['DeviceInfo']: diff --git a/qubes/ext/block.py b/qubes/ext/block.py index f19b2bb38..175b03d6f 100644 --- a/qubes/ext/block.py +++ b/qubes/ext/block.py @@ -112,7 +112,7 @@ def device_node(self): return '/dev/' + self.ident.replace('_', '/') @property - def parent_device(self) -> Optional[qubes.devices.DeviceInfo]: + def parent_device(self) -> Optional[qubes.devices.Device]: """ The parent device if any. @@ -130,13 +130,7 @@ def parent_device(self) -> Optional[qubes.devices.DeviceInfo]: parent_ident = self._sanitize(untrusted_parent) self._parent = qubes.devices.Device( self.backend_domain, parent_ident) - return self.backend_domain.devices.get( - self._parent.devclass, {}).get( - self._parent.ident, qubes.devices.UnknownDevice( - backend_domain=self._parent.backend_domain, - ident=self._parent.ident - ) - ) + return self._parent @staticmethod def _sanitize( From 471e6a5cfd41afc3e6003189dc1cb05a3b327825 Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Sat, 30 Dec 2023 01:05:01 +0100 Subject: [PATCH 03/49] q-dev: DeviceInterface --- qubes/devices.py | 194 ++++++++++++++++++++++++++++++----------- qubes/ext/block.py | 11 ++- qubes/ext/pci.py | 118 +++++++++++++++++++++++-- qubes/tests/devices.py | 6 +- 4 files changed, 268 insertions(+), 61 deletions(-) diff --git a/qubes/devices.py b/qubes/devices.py index 4faad4bb3..3f402f556 100644 --- a/qubes/devices.py +++ b/qubes/devices.py @@ -133,48 +133,146 @@ def devclass(self, devclass: str): self.__bus = devclass -class DeviceInterface(Enum): - # USB interfaces: - # https://www.usb.org/defined-class-codes#anchor_BaseClass03h - Other = "******" - USB_Audio = "01****" - USB_CDC = "02****" # Communications Device Class - USB_HID = "03****" - USB_HID_Keyboard = "03**01" - USB_HID_Mouse = "03**02" - # USB_Physical = "05****" - # USB_Still_Imaging = "06****" # Camera - USB_Printer = "07****" - USB_Mass_Storage = "08****" - USB_Hub = "09****" - USB_CDC_Data = "0a****" - USB_Smart_Card = "0b****" - # USB_Content_Security = "0d****" - USB_Video = "0e****" # Video Camera - # USB_Personal_Healthcare = "0f****" - USB_Audio_Video = "10****" - # USB_Billboard = "11****" - # USB_C_Bridge = "12****" - # and more... +class DeviceCategory(Enum): + """ + Arbitrarily selected interfaces that are important to users, + thus deserving special recognition such as a custom icon, etc. + + """ + Other = "*******" + + Communication = ("u02****", "p07****") # eg. modems + Input = ("u03****", "p09****") # HID etc. + Keyboard = ("u03**01", "p0900**") + Mouse = ("u03**02", "p0902**") + Printer = ("u07****",) + Scanner = ("p0903**",) + # Multimedia = Audio, Video, Displays etc. + Multimedia = ("u01****", "u0e****", "u06****", "u10****", "p03****", + "p04****") + Wireless = ("ue0****", "p0d****") + Bluetooth = ("ue00101", "p0d11**") + Mass_Data = ("b******", "u08****", "p01****") + Network = ("p02****",) + Memory = ("p05****",) + PCI_Bridge = ("p06****",) + Docking_Station = ("p0a****",) + Processor = ("p0b****", "p40****") + PCI_Serial_Bus = ("p0c****",) + PCI_USB = ("p0c03**",) @staticmethod - def from_str(interface_encoding: str) -> 'DeviceInterface': - result = DeviceInterface.Other + def from_str(interface_encoding: str) -> 'DeviceCategory': + result = DeviceCategory.Other + if len(interface_encoding) != len(DeviceCategory.Other.value): + return result best_score = 0 - for interface in DeviceInterface: - pattern = interface.value - score = 0 - for t, p in zip(interface_encoding, pattern): - if t == p: - score += 1 - elif p != "*": - score = -1 # inconsistent with pattern - break + for interface in DeviceCategory: + for pattern in interface.value: + score = 0 + for t, p in zip(interface_encoding, pattern): + if t == p: + score += 1 + elif p != "*": + score = -1 # inconsistent with pattern + break - if score > best_score: - best_score = score - result = interface + if score > best_score: + best_score = score + result = interface + + return result + + +class DeviceInterface: + def __init__(self, interface_encoding: str, devclass: Optional[str] = None): + ifc_padded = interface_encoding.ljust(6, '*') + if devclass: + if len(ifc_padded) > 6: + print( + f"interface_encoding is too long " + f"(is {len(interface_encoding)}, expected max. 6) " + f"for given {devclass=}", + file=sys.stderr + ) + ifc_full = devclass[0] + ifc_padded + else: + known_devclasses = {'p': 'pci', 'u': 'usb', 'b': 'block'} + devclass = known_devclasses.get(interface_encoding[0], None) + if len(ifc_padded) > 7: + print( + f"interface_encoding is too long " + f"(is {len(interface_encoding)}, expected max. 7)", + file=sys.stderr + ) + ifc_full = ifc_padded + elif len(ifc_padded) == 6: + ifc_full = ' ' + ifc_padded + else: + ifc_full = ifc_padded + + self._devclass = devclass + self._interface_encoding = ifc_full + self._category = DeviceCategory.from_str(self._interface_encoding) + + @property + def devclass(self) -> Optional[str]: + """ Immutable Device class such like: 'usb', 'pci' etc. """ + return self._devclass + + @property + def category(self) -> DeviceCategory: + """ Immutable Device category such like: 'Mouse', 'Mass_Data' etc. """ + return self._category + + @property + def unknown(self) -> 'DeviceInterface': + return DeviceInterface(" ******") + + @property + def __repr__(self): + return self._interface_encoding + + @property + def __str__(self): + if self.devclass == "block": + return "Block device" + if self.devclass in ("usb", "pci"): + self._load_classes(self.devclass).get( + self._interface_encoding[1:], + f"Unclassified {self.devclass} device") + return repr(self) + + @staticmethod + def _load_classes(bus: str): + """ + List of known device classes, subclasses and programming interfaces. + """ + # Syntax: + # C class class_name + # subclass subclass_name <-- single tab + # prog-if prog-if_name <-- two tabs + result = {} + with open(f'/usr/share/hwdata/{bus}.ids', + encoding='utf-8', errors='ignore') as pciids: + class_id = None + subclass_id = None + for line in pciids.readlines(): + line = line.rstrip() + if line.startswith('\t\t') and class_id and subclass_id: + (progif_id, _, progif_name) = line[2:].split(' ', 2) + result[class_id + subclass_id + progif_id] = \ + f"{class_name}: {subclass_name} ({progif_name})" + elif line.startswith('\t') and class_id: + (subclass_id, _, subclass_name) = line[1:].split(' ', 2) + # store both prog-if specific entry and generic one + result[class_id + subclass_id + '**'] = \ + f"{class_name}: {subclass_name}" + elif line.startswith('C '): + (_, class_id, _, class_name) = line.split(' ', 3) + result[class_id + '****'] = class_name + subclass_id = None return result @@ -312,7 +410,7 @@ def interfaces(self) -> List[DeviceInterface]: Every device should have at least one interface. """ if not self._interfaces: - return [DeviceInterface.Other] + return [DeviceInterface.unknown] return self._interfaces @property @@ -370,7 +468,7 @@ def serialize(self) -> bytes: backend_domain_name.encode('ascii')) properties += b' ' + base64.b64encode(backend_domain_prop) - interfaces = ''.join(ifc.value for ifc in self.interfaces) + interfaces = ''.join(ifc._interface_encoding for ifc in self.interfaces) interfaces_prop = b'interfaces=' + str(interfaces).encode('ascii') properties += b' ' + base64.b64encode(interfaces_prop) @@ -435,8 +533,8 @@ def _deserialize( interfaces = properties['interfaces'] interfaces = [ - DeviceInterface.from_str(interfaces[i:i + 6]) - for i in range(0, len(interfaces), 6)] + DeviceInterface(interfaces[i:i + 7]) + for i in range(0, len(interfaces), 7)] properties['interfaces'] = interfaces if 'parent' in properties: @@ -563,10 +661,9 @@ def __init__(self, vm, bus): 'qubes.devices', self._bus) async def attach(self, device_assignment: DeviceAssignment): - '''Attach (add) device to domain. - - :param DeviceInfo device: device object - ''' + """ + Attach device to domain. + """ if device_assignment.devclass is None: device_assignment.devclass = self._bus @@ -623,10 +720,9 @@ def update_persistent(self, device: DeviceInfo, persistent: bool): self._set.discard(assignment) async def detach(self, device_assignment: DeviceAssignment): - '''Detach (remove) device from domain. - - :param DeviceInfo device: device object - ''' + """ + Detach device from domain. + """ if device_assignment.devclass is None: device_assignment.devclass = self._bus diff --git a/qubes/ext/block.py b/qubes/ext/block.py index 175b03d6f..2b6112c63 100644 --- a/qubes/ext/block.py +++ b/qubes/ext/block.py @@ -22,7 +22,7 @@ import collections import re import string -from typing import Optional +from typing import Optional, List import lxml.etree @@ -111,6 +111,15 @@ def device_node(self): """Device node in backend domain""" return '/dev/' + self.ident.replace('_', '/') + @property + def interfaces(self) -> List[qubes.devices.DeviceInterface]: + """ + List of device interfaces. + + Every device should have at least one interface. + """ + return [qubes.devices.DeviceInterface("******", "block")] + @property def parent_device(self) -> Optional[qubes.devices.Device]: """ diff --git a/qubes/ext/pci.py b/qubes/ext/pci.py index 09f8839da..5f5df7988 100644 --- a/qubes/ext/pci.py +++ b/qubes/ext/pci.py @@ -24,6 +24,8 @@ import os import re import subprocess +from typing import Optional, List + import libvirt import lxml import lxml.etree @@ -83,7 +85,7 @@ def pcidev_class(dev_xmldesc): with open(sysfs_path + '/class', encoding='ascii') as f_class: class_id = f_class.read().strip() except OSError: - return "Unknown" + return "unknown" if not qubes.ext.pci.pci_classes: qubes.ext.pci.pci_classes = load_pci_classes() @@ -93,7 +95,7 @@ def pcidev_class(dev_xmldesc): # ignore prog-if return qubes.ext.pci.pci_classes[class_id[0:4]] except KeyError: - return "Unknown" + return "unknown" def attached_devices(app): @@ -166,11 +168,111 @@ def __init__(self, backend_domain, ident, libvirt_name=None): # lazy loading self._description = None + @property + def vendor(self) -> str: + """ + Device vendor from local database `/usr/share/hwdata/usb.ids` + + Could be empty string or "unknown". + + Lazy loaded. + """ + if self._vendor is None: + result = self._load_desc_from_qubesdb()["vendor"] + else: + result = self._vendor + return result + + @property + def product(self) -> str: + """ + Device name from local database `/usr/share/hwdata/usb.ids` + + Could be empty string or "unknown". + + Lazy loaded. + """ + if self._product is None: + result = self._load_desc_from_qubesdb()["product"] + else: + result = self._product + return result + + @property + def manufacturer(self) -> str: + """ + The name of the manufacturer of the device introduced by device itself + + Could be empty string or "unknown". + + Lazy loaded. + """ + if self._manufacturer is None: + result = self._load_desc_from_qubesdb()["manufacturer"] + else: + result = self._manufacturer + return result + + @property + def name(self) -> str: + """ + The name of the device it introduced itself with (could be empty string) + + Could be empty string or "unknown". + + Lazy loaded. + """ + if self._name is None: + result = self._load_desc_from_qubesdb()["name"] + else: + result = self._name + return result + + @property + def serial(self) -> str: + """ + The serial number of the device it introduced itself with. + + Could be empty string or "unknown". + + Lazy loaded. + """ + if self._serial is None: + result = self._load_desc_from_qubesdb()["serial"] + else: + result = self._serial + return result + + @property + def interfaces(self) -> List[qubes.devices.DeviceCategory]: + """ + List of device interfaces. + + Every device should have at least one interface. + """ + if self._interfaces is None: + hostdev_details = \ + self.backend_domain.app.vmm.libvirt_conn.nodeDeviceLookupByName( + self.libvirt_name + ) + self._interfaces = [pcidev_class(lxml.etree.fromstring( + hostdev_details.XMLDesc()))] + return self._interfaces + + @property + def parent_device(self) -> Optional[qubes.devices.DeviceInfo]: + """ + The parent device if any. + + PCI device has no parents. + """ + return None + @property def libvirt_name(self): # pylint: disable=no-member # noinspection PyUnresolvedReferences - return 'pci_0000_{}_{}_{}'.format(self.bus, self.device, self.function) + return f'pci_0000_{self.bus}_{self.device}_{self.function}' @property def description(self): @@ -183,11 +285,11 @@ def description(self): hostdev_details.XMLDesc())) return self._description - # @property - # def frontend_domain(self): # TODO: possibly could be removed - # # TODO: cache this - # all_attached = attached_devices(self.backend_domain.app) - # return all_attached.get(self.ident, None) + @property + def frontend_domain(self): + # TODO: cache this + all_attached = attached_devices(self.backend_domain.app) + return all_attached.get(self.ident, None) class PCIDeviceExtension(qubes.ext.Extension): diff --git a/qubes/tests/devices.py b/qubes/tests/devices.py index 562ea22ab..91e51b0c6 100644 --- a/qubes/tests/devices.py +++ b/qubes/tests/devices.py @@ -21,7 +21,7 @@ # import qubes.devices -from qubes.devices import DeviceInfo, DeviceInterface +from qubes.devices import DeviceInfo, DeviceCategory import qubes.tests @@ -245,7 +245,7 @@ def test_010_serialize(self): manufacturer="", name="Some untrusted garbage", serial=None, - interfaces=[DeviceInterface.Other, DeviceInterface.USB_HID], + interfaces=[DeviceCategory.Other, DeviceCategory.USB_HID], # additional_info="", # TODO # date="06.12.23", # TODO ) @@ -277,7 +277,7 @@ def test_020_deserialize(self): manufacturer="", name="Some untrusted garbage", serial=None, - interfaces=[DeviceInterface.Other, DeviceInterface.USB_HID], + interfaces=[DeviceCategory.Other, DeviceCategory.USB_HID], # additional_info="", # TODO # date="06.12.23", # TODO ) From 763220856f8061e4bf137f30fb038cdced4f73e9 Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Sat, 30 Dec 2023 12:10:04 +0100 Subject: [PATCH 04/49] q-dev: pci devices draft fix DeviceInterface.unknown --- qubes/api/admin.py | 27 ++--------- qubes/devices.py | 38 ++++++++++------ qubes/ext/pci.py | 109 ++++++++++++++++++++++++--------------------- 3 files changed, 86 insertions(+), 88 deletions(-) diff --git a/qubes/api/admin.py b/qubes/api/admin.py index bc0b5f484..59a53d2f4 100644 --- a/qubes/api/admin.py +++ b/qubes/api/admin.py @@ -28,6 +28,7 @@ import string import subprocess import pathlib +import sys import libvirt import lxml.etree @@ -1204,31 +1205,9 @@ async def vm_device_available(self, endpoint): # the list is empty self.enforce(len(devices) <= 1) devices = self.fire_event_for_filter(devices, devclass=devclass) - - # dev_info = {dev.ident: dev.serialize() for dev in devices} - dev_info = {} - for dev in devices: - # TODO: - if hasattr(dev, "serialize"): - properties_txt = dev.serialize().decode() - else: - non_default_attrs = set(attr for attr in dir(dev) if - not attr.startswith('_')).difference(( - 'backend_domain', 'ident', 'frontend_domain', - 'description', 'options', 'regex')) - properties_txt = ' '.join( - '{}={!s}'.format(prop, value) for prop, value - in itertools.chain( - ((key, getattr(dev, key)) for key in non_default_attrs), - # keep description as the last one, according to API - # specification - (('description', dev.description),) - )) - self.enforce('\n' not in properties_txt) - dev_info[dev.ident] = properties_txt - + dev_info = {dev.ident: dev.serialize().decode() for dev in devices} return ''.join('{} {}\n'.format(ident, dev_info[ident]) - for ident in sorted(dev_info)) + for ident in sorted(dev_info)) @qubes.api.method('admin.vm.device.{endpoint}.List', endpoints=(ep.name for ep in importlib.metadata.entry_points(group='qubes.devices')), diff --git a/qubes/devices.py b/qubes/devices.py index 3f402f556..4e0cfef5f 100644 --- a/qubes/devices.py +++ b/qubes/devices.py @@ -135,9 +135,10 @@ def devclass(self, devclass: str): class DeviceCategory(Enum): """ + Category of peripheral device. + Arbitrarily selected interfaces that are important to users, thus deserving special recognition such as a custom icon, etc. - """ Other = "*******" @@ -186,12 +187,16 @@ def from_str(interface_encoding: str) -> 'DeviceCategory': class DeviceInterface: + """ + Peripheral device interface wrapper. + """ + def __init__(self, interface_encoding: str, devclass: Optional[str] = None): ifc_padded = interface_encoding.ljust(6, '*') if devclass: if len(ifc_padded) > 6: print( - f"interface_encoding is too long " + f"{interface_encoding=} is too long " f"(is {len(interface_encoding)}, expected max. 6) " f"for given {devclass=}", file=sys.stderr @@ -202,7 +207,7 @@ def __init__(self, interface_encoding: str, devclass: Optional[str] = None): devclass = known_devclasses.get(interface_encoding[0], None) if len(ifc_padded) > 7: print( - f"interface_encoding is too long " + f"{interface_encoding=} is too long " f"(is {len(interface_encoding)}, expected max. 7)", file=sys.stderr ) @@ -226,22 +231,29 @@ def category(self) -> DeviceCategory: """ Immutable Device category such like: 'Mouse', 'Mass_Data' etc. """ return self._category - @property - def unknown(self) -> 'DeviceInterface': - return DeviceInterface(" ******") + @classmethod + def unknown(cls) -> 'DeviceInterface': + """ Value for unknown device interface. """ + return cls(" ******") - @property def __repr__(self): return self._interface_encoding - @property def __str__(self): if self.devclass == "block": return "Block device" if self.devclass in ("usb", "pci"): - self._load_classes(self.devclass).get( - self._interface_encoding[1:], - f"Unclassified {self.devclass} device") + result = self._load_classes(self.devclass).get( + self._interface_encoding[1:], None) + if result is None: + result = self._load_classes(self.devclass).get( + self._interface_encoding[1:-2] + '**', None) + if result is None: + result = self._load_classes(self.devclass).get( + self._interface_encoding[1:-4] + '****', None) + if result is None: + result = f"Unclassified {self.devclass} device" + return result return repr(self) @staticmethod @@ -410,7 +422,7 @@ def interfaces(self) -> List[DeviceInterface]: Every device should have at least one interface. """ if not self._interfaces: - return [DeviceInterface.unknown] + return [DeviceInterface.unknown()] return self._interfaces @property @@ -468,7 +480,7 @@ def serialize(self) -> bytes: backend_domain_name.encode('ascii')) properties += b' ' + base64.b64encode(backend_domain_prop) - interfaces = ''.join(ifc._interface_encoding for ifc in self.interfaces) + interfaces = ''.join(repr(ifc) for ifc in self.interfaces) interfaces_prop = b'interfaces=' + str(interfaces).encode('ascii') properties += b' ' + base64.b64encode(interfaces_prop) diff --git a/qubes/ext/pci.py b/qubes/ext/pci.py index 5f5df7988..0243dc68a 100644 --- a/qubes/ext/pci.py +++ b/qubes/ext/pci.py @@ -23,8 +23,10 @@ import functools import os import re +import string import subprocess -from typing import Optional, List +import sys +from typing import Optional, List, Dict, Tuple import libvirt import lxml @@ -98,6 +100,20 @@ def pcidev_class(dev_xmldesc): return "unknown" +def pcidev_interface(dev_xmldesc): + sysfs_path = dev_xmldesc.findtext('path') + assert sysfs_path + try: + with open(sysfs_path + '/class', encoding='ascii') as f_class: + class_id = f_class.read().strip() + except OSError: + return "000000" + + if class_id.startswith('0x'): + class_id = class_id[2:] + return class_id + + def attached_devices(app): """Return map device->domain-name for all currently attached devices""" @@ -178,7 +194,7 @@ def vendor(self) -> str: Lazy loaded. """ if self._vendor is None: - result = self._load_desc_from_qubesdb()["vendor"] + result = self._load_desc()["vendor"] else: result = self._vendor return result @@ -193,58 +209,13 @@ def product(self) -> str: Lazy loaded. """ if self._product is None: - result = self._load_desc_from_qubesdb()["product"] + result = self._load_desc()["product"] else: result = self._product return result @property - def manufacturer(self) -> str: - """ - The name of the manufacturer of the device introduced by device itself - - Could be empty string or "unknown". - - Lazy loaded. - """ - if self._manufacturer is None: - result = self._load_desc_from_qubesdb()["manufacturer"] - else: - result = self._manufacturer - return result - - @property - def name(self) -> str: - """ - The name of the device it introduced itself with (could be empty string) - - Could be empty string or "unknown". - - Lazy loaded. - """ - if self._name is None: - result = self._load_desc_from_qubesdb()["name"] - else: - result = self._name - return result - - @property - def serial(self) -> str: - """ - The serial number of the device it introduced itself with. - - Could be empty string or "unknown". - - Lazy loaded. - """ - if self._serial is None: - result = self._load_desc_from_qubesdb()["serial"] - else: - result = self._serial - return result - - @property - def interfaces(self) -> List[qubes.devices.DeviceCategory]: + def interfaces(self) -> List[qubes.devices.DeviceInterface]: """ List of device interfaces. @@ -255,8 +226,10 @@ def interfaces(self) -> List[qubes.devices.DeviceCategory]: self.backend_domain.app.vmm.libvirt_conn.nodeDeviceLookupByName( self.libvirt_name ) - self._interfaces = [pcidev_class(lxml.etree.fromstring( - hostdev_details.XMLDesc()))] + interface_encoding = pcidev_interface(lxml.etree.fromstring( + hostdev_details.XMLDesc())) + self._interfaces = [qubes.devices.DeviceInterface( + interface_encoding, devclass='pci')] return self._interfaces @property @@ -285,6 +258,40 @@ def description(self): hostdev_details.XMLDesc())) return self._description + def _load_desc(self) -> Dict[str, str]: + unknown = "unknown" + result = {"vendor": unknown, + "product": unknown, + "manufacturer": unknown, + "name": unknown, + "serial": unknown} + if not self.backend_domain.is_running(): + # don't cache this value + return result + hostdev_details = \ + self.backend_domain.app.vmm.libvirt_conn.nodeDeviceLookupByName( + self.libvirt_name + ) + hostdev_xml = lxml.etree.fromstring(hostdev_details.XMLDesc()) + self._vendor = result["vendor"] = hostdev_xml.findtext( + 'capability/vendor') + self._product = result["product"] = hostdev_xml.findtext( + 'capability/product') + return result + + @staticmethod + def _sanitize( + untrusted_device_desc: bytes, + safe_chars: str = + string.ascii_letters + string.digits + string.punctuation + ' ' + ) -> str: + # b'USB\\x202.0\\x20Camera' -> 'USB 2.0 Camera' + untrusted_device_desc = untrusted_device_desc.decode( + 'unicode_escape', errors='ignore') + return ''.join( + c if c in set(safe_chars) else '_' for c in untrusted_device_desc + ) + @property def frontend_domain(self): # TODO: cache this From 7708add143ac97f7abfb27a62ca9c2e20da9ce4c Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Mon, 8 Jan 2024 03:19:55 +0100 Subject: [PATCH 05/49] q-dev: split persistent --- Makefile | 35 +- .../90-admin-default.policy.header | 14 +- qubes/api/admin.py | 231 ++++++++-- qubes/app.py | 3 +- qubes/devices.py | 426 +++++++++++------- qubes/ext/pci.py | 2 +- qubes/tests/api_admin.py | 28 +- qubes/tests/devices.py | 60 +-- qubes/tests/integ/backup.py | 4 +- qubes/tests/integ/devices_pci.py | 108 +++-- qubes/tests/vm/init.py | 8 +- qubes/tests/vm/qubesvm.py | 7 +- qubes/vm/__init__.py | 21 +- qubes/vm/qubesvm.py | 26 +- rpm_spec/core-dom0.spec.in | 14 +- templates/libvirt/xen.xml | 4 +- 16 files changed, 652 insertions(+), 339 deletions(-) diff --git a/Makefile b/Makefile index 3bea491f0..a52eb2e29 100644 --- a/Makefile +++ b/Makefile @@ -60,26 +60,38 @@ ADMIN_API_METHODS_SIMPLE = \ admin.vm.Shutdown \ admin.vm.Start \ admin.vm.Unpause \ + admin.vm.device.pci.Assign \ + admin.vm.device.pci.Assigned \ admin.vm.device.pci.Attach \ + admin.vm.device.pci.Attached \ admin.vm.device.pci.Available \ admin.vm.device.pci.Detach \ - admin.vm.device.pci.List \ - admin.vm.device.pci.Set.persistent \ + admin.vm.device.pci.Set.assignment \ + admin.vm.device.pci.Unassign \ + admin.vm.device.block.Assign \ + admin.vm.device.block.Assigned \ admin.vm.device.block.Attach \ + admin.vm.device.block.Attached \ admin.vm.device.block.Available \ admin.vm.device.block.Detach \ - admin.vm.device.block.List \ - admin.vm.device.block.Set.persistent \ + admin.vm.device.block.Set.assignment \ + admin.vm.device.block.Unassign \ + admin.vm.device.usb.Assign \ + admin.vm.device.usb.Assigned \ admin.vm.device.usb.Attach \ + admin.vm.device.usb.Attached \ admin.vm.device.usb.Available \ admin.vm.device.usb.Detach \ - admin.vm.device.usb.List \ - admin.vm.device.usb.Set.persistent \ + admin.vm.device.usb.Set.assignment \ + admin.vm.device.usb.Unassign \ + admin.vm.device.mic.Assign \ + admin.vm.device.mic.Assigned \ admin.vm.device.mic.Attach \ + admin.vm.device.mic.Attached \ admin.vm.device.mic.Available \ admin.vm.device.mic.Detach \ - admin.vm.device.mic.List \ - admin.vm.device.mic.Set.persistent \ + admin.vm.device.mic.Set.assignment \ + admin.vm.device.mic.Unassign \ admin.vm.feature.CheckWithNetvm \ admin.vm.feature.CheckWithTemplate \ admin.vm.feature.CheckWithAdminVM \ @@ -211,8 +223,11 @@ endif admin.vm.CreateInPool.AdminVM \ admin.vm.device.testclass.Attach \ admin.vm.device.testclass.Detach \ - admin.vm.device.testclass.List \ - admin.vm.device.testclass.Set.persistent \ + admin.vm.device.testclass.Assign \ + admin.vm.device.testclass.Unassign \ + admin.vm.device.testclass.Attached \ + admin.vm.device.testclass.Assigned \ + admin.vm.device.testclass.Set.assignment \ admin.vm.device.testclass.Available install -d $(DESTDIR)/etc/qubes/policy.d/include install -m 0644 qubes-rpc-policy/admin-local-ro \ diff --git a/qubes-rpc-policy/90-admin-default.policy.header b/qubes-rpc-policy/90-admin-default.policy.header index fefdb207a..bdd5c7065 100644 --- a/qubes-rpc-policy/90-admin-default.policy.header +++ b/qubes-rpc-policy/90-admin-default.policy.header @@ -20,14 +20,20 @@ !include-service admin.vm.volume.Import * include/admin-local-rwx !include-service admin.vm.volume.ImportWithSize * include/admin-local-rwx +!include-service admin.vm.device.mic.Assign * include/admin-local-rwx +!include-service admin.vm.device.mic.Assigned * include/admin-local-ro !include-service admin.vm.device.mic.Attach * include/admin-local-rwx +!include-service admin.vm.device.mic.Attached * include/admin-local-ro !include-service admin.vm.device.mic.Available * include/admin-local-ro !include-service admin.vm.device.mic.Detach * include/admin-local-rwx -!include-service admin.vm.device.mic.List * include/admin-local-ro -!include-service admin.vm.device.mic.Set.persistent * include/admin-local-rwx +!include-service admin.vm.device.mic.Set.assignment * include/admin-local-rwx +!include-service admin.vm.device.mic.Unassign * include/admin-local-rwx +!include-service admin.vm.device.usb.Assign * include/admin-local-rwx +!include-service admin.vm.device.usb.Assigned * include/admin-local-ro !include-service admin.vm.device.usb.Attach * include/admin-local-rwx +!include-service admin.vm.device.usb.Attached * include/admin-local-ro !include-service admin.vm.device.usb.Available * include/admin-local-ro !include-service admin.vm.device.usb.Detach * include/admin-local-rwx -!include-service admin.vm.device.usb.List * include/admin-local-ro -!include-service admin.vm.device.usb.Set.persistent * include/admin-local-rwx +!include-service admin.vm.device.usb.Set.assignment * include/admin-local-rwx +!include-service admin.vm.device.usb.Unassign * include/admin-local-rwx diff --git a/qubes/api/admin.py b/qubes/api/admin.py index 59a53d2f4..d5bd50891 100644 --- a/qubes/api/admin.py +++ b/qubes/api/admin.py @@ -28,7 +28,6 @@ import string import subprocess import pathlib -import sys import libvirt import lxml.etree @@ -1156,8 +1155,8 @@ async def create_disposable(self): return dispvm.name - @qubes.api.method('admin.vm.Remove', no_payload=True, - scope='global', write=True) + @qubes.api.method( + 'admin.vm.Remove', no_payload=True, scope='global', write=True) async def vm_remove(self): self.enforce(not self.arg) @@ -1198,7 +1197,7 @@ async def deviceclass_list(self): scope='local', read=True) async def vm_device_available(self, endpoint): devclass = endpoint - devices = self.dest.devices[devclass].available() + devices = self.dest.devices[devclass].get_exposed_devices() if self.arg: devices = [dev for dev in devices if dev.ident == self.arg] # no duplicated devices, but device may not exist, in which case @@ -1209,13 +1208,13 @@ async def vm_device_available(self, endpoint): return ''.join('{} {}\n'.format(ident, dev_info[ident]) for ident in sorted(dev_info)) - @qubes.api.method('admin.vm.device.{endpoint}.List', endpoints=(ep.name + @qubes.api.method('admin.vm.device.{endpoint}.Assigned', endpoints=(ep.name for ep in importlib.metadata.entry_points(group='qubes.devices')), no_payload=True, scope='local', read=True) async def vm_device_list(self, endpoint): devclass = endpoint - device_assignments = self.dest.devices[devclass].assignments() + device_assignments = self.dest.devices[devclass].get_assigned_devices() if self.arg: select_backend, select_ident = self.arg.split('+', 1) device_assignments = [dev for dev in device_assignments @@ -1224,8 +1223,45 @@ async def vm_device_list(self, endpoint): # no duplicated devices, but device may not exist, in which case # the list is empty self.enforce(len(device_assignments) <= 1) + device_assignments = self.fire_event_for_filter( + device_assignments, devclass=devclass) + + dev_info = {} # TODO: deviceAssignment.serialization() + for dev in device_assignments: + properties_txt = ' '.join( + '{}={!s}'.format(opt, value) for opt, value + in itertools.chain( + dev.options.items(), + (('required', 'yes' if dev.required else 'no'), + ('attach_automatically', + 'yes' if dev.attach_automatically else 'no')), + )) + self.enforce('\n' not in properties_txt) + ident = '{!s}+{!s}'.format(dev.backend_domain, dev.ident) + dev_info[ident] = properties_txt + + return ''.join('{} {}\n'.format(ident, dev_info[ident]) + for ident in sorted(dev_info)) + + @qubes.api.method( + 'admin.vm.device.{endpoint}.Attached', + endpoints=( + ep.name + for ep in importlib.metadata.entry_points(group='qubes.devices')), + no_payload=True, scope='local', read=True) + async def vm_device_attached(self, endpoint): + devclass = endpoint + device_assignments = self.dest.devices[devclass].get_attached_devices() + if self.arg: + select_backend, select_ident = self.arg.split('+', 1) + device_assignments = [dev for dev in device_assignments + if (str(dev.backend_domain), dev.ident) + == (select_backend, select_ident)] + # no duplicated devices, but device may not exist, in which case + # the list is empty + self.enforce(len(device_assignments) <= 1) device_assignments = self.fire_event_for_filter(device_assignments, - devclass=devclass) + devclass=devclass) dev_info = {} for dev in device_assignments: @@ -1233,33 +1269,43 @@ async def vm_device_list(self, endpoint): '{}={!s}'.format(opt, value) for opt, value in itertools.chain( dev.options.items(), - (('persistent', 'yes' if dev.persistent else 'no'),) + (('required', 'yes' if dev.required else 'no'), + ('attach_automatically', + 'yes' if dev.attach_automatically else 'no')), )) self.enforce('\n' not in properties_txt) ident = '{!s}+{!s}'.format(dev.backend_domain, dev.ident) dev_info[ident] = properties_txt return ''.join('{} {}\n'.format(ident, dev_info[ident]) - for ident in sorted(dev_info)) + for ident in sorted(dev_info)) - # Attach/Detach action can both modify persistent state (with - # persistent=True) and volatile state of running VM (with persistent=False). - # For this reason, write=True + execute=True - @qubes.api.method('admin.vm.device.{endpoint}.Attach', endpoints=(ep.name + # Assign/Unassign action can modify only persistent state of running VM. + # For this reason, write=True + @qubes.api.method('admin.vm.device.{endpoint}.Assign', endpoints=(ep.name for ep in importlib.metadata.entry_points(group='qubes.devices')), - scope='local', write=True, execute=True) - async def vm_device_attach(self, endpoint, untrusted_payload): + scope='local', write=True) + async def vm_device_assign(self, endpoint, untrusted_payload): devclass = endpoint + # TODO: deserialize options = {} - persistent = False - for untrusted_option in untrusted_payload.decode('ascii', - 'strict').split(): + attach_automatically = False + required = False + for untrusted_option in untrusted_payload.decode( + 'ascii', 'strict').split(): try: untrusted_key, untrusted_value = untrusted_option.split('=', 1) except ValueError: raise qubes.api.ProtocolError('Invalid options format') - if untrusted_key == 'persistent': - persistent = qubes.property.bool(None, None, untrusted_value) + if untrusted_key == 'attach_automatically': + attach_automatically = qubes.property.bool( + None, None, untrusted_value) + + self.enforce(attach_automatically) + + elif untrusted_key == 'required': + required = qubes.property.bool( + None, None, untrusted_value) else: allowed_chars_key = string.digits + string.ascii_letters + '-_.' allowed_chars_value = allowed_chars_key + ',+:' @@ -1276,23 +1322,113 @@ async def vm_device_attach(self, endpoint, untrusted_payload): # may raise KeyError, either on domain or ident dev = self.app.domains[backend_domain].devices[devclass][ident] - self.fire_event_for_permission(device=dev, - devclass=devclass, persistent=persistent, - options=options) + self.fire_event_for_permission( + device=dev, devclass=devclass, + required=required, attach_automatically=attach_automatically, + options=options + ) assignment = qubes.devices.DeviceAssignment( dev.backend_domain, dev.ident, - options=options, persistent=persistent) - await self.dest.devices[devclass].attach(assignment) + required=required, attach_automatically=attach_automatically, + options=options + ) + await self.dest.devices[devclass].assign(assignment) self.app.save() - # Attach/Detach action can both modify persistent state (with - # persistent=True) and volatile state of running VM (with persistent=False). - # For this reason, write=True + execute=True - @qubes.api.method('admin.vm.device.{endpoint}.Detach', endpoints=(ep.name + # Assign/Unassign action can modify only persistent state of running VM. + # For this reason, write=True + @qubes.api.method( + 'admin.vm.device.{endpoint}.Unassign', + endpoints=( + ep.name + for ep in importlib.metadata.entry_points(group='qubes.devices')), no_payload=True, scope='local', write=True) + async def vm_device_unassign(self, endpoint): + devclass = endpoint + + # qrexec already verified that no strange characters are in self.arg + backend_domain, ident = self.arg.split('+', 1) + # may raise KeyError; if device isn't found, it will be UnknownDevice + # instance - but allow it, otherwise it will be impossible to detach + # already removed device + dev = self.app.domains[backend_domain].devices[devclass][ident] + + self.fire_event_for_permission(device=dev, devclass=devclass) + + assignment = qubes.devices.DeviceAssignment( + dev.backend_domain, dev.ident) + await self.dest.devices[devclass].unassign(assignment) + self.app.save() + + # Attach/Detach action can modify only volatile state of running VM. + # For this reason, execute=True + @qubes.api.method( + 'admin.vm.device.{endpoint}.Attach', + endpoints=( + ep.name for ep in importlib.metadata.entry_points(group='qubes.devices')), - no_payload=True, - scope='local', write=True, execute=True) + scope='local', execute=True) + async def vm_device_attach(self, endpoint, untrusted_payload): + devclass = endpoint + # TODO: deserialize + options = {} + attach_automatically = False + required = False + for untrusted_option in untrusted_payload.decode( + 'ascii', 'strict').split(): + try: + untrusted_key, untrusted_value = untrusted_option.split('=', + 1) + except ValueError: + raise qubes.api.ProtocolError('Invalid options format') + if untrusted_key == 'attach_automatically': + attach_automatically = qubes.property.bool( + None, None, untrusted_value) + + self.enforce(not attach_automatically) + + elif untrusted_key == 'required': + required = qubes.property.bool( + None, None, untrusted_value) + else: + allowed_chars_key = string.digits + string.ascii_letters + '-_.' + allowed_chars_value = allowed_chars_key + ',+:' + if any(x not in allowed_chars_key for x in untrusted_key): + raise qubes.api.ProtocolError( + 'Invalid chars in option name') + if any(x not in allowed_chars_value for x in + untrusted_value): + raise qubes.api.ProtocolError( + 'Invalid chars in option value') + options[untrusted_key] = untrusted_value + + # qrexec already verified that no strange characters are in self.arg + backend_domain, ident = self.arg.split('+', 1) + # may raise KeyError, either on domain or ident + dev = self.app.domains[backend_domain].devices[devclass][ident] + + self.fire_event_for_permission( + device=dev, devclass=devclass, + required=required, attach_automatically=attach_automatically, + options=options + ) + + assignment = qubes.devices.DeviceAssignment( + dev.backend_domain, dev.ident, + required=required, attach_automatically=attach_automatically, + options=options + ) + await self.dest.devices[devclass].attach(assignment) + self.app.save() # not needed? + + # Attach/Detach action can modify only volatile state of running VM. + # For this reason, execute=True + @qubes.api.method( + 'admin.vm.device.{endpoint}.Detach', + endpoints=( + ep.name + for ep in importlib.metadata.entry_points(group='qubes.devices')), + no_payload=True, scope='local', execute=True) async def vm_device_detach(self, endpoint): devclass = endpoint @@ -1303,41 +1439,48 @@ async def vm_device_detach(self, endpoint): # already removed device dev = self.app.domains[backend_domain].devices[devclass][ident] - self.fire_event_for_permission(device=dev, - devclass=devclass) + self.fire_event_for_permission(device=dev, devclass=devclass) assignment = qubes.devices.DeviceAssignment( dev.backend_domain, dev.ident) await self.dest.devices[devclass].detach(assignment) - self.app.save() + self.app.save() # TODO: not needed # Attach/Detach action can both modify persistent state (with - # persistent=True) and volatile state of running VM (with persistent=False). - # For this reason, write=True + execute=True - @qubes.api.method('admin.vm.device.{endpoint}.Set.persistent', + # required=True or required=False) and volatile state of running VM + # (with required=None). For this reason, write=True + execute=True + @qubes.api.method('admin.vm.device.{endpoint}.Set.assignment', endpoints=(ep.name for ep in importlib.metadata.entry_points(group='qubes.devices')), scope='local', write=True, execute=True) - async def vm_device_set_persistent(self, endpoint, untrusted_payload): + async def vm_device_set_assignment(self, endpoint, untrusted_payload): + """ + Update assignment of already attached device. + + Payload: + `None` -> unassign device from qube + `False` -> device will be auto-attached to qube + `True` -> device is required to start qube + """ devclass = endpoint - self.enforce(untrusted_payload in (b'True', b'False')) - persistent = untrusted_payload == b'True' + self.enforce(untrusted_payload in (b'True', b'False', b'None')) + # now is safe to eval, since value of untrusted_payload is trusted + assignment = eval(untrusted_payload) del untrusted_payload # qrexec already verified that no strange characters are in self.arg backend_domain, ident = self.arg.split('+', 1) # device must be already attached matching_devices = [dev for dev - in self.dest.devices[devclass].attached() + in self.dest.devices[devclass].get_attached_devices() if dev.backend_domain.name == backend_domain and dev.ident == ident] self.enforce(len(matching_devices) == 1) dev = matching_devices[0] - self.fire_event_for_permission(device=dev, - persistent=persistent) + self.fire_event_for_permission(device=dev, assignment=assignment) - self.dest.devices[devclass].update_persistent(dev, persistent) + self.dest.devices[devclass].update_assignment(dev, assignment) self.app.save() @qubes.api.method('admin.vm.firewall.Get', no_payload=True, diff --git a/qubes/app.py b/qubes/app.py index d8bd77a7a..df0b98071 100644 --- a/qubes/app.py +++ b/qubes/app.py @@ -1509,8 +1509,7 @@ def on_domain_pre_deleted(self, event, vm): assignment.ident for assignment in assignments) raise qubes.exc.QubesVMInUseError( vm, - 'VM has devices attached persistently to other VMs: ' + # TODO - desc) + 'VM has devices assigned to other VMs: ' + desc) @qubes.events.handler('domain-delete') def on_domain_deleted(self, event, vm): diff --git a/qubes/devices.py b/qubes/devices.py index 4e0cfef5f..18d035af2 100644 --- a/qubes/devices.py +++ b/qubes/devices.py @@ -55,22 +55,37 @@ `domain-qdb-change:path`) to detect changes and fire `device-list-change:class` event. """ -import itertools import base64 +import itertools import sys from enum import Enum -from typing import Optional, List, Type +from typing import Optional, List, Type, Dict, Any, Iterable import qubes.utils -from qubes.api import PermissionDenied class DeviceNotAttached(qubes.exc.QubesException, KeyError): - """Trying to detach not attached device""" + """ + Trying to detach not attached device. + """ + + +class DeviceNotAssigned(qubes.exc.QubesException, KeyError): + """ + Trying to unassign not assigned device. + """ class DeviceAlreadyAttached(qubes.exc.QubesException, KeyError): - """Trying to attach already attached device""" + """ + Trying to attach already attached device. + """ + + +class DeviceAlreadyAssigned(qubes.exc.QubesException, KeyError): + """ + Trying to assign already assigned device. + """ class Device: @@ -90,8 +105,8 @@ def __eq__(self, other): def __lt__(self, other): if isinstance(other, Device): - return (self.backend_domain, self.ident) < \ - (other.backend_domain, other.ident) + return (self.backend_domain.name, self.ident) < \ + (other.backend_domain.name, other.ident) return NotImplemented def __repr__(self): @@ -110,7 +125,7 @@ def ident(self) -> str: return self.__ident @property - def backend_domain(self) -> 'qubesadmin.vm.QubesVM': + def backend_domain(self) -> 'qubes.vm.BaseVM': """ Which domain provides this device. (immutable)""" return self.__backend_domain @@ -149,6 +164,7 @@ class DeviceCategory(Enum): Printer = ("u07****",) Scanner = ("p0903**",) # Multimedia = Audio, Video, Displays etc. + Microphone = ("m******",) Multimedia = ("u01****", "u0e****", "u06****", "u10****", "p03****", "p04****") Wireless = ("ue0****", "p0d****") @@ -203,7 +219,8 @@ def __init__(self, interface_encoding: str, devclass: Optional[str] = None): ) ifc_full = devclass[0] + ifc_padded else: - known_devclasses = {'p': 'pci', 'u': 'usb', 'b': 'block'} + known_devclasses = { + 'p': 'pci', 'u': 'usb', 'b': 'block', 'm': 'mic'} devclass = known_devclasses.get(interface_encoding[0], None) if len(ifc_padded) > 7: print( @@ -254,6 +271,8 @@ def __str__(self): if result is None: result = f"Unclassified {self.devclass} device" return result + if self.devclass == 'mic': + return "Microphone" return repr(self) @staticmethod @@ -295,7 +314,7 @@ class DeviceInfo(Device): # pylint: disable=too-few-public-methods def __init__( self, - backend_domain: 'qubes.vm.qubesvm.QubesVM', # TODO + backend_domain: 'qubes.vm.BaseVM', ident: str, devclass: Optional[str] = None, vendor: Optional[str] = None, @@ -446,13 +465,6 @@ def subdevices(self) -> List['DeviceInfo']: return [dev for dev in self.backend_domain.devices[self.devclass] if dev.parent_device.ident == self.ident] - # @property - # def port_id(self) -> str: - # """ - # Which port the device is connected to. - # """ - # return self.ident # TODO: ??? - @property def attachments(self) -> List['DeviceAssignment']: """ @@ -501,7 +513,7 @@ def serialize(self) -> bytes: def deserialize( cls, serialization: bytes, - expected_backend_domain: 'qubes.vm.qubesvm.QubesVM', + expected_backend_domain: 'qubes.vm.BaseVM', expected_devclass: Optional[str] = None, ) -> 'DeviceInfo': try: @@ -522,7 +534,7 @@ def deserialize( def _deserialize( cls: Type, serialization: bytes, - expected_backend_domain: 'qubes.vm.qubesvm.QubesVM', + expected_backend_domain: 'qubes.vm.BaseVM', expected_devclass: Optional[str] = None, ) -> 'DeviceInfo': properties_str = [ @@ -570,24 +582,44 @@ def __init__(self, backend_domain, devclass, ident, **kwargs): super().__init__(backend_domain, ident, devclass=devclass, **kwargs) -class DeviceAssignment(Device): # pylint: disable=too-few-public-methods - """ Maps a device to a frontend_domain. """ +class DeviceAssignment(Device): + """ Maps a device to a frontend_domain. + + There are 3 flags `attached`, `automatically_attached` and `required`. + The meaning of valid combinations is as follows: + 1. (True, False, False) -> domain is running, device is manually attached + and could be manually detach any time. + 2. (True, True, False) -> domain is running, device is attached + and could be manually detach any time (see 4.), + but in the future will be auto-attached again. + 3. (True, True, True) -> domain is running, device is attached + and couldn't be detached. + 4. (False, Ture, False) -> device is assigned to domain, but not attached + because either domain is halted + or device manually detached. + 5. (False, True, True) -> domain is halted, device assigned to domain + and required to start domain. + """ - def __init__( - self, backend_domain, ident, options=None, persistent=False, bus=None - ): - super().__init__(backend_domain, ident, bus) # TODO - self.options = options or {} - self.persistent = persistent # TODO + def __init__(self, backend_domain, ident, options=None, + frontend_domain=None, devclass=None, + required=False, attach_automatically=False): + super().__init__(backend_domain, ident, devclass) + self.__options = options or {} + self.__required = required + self.__attach_automatically = attach_automatically + self.__frontend_domain = frontend_domain def clone(self): """Clone object instance""" return self.__class__( - self.backend_domain, - self.ident, - self.options, - self.persistent, - self.devclass, + backend_domain=self.backend_domain, + ident=self.ident, + options=self.options, + required=self.required, + attach_automatically=self.attach_automatically, + frontend_domain=self.frontend_domain, + devclass=self.devclass, ) @property @@ -595,6 +627,60 @@ def device(self) -> DeviceInfo: """Get DeviceInfo object corresponding to this DeviceAssignment""" return self.backend_domain.devices[self.devclass][self.ident] + @property + def frontend_domain(self) -> Optional['qubes.vm.qubesvm.QubesVM']: + """ Which domain the device is attached to. """ + return self.__frontend_domain + + @frontend_domain.setter + def frontend_domain( + self, frontend_domain: Optional['qubes.vm.qubesvm.QubesVM'] + ): + """ Which domain the device is attached to. """ + self.__frontend_domain = frontend_domain + + @property + def attached(self) -> bool: + """ + Is the device already attached to the fronted domain? + """ + return (self.frontend_domain is not None + and self.frontend_domain.is_running()) + + @property + def required(self) -> bool: + """ + Is the presence of this device required for the domain to start? If yes, + it will be attached automatically. + """ + return self.__required + + @required.setter + def required(self, required: bool): + self.__required = required + + @property + def attach_automatically(self) -> bool: + """ + Should this device automatically connect to the frontend domain when + available and not connected to other qubes? + """ + return self.__attach_automatically + + @attach_automatically.setter + def attach_automatically(self, attach_automatically: bool): + self.__attach_automatically = attach_automatically + + @property + def options(self) -> Dict[str, Any]: + """ Device options (same as in the legacy API). """ + return self.__options + + @options.setter + def options(self, options: Optional[Dict[str, Any]]): + """ Device options (same as in the legacy API). """ + self.__options = options or {} + class DeviceCollection: """Bag for devices. @@ -604,7 +690,7 @@ class DeviceCollection: :param vm: VM for which we manage devices :param bus: device bus - This class emits following events on VM object: + This class emits following events on VM object: # TODO pre-assign, assign, pre-unassign, unassign .. event:: device-added: (device) @@ -656,7 +742,7 @@ class DeviceCollection: :py:class:`DeviceInfo`, or :py:obj:`None`. Especially should not raise :py:class:`exceptions.KeyError`. - .. event:: device-list-attached: (persistent) + .. event:: device-list-attached: Fired to get list of currently attached devices to a VM. Handlers of this event should return list of devices actually attached to @@ -667,7 +753,7 @@ class DeviceCollection: def __init__(self, vm, bus): self._vm = vm self._bus = bus - self._set = PersistentCollection() # TODO + self._set = AssignedCollection() self.devclass = qubes.utils.get_entry_point_one( 'qubes.devices', self._bus) @@ -683,41 +769,78 @@ async def attach(self, device_assignment: DeviceAssignment): raise ValueError( 'Trying to attach DeviceAssignment of a different device class') - if not device_assignment.persistent and self._vm.is_halted(): # TODO - raise qubes.exc.QubesVMNotRunningError(self._vm, - "VM not running, can only attach device with persistent flag") + if self._vm.is_halted(): + raise qubes.exc.QubesVMNotRunningError( + self._vm,"VM not running, cannot attach device," + " did you mean `assign`?") device = device_assignment.device - if device in self.assignments(): + if device in self.get_attached_devices(): raise DeviceAlreadyAttached( 'device {!s} of class {} already attached to {!s}'.format( device, self._bus, self._vm)) - await self._vm.fire_event_async('device-pre-attach:' + self._bus, - pre_event=True, + await self._vm.fire_event_async( + 'device-pre-attach:' + self._bus, + pre_event=True, device=device, options=device_assignment.options) + + await self._vm.fire_event_async( + 'device-attach:' + self._bus, device=device, options=device_assignment.options) - if device_assignment.persistent: - self._set.add(device_assignment) - await self._vm.fire_event_async('device-attach:' + self._bus, + + async def assign(self, device_assignment: DeviceAssignment): + """ + Assign device to domain. + """ + if device_assignment.devclass is None: + device_assignment.devclass = self._bus + elif device_assignment.devclass != self._bus: + raise ValueError( + 'Trying to assign DeviceAssignment of a different device class') + + device = device_assignment.device + if device in self.get_assigned_devices(): + raise DeviceAlreadyAssigned( + 'device {!s} of class {} already attached to {!s}'.format( + device, self._bus, self._vm)) + + # TODO: check if needed + await self._vm.fire_event_async( + 'device-pre-assign:' + self._bus, + pre_event=True, device=device, options=device_assignment.options) + + self._set.add(device_assignment) + + await self._vm.fire_event_async( + 'device-assign:' + self._bus, device=device, options=device_assignment.options) - def load_persistent(self, device_assignment: DeviceAssignment): - '''Load DeviceAssignment retrieved from qubes.xml + def load_assignment(self, device_assignment: DeviceAssignment): + """Load DeviceAssignment retrieved from qubes.xml This can be used only for loading qubes.xml, when VM events are not enabled yet. - ''' + """ assert not self._vm.events_enabled - assert device_assignment.persistent + assert device_assignment.attach_automatically device_assignment.devclass = self._bus self._set.add(device_assignment) - def update_persistent(self, device: DeviceInfo, persistent: bool): - '''Update `persistent` flag of already attached device. - ''' + def update_assignment(self, device: DeviceInfo, required: Optional[bool]): + """ + Update assignment of already attached device. + :param DeviceInfo device: device for which change required flag + :param bool required: new assignment: + `None` -> unassign device from qube + `False` -> device will be auto-attached to qube + `True` -> device is required to start qube + """ if self._vm.is_halted(): - raise qubes.exc.QubesVMNotStartedError(self._vm, - 'VM must be running to modify device persistence flag') - assignments = [a for a in self.assignments() if a.device == device] + raise qubes.exc.QubesVMNotStartedError( + self._vm, + 'VM must be running to modify device assignment' + ) + assignments = [a for a in self.get_assigned_devices() + if a.device == device] if not assignments: raise qubes.exc.QubesValueError('Device not assigned') assert len(assignments) == 1 @@ -725,98 +848,106 @@ def update_persistent(self, device: DeviceInfo, persistent: bool): # be careful to use already present assignment, not the provided one # - to not change options as a side effect - if persistent and device not in self._set: - assignment.persistent = True + if required is not None and device not in self._set: + assignment.attach_automatically = True + assignment.required = required self._set.add(assignment) - elif not persistent and device in self._set: + elif required is None and device in self._set: self._set.discard(assignment) - async def detach(self, device_assignment: DeviceAssignment): + async def detach(self, device_assignment: DeviceAssignment): # TODO: argument should be just device """ Detach device from domain. """ + for assignment in self.get_attached_devices(): + if device_assignment == assignment: + # load all options + device_assignment = assignment + break + else: + raise DeviceNotAssigned( + f'device {device_assignment.ident!s} of class {self._bus} not ' + f'assigned to {self._vm!s}') - if device_assignment.devclass is None: - device_assignment.devclass = self._bus + if device_assignment.required and not self._vm.is_halted(): + raise qubes.exc.QubesVMNotHaltedError( + self._vm, + "Can not remove a required device from a non halted qube." + "You need to unassign device first.") + + device = device_assignment.device + await self._vm.fire_event_async( + 'device-pre-detach:' + self._bus, pre_event=True, device=device) + + await self._vm.fire_event_async( + 'device-detach:' + self._bus, device=device) + + async def unassign(self, device_assignment: DeviceAssignment): + """ + Unassign device from domain. + """ + for assignment in self.get_assigned_devices(): + if device_assignment == assignment: + # load all options + device_assignment = assignment + break else: - assert device_assignment.devclass == self._bus, \ - "Trying to attach DeviceAssignment of a different device class" + raise DeviceNotAssigned( + f'device {device_assignment.ident!s} of class {self._bus} not ' + f'assigned to {self._vm!s}') - if device_assignment in self._set and not self._vm.is_halted(): - raise qubes.exc.QubesVMNotHaltedError(self._vm, - "Can not remove a persistent attachment from a non halted vm") - if device_assignment not in self.assignments(): - raise DeviceNotAttached( - 'device {!s} of class {} not attached to {!s}'.format( - device_assignment.ident, self._bus, self._vm)) + if not self._vm.is_halted(): + raise qubes.exc.QubesVMNotHaltedError( + self._vm, + "Can not remove an assignment from a non halted qube.") device = device_assignment.device - await self._vm.fire_event_async('device-pre-detach:' + self._bus, - pre_event=True, device=device) - if device in self._set: - device_assignment.persistent = True - self._set.discard(device_assignment) - - await self._vm.fire_event_async('device-detach:' + self._bus, - device=device) - - def attached(self): - '''List devices which are (or may be) attached to this vm ''' - attached = self._vm.fire_event('device-list-attached:' + self._bus, - persistent=None) - if attached: - return [dev for dev, _ in attached] - - return [] - - def persistent(self): - ''' Devices persistently attached and safe to access before libvirt - bootstrap. - ''' - return [a.device for a in self._set] + # TODO: check if needed + await self._vm.fire_event_async( + 'device-pre-detach:' + self._bus, pre_event=True, device=device) - def assignments(self, persistent: Optional[bool]=None): - '''List assignments for devices which are (or may be) attached to the - vm. + self._set.discard(device_assignment) - Devices may be attached persistently (so they are included in - :file:`qubes.xml`) or not. Device can also be in :file:`qubes.xml`, - but be temporarily detached. + await self._vm.fire_event_async( + 'device-detach:' + self._bus, device=device) - :param Optional[bool] persistent: only include devices which are or are - not attached persistently. - ''' + def get_dedicated_devices(self) -> Iterable[DeviceAssignment]: + """ + List devices which are attached or assigned to this vm. + """ + dedicated = {dev for dev in itertools.chain( + self.get_attached_devices(), self.get_assigned_devices())} + for dev in dedicated: + yield dev - try: - devices = self._vm.fire_event('device-list-attached:' + self._bus, - persistent=persistent) - except Exception: # pylint: disable=broad-except - self._vm.log.exception('Failed to list {} devices'.format( - self._bus)) - if persistent is True: - # don't break app.save() - return list(self._set) - raise - result = [] - if persistent is not False: # None or True - result.extend(self._set) - if not persistent: # None or False - for dev, options in devices: - if dev not in self._set: - result.append( - DeviceAssignment( - backend_domain=dev.backend_domain, - ident=dev.ident, options=options, - bus=self._bus)) - return result + def get_attached_devices(self) -> Iterable[DeviceAssignment]: + """ + List devices which are attached to this vm. + """ + attached = self._vm.fire_event('device-list-attached:' + self._bus) + return [dev for dev, _ in attached] - def available(self): - '''List devices exposed by this vm''' + def get_assigned_devices( + self, required_only: bool = False + ) -> Iterable[DeviceAssignment]: + """ + Devices assigned to this vm (included in :file:`qubes.xml`). + + Safe to access before libvirt bootstrap. + """ + for dev in self._set: + if required_only and not dev.required: + continue + yield dev + + def get_exposed_devices(self) -> Iterable[DeviceInfo]: + """ + List devices exposed by this vm. + """ devices = self._vm.fire_event('device-list:' + self._bus) return devices - def __iter__(self): - return iter(self.available()) + __iter__ = get_exposed_devices def __getitem__(self, ident): '''Get device object with given ident. @@ -840,10 +971,11 @@ def __getitem__(self, ident): class DeviceManager(dict): - '''Device manager that hold all devices by their classes. + """ + Device manager that hold all devices by their classes. :param vm: VM for which we manage devices - ''' + """ def __init__(self, vm): super().__init__() @@ -854,28 +986,17 @@ def __missing__(self, key): return self[key] -class UnknownDevice(DeviceInfo): - # pylint: disable=too-few-public-methods - '''Unknown device - for example exposed by domain not running currently''' - - def __init__(self, backend_domain, ident, description=None, - frontend_domain=None): - if description is None: - description = "Unknown device" - super().__init__(backend_domain, ident, description, frontend_domain) - - -class PersistentCollection: - - ''' Helper object managing persistent `DeviceAssignment`s. - ''' +class AssignedCollection: + """ + Helper object managing assigned devices. + """ def __init__(self): self._dict = {} def add(self, assignment: DeviceAssignment): - ''' Add assignment to collection ''' - assert assignment.persistent + """ Add assignment to collection """ + assert assignment.attach_automatically vm = assignment.backend_domain ident = assignment.ident key = (vm, ident) @@ -883,9 +1004,11 @@ def add(self, assignment: DeviceAssignment): self._dict[key] = assignment - def discard(self, assignment): - ''' Discard assignment from collection ''' - assert assignment.persistent + def discard(self, assignment: DeviceAssignment): + """ + Discard assignment from collection. + """ + assert assignment.attach_automatically vm = assignment.backend_domain ident = assignment.ident key = (vm, ident) @@ -897,8 +1020,9 @@ def __contains__(self, device) -> bool: return (device.backend_domain, device.ident) in self._dict def get(self, device: DeviceInfo) -> DeviceAssignment: - ''' Returns the corresponding `qubes.devices.DeviceAssignment` for the - device. ''' + """ + Returns the corresponding `DeviceAssignment` for the device. + """ return self._dict[(device.backend_domain, device.ident)] def __iter__(self): diff --git a/qubes/ext/pci.py b/qubes/ext/pci.py index 0243dc68a..dfc2b0900 100644 --- a/qubes/ext/pci.py +++ b/qubes/ext/pci.py @@ -423,7 +423,7 @@ def on_device_pre_detached_pci(self, vm, event, device): @qubes.ext.handler('domain-pre-start') def on_domain_pre_start(self, vm, _event, **_kwargs): # Bind pci devices to pciback driver - for assignment in vm.devices['pci'].persistent(): + for assignment in vm.devices['pci'].get_assigned_devices(): device = _cache_get(assignment.backend_domain, assignment.ident) self.bind_pci_to_pciback(vm.app, device) diff --git a/qubes/tests/api_admin.py b/qubes/tests/api_admin.py index c24d29f12..db9a384eb 100644 --- a/qubes/tests/api_admin.py +++ b/qubes/tests/api_admin.py @@ -1803,7 +1803,7 @@ def test_480_vm_device_attach(self): mock_attach.assert_called_once_with(self.vm, 'device-attach:testclass', device=self.vm.devices['testclass']['1234'], options={}) - self.assertEqual(len(self.vm.devices['testclass'].persistent()), 0) + self.assertEqual(len(self.vm.devices['testclass'].get_assigned_devices()), 0) self.app.save.assert_called_once_with() def test_481_vm_device_attach(self): @@ -1820,7 +1820,7 @@ def test_481_vm_device_attach(self): mock_attach.assert_called_once_with(self.vm, 'device-attach:testclass', device=self.vm.devices['testclass']['1234'], options={}) - self.assertEqual(len(self.vm.devices['testclass'].persistent()), 0) + self.assertEqual(len(self.vm.devices['testclass'].get_assigned_devices()), 0) self.app.save.assert_called_once_with() def test_482_vm_device_attach_not_running(self): @@ -1832,7 +1832,7 @@ def test_482_vm_device_attach_not_running(self): self.call_mgmt_func(b'admin.vm.device.testclass.Attach', b'test-vm1', b'test-vm1+1234') self.assertFalse(mock_attach.called) - self.assertEqual(len(self.vm.devices['testclass'].persistent()), 0) + self.assertEqual(len(self.vm.devices['testclass'].get_assigned_devices()), 0) self.assertFalse(self.app.save.called) def test_483_vm_device_attach_persistent(self): @@ -1850,7 +1850,7 @@ def test_483_vm_device_attach_persistent(self): mock_attach.assert_called_once_with(self.vm, 'device-attach:testclass', device=dev, options={}) - self.assertIn(dev, self.vm.devices['testclass'].persistent()) + self.assertIn(dev, self.vm.devices['testclass'].get_assigned_devices()) self.app.save.assert_called_once_with() def test_484_vm_device_attach_persistent_not_running(self): @@ -1866,7 +1866,7 @@ def test_484_vm_device_attach_persistent_not_running(self): mock_attach.assert_called_once_with(self.vm, 'device-attach:testclass', device=dev, options={}) - self.assertIn(dev, self.vm.devices['testclass'].persistent()) + self.assertIn(dev, self.vm.devices['testclass'].get_assigned_devices()) self.app.save.assert_called_once_with() def test_485_vm_device_attach_options(self): @@ -2643,7 +2643,7 @@ def test_650_vm_device_set_persistent_true(self): b'test-vm1', b'test-vm1+1234', b'True') self.assertIsNone(value) dev = qubes.devices.DeviceInfo(self.vm, '1234') - self.assertIn(dev, self.vm.devices['testclass'].persistent()) + self.assertIn(dev, self.vm.devices['testclass'].get_assigned_devices()) self.app.save.assert_called_once_with() def test_651_vm_device_set_persistent_false_unchanged(self): @@ -2658,7 +2658,7 @@ def test_651_vm_device_set_persistent_false_unchanged(self): b'test-vm1', b'test-vm1+1234', b'False') self.assertIsNone(value) dev = qubes.devices.DeviceInfo(self.vm, '1234') - self.assertNotIn(dev, self.vm.devices['testclass'].persistent()) + self.assertNotIn(dev, self.vm.devices['testclass'].get_assigned_devices()) self.app.save.assert_called_once_with() def test_652_vm_device_set_persistent_false(self): @@ -2671,15 +2671,15 @@ def test_652_vm_device_set_persistent_false(self): self.vm.add_handler('device-list-attached:testclass', self.device_list_attached_testclass) dev = qubes.devices.DeviceInfo(self.vm, '1234') - self.assertIn(dev, self.vm.devices['testclass'].persistent()) + self.assertIn(dev, self.vm.devices['testclass'].get_assigned_devices()) with unittest.mock.patch.object(qubes.vm.qubesvm.QubesVM, 'is_halted', lambda _: False): value = self.call_mgmt_func( b'admin.vm.device.testclass.Set.persistent', b'test-vm1', b'test-vm1+1234', b'False') self.assertIsNone(value) - self.assertNotIn(dev, self.vm.devices['testclass'].persistent()) - self.assertIn(dev, self.vm.devices['testclass'].attached()) + self.assertNotIn(dev, self.vm.devices['testclass'].get_assigned_devices()) + self.assertIn(dev, self.vm.devices['testclass'].get_attached_devices()) self.app.save.assert_called_once_with() def test_653_vm_device_set_persistent_true_unchanged(self): @@ -2698,8 +2698,8 @@ def test_653_vm_device_set_persistent_true_unchanged(self): b'test-vm1', b'test-vm1+1234', b'True') self.assertIsNone(value) dev = qubes.devices.DeviceInfo(self.vm, '1234') - self.assertIn(dev, self.vm.devices['testclass'].persistent()) - self.assertIn(dev, self.vm.devices['testclass'].attached()) + self.assertIn(dev, self.vm.devices['testclass'].get_assigned_devices()) + self.assertIn(dev, self.vm.devices['testclass'].get_attached_devices()) self.app.save.assert_called_once_with() def test_654_vm_device_set_persistent_not_attached(self): @@ -2712,7 +2712,7 @@ def test_654_vm_device_set_persistent_not_attached(self): b'admin.vm.device.testclass.Set.persistent', b'test-vm1', b'test-vm1+1234', b'True') dev = qubes.devices.DeviceInfo(self.vm, '1234') - self.assertNotIn(dev, self.vm.devices['testclass'].persistent()) + self.assertNotIn(dev, self.vm.devices['testclass'].get_assigned_devices()) self.assertFalse(self.app.save.called) def test_655_vm_device_set_persistent_invalid_value(self): @@ -2725,7 +2725,7 @@ def test_655_vm_device_set_persistent_invalid_value(self): b'admin.vm.device.testclass.Set.persistent', b'test-vm1', b'test-vm1+1234', b'maybe') dev = qubes.devices.DeviceInfo(self.vm, '1234') - self.assertNotIn(dev, self.vm.devices['testclass'].persistent()) + self.assertNotIn(dev, self.vm.devices['testclass'].get_assigned_devices()) self.assertFalse(self.app.save.called) def test_660_pool_set_revisions_to_keep(self): diff --git a/qubes/tests/devices.py b/qubes/tests/devices.py index 91e51b0c6..ea23480a4 100644 --- a/qubes/tests/devices.py +++ b/qubes/tests/devices.py @@ -133,12 +133,12 @@ def test_012_double_detach(self): self.collection.detach(self.assignment)) def test_013_list_attached_persistent(self): - self.assertEqual(set([]), set(self.collection.persistent())) + self.assertEqual(set([]), set(self.collection.get_assigned_devices())) self.loop.run_until_complete(self.collection.attach(self.assignment)) self.assertEventFired(self.emitter, 'device-list-attached:testclass') - self.assertEqual({self.device}, set(self.collection.persistent())) + self.assertEqual({self.device}, set(self.collection.get_assigned_devices())) self.assertEqual(set([]), - set(self.collection.attached())) + set(self.collection.get_attached_devices())) def test_014_list_attached_non_persistent(self): self.assignment.persistent = False @@ -147,11 +147,11 @@ def test_014_list_attached_non_persistent(self): # device-attach event not implemented, so manipulate object manually self.device.data['test_frontend_domain'] = self.emitter self.assertEqual({self.device}, - set(self.collection.attached())) + set(self.collection.get_attached_devices())) self.assertEqual(set([]), - set(self.collection.persistent())) + set(self.collection.get_assigned_devices())) self.assertEqual({self.device}, - set(self.collection.attached())) + set(self.collection.get_attached_devices())) self.assertEventFired(self.emitter, 'device-list-attached:testclass') def test_015_list_available(self): @@ -160,49 +160,49 @@ def test_015_list_available(self): def test_020_update_persistent_to_false(self): self.emitter.running = True - self.assertEqual(set([]), set(self.collection.persistent())) + self.assertEqual(set([]), set(self.collection.get_assigned_devices())) self.loop.run_until_complete(self.collection.attach(self.assignment)) # device-attach event not implemented, so manipulate object manually self.device.data['test_frontend_domain'] = self.emitter - self.assertEqual({self.device}, set(self.collection.persistent())) - self.assertEqual({self.device}, set(self.collection.attached())) - self.assertEqual({self.device}, set(self.collection.persistent())) - self.assertEqual({self.device}, set(self.collection.attached())) - self.collection.update_persistent(self.device, False) - self.assertEqual(set(), set(self.collection.persistent())) - self.assertEqual({self.device}, set(self.collection.attached())) + self.assertEqual({self.device}, set(self.collection.get_assigned_devices())) + self.assertEqual({self.device}, set(self.collection.get_attached_devices())) + self.assertEqual({self.device}, set(self.collection.get_assigned_devices())) + self.assertEqual({self.device}, set(self.collection.get_attached_devices())) + self.collection.update_assignment(self.device, False) + self.assertEqual(set(), set(self.collection.get_assigned_devices())) + self.assertEqual({self.device}, set(self.collection.get_attached_devices())) def test_021_update_persistent_to_true(self): self.assignment.persistent = False self.emitter.running = True - self.assertEqual(set([]), set(self.collection.persistent())) + self.assertEqual(set([]), set(self.collection.get_assigned_devices())) self.loop.run_until_complete(self.collection.attach(self.assignment)) # device-attach event not implemented, so manipulate object manually self.device.data['test_frontend_domain'] = self.emitter - self.assertEqual(set(), set(self.collection.persistent())) - self.assertEqual({self.device}, set(self.collection.attached())) - self.assertEqual(set(), set(self.collection.persistent())) - self.assertEqual({self.device}, set(self.collection.attached())) - self.collection.update_persistent(self.device, True) - self.assertEqual({self.device}, set(self.collection.persistent())) - self.assertEqual({self.device}, set(self.collection.attached())) + self.assertEqual(set(), set(self.collection.get_assigned_devices())) + self.assertEqual({self.device}, set(self.collection.get_attached_devices())) + self.assertEqual(set(), set(self.collection.get_assigned_devices())) + self.assertEqual({self.device}, set(self.collection.get_attached_devices())) + self.collection.update_assignment(self.device, True) + self.assertEqual({self.device}, set(self.collection.get_assigned_devices())) + self.assertEqual({self.device}, set(self.collection.get_attached_devices())) def test_022_update_persistent_reject_not_running(self): - self.assertEqual(set([]), set(self.collection.persistent())) + self.assertEqual(set([]), set(self.collection.get_assigned_devices())) self.loop.run_until_complete(self.collection.attach(self.assignment)) - self.assertEqual({self.device}, set(self.collection.persistent())) - self.assertEqual(set(), set(self.collection.attached())) + self.assertEqual({self.device}, set(self.collection.get_assigned_devices())) + self.assertEqual(set(), set(self.collection.get_attached_devices())) with self.assertRaises(qubes.exc.QubesVMNotStartedError): - self.collection.update_persistent(self.device, False) + self.collection.update_assignment(self.device, False) def test_023_update_persistent_reject_not_attached(self): - self.assertEqual(set(), set(self.collection.persistent())) - self.assertEqual(set(), set(self.collection.attached())) + self.assertEqual(set(), set(self.collection.get_assigned_devices())) + self.assertEqual(set(), set(self.collection.get_attached_devices())) self.emitter.running = True with self.assertRaises(qubes.exc.QubesValueError): - self.collection.update_persistent(self.device, True) + self.collection.update_assignment(self.device, True) with self.assertRaises(qubes.exc.QubesValueError): - self.collection.update_persistent(self.device, False) + self.collection.update_assignment(self.device, False) class TC_01_DeviceManager(qubes.tests.QubesTestCase): diff --git a/qubes/tests/integ/backup.py b/qubes/tests/integ/backup.py index 42b40f02c..d62c6d3f7 100644 --- a/qubes/tests/integ/backup.py +++ b/qubes/tests/integ/backup.py @@ -303,7 +303,7 @@ def get_vms_info(self, vms): vm_info['default'][prop] = vm.property_is_default(prop) for dev_class in vm.devices.keys(): vm_info['devices'][dev_class] = {} - for dev_ass in vm.devices[dev_class].assignments(): + for dev_ass in vm.devices[dev_class].get_assigned_devices(): vm_info['devices'][dev_class][str(dev_ass.device)] = \ dev_ass.options vms_info[vm.name] = vm_info @@ -338,7 +338,7 @@ def assertCorrectlyRestored(self, vms_info, orig_hashes): for dev in vm_info['devices'][dev_class]: found = False for restored_dev_ass in restored_vm.devices[ - dev_class].assignments(): + dev_class].get_assigned_devices(): if str(restored_dev_ass.device) == dev: found = True self.assertEqual(vm_info['devices'][dev_class][dev], diff --git a/qubes/tests/integ/devices_pci.py b/qubes/tests/integ/devices_pci.py index 290607e3b..96c6eea3b 100644 --- a/qubes/tests/integ/devices_pci.py +++ b/qubes/tests/integ/devices_pci.py @@ -40,7 +40,14 @@ def setUp(self): self.assignment = qubes.devices.DeviceAssignment( backend_domain=self.dev.backend_domain, ident=self.dev.ident, - persistent=True) + attach_automatically=True, + ) + self.required_assignment = qubes.devices.DeviceAssignment( + backend_domain=self.dev.backend_domain, + ident=self.dev.ident, + attach_automatically=True, + required=True, + ) if isinstance(self.dev, qubes.devices.UnknownDevice): self.skipTest('Specified device {} does not exists'.format(pcidev)) self.init_default_template() @@ -73,27 +80,35 @@ def test_000_list(self): self.fail('Not all devices listed, missing: {}'.format( actual_devices)) - def assertDeviceNotInCollection(self, dev, dev_col): - self.assertNotIn(dev, dev_col.attached()) - self.assertNotIn(dev, dev_col.persistent()) - self.assertNotIn(dev, dev_col.assignments()) - self.assertNotIn(dev, dev_col.assignments(persistent=True)) - - def test_010_attach_offline_persistent(self): + def assertDeviceIs( + self, device, *, attached: bool, assigned: bool, required: bool + ): dev_col = self.vm.devices['pci'] - self.assertDeviceNotInCollection(self.dev, dev_col) - self.loop.run_until_complete( - dev_col.attach(self.assignment)) + if required: + assert assigned + self.assertTrue(attached == device in dev_col.get_attached_devices()) + self.assertTrue(assigned == device in dev_col.get_assigned_devices()) + self.assertTrue( + required == device in dev_col.get_assigned_devices( + required_only=True) + ) + dedicated = assigned or attached + self.assertTrue(dedicated == device in dev_col.get_dedicated_devices()) + + def test_010_assign_offline(self): # TODO required + dev_col = self.vm.devices['pci'] + self.assertDeviceIs( + self.dev, attached=False, assigned=False, required=False) + + self.loop.run_until_complete(dev_col.assign(self.assignment)) self.app.save() - self.assertNotIn(self.dev, dev_col.attached()) - self.assertIn(self.dev, dev_col.persistent()) - self.assertIn(self.dev, dev_col.assignments()) - self.assertIn(self.dev, dev_col.assignments(persistent=True)) - self.assertNotIn(self.dev, dev_col.assignments(persistent=False)) + self.assertDeviceIs( + self.dev, attached=False, assigned=True, required=False) self.loop.run_until_complete(self.vm.start()) + self.assertDeviceIs( + self.dev, attached=True, assigned=False, required=False) - self.assertIn(self.dev, dev_col.attached()) (stdout, _) = self.loop.run_until_complete( self.vm.run_for_stdio('lspci')) self.assertIn(self.dev.description, stdout.decode()) @@ -101,26 +116,24 @@ def test_010_attach_offline_persistent(self): def test_011_attach_offline_temp_fail(self): dev_col = self.vm.devices['pci'] - self.assertDeviceNotInCollection(self.dev, dev_col) + self.assertDeviceIs( + self.dev, attached=False, assigned=False, required=False) self.assignment.persistent = False with self.assertRaises(qubes.exc.QubesVMNotRunningError): self.loop.run_until_complete( dev_col.attach(self.assignment)) - - def test_020_attach_online_persistent(self): + def test_020_attach_online_persistent(self): # TODO: required self.loop.run_until_complete( self.vm.start()) dev_col = self.vm.devices['pci'] - self.assertDeviceNotInCollection(self.dev, dev_col) + self.assertDeviceIs( + self.dev, attached=False, assigned=False, required=False) + self.loop.run_until_complete( dev_col.attach(self.assignment)) - - self.assertIn(self.dev, dev_col.attached()) - self.assertIn(self.dev, dev_col.persistent()) - self.assertIn(self.dev, dev_col.assignments()) - self.assertIn(self.dev, dev_col.assignments(persistent=True)) - self.assertNotIn(self.dev, dev_col.assignments(persistent=False)) + self.assertDeviceIs( + self.dev, attached=True, assigned=True, required=False) # give VM kernel some time to discover new device time.sleep(1) @@ -131,7 +144,8 @@ def test_020_attach_online_persistent(self): def test_021_persist_detach_online_fail(self): dev_col = self.vm.devices['pci'] - self.assertDeviceNotInCollection(self.dev, dev_col) + self.assertDeviceIs( + self.dev, attached=False, assigned=False, required=False) self.loop.run_until_complete( dev_col.attach(self.assignment)) self.app.save() @@ -141,37 +155,34 @@ def test_021_persist_detach_online_fail(self): self.loop.run_until_complete( self.vm.devices['pci'].detach(self.assignment)) - def test_030_persist_attach_detach_offline(self): + def test_030_persist_attach_detach_offline(self): # TODO: required dev_col = self.vm.devices['pci'] - self.assertDeviceNotInCollection(self.dev, dev_col) + self.assertDeviceIs( + self.dev, attached=False, assigned=False, required=False) + self.loop.run_until_complete( dev_col.attach(self.assignment)) self.app.save() - self.assertNotIn(self.dev, dev_col.attached()) - self.assertIn(self.dev, dev_col.persistent()) - self.assertIn(self.dev, dev_col.assignments()) - self.assertIn(self.dev, dev_col.assignments(persistent=True)) - self.assertNotIn(self.dev, dev_col.assignments(persistent=False)) + self.assertDeviceIs( + self.dev, attached=False, assigned=True, required=False) + self.loop.run_until_complete( dev_col.detach(self.assignment)) - self.assertDeviceNotInCollection(self.dev, dev_col) + self.assertDeviceIs( + self.dev, attached=False, assigned=False, required=False) - def test_031_attach_detach_online_temp(self): + def test_031_attach_detach_online_temp(self): # TODO: requiured dev_col = self.vm.devices['pci'] self.loop.run_until_complete( self.vm.start()) - self.assignment.persistent = False - self.assertDeviceNotInCollection(self.dev, dev_col) + self.assignment.assigned = False + self.assertDeviceIs( + self.dev, attached=False, assigned=False, required=False) + self.loop.run_until_complete( dev_col.attach(self.assignment)) - - self.assertIn(self.dev, dev_col.attached()) - self.assertNotIn(self.dev, dev_col.persistent()) - self.assertIn(self.dev, dev_col.assignments()) - self.assertIn(self.dev, dev_col.assignments(persistent=False)) - self.assertNotIn(self.dev, dev_col.assignments(persistent=True)) - self.assertIn(self.dev, dev_col.assignments(persistent=False)) - + self.assertDeviceIs( + self.dev, attached=True, assigned=False, required=False) # give VM kernel some time to discover new device time.sleep(1) @@ -181,7 +192,8 @@ def test_031_attach_detach_online_temp(self): self.assertIn(self.dev.description, stdout.decode()) self.loop.run_until_complete( dev_col.detach(self.assignment)) - self.assertDeviceNotInCollection(self.dev, dev_col) + self.assertDeviceIs( + self.dev, attached=False, assigned=False, required=False) (stdout, _) = self.loop.run_until_complete( self.vm.run_for_stdio('lspci')) diff --git a/qubes/tests/vm/init.py b/qubes/tests/vm/init.py index bf1f9ef21..761ff9953 100644 --- a/qubes/tests/vm/init.py +++ b/qubes/tests/vm/init.py @@ -110,13 +110,13 @@ def test_000_load(self): }) self.assertCountEqual(vm.devices.keys(), ('pci',)) - self.assertCountEqual(list(vm.devices['pci'].persistent()), - [qubes.ext.pci.PCIDevice(vm, '00_11.22')]) + self.assertCountEqual(list(vm.devices['pci'].get_assigned_devices()), + [qubes.ext.pci.PCIDevice(vm, '00_11.22')]) - assignments = list(vm.devices['pci'].assignments()) + assignments = list(vm.devices['pci'].get_assigned_devices()) self.assertEqual(len(assignments), 1) self.assertEqual(assignments[0].options, {'no-strict-reset': 'True'}) - self.assertEqual(assignments[0].persistent, True) + self.assertEqual(assignments[0].attach_automatically, True) self.assertXMLIsValid(vm.__xml__(), 'relaxng/domain.rng') diff --git a/qubes/tests/vm/qubesvm.py b/qubes/tests/vm/qubesvm.py index b07fdb2b0..f9449c932 100644 --- a/qubes/tests/vm/qubesvm.py +++ b/qubes/tests/vm/qubesvm.py @@ -1886,10 +1886,11 @@ def test_615_libvirt_xml_block_devices(self): 'device.backend_domain.features.check_with_template.return_value': '4.2', }), ] - vm.devices['block'].assignments = lambda persistent: assignments + vm.devices['block'].get_assigned_devices = \ + lambda required_only: assignments libvirt_xml = vm.create_config_file() - self.assertXMLEqual(lxml.etree.XML(libvirt_xml), - lxml.etree.XML(expected)) + self.assertXMLEqual( + lxml.etree.XML(libvirt_xml), lxml.etree.XML(expected)) @unittest.mock.patch('qubes.utils.get_timezone') @unittest.mock.patch('qubes.utils.urandom') diff --git a/qubes/vm/__init__.py b/qubes/vm/__init__.py index 9c2b09f3a..926c717a7 100644 --- a/qubes/vm/__init__.py +++ b/qubes/vm/__init__.py @@ -27,6 +27,7 @@ import re import string import uuid +from typing import List import lxml.etree @@ -279,10 +280,10 @@ def load_extras(self): self.app.domains[node.get('backend-domain')], node.get('id'), options, - persistent=True, - # TODO: required=node.get('required') + attach_automatically=True, + required=node.get('required'), ) - self.devices[devclass].load_persistent(device_assignment) + self.devices[devclass].load_assignment(device_assignment) except KeyError: msg = "{}: Cannot find backend domain '{}' " \ "for device type {} '{}'".format(self.name, node.get( @@ -296,15 +297,18 @@ def load_extras(self): # SEE:1815 firewall, policy. - def get_provided_assignments(self): - '''List of persistent device assignments from this VM.''' + def get_provided_assignments(self, required_only: bool)\ + -> List['qubes.devices.DeviceAssignment']: + """ + List device assignments from this VM. + """ assignments = [] for domain in self.app.domains: if domain == self: continue for device_collection in domain.devices.values(): - for assignment in device_collection.persistent(): + for assignment in device_collection.get_assigned_devices(): if assignment.backend_domain == self: assignments.append(assignment) return assignments @@ -330,11 +334,12 @@ def __xml__(self): for devclass in self.devices: devices = lxml.etree.Element('devices') devices.set('class', devclass) - for device in self.devices[devclass].assignments(persistent=True): + for device in self.devices[devclass].get_assigned_devices(): node = lxml.etree.Element('device') node.set('backend-domain', device.backend_domain.name) node.set('id', device.ident) - # TODO: node.set('required', device.required) + node.set('required', device.required) + # TODO: serial for key, val in device.options.items(): option_node = lxml.etree.Element('option') option_node.set('name', key) diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index bab1d3979..1b1fd952b 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -132,10 +132,10 @@ def _setter_virt_mode(self, prop, value): raise qubes.exc.QubesPropertyValueError( self, prop, value, 'Invalid virtualization mode, supported values: hvm, pv, pvh') - if value == 'pvh' and list(self.devices['pci'].persistent()): + if value == 'pvh' and list(self.devices['pci'].get_assigned_devices()): raise qubes.exc.QubesPropertyValueError( self, prop, value, - "pvh mode can't be set if pci devices are attached") + "pvh mode can't be set if pci devices are assigned") return value @@ -169,7 +169,7 @@ def _setter_kbd_layout(self, prop, value): def _default_virt_mode(self): - if self.devices['pci'].persistent(): + if self.devices['pci'].get_assigned_devices(): return 'hvm' try: return self.template.virt_mode @@ -215,7 +215,7 @@ def _default_maxmem(self): def _default_kernelopts(self): """ Return default kernel options for the given kernel. If kernel directory - contains 'default-kernelopts-{pci,nopci}.txt' file, use that. Otherwise + contains 'default-kernelopts-{pci,nopci}.txt' file, use that. Otherwise, use built-in defaults. For qubes without PCI devices, kernelopts of qube's template are considered (for template-based qubes). @@ -228,9 +228,9 @@ def _default_kernelopts(self): kernels_dir = os.path.join( qubes.config.system_path['qubes_kernels_base_dir'], self.kernel) - pci = bool(list(self.devices['pci'].persistent())) + any_pci_assigned = bool(list(self.devices['pci'].get_assigned_pci())) extra_opts = "" - if pci: + if any_pci_assigned: path = os.path.join(kernels_dir, 'default-kernelopts-pci.txt') if self.app.domains[0].features.get('suspend-s0ix', False): extra_opts = " qubes_exp_pm_use_suspend=1" @@ -244,8 +244,9 @@ def _default_kernelopts(self): with open(path, encoding='ascii') as f_kernelopts: return f_kernelopts.read().strip() + extra_opts else: - return (qubes.config.defaults['kernelopts_pcidevs'] if pci else - qubes.config.defaults['kernelopts']) + extra_opts + return (qubes.config.defaults['kernelopts_pcidevs'] + if any_pci_assigned else qubes.config.defaults['kernelopts'] + ) + extra_opts class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): @@ -1169,7 +1170,7 @@ async def start(self, start_guid=True, notify_function=None, qmemman_client = None try: for devclass in self.devices: - for dev in self.devices[devclass].persistent(): + for dev in self.devices[devclass].get_assigned_devices(): if isinstance(dev, qubes.devices.UnknownDevice): raise qubes.exc.QubesException( '{} device {} not available'.format( @@ -1216,8 +1217,9 @@ async def start(self, start_guid=True, notify_function=None, except libvirt.libvirtError as exc: # missing IOMMU? if self.virt_mode == 'hvm' and \ - list(self.devices['pci'].persistent()) and \ - not self.app.host.is_iommu_supported(): + list(self.devices['pci'].get_assigned_devices( + required_only=True) + ) and not self.app.host.is_iommu_supported(): exc = qubes.exc.QubesException( 'Failed to start an HVM qube with PCI devices assigned ' '- hardware does not support IOMMU/VT-d/AMD-Vi') @@ -1628,7 +1630,7 @@ def is_memory_balancing_possible(self): include balloon driver) and lack of qrexec/meminfo-writer service support (no qubes tools installed). """ - if list(self.devices['pci'].persistent()): + if list(self.devices['pci'].get_assigned_devices()): return False if self.virt_mode == 'hvm': # if VM announce any supported service diff --git a/rpm_spec/core-dom0.spec.in b/rpm_spec/core-dom0.spec.in index 6ac367dff..0ba6008cd 100644 --- a/rpm_spec/core-dom0.spec.in +++ b/rpm_spec/core-dom0.spec.in @@ -254,16 +254,22 @@ admin.vm.Shutdown admin.vm.Start admin.vm.Stats admin.vm.Unpause +admin.vm.device.block.Assign +admin.vm.device.block.Assigned admin.vm.device.block.Attach +admin.vm.device.block.Attached admin.vm.device.block.Available admin.vm.device.block.Detach -admin.vm.device.block.List -admin.vm.device.block.Set.persistent +admin.vm.device.block.Set.assignment +admin.vm.device.block.Unassign +admin.vm.device.pci.Assign +admin.vm.device.pci.Assigned admin.vm.device.pci.Attach +admin.vm.device.pci.Attached admin.vm.device.pci.Available admin.vm.device.pci.Detach -admin.vm.device.pci.List -admin.vm.device.pci.Set.persistent +admin.vm.device.pci.Set.assignment +admin.vm.device.pci.Unassign admin.vm.feature.CheckWithAdminVM admin.vm.feature.CheckWithNetvm admin.vm.feature.CheckWithTemplate diff --git a/templates/libvirt/xen.xml b/templates/libvirt/xen.xml index 40af09f33..e752b344f 100644 --- a/templates/libvirt/xen.xml +++ b/templates/libvirt/xen.xml @@ -7,7 +7,7 @@ {% block basic %} {{ vm.name }} {{ vm.uuid }} - {% if ((vm.virt_mode == 'hvm' and vm.devices['pci'].persistent() | list) + {% if ((vm.virt_mode == 'hvm' and vm.devices['pci'].get_assigned_devices() | list) or vm.maxmem == 0) -%} {{ vm.memory }} {% elif vm.use_memory_hotplug %} @@ -79,7 +79,7 @@ {% endif %} - {% if vm.devices['pci'].persistent() | list + {% if vm.devices['pci'].get_assigned_devices() | list and vm.features.get('pci-e820-host', True) %} From d1f333f7d0c45091663139ab2ed4418d98ed7e0a Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Sun, 21 Jan 2024 18:03:00 +0100 Subject: [PATCH 06/49] q-dev: return DeviceAssignment in get_attached_devices use required in qubes.xml --- qubes/api/admin.py | 11 +++------ qubes/devices.py | 49 +++++++++++++++++++++++++++------------ qubes/vm/__init__.py | 6 +++-- qubes/vm/qubesvm.py | 4 ++-- templates/libvirt/xen.xml | 4 ++-- 5 files changed, 45 insertions(+), 29 deletions(-) diff --git a/qubes/api/admin.py b/qubes/api/admin.py index d5bd50891..019dc3fbc 100644 --- a/qubes/api/admin.py +++ b/qubes/api/admin.py @@ -1289,7 +1289,7 @@ async def vm_device_assign(self, endpoint, untrusted_payload): devclass = endpoint # TODO: deserialize options = {} - attach_automatically = False + attach_automatically = True required = False for untrusted_option in untrusted_payload.decode( 'ascii', 'strict').split(): @@ -1297,13 +1297,8 @@ async def vm_device_assign(self, endpoint, untrusted_payload): untrusted_key, untrusted_value = untrusted_option.split('=', 1) except ValueError: raise qubes.api.ProtocolError('Invalid options format') - if untrusted_key == 'attach_automatically': - attach_automatically = qubes.property.bool( - None, None, untrusted_value) - - self.enforce(attach_automatically) - elif untrusted_key == 'required': + if untrusted_key == 'required': required = qubes.property.bool( None, None, untrusted_value) else: @@ -1480,7 +1475,7 @@ async def vm_device_set_assignment(self, endpoint, untrusted_payload): self.fire_event_for_permission(device=dev, assignment=assignment) - self.dest.devices[devclass].update_assignment(dev, assignment) + await self.dest.devices[devclass].update_assignment(dev, assignment) self.app.save() @qubes.api.method('admin.vm.firewall.Get', no_payload=True, diff --git a/qubes/devices.py b/qubes/devices.py index 18d035af2..e64b70624 100644 --- a/qubes/devices.py +++ b/qubes/devices.py @@ -578,7 +578,7 @@ class UnknownDevice(DeviceInfo): # pylint: disable=too-few-public-methods """Unknown device - for example exposed by domain not running currently""" - def __init__(self, backend_domain, devclass, ident, **kwargs): + def __init__(self, backend_domain, ident, *, devclass, **kwargs): super().__init__(backend_domain, ident, devclass=devclass, **kwargs) @@ -799,7 +799,7 @@ async def assign(self, device_assignment: DeviceAssignment): device = device_assignment.device if device in self.get_assigned_devices(): raise DeviceAlreadyAssigned( - 'device {!s} of class {} already attached to {!s}'.format( + 'device {!s} of class {} already assigned to {!s}'.format( device, self._bus, self._vm)) # TODO: check if needed @@ -824,7 +824,8 @@ def load_assignment(self, device_assignment: DeviceAssignment): device_assignment.devclass = self._bus self._set.add(device_assignment) - def update_assignment(self, device: DeviceInfo, required: Optional[bool]): + async def update_assignment( + self, device: DeviceInfo, required: Optional[bool]): """ Update assignment of already attached device. @@ -848,12 +849,15 @@ def update_assignment(self, device: DeviceInfo, required: Optional[bool]): # be careful to use already present assignment, not the provided one # - to not change options as a side effect - if required is not None and device not in self._set: - assignment.attach_automatically = True + if required is not None: + if assignment.required == required: + return + assignment.required = required - self._set.add(assignment) - elif required is None and device in self._set: - self._set.discard(assignment) + await self._vm.fire_event_async( + 'device-assignment-changed:' + self._bus, device=device) + else: + await self.detach(assignment) async def detach(self, device_assignment: DeviceAssignment): # TODO: argument should be just device """ @@ -867,12 +871,12 @@ async def detach(self, device_assignment: DeviceAssignment): # TODO: argument s else: raise DeviceNotAssigned( f'device {device_assignment.ident!s} of class {self._bus} not ' - f'assigned to {self._vm!s}') + f'attached to {self._vm!s}') if device_assignment.required and not self._vm.is_halted(): raise qubes.exc.QubesVMNotHaltedError( self._vm, - "Can not remove a required device from a non halted qube." + "Can not detach a required device from a non halted qube. " "You need to unassign device first.") device = device_assignment.device @@ -904,12 +908,12 @@ async def unassign(self, device_assignment: DeviceAssignment): device = device_assignment.device # TODO: check if needed await self._vm.fire_event_async( - 'device-pre-detach:' + self._bus, pre_event=True, device=device) + 'device-pre-unassign:' + self._bus, pre_event=True, device=device) self._set.discard(device_assignment) await self._vm.fire_event_async( - 'device-detach:' + self._bus, device=device) + 'device-unassign:' + self._bus, device=device) def get_dedicated_devices(self) -> Iterable[DeviceAssignment]: """ @@ -925,7 +929,21 @@ def get_attached_devices(self) -> Iterable[DeviceAssignment]: List devices which are attached to this vm. """ attached = self._vm.fire_event('device-list-attached:' + self._bus) - return [dev for dev, _ in attached] + for dev, options in attached: + for assignment in self._set: + if dev == assignment: + yield assignment + break + else: + yield DeviceAssignment( + backend_domain=dev.backend_domain, + ident=dev.ident, + options=options, + frontend_domain=dev.frontend_domain, + devclass=dev.devclass, + attach_automatically=False, + required=False, + ) def get_assigned_devices( self, required_only: bool = False @@ -945,7 +963,8 @@ def get_exposed_devices(self) -> Iterable[DeviceInfo]: List devices exposed by this vm. """ devices = self._vm.fire_event('device-list:' + self._bus) - return devices + for device in devices: + yield device __iter__ = get_exposed_devices @@ -967,7 +986,7 @@ def __getitem__(self, ident): assert len(dev) == 1 return dev[0] - return UnknownDevice(self._vm, ident) + return UnknownDevice(self._vm, ident, devclass=self._bus) class DeviceManager(dict): diff --git a/qubes/vm/__init__.py b/qubes/vm/__init__.py index 926c717a7..25675ecd4 100644 --- a/qubes/vm/__init__.py +++ b/qubes/vm/__init__.py @@ -281,7 +281,9 @@ def load_extras(self): node.get('id'), options, attach_automatically=True, - required=node.get('required'), + # backward compatibility: persistent~>required=True + required=qubes.property.bool( + None, None, node.get('required', 'yes')), ) self.devices[devclass].load_assignment(device_assignment) except KeyError: @@ -338,7 +340,7 @@ def __xml__(self): node = lxml.etree.Element('device') node.set('backend-domain', device.backend_domain.name) node.set('id', device.ident) - node.set('required', device.required) + node.set('required', 'yes' if device.required else 'no') # TODO: serial for key, val in device.options.items(): option_node = lxml.etree.Element('option') diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 1b1fd952b..c06f8587b 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -228,7 +228,7 @@ def _default_kernelopts(self): kernels_dir = os.path.join( qubes.config.system_path['qubes_kernels_base_dir'], self.kernel) - any_pci_assigned = bool(list(self.devices['pci'].get_assigned_pci())) + any_pci_assigned = bool(list(self.devices['pci'].get_assigned_devices())) extra_opts = "" if any_pci_assigned: path = os.path.join(kernels_dir, 'default-kernelopts-pci.txt') @@ -1626,7 +1626,7 @@ def is_memory_balancing_possible(self): - balloon driver not present We don't have reliable way to detect the second point, but good - heuristic is HVM virt_mode (PV and PVH require OS support and it does + heuristic is HVM virt_mode (PV and PVH require OS support, and it does include balloon driver) and lack of qrexec/meminfo-writer service support (no qubes tools installed). """ diff --git a/templates/libvirt/xen.xml b/templates/libvirt/xen.xml index e752b344f..95b59a0b6 100644 --- a/templates/libvirt/xen.xml +++ b/templates/libvirt/xen.xml @@ -156,7 +156,7 @@ {# start external devices from xvdi #} {% set counter = {'i': 4} %} - {% for assignment in vm.devices.block.assignments(True) %} + {% for assignment in vm.devices.block.get_assigned_devices(True) %} {% set device = assignment.device %} {% set options = assignment.options %} {% include 'libvirt/devices/block.xml' %} @@ -166,7 +166,7 @@ {% include 'libvirt/devices/net.xml' with context %} {% endif %} - {% for assignment in vm.devices.pci.assignments(True) %} + {% for assignment in vm.devices.pci.get_assigned_devices(True) %} {% set device = assignment.device %} {% set options = assignment.options %} {% set power_mgmt = From a113b12075c41137e3c537faafc16470118bc862 Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Tue, 23 Jan 2024 13:20:45 +0100 Subject: [PATCH 07/49] q-dev: readable serialization --- qubes/devices.py | 97 +++++++++++++++++++++++++++++++----------------- 1 file changed, 62 insertions(+), 35 deletions(-) diff --git a/qubes/devices.py b/qubes/devices.py index e64b70624..c0c223b6d 100644 --- a/qubes/devices.py +++ b/qubes/devices.py @@ -55,7 +55,6 @@ `domain-qdb-change:path`) to detect changes and fire `device-list-change:class` event. """ -import base64 import itertools import sys from enum import Enum @@ -88,6 +87,12 @@ class DeviceAlreadyAssigned(qubes.exc.QubesException, KeyError): """ +class UnexpectedDeviceProperty(qubes.exc.QubesException, ValueError): + """ + Device has unexpected property such as backend_domain, devclass etc. + """ + + class Device: def __init__(self, backend_domain, ident, devclass=None): self.__backend_domain = backend_domain @@ -230,7 +235,7 @@ def __init__(self, interface_encoding: str, devclass: Optional[str] = None): ) ifc_full = ifc_padded elif len(ifc_padded) == 6: - ifc_full = ' ' + ifc_padded + ifc_full = '?' + ifc_padded else: ifc_full = ifc_padded @@ -251,7 +256,7 @@ def category(self) -> DeviceCategory: @classmethod def unknown(cls) -> 'DeviceInterface': """ Value for unknown device interface. """ - return cls(" ******") + return cls("?******") def __repr__(self): return self._interface_encoding @@ -482,26 +487,27 @@ def serialize(self) -> bytes: 'ident', 'devclass', 'vendor', 'product', 'manufacturer', 'name', 'serial'} properties = b' '.join( - base64.b64encode(f'{prop}={value!s}'.encode('ascii')) + f'{prop}={self.serialize_str(value)}'.encode('ascii') for prop, value in ( (key, getattr(self, key)) for key in default_attrs) ) - backend_domain_name = self.backend_domain.name - backend_domain_prop = (b'backend_domain=' + - backend_domain_name.encode('ascii')) - properties += b' ' + base64.b64encode(backend_domain_prop) + back_name = self.serialize_str(self.backend_domain.name) + backend_domain_prop = (b"backend_domain=" + back_name.encode('ascii')) + properties += b' ' + backend_domain_prop - interfaces = ''.join(repr(ifc) for ifc in self.interfaces) - interfaces_prop = b'interfaces=' + str(interfaces).encode('ascii') - properties += b' ' + base64.b64encode(interfaces_prop) + interfaces = self.serialize_str( + ''.join(repr(ifc) for ifc in self.interfaces)) + interfaces_prop = (b'interfaces=' + interfaces.encode('ascii')) + properties += b' ' + interfaces_prop if self.parent_device is not None: - parent_prop = b'parent=' + self.parent_device.ident.encode('ascii') - properties += b' ' + base64.b64encode(parent_prop) + parent_ident = self.serialize_str(self.parent_device.ident) + parent_prop = (b'parent=' + parent_ident.encode('ascii')) + properties += b' ' + parent_prop data = b' '.join( - base64.b64encode(f'_{prop}={value!s}'.encode('ascii')) + f'_{prop}={self.serialize_str(value)}'.encode('ascii') for prop, value in ((key, self.data[key]) for key in self.data) ) if data: @@ -520,7 +526,7 @@ def deserialize( result = DeviceInfo._deserialize( cls, serialization, expected_backend_domain, expected_devclass) except Exception as exc: - print(exc, file=sys.stderr) # TODO + print(exc, file=sys.stderr) ident = serialization.split(b' ')[0].decode( 'ascii', errors='ignore') result = UnknownDevice( @@ -537,23 +543,37 @@ def _deserialize( expected_backend_domain: 'qubes.vm.BaseVM', expected_devclass: Optional[str] = None, ) -> 'DeviceInfo': - properties_str = [ - base64.b64decode(line).decode('ascii', errors='ignore') - for line in serialization.split(b' ')[1:]] + decoded = serialization.decode('ascii', errors='ignore') + _ident, _, rest = decoded.partition(' ') + keys = [] + values = [] + key, _, rest = rest.partition("='") + keys.append(key) + while "='" in rest: + value_key, _, rest = rest.partition("='") + value, _, key = value_key.rpartition("' ") + values.append(DeviceInfo.deserialize_str(value)) + keys.append(key) + value = rest[:-1] # ending ' + values.append(DeviceInfo.deserialize_str(value)) properties = dict() - for line in properties_str: - key, _, param = line.partition("=") + for key, value in zip(keys, values): if key.startswith("_"): - properties[key[1:]] = param + # it's handled in cls.__init__ + properties[key[1:]] = value else: - properties[key] = param + properties[key] = value if properties['backend_domain'] != expected_backend_domain.name: - raise ValueError("TODO") # TODO + raise UnexpectedDeviceProperty( + f"Got device exposed by {properties['backend_domain']}" + f"when expected devices from {expected_backend_domain.name}.") properties['backend_domain'] = expected_backend_domain - # if expected_devclass and properties['devclass'] != expected_devclass: - # raise ValueError("TODO") # TODO + if expected_devclass and properties['devclass'] != expected_devclass: + raise UnexpectedDeviceProperty( + f"Got {properties['devclass']} device " + f"when expected {expected_devclass}.") interfaces = properties['interfaces'] interfaces = [ @@ -569,6 +589,14 @@ def _deserialize( return cls(**properties) + @staticmethod + def serialize_str(string: str): + return repr(string) + + @staticmethod + def deserialize_str(string: str): + return string.replace("\\\'", "'") + @property def frontend_domain(self): return self.data.get("frontend_domain", None) @@ -690,7 +718,7 @@ class DeviceCollection: :param vm: VM for which we manage devices :param bus: device bus - This class emits following events on VM object: # TODO pre-assign, assign, pre-unassign, unassign + This class emits following events on VM object: .. event:: device-added: (device) @@ -802,7 +830,6 @@ async def assign(self, device_assignment: DeviceAssignment): 'device {!s} of class {} already assigned to {!s}'.format( device, self._bus, self._vm)) - # TODO: check if needed await self._vm.fire_event_async( 'device-pre-assign:' + self._bus, pre_event=True, device=device, options=device_assignment.options) @@ -859,27 +886,28 @@ async def update_assignment( else: await self.detach(assignment) - async def detach(self, device_assignment: DeviceAssignment): # TODO: argument should be just device + async def detach(self, device: Device): """ Detach device from domain. """ - for assignment in self.get_attached_devices(): - if device_assignment == assignment: + for assign in self.get_attached_devices(): + if device == assign: # load all options - device_assignment = assignment + assignment = assign break else: raise DeviceNotAssigned( - f'device {device_assignment.ident!s} of class {self._bus} not ' + f'device {device.ident!s} of class {self._bus} not ' f'attached to {self._vm!s}') - if device_assignment.required and not self._vm.is_halted(): + if assignment.required and not self._vm.is_halted(): raise qubes.exc.QubesVMNotHaltedError( self._vm, "Can not detach a required device from a non halted qube. " "You need to unassign device first.") - device = device_assignment.device + # use local object + device = assignment.device await self._vm.fire_event_async( 'device-pre-detach:' + self._bus, pre_event=True, device=device) @@ -906,7 +934,6 @@ async def unassign(self, device_assignment: DeviceAssignment): "Can not remove an assignment from a non halted qube.") device = device_assignment.device - # TODO: check if needed await self._vm.fire_event_async( 'device-pre-unassign:' + self._bus, pre_event=True, device=device) From 81c8c3e9956f5a22fa8dca98a49baabeb37ceaaf Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Thu, 25 Jan 2024 19:32:34 +0100 Subject: [PATCH 08/49] q-dev: device-{added,removed}:block --- qubes/ext/block.py | 59 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 49 insertions(+), 10 deletions(-) diff --git a/qubes/ext/block.py b/qubes/ext/block.py index 2b6112c63..34a86f3db 100644 --- a/qubes/ext/block.py +++ b/qubes/ext/block.py @@ -19,6 +19,7 @@ # License along with this library; if not, see . """ Qubes block devices extensions """ +import asyncio import collections import re import string @@ -183,18 +184,56 @@ def on_qdb_change(self, vm, event, path): current_devices = dict((dev.ident, dev.frontend_domain) for dev in self.on_device_list_block(vm, None)) - devices_cache_for_vm = self.devices_cache[vm.name] - for dev_id, connected_to in current_devices.items(): - if dev_id not in devices_cache_for_vm: - device = BlockDevice(vm, dev_id) - vm.fire_event('device-added:block', device=device) - for dev_id, connected_to in devices_cache_for_vm.items(): + # send events about devices detached/attached outside by themselves + # (like device pulled out or manual qubes.USB qrexec call) + # compare cached devices and current devices, collect: + # - newly appeared devices (ident) + # - devices attached from a vm to frontend vm (ident: frontend_vm) + # - devices detached from frontend vm (ident: frontend_vm) + # - disappeared devices, e.g. plugged out (ident) + added = set() + attached = dict() + detached = dict() + removed = set() + cache = self.devices_cache[vm.name] + for dev_id, front_vm in current_devices.items(): + if dev_id not in cache: + added.add(dev_id) + if front_vm is not None: + attached[dev_id] = front_vm + elif cache[dev_id] != front_vm: + cached_front = cache[dev_id] + if front_vm is None: + detached[dev_id] = cached_front + elif cached_front is None: + attached[dev_id] = front_vm + else: + # front changed from one to another, so we signal it as: + # detach from first one and attach to the second one. + detached[dev_id] = cached_front + attached[dev_id] = front_vm + + for dev_id, cached_front in cache.items(): if dev_id not in current_devices: - device = BlockDevice(vm, dev_id) - vm.fire_event('device-removed:block', device=device) + removed.add(dev_id) + if cached_front is not None: + detached[dev_id] = cached_front + + for dev_id, front_vm in detached.items(): + dev = BlockDevice(vm, dev_id) + asyncio.ensure_future(front_vm.fire_event_async( + 'device-detach:block', device=dev)) + for dev_id in removed: + device = BlockDevice(vm, dev_id) + vm.fire_event('device-removed:block', device=device) + for dev_id in added: + device = BlockDevice(vm, dev_id) + vm.fire_event('device-added:block', device=device) + for dev_ident, front_vm in attached.items(): + dev = BlockDevice(vm, dev_ident) + asyncio.ensure_future(front_vm.fire_event_async( + 'device-attach:block', device=dev, options={})) - # TODO: if not removed and not added - # TODO attach/detach self.devices_cache[vm.name] = current_devices def device_get(self, vm, ident): From 7febeffee4ffcdfe4beea5268ba5d18ad3451e37 Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Mon, 29 Jan 2024 20:59:59 +0100 Subject: [PATCH 09/49] q-dev: DeviceAssignment serialization --- qubes/api/admin.py | 125 ++++++++---------------------- qubes/devices.py | 185 ++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 197 insertions(+), 113 deletions(-) diff --git a/qubes/api/admin.py b/qubes/api/admin.py index 019dc3fbc..b3532026e 100644 --- a/qubes/api/admin.py +++ b/qubes/api/admin.py @@ -1226,19 +1226,10 @@ async def vm_device_list(self, endpoint): device_assignments = self.fire_event_for_filter( device_assignments, devclass=devclass) - dev_info = {} # TODO: deviceAssignment.serialization() - for dev in device_assignments: - properties_txt = ' '.join( - '{}={!s}'.format(opt, value) for opt, value - in itertools.chain( - dev.options.items(), - (('required', 'yes' if dev.required else 'no'), - ('attach_automatically', - 'yes' if dev.attach_automatically else 'no')), - )) - self.enforce('\n' not in properties_txt) - ident = '{!s}+{!s}'.format(dev.backend_domain, dev.ident) - dev_info[ident] = properties_txt + dev_info = { + f'{assignment.backend_domain}+{assignment.ident}': + assignment.serialize().decode('ascii', errors="ignore") + for assignment in device_assignments} return ''.join('{} {}\n'.format(ident, dev_info[ident]) for ident in sorted(dev_info)) @@ -1263,19 +1254,10 @@ async def vm_device_attached(self, endpoint): device_assignments = self.fire_event_for_filter(device_assignments, devclass=devclass) - dev_info = {} - for dev in device_assignments: - properties_txt = ' '.join( - '{}={!s}'.format(opt, value) for opt, value - in itertools.chain( - dev.options.items(), - (('required', 'yes' if dev.required else 'no'), - ('attach_automatically', - 'yes' if dev.attach_automatically else 'no')), - )) - self.enforce('\n' not in properties_txt) - ident = '{!s}+{!s}'.format(dev.backend_domain, dev.ident) - dev_info[ident] = properties_txt + dev_info = { + f'{assignment.backend_domain}+{assignment.ident}': + assignment.serialize().decode('ascii', errors="ignore") + for assignment in device_assignments} return ''.join('{} {}\n'.format(ident, dev_info[ident]) for ident in sorted(dev_info)) @@ -1287,47 +1269,26 @@ async def vm_device_attached(self, endpoint): scope='local', write=True) async def vm_device_assign(self, endpoint, untrusted_payload): devclass = endpoint - # TODO: deserialize - options = {} - attach_automatically = True - required = False - for untrusted_option in untrusted_payload.decode( - 'ascii', 'strict').split(): - try: - untrusted_key, untrusted_value = untrusted_option.split('=', 1) - except ValueError: - raise qubes.api.ProtocolError('Invalid options format') - - if untrusted_key == 'required': - required = qubes.property.bool( - None, None, untrusted_value) - else: - allowed_chars_key = string.digits + string.ascii_letters + '-_.' - allowed_chars_value = allowed_chars_key + ',+:' - if any(x not in allowed_chars_key for x in untrusted_key): - raise qubes.api.ProtocolError( - 'Invalid chars in option name') - if any(x not in allowed_chars_value for x in untrusted_value): - raise qubes.api.ProtocolError( - 'Invalid chars in option value') - options[untrusted_key] = untrusted_value # qrexec already verified that no strange characters are in self.arg backend_domain, ident = self.arg.split('+', 1) # may raise KeyError, either on domain or ident dev = self.app.domains[backend_domain].devices[devclass][ident] + assignment = qubes.devices.DeviceAssignment.deserialize( + untrusted_payload, + expected_backend_domain=dev.backend_domain, + expected_ident=ident, + expected_devclass=devclass + ) + self.fire_event_for_permission( device=dev, devclass=devclass, - required=required, attach_automatically=attach_automatically, - options=options + required=assignment.required, + attach_automatically=assignment.attach_automatically, + options=assignment.options ) - assignment = qubes.devices.DeviceAssignment( - dev.backend_domain, dev.ident, - required=required, attach_automatically=attach_automatically, - options=options - ) await self.dest.devices[devclass].assign(assignment) self.app.save() @@ -1339,6 +1300,7 @@ async def vm_device_assign(self, endpoint, untrusted_payload): ep.name for ep in importlib.metadata.entry_points(group='qubes.devices')), no_payload=True, scope='local', write=True) async def vm_device_unassign(self, endpoint): + # TODO DeviceAssignment.deserialize() ? Device devclass = endpoint # qrexec already verified that no strange characters are in self.arg @@ -1365,54 +1327,26 @@ async def vm_device_unassign(self, endpoint): scope='local', execute=True) async def vm_device_attach(self, endpoint, untrusted_payload): devclass = endpoint - # TODO: deserialize - options = {} - attach_automatically = False - required = False - for untrusted_option in untrusted_payload.decode( - 'ascii', 'strict').split(): - try: - untrusted_key, untrusted_value = untrusted_option.split('=', - 1) - except ValueError: - raise qubes.api.ProtocolError('Invalid options format') - if untrusted_key == 'attach_automatically': - attach_automatically = qubes.property.bool( - None, None, untrusted_value) - - self.enforce(not attach_automatically) - - elif untrusted_key == 'required': - required = qubes.property.bool( - None, None, untrusted_value) - else: - allowed_chars_key = string.digits + string.ascii_letters + '-_.' - allowed_chars_value = allowed_chars_key + ',+:' - if any(x not in allowed_chars_key for x in untrusted_key): - raise qubes.api.ProtocolError( - 'Invalid chars in option name') - if any(x not in allowed_chars_value for x in - untrusted_value): - raise qubes.api.ProtocolError( - 'Invalid chars in option value') - options[untrusted_key] = untrusted_value # qrexec already verified that no strange characters are in self.arg backend_domain, ident = self.arg.split('+', 1) # may raise KeyError, either on domain or ident dev = self.app.domains[backend_domain].devices[devclass][ident] + assignment = qubes.devices.DeviceAssignment.deserialize( + untrusted_payload, + expected_backend_domain=dev.backend_domain, + expected_ident=ident, + expected_devclass=devclass + ) + self.fire_event_for_permission( device=dev, devclass=devclass, - required=required, attach_automatically=attach_automatically, - options=options + required=assignment.required, + attach_automatically=assignment.attach_automatically, + options=assignment.options ) - assignment = qubes.devices.DeviceAssignment( - dev.backend_domain, dev.ident, - required=required, attach_automatically=attach_automatically, - options=options - ) await self.dest.devices[devclass].attach(assignment) self.app.save() # not needed? @@ -1425,6 +1359,7 @@ async def vm_device_attach(self, endpoint, untrusted_payload): for ep in importlib.metadata.entry_points(group='qubes.devices')), no_payload=True, scope='local', execute=True) async def vm_device_detach(self, endpoint): + # TODO DeviceAssignment.deserialize() ? Device devclass = endpoint # qrexec already verified that no strange characters are in self.arg diff --git a/qubes/devices.py b/qubes/devices.py index c0c223b6d..8995bde38 100644 --- a/qubes/devices.py +++ b/qubes/devices.py @@ -56,11 +56,13 @@ `device-list-change:class` event. """ import itertools +import string import sys from enum import Enum from typing import Optional, List, Type, Dict, Any, Iterable import qubes.utils +from qubes.api import ProtocolError class DeviceNotAttached(qubes.exc.QubesException, KeyError): @@ -487,27 +489,27 @@ def serialize(self) -> bytes: 'ident', 'devclass', 'vendor', 'product', 'manufacturer', 'name', 'serial'} properties = b' '.join( - f'{prop}={self.serialize_str(value)}'.encode('ascii') + f'{prop}={serialize_str(value)}'.encode('ascii') for prop, value in ( (key, getattr(self, key)) for key in default_attrs) ) - back_name = self.serialize_str(self.backend_domain.name) + back_name = serialize_str(self.backend_domain.name) backend_domain_prop = (b"backend_domain=" + back_name.encode('ascii')) properties += b' ' + backend_domain_prop - interfaces = self.serialize_str( + interfaces = serialize_str( ''.join(repr(ifc) for ifc in self.interfaces)) interfaces_prop = (b'interfaces=' + interfaces.encode('ascii')) properties += b' ' + interfaces_prop if self.parent_device is not None: - parent_ident = self.serialize_str(self.parent_device.ident) + parent_ident = serialize_str(self.parent_device.ident) parent_prop = (b'parent=' + parent_ident.encode('ascii')) properties += b' ' + parent_prop data = b' '.join( - f'_{prop}={self.serialize_str(value)}'.encode('ascii') + f'_{prop}={serialize_str(value)}'.encode('ascii') for prop, value in ((key, self.data[key]) for key in self.data) ) if data: @@ -544,7 +546,7 @@ def _deserialize( expected_devclass: Optional[str] = None, ) -> 'DeviceInfo': decoded = serialization.decode('ascii', errors='ignore') - _ident, _, rest = decoded.partition(' ') + ident, _, rest = decoded.partition(' ') keys = [] values = [] key, _, rest = rest.partition("='") @@ -552,10 +554,10 @@ def _deserialize( while "='" in rest: value_key, _, rest = rest.partition("='") value, _, key = value_key.rpartition("' ") - values.append(DeviceInfo.deserialize_str(value)) + values.append(deserialize_str(value)) keys.append(key) value = rest[:-1] # ending ' - values.append(DeviceInfo.deserialize_str(value)) + values.append(deserialize_str(value)) properties = dict() for key, value in zip(keys, values): @@ -570,11 +572,17 @@ def _deserialize( f"Got device exposed by {properties['backend_domain']}" f"when expected devices from {expected_backend_domain.name}.") properties['backend_domain'] = expected_backend_domain + if expected_devclass and properties['devclass'] != expected_devclass: raise UnexpectedDeviceProperty( f"Got {properties['devclass']} device " f"when expected {expected_devclass}.") + if properties["ident"] != ident: + raise UnexpectedDeviceProperty( + f"Got device with id: {properties['ident']} " + f"when expected id: {ident}.") + interfaces = properties['interfaces'] interfaces = [ DeviceInterface(interfaces[i:i + 7]) @@ -589,19 +597,44 @@ def _deserialize( return cls(**properties) - @staticmethod - def serialize_str(string: str): - return repr(string) - - @staticmethod - def deserialize_str(string: str): - return string.replace("\\\'", "'") - @property def frontend_domain(self): return self.data.get("frontend_domain", None) +def serialize_str(value: str): + return repr(str(value)) + + +def deserialize_str(value: str): + return value.replace("\\\'", "'") + + +def sanitize_str( + untrusted_value: str, + allowed_chars: str, + replace_char: str = None, + error_message: str = "" +) -> str: + """ + Sanitize given untrusted string. + + If `replace_char` is not None, ignore `error_message` and replace invalid + characters with the string. + """ + if replace_char is None: + if any(x not in allowed_chars for x in untrusted_value): + raise qubes.api.ProtocolError(error_message) + return untrusted_value + result = "" + for char in untrusted_value: + if char in allowed_chars: + result += char + else: + result += replace_char + return result + + class UnknownDevice(DeviceInfo): # pylint: disable=too-few-public-methods """Unknown device - for example exposed by domain not running currently""" @@ -657,14 +690,14 @@ def device(self) -> DeviceInfo: @property def frontend_domain(self) -> Optional['qubes.vm.qubesvm.QubesVM']: - """ Which domain the device is attached to. """ + """ Which domain the device is attached/assigned to. """ return self.__frontend_domain @frontend_domain.setter def frontend_domain( self, frontend_domain: Optional['qubes.vm.qubesvm.QubesVM'] ): - """ Which domain the device is attached to. """ + """ Which domain the device is attached/assigned to. """ self.__frontend_domain = frontend_domain @property @@ -709,6 +742,122 @@ def options(self, options: Optional[Dict[str, Any]]): """ Device options (same as in the legacy API). """ self.__options = options or {} + def serialize(self) -> bytes: + properties = b' '.join( + f'{prop}={serialize_str(value)}'.encode('ascii') + for prop, value in ( + ('required', 'yes' if self.required else 'no'), + ('attach_automatically', + 'yes' if self.attach_automatically else 'no'), + ('ident', self.ident), + ('devclass', self.devclass) + ) + ) + + back_name = serialize_str(self.backend_domain.name) + backend_domain_prop = (b"backend_domain=" + back_name.encode('ascii')) + properties += b' ' + backend_domain_prop + + if self.frontend_domain is not None: + front_name = serialize_str(self.frontend_domain.name) + frontend_domain_prop = ( + b"frontend_domain=" + front_name.encode('ascii')) + properties += b' ' + frontend_domain_prop + + properties += b' ' + b' '.join( + f'_{prop}={serialize_str(value)}'.encode('ascii') + for prop, value in self.options.items() + ) + + return properties + + @classmethod + def deserialize( + cls, + serialization: bytes, + expected_backend_domain: 'qubes.vm.BaseVM', + expected_ident: str, + expected_devclass: Optional[str] = None, + ) -> 'DeviceAssignment': + try: + result = DeviceAssignment._deserialize( + cls, serialization, + expected_backend_domain, expected_ident, expected_devclass + ) + except Exception as exc: + raise ProtocolError() from exc + return result + + @staticmethod + def _deserialize( + cls: Type, + untrusted_serialization: bytes, + expected_backend_domain: 'qubes.vm.BaseVM', + expected_ident: str, + expected_devclass: Optional[str] = None, + ) -> 'DeviceAssignment': + options = {} + allowed_chars_key = string.digits + string.ascii_letters + '-_.' + allowed_chars_value = allowed_chars_key + ',+:' + + untrusted_decoded = untrusted_serialization.decode('ascii', 'strict') + keys = [] + values = [] + untrusted_key, _, untrusted_rest = untrusted_decoded.partition("='") + + key = sanitize_str( + untrusted_key, allowed_chars_key, + error_message='Invalid chars in property name') + keys.append(key) + while "='" in untrusted_rest: + ut_value_key, _, untrusted_rest = untrusted_rest.partition("='") + untrusted_value, _, untrusted_key = ut_value_key.rpartition("' ") + value = sanitize_str( + deserialize_str(untrusted_value), allowed_chars_value, + error_message='Invalid chars in property value') + values.append(value) + key = sanitize_str( + untrusted_key, allowed_chars_key, + error_message='Invalid chars in property name') + keys.append(key) + untrusted_value = untrusted_rest[:-1] # ending ' + value = sanitize_str( + deserialize_str(untrusted_value), allowed_chars_value, + error_message='Invalid chars in property value') + values.append(value) + + properties = dict() + for key, value in zip(keys, values): + if key.startswith("_"): + options[key[1:]] = value + else: + properties[key] = value + + properties['options'] = options + + if properties['backend_domain'] != expected_backend_domain.name: + raise UnexpectedDeviceProperty( + f"Got device exposed by {properties['backend_domain']}" + f"when expected devices from {expected_backend_domain.name}.") + properties['backend_domain'] = expected_backend_domain + + if properties["ident"] != expected_ident: + raise UnexpectedDeviceProperty( + f"Got device with id: {properties['ident']} " + f"when expected id: {expected_ident}.") + + if expected_devclass and properties['devclass'] != expected_devclass: + raise UnexpectedDeviceProperty( + f"Got {properties['devclass']} device " + f"when expected {expected_devclass}.") + + properties['attach_automatically'] = qubes.property.bool( + None, None, properties['attach_automatically']) + properties['required'] = qubes.property.bool( + None, None, properties['required']) + + return cls(**properties) + class DeviceCollection: """Bag for devices. From 2b033d4eeee32d39c84daca1942304ce88d8e0e0 Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Tue, 30 Jan 2024 21:02:34 +0100 Subject: [PATCH 10/49] q-dev: full_identity of device --- qubes/api/admin.py | 2 +- qubes/devices.py | 66 ++++++++++++++++++++++++++++++++++---------- qubes/vm/__init__.py | 1 - qubes/vm/qubesvm.py | 2 +- 4 files changed, 53 insertions(+), 18 deletions(-) diff --git a/qubes/api/admin.py b/qubes/api/admin.py index b3532026e..d6b4842bf 100644 --- a/qubes/api/admin.py +++ b/qubes/api/admin.py @@ -23,7 +23,6 @@ import asyncio import functools -import itertools import os import string import subprocess @@ -38,6 +37,7 @@ import qubes.backup import qubes.config import qubes.devices +import qubes.ext import qubes.firewall import qubes.storage import qubes.utils diff --git a/qubes/devices.py b/qubes/devices.py index 8995bde38..103f13b30 100644 --- a/qubes/devices.py +++ b/qubes/devices.py @@ -601,6 +601,27 @@ def _deserialize( def frontend_domain(self): return self.data.get("frontend_domain", None) + @property + def full_identity(self) -> str: + """ + Get user understandable identification of device not related to ports. + + In addition to the description returns presented interfaces. + It is used to auto-attach usb devices, so an attacking device needs to + mimic not only a name, but also interfaces of trusted device (and have + to be plugged to the same port). For a common user it is all the data + she uses to recognize the device. + """ + allowed_chars = string.digits + string.ascii_letters + '-_.' + description = "" + for char in self.description: + if char in allowed_chars: + description += char + else: + description += "_" + interfaces = ''.join(repr(ifc) for ifc in self.interfaces) + return f'{description}:{interfaces}' + def serialize_str(value: str): return repr(str(value)) @@ -624,6 +645,7 @@ def sanitize_str( """ if replace_char is None: if any(x not in allowed_chars for x in untrusted_value): + print(untrusted_value, file=sys.stderr) # TODO raise qubes.api.ProtocolError(error_message) return untrusted_value result = "" @@ -759,15 +781,19 @@ def serialize(self) -> bytes: properties += b' ' + backend_domain_prop if self.frontend_domain is not None: - front_name = serialize_str(self.frontend_domain.name) + if isinstance(self.frontend_domain, str): + front_name = serialize_str(self.frontend_domain) # TODO + else: + front_name = serialize_str(self.frontend_domain.name) frontend_domain_prop = ( b"frontend_domain=" + front_name.encode('ascii')) properties += b' ' + frontend_domain_prop - properties += b' ' + b' '.join( - f'_{prop}={serialize_str(value)}'.encode('ascii') - for prop, value in self.options.items() - ) + if self.options: + properties += b' ' + b' '.join( + f'_{prop}={serialize_str(value)}'.encode('ascii') + for prop, value in self.options.items() + ) return properties @@ -800,7 +826,8 @@ def _deserialize( allowed_chars_key = string.digits + string.ascii_letters + '-_.' allowed_chars_value = allowed_chars_key + ',+:' - untrusted_decoded = untrusted_serialization.decode('ascii', 'strict') + untrusted_decoded = untrusted_serialization.decode( + 'ascii', 'strict').strip() keys = [] values = [] untrusted_key, _, untrusted_rest = untrusted_decoded.partition("='") @@ -837,7 +864,7 @@ def _deserialize( if properties['backend_domain'] != expected_backend_domain.name: raise UnexpectedDeviceProperty( - f"Got device exposed by {properties['backend_domain']}" + f"Got device exposed by {properties['backend_domain']} " f"when expected devices from {expected_backend_domain.name}.") properties['backend_domain'] = expected_backend_domain @@ -963,31 +990,40 @@ async def attach(self, device_assignment: DeviceAssignment): 'device-attach:' + self._bus, device=device, options=device_assignment.options) - async def assign(self, device_assignment: DeviceAssignment): + async def assign(self, assignment: DeviceAssignment): """ Assign device to domain. """ - if device_assignment.devclass is None: - device_assignment.devclass = self._bus - elif device_assignment.devclass != self._bus: + if assignment.devclass is None: + assignment.devclass = self._bus + elif assignment.devclass != self._bus: raise ValueError( 'Trying to assign DeviceAssignment of a different device class') - device = device_assignment.device + device = assignment.device if device in self.get_assigned_devices(): raise DeviceAlreadyAssigned( 'device {!s} of class {} already assigned to {!s}'.format( device, self._bus, self._vm)) + if (assignment.devclass not in ('pci', 'testclass') + and assignment.required): + raise qubes.exc.QubesValueError( + "Only pci devices can be set as required.") + if (assignment.devclass not in ('pci', 'testclass', 'mic', 'usb') + and assignment.attach_automatically): + raise qubes.exc.QubesValueError( + "Only pci, mic and usb devices can be automatically attached.") + await self._vm.fire_event_async( 'device-pre-assign:' + self._bus, - pre_event=True, device=device, options=device_assignment.options) + pre_event=True, device=device, options=assignment.options) - self._set.add(device_assignment) + self._set.add(assignment) await self._vm.fire_event_async( 'device-assign:' + self._bus, - device=device, options=device_assignment.options) + device=device, options=assignment.options) def load_assignment(self, device_assignment: DeviceAssignment): """Load DeviceAssignment retrieved from qubes.xml diff --git a/qubes/vm/__init__.py b/qubes/vm/__init__.py index 25675ecd4..4d5ed777a 100644 --- a/qubes/vm/__init__.py +++ b/qubes/vm/__init__.py @@ -341,7 +341,6 @@ def __xml__(self): node.set('backend-domain', device.backend_domain.name) node.set('id', device.ident) node.set('required', 'yes' if device.required else 'no') - # TODO: serial for key, val in device.options.items(): option_node = lxml.etree.Element('option') option_node.set('name', key) diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index c06f8587b..4646775bc 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -1170,7 +1170,7 @@ async def start(self, start_guid=True, notify_function=None, qmemman_client = None try: for devclass in self.devices: - for dev in self.devices[devclass].get_assigned_devices(): + for dev in self.devices[devclass].get_assigned_devices(): # TODO: fix if isinstance(dev, qubes.devices.UnknownDevice): raise qubes.exc.QubesException( '{} device {} not available'.format( From 33bb3f2cd2955e0d33715557b75bd1a37cc80c8d Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Wed, 31 Jan 2024 15:40:13 +0100 Subject: [PATCH 11/49] q-dev: development of auto-attach and required flags mic cannot be auto-attached since we have to wait to pulseaudio client to start first make auto-attach and required as immutable properties unassign non-required device from running vm --- qubes/devices.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/qubes/devices.py b/qubes/devices.py index 103f13b30..f0d865811 100644 --- a/qubes/devices.py +++ b/qubes/devices.py @@ -738,10 +738,6 @@ def required(self) -> bool: """ return self.__required - @required.setter - def required(self, required: bool): - self.__required = required - @property def attach_automatically(self) -> bool: """ @@ -750,10 +746,6 @@ def attach_automatically(self) -> bool: """ return self.__attach_automatically - @attach_automatically.setter - def attach_automatically(self, attach_automatically: bool): - self.__attach_automatically = attach_automatically - @property def options(self) -> Dict[str, Any]: """ Device options (same as in the legacy API). """ @@ -1009,11 +1001,12 @@ async def assign(self, assignment: DeviceAssignment): if (assignment.devclass not in ('pci', 'testclass') and assignment.required): raise qubes.exc.QubesValueError( - "Only pci devices can be set as required.") - if (assignment.devclass not in ('pci', 'testclass', 'mic', 'usb') + "Only pci devices can be assigned as required.") + if (assignment.devclass not in ('pci', 'testclass', 'usb') and assignment.attach_automatically): raise qubes.exc.QubesValueError( - "Only pci, mic and usb devices can be automatically attached.") + "Only pci and usb devices can be assigned " + "to be automatically attached.") await self._vm.fire_event_async( 'device-pre-assign:' + self._bus, @@ -1113,10 +1106,11 @@ async def unassign(self, device_assignment: DeviceAssignment): f'device {device_assignment.ident!s} of class {self._bus} not ' f'assigned to {self._vm!s}') - if not self._vm.is_halted(): + if not self._vm.is_halted() and assignment.required: raise qubes.exc.QubesVMNotHaltedError( self._vm, - "Can not remove an assignment from a non halted qube.") + "Can not remove an required assignment from " + "a non halted qube.") device = device_assignment.device await self._vm.fire_event_async( From 4a426e6d8596d3b2196865445018456670d7d06d Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Thu, 1 Feb 2024 09:38:26 +0100 Subject: [PATCH 12/49] q-dev: implementation of DeviceInfo.attachment --- qubes/api/admin.py | 10 ++---- qubes/devices.py | 87 +++++++++++++++++++++++++++++----------------- qubes/ext/block.py | 4 +-- 3 files changed, 61 insertions(+), 40 deletions(-) diff --git a/qubes/api/admin.py b/qubes/api/admin.py index d6b4842bf..c796dbd4a 100644 --- a/qubes/api/admin.py +++ b/qubes/api/admin.py @@ -1300,7 +1300,6 @@ async def vm_device_assign(self, endpoint, untrusted_payload): ep.name for ep in importlib.metadata.entry_points(group='qubes.devices')), no_payload=True, scope='local', write=True) async def vm_device_unassign(self, endpoint): - # TODO DeviceAssignment.deserialize() ? Device devclass = endpoint # qrexec already verified that no strange characters are in self.arg @@ -1359,7 +1358,6 @@ async def vm_device_attach(self, endpoint, untrusted_payload): for ep in importlib.metadata.entry_points(group='qubes.devices')), no_payload=True, scope='local', execute=True) async def vm_device_detach(self, endpoint): - # TODO DeviceAssignment.deserialize() ? Device devclass = endpoint # qrexec already verified that no strange characters are in self.arg @@ -1374,15 +1372,13 @@ async def vm_device_detach(self, endpoint): assignment = qubes.devices.DeviceAssignment( dev.backend_domain, dev.ident) await self.dest.devices[devclass].detach(assignment) - self.app.save() # TODO: not needed - # Attach/Detach action can both modify persistent state (with - # required=True or required=False) and volatile state of running VM - # (with required=None). For this reason, write=True + execute=True + # Assign/Unassign action can modify only persistent state of running VM. + # For this reason, write=True @qubes.api.method('admin.vm.device.{endpoint}.Set.assignment', endpoints=(ep.name for ep in importlib.metadata.entry_points(group='qubes.devices')), - scope='local', write=True, execute=True) + scope='local', write=True) async def vm_device_set_assignment(self, endpoint, untrusted_payload): """ Update assignment of already attached device. diff --git a/qubes/devices.py b/qubes/devices.py index f0d865811..18aa46ded 100644 --- a/qubes/devices.py +++ b/qubes/devices.py @@ -318,7 +318,6 @@ def _load_classes(bus: str): class DeviceInfo(Device): """ Holds all information about a device """ - # pylint: disable=too-few-public-methods def __init__( self, backend_domain: 'qubes.vm.BaseVM', @@ -331,6 +330,7 @@ def __init__( serial: Optional[str] = None, interfaces: Optional[List[DeviceInterface]] = None, parent: Optional[Device] = None, + attachment: Optional['qubes.vm.BaseVM'] = None, **kwargs ): super().__init__(backend_domain, ident, devclass) @@ -342,6 +342,7 @@ def __init__( self._serial = serial self._interfaces = interfaces self._parent = parent + self._attachment = attachment self.data = kwargs @@ -473,17 +474,17 @@ def subdevices(self) -> List['DeviceInfo']: if dev.parent_device.ident == self.ident] @property - def attachments(self) -> List['DeviceAssignment']: + def attachment(self) -> Optional['qubes.vm.BaseVM']: """ - Device attachments + VM to which device is attached (frontend domain). """ - return [] # TODO + return self._attachment def serialize(self) -> bytes: """ Serialize object to be transmitted via Qubes API. """ - # 'backend_domain', 'interfaces', 'data', 'parent_device' + # 'backend_domain', 'attachment', 'interfaces', 'data', 'parent_device' # are not string, so they need special treatment default_attrs = { 'ident', 'devclass', 'vendor', 'product', 'manufacturer', 'name', @@ -494,9 +495,14 @@ def serialize(self) -> bytes: (key, getattr(self, key)) for key in default_attrs) ) - back_name = serialize_str(self.backend_domain.name) - backend_domain_prop = (b"backend_domain=" + back_name.encode('ascii')) - properties += b' ' + backend_domain_prop + qname = serialize_str(self.backend_domain.name) + backend_prop = (b"backend_domain=" + qname.encode('ascii')) + properties += b' ' + backend_prop + + if self.attachment: + qname = serialize_str(self.attachment.name) + attachment_prop = (b"attachment=" + qname.encode('ascii')) + properties += b' ' + attachment_prop interfaces = serialize_str( ''.join(repr(ifc) for ifc in self.interfaces)) @@ -573,6 +579,13 @@ def _deserialize( f"when expected devices from {expected_backend_domain.name}.") properties['backend_domain'] = expected_backend_domain + if 'attachment' not in properties or not properties['attachment']: + properties['attachment'] = None + else: + app = expected_backend_domain.app + properties['attachment'] = app.domains.get_blind( + properties['attachment']) + if expected_devclass and properties['devclass'] != expected_devclass: raise UnexpectedDeviceProperty( f"Got {properties['devclass']} device " @@ -597,10 +610,6 @@ def _deserialize( return cls(**properties) - @property - def frontend_domain(self): - return self.data.get("frontend_domain", None) - @property def full_identity(self) -> str: """ @@ -645,7 +654,6 @@ def sanitize_str( """ if replace_char is None: if any(x not in allowed_chars for x in untrusted_value): - print(untrusted_value, file=sys.stderr) # TODO raise qubes.api.ProtocolError(error_message) return untrusted_value result = "" @@ -678,8 +686,9 @@ class DeviceAssignment(Device): 3. (True, True, True) -> domain is running, device is attached and couldn't be detached. 4. (False, Ture, False) -> device is assigned to domain, but not attached - because either domain is halted - or device manually detached. + because either (i) domain is halted, + device (ii) manually detached or + (iii) attach to different domain. 5. (False, True, True) -> domain is halted, device assigned to domain and required to start domain. """ @@ -725,10 +734,11 @@ def frontend_domain( @property def attached(self) -> bool: """ - Is the device already attached to the fronted domain? + Is the device attached to the fronted domain? + + Returns False if device is attached to different domain """ - return (self.frontend_domain is not None - and self.frontend_domain.is_running()) + return self.device.attachment == self.frontend_domain @property def required(self) -> bool: @@ -738,6 +748,10 @@ def required(self) -> bool: """ return self.__required + @required.setter + def required(self, required: bool): + self.__required = required + @property def attach_automatically(self) -> bool: """ @@ -746,6 +760,10 @@ def attach_automatically(self) -> bool: """ return self.__attach_automatically + @attach_automatically.setter + def attach_automatically(self, attach_automatically: bool): + self.__attach_automatically = attach_automatically + @property def options(self) -> Dict[str, Any]: """ Device options (same as in the legacy API). """ @@ -773,10 +791,7 @@ def serialize(self) -> bytes: properties += b' ' + backend_domain_prop if self.frontend_domain is not None: - if isinstance(self.frontend_domain, str): - front_name = serialize_str(self.frontend_domain) # TODO - else: - front_name = serialize_str(self.frontend_domain.name) + front_name = serialize_str(self.frontend_domain.name) frontend_domain_prop = ( b"frontend_domain=" + front_name.encode('ascii')) properties += b' ' + frontend_domain_prop @@ -925,6 +940,23 @@ class DeviceCollection: :param device: :py:class:`DeviceInfo` object to be attached + .. event:: device-assign: (device, options) + + Fired when device is assigned to a VM. + + Handler for this event may be asynchronous. + + :param device: :py:class:`DeviceInfo` object to be assigned + :param options: :py:class:`dict` of assignment options + + .. event:: device-unassign: (device) + + Fired when device is unassigned from a VM. + + Handler for this event can be asynchronous (a coroutine). + + :param device: :py:class:`DeviceInfo` object to be unassigned + .. event:: device-list: Fired to get list of devices exposed by a VM. Handlers of this @@ -1008,10 +1040,6 @@ async def assign(self, assignment: DeviceAssignment): "Only pci and usb devices can be assigned " "to be automatically attached.") - await self._vm.fire_event_async( - 'device-pre-assign:' + self._bus, - pre_event=True, device=device, options=assignment.options) - self._set.add(assignment) await self._vm.fire_event_async( @@ -1112,12 +1140,9 @@ async def unassign(self, device_assignment: DeviceAssignment): "Can not remove an required assignment from " "a non halted qube.") - device = device_assignment.device - await self._vm.fire_event_async( - 'device-pre-unassign:' + self._bus, pre_event=True, device=device) - self._set.discard(device_assignment) + device = device_assignment.device await self._vm.fire_event_async( 'device-unassign:' + self._bus, device=device) @@ -1145,7 +1170,7 @@ def get_attached_devices(self) -> Iterable[DeviceAssignment]: backend_domain=dev.backend_domain, ident=dev.ident, options=options, - frontend_domain=dev.frontend_domain, + frontend_domain=self._vm, devclass=dev.devclass, attach_automatically=False, required=False, diff --git a/qubes/ext/block.py b/qubes/ext/block.py index 34a86f3db..d56809cb0 100644 --- a/qubes/ext/block.py +++ b/qubes/ext/block.py @@ -169,7 +169,7 @@ def on_domain_init_load(self, vm, event): # avoid building a cache on domain-init, as it isn't fully set yet, # and definitely isn't running yet current_devices = { - dev.ident: dev.frontend_domain + dev.ident: dev.attachment for dev in self.on_device_list_block(vm, None) } self.devices_cache[vm.name] = current_devices @@ -181,7 +181,7 @@ def on_qdb_change(self, vm, event, path): """A change in QubesDB means a change in device list.""" # pylint: disable=unused-argument vm.fire_event('device-list-change:block') - current_devices = dict((dev.ident, dev.frontend_domain) + current_devices = dict((dev.ident, dev.attachment) for dev in self.on_device_list_block(vm, None)) # send events about devices detached/attached outside by themselves From cb8837b01328e74ebf651a08068f48baef49f24e Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Mon, 12 Feb 2024 12:06:40 +0100 Subject: [PATCH 13/49] q-dev: move preventing of non-pci device as required to client --- qubes/devices.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/qubes/devices.py b/qubes/devices.py index 18aa46ded..02664aa24 100644 --- a/qubes/devices.py +++ b/qubes/devices.py @@ -1030,16 +1030,6 @@ async def assign(self, assignment: DeviceAssignment): 'device {!s} of class {} already assigned to {!s}'.format( device, self._bus, self._vm)) - if (assignment.devclass not in ('pci', 'testclass') - and assignment.required): - raise qubes.exc.QubesValueError( - "Only pci devices can be assigned as required.") - if (assignment.devclass not in ('pci', 'testclass', 'usb') - and assignment.attach_automatically): - raise qubes.exc.QubesValueError( - "Only pci and usb devices can be assigned " - "to be automatically attached.") - self._set.add(assignment) await self._vm.fire_event_async( From ab9b205a99642eecbd08707bb5661e3862e5a60d Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Sat, 17 Feb 2024 06:44:07 +0100 Subject: [PATCH 14/49] q-dev: rename full_identity -> self_identity +implementation for block and pci devices make DeviceAssignment.frontend_domain a VM instead of str add vendor and product id to pci devices --- qubes/devices.py | 56 +++++++++++++++++++++++++++------------------- qubes/ext/block.py | 21 ++++++++++++++--- qubes/ext/pci.py | 31 ++++++++++++++++++++++++- 3 files changed, 81 insertions(+), 27 deletions(-) diff --git a/qubes/devices.py b/qubes/devices.py index 02664aa24..cb7791d86 100644 --- a/qubes/devices.py +++ b/qubes/devices.py @@ -59,7 +59,7 @@ import string import sys from enum import Enum -from typing import Optional, List, Type, Dict, Any, Iterable +from typing import Optional, List, Type, Dict, Any, Iterable, Union import qubes.utils from qubes.api import ProtocolError @@ -94,6 +94,11 @@ class UnexpectedDeviceProperty(qubes.exc.QubesException, ValueError): Device has unexpected property such as backend_domain, devclass etc. """ +class UnrecognizedDevice(qubes.exc.QubesException, ValueError): + """ + Device identity is not as expected. + """ + class Device: def __init__(self, backend_domain, ident, devclass=None): @@ -331,6 +336,7 @@ def __init__( interfaces: Optional[List[DeviceInterface]] = None, parent: Optional[Device] = None, attachment: Optional['qubes.vm.BaseVM'] = None, + self_identity: Optional[str] = None, **kwargs ): super().__init__(backend_domain, ident, devclass) @@ -343,6 +349,7 @@ def __init__( self._interfaces = interfaces self._parent = parent self._attachment = attachment + self._self_identity = self_identity self.data = kwargs @@ -488,7 +495,7 @@ def serialize(self) -> bytes: # are not string, so they need special treatment default_attrs = { 'ident', 'devclass', 'vendor', 'product', 'manufacturer', 'name', - 'serial'} + 'serial', 'self_identity'} properties = b' '.join( f'{prop}={serialize_str(value)}'.encode('ascii') for prop, value in ( @@ -611,9 +618,14 @@ def _deserialize( return cls(**properties) @property - def full_identity(self) -> str: + def self_identity(self) -> str: """ - Get user understandable identification of device not related to ports. + Get additional identification of device presented by device itself. + + For pci/usb we expect: + ::: + For block devices: + : In addition to the description returns presented interfaces. It is used to auto-attach usb devices, so an attacking device needs to @@ -621,15 +633,9 @@ def full_identity(self) -> str: to be plugged to the same port). For a common user it is all the data she uses to recognize the device. """ - allowed_chars = string.digits + string.ascii_letters + '-_.' - description = "" - for char in self.description: - if char in allowed_chars: - description += char - else: - description += "_" - interfaces = ''.join(repr(ifc) for ifc in self.interfaces) - return f'{description}:{interfaces}' + if not self._self_identity: + return "0000:0000::?******" + return self._self_identity def serialize_str(value: str): @@ -700,7 +706,7 @@ def __init__(self, backend_domain, ident, options=None, self.__options = options or {} self.__required = required self.__attach_automatically = attach_automatically - self.__frontend_domain = frontend_domain + self.frontend_domain = frontend_domain def clone(self): """Clone object instance""" @@ -726,9 +732,11 @@ def frontend_domain(self) -> Optional['qubes.vm.qubesvm.QubesVM']: @frontend_domain.setter def frontend_domain( - self, frontend_domain: Optional['qubes.vm.qubesvm.QubesVM'] + self, frontend_domain: Optional[Union[str, 'qubes.vm.qubesvm.QubesVM']] ): """ Which domain the device is attached/assigned to. """ + if isinstance(frontend_domain, str): + frontend_domain = self.backend_domain.app.domains[frontend_domain] self.__frontend_domain = frontend_domain @property @@ -986,33 +994,35 @@ def __init__(self, vm, bus): self.devclass = qubes.utils.get_entry_point_one( 'qubes.devices', self._bus) - async def attach(self, device_assignment: DeviceAssignment): + async def attach(self, assignment: DeviceAssignment): """ Attach device to domain. """ - if device_assignment.devclass is None: - device_assignment.devclass = self._bus - elif device_assignment.devclass != self._bus: + if assignment.devclass is None: + assignment.devclass = self._bus + elif assignment.devclass != self._bus: raise ValueError( 'Trying to attach DeviceAssignment of a different device class') if self._vm.is_halted(): raise qubes.exc.QubesVMNotRunningError( self._vm,"VM not running, cannot attach device," - " did you mean `assign`?") - device = device_assignment.device + " do you mean `assign`?") + + device = assignment.device if device in self.get_attached_devices(): raise DeviceAlreadyAttached( 'device {!s} of class {} already attached to {!s}'.format( device, self._bus, self._vm)) + await self._vm.fire_event_async( 'device-pre-attach:' + self._bus, - pre_event=True, device=device, options=device_assignment.options) + pre_event=True, device=device, options=assignment.options) await self._vm.fire_event_async( 'device-attach:' + self._bus, - device=device, options=device_assignment.options) + device=device, options=assignment.options) async def assign(self, assignment: DeviceAssignment): """ diff --git a/qubes/ext/block.py b/qubes/ext/block.py index d56809cb0..c5a4323bb 100644 --- a/qubes/ext/block.py +++ b/qubes/ext/block.py @@ -51,8 +51,11 @@ def __init__(self, backend_domain, ident): super().__init__( backend_domain=backend_domain, ident=ident, devclass="block") + # lazy loading self._mode = None self._size = None + self._description = None + self._interface_num = None @property def description(self): @@ -132,16 +135,28 @@ def parent_device(self) -> Optional[qubes.devices.Device]: if self._parent is None: if not self.backend_domain.is_running(): return None - untrusted_parent: bytes = self.backend_domain.untrusted_qdb.read( + untrusted_parent_info = self.backend_domain.untrusted_qdb.read( f'/qubes-block-devices/{self.ident}/parent') - if untrusted_parent is None: + if untrusted_parent_info is None: return None else: - parent_ident = self._sanitize(untrusted_parent) + # '4-4.1:1.0' -> parent_ident='4-4.1', interface_num='1.0' + # 'sda' -> parent_ident='sda', interface_num='' + parent_ident, _, interface_num = self._sanitize( + untrusted_parent_info).partition(":") self._parent = qubes.devices.Device( self.backend_domain, parent_ident) + self._interface_num = interface_num return self._parent + @property + def self_identity(self) -> str: + """ + Get identification of device not related to port. + """ + parent_ident = self.parent_device.ident if self._parent else '' + return f'{parent_ident}:{self._interface_num}' + @staticmethod def _sanitize( untrusted_parent: bytes, diff --git a/qubes/ext/pci.py b/qubes/ext/pci.py index dfc2b0900..787bc9096 100644 --- a/qubes/ext/pci.py +++ b/qubes/ext/pci.py @@ -183,6 +183,8 @@ def __init__(self, backend_domain, ident, libvirt_name=None): # lazy loading self._description = None + self._vendor_id = None + self._product_id = None @property def vendor(self) -> str: @@ -258,25 +260,52 @@ def description(self): hostdev_details.XMLDesc())) return self._description + @property + def self_identity(self) -> str: + """ + Get identification of device not related to port. + """ + allowed_chars = string.digits + string.ascii_letters + '-_.' + if self._vendor_id is None: + vendor_id = self._load_desc()["vendor ID"] + self._vendor_id = ''.join( + c if c in set(allowed_chars) else '_' for c in vendor_id) + if self._product_id is None: + product_id = self._load_desc()["product ID"] + self._product_id = ''.join( + c if c in set(allowed_chars) else '_' for c in product_id) + interfaces = ''.join(repr(ifc) for ifc in self.interfaces) + serial = self._serial if self._serial else "" + return \ + f'{self._vendor_id}:{self._product_id}:{serial}:{interfaces}' + def _load_desc(self) -> Dict[str, str]: unknown = "unknown" result = {"vendor": unknown, + "vendor ID": "0000", "product": unknown, + "product ID": "0000", "manufacturer": unknown, "name": unknown, "serial": unknown} if not self.backend_domain.is_running(): - # don't cache this value + # don't cache these values return result hostdev_details = \ self.backend_domain.app.vmm.libvirt_conn.nodeDeviceLookupByName( self.libvirt_name ) + + # Data successfully loaded, cache these values hostdev_xml = lxml.etree.fromstring(hostdev_details.XMLDesc()) self._vendor = result["vendor"] = hostdev_xml.findtext( 'capability/vendor') + self._vendor_id = result["vendor ID"] = hostdev_xml.xpath( + "//vendor/@id") self._product = result["product"] = hostdev_xml.findtext( 'capability/product') + self._product_id = result["product ID"] = hostdev_xml.xpath( + "//product/@id") return result @staticmethod From 6735ea35b21496abc99ed8e4505d57dea771662c Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Mon, 19 Feb 2024 05:15:40 +0100 Subject: [PATCH 15/49] q-dev: block devices port assignment better block device description fix extracting partition number serialize parent devclass block auto-attach --- qubes/devices.py | 32 +++++-- qubes/ext/block.py | 142 +++++++++++++++++++++++++---- qubes/tests/api_admin.py | 20 ++-- qubes/tests/devices.py | 5 +- qubes/tests/devices_block.py | 10 +- qubes/tests/integ/devices_block.py | 24 ++--- qubes/tests/vm/qubesvm.py | 10 +- qubes/vm/qubesvm.py | 10 +- 8 files changed, 189 insertions(+), 64 deletions(-) diff --git a/qubes/devices.py b/qubes/devices.py index cb7791d86..ca7353423 100644 --- a/qubes/devices.py +++ b/qubes/devices.py @@ -434,8 +434,6 @@ def description(self) -> str: prod = self.name elif self.serial and self.serial != "unknown": prod = self.serial - elif self.parent_device is not None: - return f"sub-device of {self.parent_device}" else: prod = f"unknown {self.devclass if self.devclass else ''} device" @@ -517,9 +515,12 @@ def serialize(self) -> bytes: properties += b' ' + interfaces_prop if self.parent_device is not None: - parent_ident = serialize_str(self.parent_device.ident) - parent_prop = (b'parent=' + parent_ident.encode('ascii')) - properties += b' ' + parent_prop + ident = serialize_str(self.parent_device.ident) + ident_prop = (b'parent_ident=' + ident.encode('ascii')) + properties += b' ' + ident_prop + devclass = serialize_str(self.parent_device.devclass) + devclass_prop = (b'parent_devclass=' + devclass.encode('ascii')) + properties += b' ' + devclass_prop data = b' '.join( f'_{prop}={serialize_str(value)}'.encode('ascii') @@ -609,11 +610,14 @@ def _deserialize( for i in range(0, len(interfaces), 7)] properties['interfaces'] = interfaces - if 'parent' in properties: + if 'parent_ident' in properties: properties['parent'] = Device( backend_domain=expected_backend_domain, - ident=properties['parent'] + ident=properties['parent_ident'], + devclass=properties['parent_devclass'], ) + del properties['parent_ident'] + del properties['parent_devclass'] return cls(**properties) @@ -704,6 +708,8 @@ def __init__(self, backend_domain, ident, options=None, required=False, attach_automatically=False): super().__init__(backend_domain, ident, devclass) self.__options = options or {} + if required: + assert attach_automatically self.__required = required self.__attach_automatically = attach_automatically self.frontend_domain = frontend_domain @@ -720,6 +726,18 @@ def clone(self): devclass=self.devclass, ) + @classmethod + def from_device(cls, device: Device, **kwargs) -> 'DeviceAssignment': + """ + Get assignment of the device. + """ + return cls( + backend_domain=device.backend_domain, + ident=device.ident, + devclass=device.devclass, + **kwargs + ) + @property def device(self) -> DeviceInfo: """Get DeviceInfo object corresponding to this DeviceAssignment""" diff --git a/qubes/ext/block.py b/qubes/ext/block.py index c5a4323bb..318d2a920 100644 --- a/qubes/ext/block.py +++ b/qubes/ext/block.py @@ -23,6 +23,7 @@ import collections import re import string +import sys from typing import Optional, List import lxml.etree @@ -54,25 +55,61 @@ def __init__(self, backend_domain, ident): # lazy loading self._mode = None self._size = None - self._description = None self._interface_num = None @property - def description(self): - """Human readable device description""" - if self._description is None: - if not self.backend_domain.is_running(): - return self.ident - safe_set = {ord(c) for c in - string.ascii_letters + string.digits + '()+,-.:=_/ '} - untrusted_desc = self.backend_domain.untrusted_qdb.read( - '/qubes-block-devices/{}/desc'.format(self.ident)) - if not untrusted_desc: - return '' - desc = ''.join((chr(c) if c in safe_set else '_') - for c in untrusted_desc) - self._description = desc - return self._description + def name(self): + """ + The name of the device it introduced itself with. + + Could be empty string or "unknown". + """ + if self._name is None: + name, _ = self._load_lazily_name_and_serial() + return name + return self._name + + @property + def serial(self) -> str: + """ + The serial number of the device it introduced itself with. + + Could be empty string or "unknown". + + Override this method to return proper name directly from device itself. + """ + if self._serial is None: + _, serial = self._load_lazily_name_and_serial() + return serial + return self._serial + + def _load_lazily_name_and_serial(self): + if not self.backend_domain.is_running(): + return "unknown", "unknown" + untrusted_desc = self.backend_domain.untrusted_qdb.read( + f'/qubes-block-devices/{self.ident}/desc') + if not untrusted_desc: + return "unknown", "unknown" + desc = BlockDevice._sanitize( + untrusted_desc, + string.ascii_letters + string.digits + '()+,-.:=_/ ') + model, _, label = desc.partition(' ') + if model: + serial = self._serial = model.replace('_', ' ').strip() + else: + serial = "unknown" + # label: '(EXAMPLE)' or '()' + if label[1:-1]: + name = self._name = label.replace('_', ' ')[1:-1].strip() + else: + name = "unknown" + return name, serial + + @property + def manufacturer(self) -> str: + if self.parent_device: + return f"sub-device of {self.parent_device}" + return f"hosted by {self.backend_domain!s}" @property def mode(self): @@ -142,10 +179,13 @@ def parent_device(self) -> Optional[qubes.devices.Device]: else: # '4-4.1:1.0' -> parent_ident='4-4.1', interface_num='1.0' # 'sda' -> parent_ident='sda', interface_num='' - parent_ident, _, interface_num = self._sanitize( + parent_ident, sep, interface_num = self._sanitize( untrusted_parent_info).partition(":") + devclass = 'usb' if sep == ':' else 'block' + if not parent_ident: + return None self._parent = qubes.devices.Device( - self.backend_domain, parent_ident) + self.backend_domain, parent_ident, devclass=devclass) self._interface_num = interface_num return self._parent @@ -154,8 +194,29 @@ def self_identity(self) -> str: """ Get identification of device not related to port. """ - parent_ident = self.parent_device.ident if self._parent else '' - return f'{parent_ident}:{self._interface_num}' + parent_identity = '' + p = self.parent_device + if p is not None: + p_info = p.backend_domain.devices[p.devclass][p.ident] + parent_identity = p_info.self_identity + if p.devclass == 'usb': + parent_identity = f'{p.ident}:{parent_identity}' + if self._interface_num: + # device interface number (not partition) + self_id = self._interface_num + else: + self_id = self._get_possible_partition_number() + return f'{parent_identity}:{self_id}' + + def _get_possible_partition_number(self) -> Optional[int]: + """ + If the device is partition return partition number. + + The behavior is undefined for the rest block devices. + """ + # partition number: 'xxxxx12' -> '12' (partition) + numbers = re.findall(r'\d+$', self.ident) + return int(numbers[-1]) if numbers else None @staticmethod def _sanitize( @@ -251,6 +312,17 @@ def on_qdb_change(self, vm, event, path): self.devices_cache[vm.name] = current_devices + for front_vm in vm.app.domains: + if not front_vm.is_running(): + continue + for assignment in front_vm.devices['block'].get_assigned_devices(): + if (assignment.backend_domain == vm + and assignment.ident in added + and assignment.ident not in attached + ): + asyncio.ensure_future(self._attach_and_notify( + front_vm, assignment.device, assignment.options)) + def device_get(self, vm, ident): '''Read information about device from QubesDB @@ -371,6 +443,10 @@ def find_unused_frontend(self, vm, devtype='disk'): @qubes.ext.handler('device-pre-attach:block') def on_device_pre_attached_block(self, vm, event, device, options): # pylint: disable=unused-argument + if isinstance(device, qubes.devices.UnknownDevice): + print(f'{device.devclass.capitalize()} device {device} ' + 'not available, skipping.', file=sys.stderr) + return # validate options for option, value in options.items(): @@ -386,6 +462,15 @@ def on_device_pre_attached_block(self, vm, event, device, options): raise qubes.exc.QubesValueError( 'devtype option can only have ' '\'disk\' or \'cdrom\' value') + elif option == 'identity': + identity = value + if identity != 'any' and device.self_identity != identity: + print("Unrecognized identity, skipping attachment of" + f" {device}", file=sys.stderr) + raise qubes.devices.UnrecognizedDevice( + f"Device presented identity {device.self_identity} " + f"does not match expected {identity}" + ) else: raise qubes.exc.QubesValueError( 'Unsupported option {}'.format(option)) @@ -412,6 +497,23 @@ def on_device_pre_attached_block(self, vm, event, device, options): vm.app.env.get_template('libvirt/devices/block.xml').render( device=device, vm=vm, options=options)) + @qubes.ext.handler('domain-start') + async def on_domain_start(self, vm, _event, **_kwargs): + # pylint: disable=unused-argument + for assignment in vm.devices['block'].get_assigned_devices(): + asyncio.ensure_future(self._attach_and_notify( + vm, assignment.device, assignment.options)) + + async def _attach_and_notify(self, vm, device, options): + # bypass DeviceCollection logic preventing double attach + try: + self.on_device_pre_attached_block( + vm, 'device-pre-attach:block', device, options) + except qubes.devices.UnrecognizedDevice: + return + await vm.fire_event_async( + 'device-attach:block', device=device, options=options) + @qubes.ext.handler('device-pre-detach:block') def on_device_pre_detached_block(self, vm, event, device): # pylint: disable=unused-argument diff --git a/qubes/tests/api_admin.py b/qubes/tests/api_admin.py index db9a384eb..5ac70d59a 100644 --- a/qubes/tests/api_admin.py +++ b/qubes/tests/api_admin.py @@ -1722,7 +1722,7 @@ def test_462_vm_device_available_invalid(self): def test_470_vm_device_list_persistent(self): assignment = qubes.devices.DeviceAssignment(self.vm, '1234', - persistent=True) + attach_automatically=True, required=True) self.loop.run_until_complete( self.vm.devices['testclass'].attach(assignment)) value = self.call_mgmt_func(b'admin.vm.device.testclass.List', @@ -1733,11 +1733,11 @@ def test_470_vm_device_list_persistent(self): def test_471_vm_device_list_persistent_options(self): assignment = qubes.devices.DeviceAssignment(self.vm, '1234', - persistent=True, options={'opt1': 'value'}) + attach_automatically=True, required=True, options={'opt1': 'value'}) self.loop.run_until_complete( self.vm.devices['testclass'].attach(assignment)) assignment = qubes.devices.DeviceAssignment(self.vm, '4321', - persistent=True) + attach_automatically=True, required=True) self.loop.run_until_complete( self.vm.devices['testclass'].attach(assignment)) value = self.call_mgmt_func(b'admin.vm.device.testclass.List', @@ -1766,7 +1766,7 @@ def test_473_vm_device_list_mixed(self): self.vm.add_handler('device-list-attached:testclass', self.device_list_attached_testclass) assignment = qubes.devices.DeviceAssignment(self.vm, '4321', - persistent=True) + attach_automatically=True, required=True) self.loop.run_until_complete( self.vm.devices['testclass'].attach(assignment)) value = self.call_mgmt_func(b'admin.vm.device.testclass.List', @@ -1780,7 +1780,7 @@ def test_474_vm_device_list_specific(self): self.vm.add_handler('device-list-attached:testclass', self.device_list_attached_testclass) assignment = qubes.devices.DeviceAssignment(self.vm, '4321', - persistent=True) + attach_automatically=True, required=True) self.loop.run_until_complete( self.vm.devices['testclass'].attach(assignment)) value = self.call_mgmt_func(b'admin.vm.device.testclass.List', @@ -1944,7 +1944,7 @@ def test_501_vm_remove_running(self, mock_rmtree, mock_remove): def test_502_vm_remove_attached(self, mock_rmtree, mock_remove): self.setup_for_clone() assignment = qubes.devices.DeviceAssignment( - self.vm, '1234', persistent=True) + self.vm, '1234', attach_automatically=True, required=True) self.loop.run_until_complete( self.vm2.devices['testclass'].attach(assignment)) @@ -2665,7 +2665,7 @@ def test_652_vm_device_set_persistent_false(self): self.vm.add_handler('device-list:testclass', self.device_list_testclass) assignment = qubes.devices.DeviceAssignment(self.vm, '1234', {}, - True) + attach_automatically=True, required=True) self.loop.run_until_complete( self.vm.devices['testclass'].attach(assignment)) self.vm.add_handler('device-list-attached:testclass', @@ -2675,7 +2675,7 @@ def test_652_vm_device_set_persistent_false(self): with unittest.mock.patch.object(qubes.vm.qubesvm.QubesVM, 'is_halted', lambda _: False): value = self.call_mgmt_func( - b'admin.vm.device.testclass.Set.persistent', + b'admin.vm.device.testclass.Set.persistent', # TODO b'test-vm1', b'test-vm1+1234', b'False') self.assertIsNone(value) self.assertNotIn(dev, self.vm.devices['testclass'].get_assigned_devices()) @@ -2686,7 +2686,7 @@ def test_653_vm_device_set_persistent_true_unchanged(self): self.vm.add_handler('device-list:testclass', self.device_list_testclass) assignment = qubes.devices.DeviceAssignment(self.vm, '1234', {}, - True) + attach_automatically=True, required=True) self.loop.run_until_complete( self.vm.devices['testclass'].attach(assignment)) self.vm.add_handler('device-list-attached:testclass', @@ -2694,7 +2694,7 @@ def test_653_vm_device_set_persistent_true_unchanged(self): with unittest.mock.patch.object(qubes.vm.qubesvm.QubesVM, 'is_halted', lambda _: False): value = self.call_mgmt_func( - b'admin.vm.device.testclass.Set.persistent', + b'admin.vm.device.testclass.Set.persistent', # TODO b'test-vm1', b'test-vm1+1234', b'True') self.assertIsNone(value) dev = qubes.devices.DeviceInfo(self.vm, '1234') diff --git a/qubes/tests/devices.py b/qubes/tests/devices.py index ea23480a4..7850c022c 100644 --- a/qubes/tests/devices.py +++ b/qubes/tests/devices.py @@ -91,7 +91,8 @@ def setUp(self): self.assignment = qubes.devices.DeviceAssignment( backend_domain=self.device.backend_domain, ident=self.device.ident, - persistent=True + attach_automatically=True, + required=True, ) def test_000_init(self): @@ -220,7 +221,7 @@ def test_001_missing(self): assignment = qubes.devices.DeviceAssignment( backend_domain=device.backend_domain, ident=device.ident, - persistent=True) + attach_automatically=True, required=True) self.loop.run_until_complete( self.manager['testclass'].attach(assignment)) self.assertEventFired(self.emitter, 'device-attach:testclass') diff --git a/qubes/tests/devices_block.py b/qubes/tests/devices_block.py index aa0f83526..1c495fb0c 100644 --- a/qubes/tests/devices_block.py +++ b/qubes/tests/devices_block.py @@ -155,8 +155,8 @@ def test_000_device_get(self): self.assertIsInstance(device_info, qubes.ext.block.BlockDevice) self.assertEqual(device_info.backend_domain, vm) self.assertEqual(device_info.ident, 'sda') - self.assertEqual(device_info.description, 'Test device') - self.assertEqual(device_info._description, 'Test device') + self.assertEqual(device_info.name, 'Test device') + self.assertEqual(device_info._name, 'Test device') self.assertEqual(device_info.size, 1024000) self.assertEqual(device_info.mode, 'w') self.assertEqual( @@ -174,8 +174,8 @@ def test_001_device_get_other_node(self): self.assertIsInstance(device_info, qubes.ext.block.BlockDevice) self.assertEqual(device_info.backend_domain, vm) self.assertEqual(device_info.ident, 'mapper_dmroot') - self.assertEqual(device_info.description, 'Test device') - self.assertEqual(device_info._description, 'Test device') + self.assertEqual(device_info.name, 'Test device') + self.assertEqual(device_info._name, 'Test device') self.assertEqual(device_info.size, 1024000) self.assertEqual(device_info.mode, 'w') self.assertEqual( @@ -190,7 +190,7 @@ def test_002_device_get_invalid_desc(self): '/qubes-block-devices/sda/mode': b'w', }) device_info = self.ext.device_get(vm, 'sda') - self.assertEqual(device_info.description, 'Test device__za__abc') + self.assertEqual(device_info.name, 'Test device__za__abc') def test_003_device_get_invalid_size(self): vm = TestVM({ diff --git a/qubes/tests/integ/devices_block.py b/qubes/tests/integ/devices_block.py index 5eea5aff8..c8d39ffbf 100644 --- a/qubes/tests/integ/devices_block.py +++ b/qubes/tests/integ/devices_block.py @@ -87,7 +87,7 @@ def test_000_list_loop(self): dev_list = list(self.vm.devices['block']) found = False for dev in dev_list: - if dev.description == self.img_path: + if dev.name == self.img_path: self.assertTrue(dev.ident.startswith('loop')) self.assertEqual(dev.mode, 'w') self.assertEqual(dev.size, 1024 * 1024 * 128) @@ -113,7 +113,7 @@ def test_001_list_loop_mounted(self): dev_list = list(self.vm.devices['block']) for dev in dev_list: - if dev.description == self.img_path: + if dev.name == self.img_path: self.fail( 'Device {} ({}) should not be listed because is mounted' .format(dev, self.img_path)) @@ -132,11 +132,11 @@ def test_010_list_dm(self): found = False for dev in dev_list: if dev.ident.startswith('loop'): - self.assertNotEquals(dev.description, self.img_path, + self.assertNotEquals(dev.name, self.img_path, "Device {} ({}) should not be listed as it is used in " "device-mapper".format(dev, self.img_path) ) - elif dev.description == 'test-dm': + elif dev.name == 'test-dm': self.assertEqual(dev.mode, 'w') self.assertEqual(dev.size, 1024 * 1024 * 128) found = True @@ -163,12 +163,12 @@ def test_011_list_dm_mounted(self): dev_list = list(self.vm.devices['block']) for dev in dev_list: if dev.ident.startswith('loop'): - self.assertNotEquals(dev.description, self.img_path, + self.assertNotEquals(dev.name, self.img_path, "Device {} ({}) should not be listed as it is used in " "device-mapper".format(dev, self.img_path) ) else: - self.assertNotEquals(dev.description, 'test-dm', + self.assertNotEquals(dev.name, 'test-dm', "Device {} ({}) should not be listed as it is " "mounted".format(dev, 'test-dm') ) @@ -188,11 +188,11 @@ def test_012_list_dm_delayed(self): found = False for dev in dev_list: if dev.ident.startswith('loop'): - self.assertNotEquals(dev.description, self.img_path, + self.assertNotEquals(dev.name, self.img_path, "Device {} ({}) should not be listed as it is used in " "device-mapper".format(dev, self.img_path) ) - elif dev.description == 'test-dm': + elif dev.name == 'test-dm': self.assertEqual(dev.mode, 'w') self.assertEqual(dev.size, 1024 * 1024 * 128) found = True @@ -218,7 +218,7 @@ def test_013_list_dm_removed(self): dev_list = list(self.vm.devices['block']) found = False for dev in dev_list: - if dev.description == self.img_path: + if dev.name == self.img_path: self.assertTrue(dev.ident.startswith('loop')) self.assertEqual(dev.mode, 'w') self.assertEqual(dev.size, 1024 * 1024 * 128) @@ -242,7 +242,7 @@ def test_020_list_loop_partition(self): dev_list = list(self.vm.devices['block']) found = False for dev in dev_list: - if dev.description == self.img_path: + if dev.name == self.img_path: self.assertTrue(dev.ident.startswith('loop')) self.assertEqual(dev.mode, 'w') self.assertEqual(dev.size, 1024 * 1024 * 128) @@ -271,7 +271,7 @@ def test_021_list_loop_partition_mounted(self): dev_list = list(self.vm.devices['block']) for dev in dev_list: - if dev.description == self.img_path: + if dev.name == self.img_path: self.fail( 'Device {} ({}) should not be listed because its ' 'partition is mounted' @@ -318,7 +318,7 @@ def setUp(self): "udevadm settle".format(path=self.img_path), user="root")) dev_list = list(self.backend.devices['block']) for dev in dev_list: - if dev.description == self.img_path: + if dev.name == self.img_path: self.device = dev self.device_ident = dev.ident break diff --git a/qubes/tests/vm/qubesvm.py b/qubes/tests/vm/qubesvm.py index f9449c932..c818afbc2 100644 --- a/qubes/tests/vm/qubesvm.py +++ b/qubes/tests/vm/qubesvm.py @@ -1395,8 +1395,8 @@ def test_600_libvirt_xml_hvm_pcidev_s0ix(self): vm, # this is violation of API, but for PCI the argument # is unused '00_00.0', - bus='pci', - persistent=True) + devclass='pci', + attach_automatically=True, required=True) vm.devices['pci']._set.add( assignment) libvirt_xml = vm.create_config_file() @@ -1479,7 +1479,8 @@ def test_600_libvirt_xml_hvm_cdrom_boot(self): self.app.vmm.offline_mode = False dev = qubes.devices.DeviceAssignment( dom0, 'sda', - {'devtype': 'cdrom', 'read-only': 'yes'}, persistent=True) + {'devtype': 'cdrom', 'read-only': 'yes'}, + attach_automatically=True, required=True) self.loop.run_until_complete(vm.devices['block'].attach(dev)) libvirt_xml = vm.create_config_file() self.assertXMLEqual(lxml.etree.XML(libvirt_xml), @@ -1583,7 +1584,8 @@ def test_600_libvirt_xml_hvm_cdrom_dom0_kernel_boot(self): self.app.vmm.offline_mode = False dev = qubes.devices.DeviceAssignment( dom0, 'sda', - {'devtype': 'cdrom', 'read-only': 'yes'}, persistent=True) + {'devtype': 'cdrom', 'read-only': 'yes'}, + attach_automatically=True, required=True) self.loop.run_until_complete(vm.devices['block'].attach(dev)) libvirt_xml = vm.create_config_file() self.assertXMLEqual(lxml.etree.XML(libvirt_xml), diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 4646775bc..71ecfe12c 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -1170,11 +1170,13 @@ async def start(self, start_guid=True, notify_function=None, qmemman_client = None try: for devclass in self.devices: - for dev in self.devices[devclass].get_assigned_devices(): # TODO: fix - if isinstance(dev, qubes.devices.UnknownDevice): + for ass in self.devices[devclass].get_assigned_devices(): + if isinstance(ass.device, qubes.devices.UnknownDevice) \ + and ass.required: raise qubes.exc.QubesException( - '{} device {} not available'.format( - devclass, dev)) + f'{devclass.capitalize()} device {ass} ' + f'not available' + ) if self.virt_mode == 'pvh' and not self.kernel: raise qubes.exc.QubesException( From d430f7f9152c7b4402dad5eb673104c209be1596 Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Wed, 21 Feb 2024 03:23:32 +0100 Subject: [PATCH 16/49] q-dev: update docs --- doc/qubes-devices.rst | 125 ++++++++++++++++++++++++++++++++++++++++++ qubes/devices.py | 13 ++++- 2 files changed, 135 insertions(+), 3 deletions(-) diff --git a/doc/qubes-devices.rst b/doc/qubes-devices.rst index 61404c08c..5879a077d 100644 --- a/doc/qubes-devices.rst +++ b/doc/qubes-devices.rst @@ -1,6 +1,131 @@ :py:mod:`qubes.devices` -- Devices =================================== +Main concept is that some domain (backend) may expose (potentially multiple) +devices, which can be attached to other domains (frontend). Devices can be of +different buses (like 'pci', 'usb', etc.). Each device bus is implemented by +an extension (see :py:mod:`qubes.ext`). + +Devices are identified by pair of (backend domain, `ident`), where `ident` is +:py:class:`str` and can contain only characters from `[a-zA-Z0-9._-]` set. + + +Device Assignment vs Attachment +------------------------------- + +:py:class:`qubes.devices.DeviceAssignment` describes the assignment of a device +to a frontend VM. For clarity let's us introduce two types of assignments: +*potential* and *real* (attachment). Attachment indicates that the device +has been attached by the Qubes backend to its frontend VM and is visible +from its perspective. Potential assignment, on the other hand, +has two additional options: `automatically_attach` and `required`. +For detailed descriptions, refer to the `DeviceAssignment` documentation. +In general we refer to potential assignment as assignment +and real assignment as attachment. To check whether the device is currently +attached, we check :py:meth:`qubes.devices.DeviceAssignment.attached`, +while to check whether an (potential) assignment exists, +we check :py:meth:`qubes.devices.DeviceAssignment.attach_automatically`. +Potential and real connections may coexist at the same time, +in which case both values will be true. + + +Actions +------- + +The `assign` action signifies that a device will be assigned to the frontend VM +in a potential form (this does not change the current system state). +This will result in an attempt to automatically attach the device +upon the next VM startup. If `required=True`, and the device cannot be attached, +the VM startup will fail. Additionally, upon device detection (`device-added`), +an attempt will be made to attach the device. However, at any time +(unless `required=True`), the user can manually modify this state by performing +`attach` or `detach` on the device, changing the current system state. +This will not alter the assignment, and automatic attachment attempts +will still be made in the future. To remove the assignment the user +need to perform `unassign` (see next section). + +Assignment Management +--------------------- + +Assignments can be edited at any time: regardless of whether the VM is running +or the device is currently attached. An exception is `required=True`, +in which case the VM must be shut down. Removing the assignment does not change the real system state, so if the device is currently attached +and the user remove the assignment, it will not be detached, +but it will not be automatically attached in the future. +Similarly, it works the other way around with `assign`. + +Proper Assignment States +------------------------ + +In short, we can think of device assignment in terms of three flags: +#. `attached` - indicating whether the device is currently assigned, +#. `attach_automatically` - indicating whether the device will be +automatically attached by the system daemon, +#. `required` - determining whether the failure of automatic attachment should +result in the domain startup being interrupted. + +Then the proper states of assignment +(`attached`, `automatically_attached`, `required`) are as follow: +#. `(True, False, False)` -> domain is running, device is manually attached +and could be manually detach any time. +#. `(True, True, False)` -> domain is running, device is attached +and could be manually detach any time (see 4.), +but in the future will be auto-attached again. +#. `(True, True, True)` -> domain is running, device is attached +and couldn't be detached. +#. `(False, Ture, False)` -> device is assigned to domain, but not attached +because either (i) domain is halted, device (ii) manually detached or +(iii) attach to different domain. +#. `(False, True, True)` -> domain is halted, device assigned to domain +and required to start domain. + + +PCI Devices +----------- + +PCI devices cannot be manually attached to a VM at any time. +We must first create an assignment (`assign`) as required +(in client we can use `--required` flag) while the VM is turned off. +Then, it will be automatically attached upon each VM startup. +However, if a PCI device is currently in use by another VM, +the startup of the second VM will fail. +PCI devices can only be assigned with the `required=True`, which does not +allow for manual modification of the state during VM operation (attach/detach). + +Microphone +---------- + +The microphone cannot be assigned (potentially) to any VM (attempting to attach the microphone during VM startup fails). + +Understanding Device Self Identity +---------------------------------- + +It is important to understand that :py:class:`qubes.devices.Device` does not +correspond to the device itself, but rather to the *port* to which the device +is connected. Therefore, when assigning a device to a VM, such as +`sys-usb:1-1.1`, the port `1-1.1` is actually assigned, and thus +*every* devices connected to it will be automatically attached. +Similarly, when assigning `vm:sda`, every block device with the name `sda` +will be automatically attached. We can limit this using :py:meth:`qubes.devices.DeviceInfo.self_identity`, which returns a string containing information +presented by the device, such as, `vendor_id`, `product_id`, `serial_number`, +and encoded interfaces. In the case of block devices, `self_identity` +consists of the parent port to which the device is connected (if any), +the parent's `self_identity`, and the interface/partition number. +In practice, this means that, a partition on a USB drive will only be +automatically attached to a frontend domain if the parent presents +the correct serial number etc., and is connected to a specific port. + +Port Assignment +--------------- + +It is possible to not assign a specific device but rather a port, +(e.g., we can use the `--port` flag in the client). In this case, +the value `any` will appear in the `identity` field of the `qubes.xml` file. +This indicates that the identity presented by the devices will be ignored, +and all connected devices will be automatically attached. Note that to create +an assignment, *any* device must currently be connected to the port. + + .. automodule:: qubes.devices :members: :show-inheritance: diff --git a/qubes/devices.py b/qubes/devices.py index ca7353423..ea86945fd 100644 --- a/qubes/devices.py +++ b/qubes/devices.py @@ -4,6 +4,8 @@ # Copyright (C) 2010-2016 Joanna Rutkowska # Copyright (C) 2015-2016 Wojtek Porczyk # Copyright (C) 2016 Bahtiar `kalkin-` Gadimov +# Copyright (C) 2024 Piotr Bartman-Szwarc +# # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -41,9 +43,10 @@ domain; it should return a list of appropriate DeviceInfo objects - handle `device-get:bus` event - get one device object exposed by this domain of given identifier - - handle `device-list-attached:class` event - list devices currently attached + - handle `device-list-attached:bus` event - list devices currently attached to this domain - - fire `device-list-change:class` event when a device list change is detected + - fire `device-list-change:bus` and following `device-added:bus` or + `device-removed:bus` events when a device list change is detected (new/removed device) Note that device-listing event handlers cannot be asynchronous. This for @@ -931,7 +934,11 @@ class DeviceCollection: .. event:: device-added: (device) - Fired when new device is discovered to a VM. + Fired when new device is discovered. + + .. event:: device-removed: (device) + + Fired when device is no longer exposed by a backend VM. .. event:: device-attach: (device, options) From eae86a6e37350d6a5739fc5aedc809cec86316a0 Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Wed, 21 Feb 2024 07:59:39 +0100 Subject: [PATCH 17/49] q-dev: update tests --- qubes/devices.py | 8 +- qubes/tests/api_admin.py | 2 +- qubes/tests/devices.py | 344 ++++++++++++++++++++++++++--------- qubes/tests/devices_block.py | 65 +++---- 4 files changed, 287 insertions(+), 132 deletions(-) diff --git a/qubes/devices.py b/qubes/devices.py index ea86945fd..6b269950c 100644 --- a/qubes/devices.py +++ b/qubes/devices.py @@ -68,12 +68,6 @@ from qubes.api import ProtocolError -class DeviceNotAttached(qubes.exc.QubesException, KeyError): - """ - Trying to detach not attached device. - """ - - class DeviceNotAssigned(qubes.exc.QubesException, KeyError): """ Trying to unassign not assigned device. @@ -1115,7 +1109,7 @@ async def update_assignment( await self._vm.fire_event_async( 'device-assignment-changed:' + self._bus, device=device) else: - await self.detach(assignment) + await self.unassign(assignment) async def detach(self, device: Device): """ diff --git a/qubes/tests/api_admin.py b/qubes/tests/api_admin.py index 5ac70d59a..d4781a5c2 100644 --- a/qubes/tests/api_admin.py +++ b/qubes/tests/api_admin.py @@ -1910,7 +1910,7 @@ def test_491_vm_device_detach_not_attached(self): self.vm.add_handler('device-detach:testclass', mock_detach) with unittest.mock.patch.object(qubes.vm.qubesvm.QubesVM, 'is_halted', lambda _: False): - with self.assertRaises(qubes.devices.DeviceNotAttached): + with self.assertRaises(qubes.devices.DeviceNotAssigned): self.call_mgmt_func(b'admin.vm.device.testclass.Detach', b'test-vm1', b'test-vm1+1234') self.assertFalse(mock_detach.called) diff --git a/qubes/tests/devices.py b/qubes/tests/devices.py index 7850c022c..7d9534007 100644 --- a/qubes/tests/devices.py +++ b/qubes/tests/devices.py @@ -21,10 +21,11 @@ # import qubes.devices -from qubes.devices import DeviceInfo, DeviceCategory +from qubes.devices import DeviceInfo import qubes.tests + class TestDevice(qubes.devices.DeviceInfo): # pylint: disable=too-few-public-methods pass @@ -46,7 +47,7 @@ def __init__(self, app, name, *args, **kwargs): super(TestVM, self).__init__(*args, **kwargs) self.app = app self.name = name - self.device = TestDevice(self, 'testdev', 'Description') + self.device = TestDevice(self, 'testdev', 'testclass') self.events_enabled = True self.devices = { 'testclass': qubes.devices.DeviceCollection(self, 'testclass') @@ -59,7 +60,7 @@ def __str__(self): return self.name @qubes.events.handler('device-list-attached:testclass') - def dev_testclass_list_attached(self, event, persistent = False): + def dev_testclass_list_attached(self, event, persistent=False): for vm in self.app.domains: if vm.device.data.get('test_frontend_domain', None) == self: yield (vm.device, {}) @@ -95,115 +96,252 @@ def setUp(self): required=True, ) + def attach(self): + self.emitter.running = True + # device-attach event not implemented, so manipulate object manually + self.device.data['test_frontend_domain'] = self.emitter + + def detach(self): + # device-detach event not implemented, so manipulate object manually + del self.device.data['test_frontend_domain'] + def test_000_init(self): self.assertFalse(self.collection._set) def test_001_attach(self): + self.emitter.running = True self.loop.run_until_complete(self.collection.attach(self.assignment)) self.assertEventFired(self.emitter, 'device-pre-attach:testclass') self.assertEventFired(self.emitter, 'device-attach:testclass') self.assertEventNotFired(self.emitter, 'device-pre-detach:testclass') self.assertEventNotFired(self.emitter, 'device-detach:testclass') - def test_002_detach(self): - self.loop.run_until_complete(self.collection.attach(self.assignment)) + def test_002_attach_to_halted(self): + with self.assertRaises(qubes.exc.QubesVMNotRunningError): + self.loop.run_until_complete( + self.collection.attach(self.assignment)) + + def test_003_detach(self): + self.attach() self.loop.run_until_complete(self.collection.detach(self.assignment)) - self.assertEventFired(self.emitter, 'device-pre-attach:testclass') - self.assertEventFired(self.emitter, 'device-attach:testclass') self.assertEventFired(self.emitter, 'device-pre-detach:testclass') self.assertEventFired(self.emitter, 'device-detach:testclass') + def test_004_detach_from_halted(self): + with self.assertRaises(LookupError): + self.loop.run_until_complete( + self.collection.detach(self.assignment)) + def test_010_empty_detach(self): + self.emitter.running = True with self.assertRaises(LookupError): self.loop.run_until_complete( self.collection.detach(self.assignment)) - def test_011_double_attach(self): - self.loop.run_until_complete(self.collection.attach(self.assignment)) + def test_011_empty_unassign(self): + for _ in range(2): + with self.assertRaises(LookupError): + self.loop.run_until_complete( + self.collection.unassign(self.assignment)) + self.emitter.running = True + def test_012_double_attach(self): + self.attach() with self.assertRaises(qubes.devices.DeviceAlreadyAttached): self.loop.run_until_complete( self.collection.attach(self.assignment)) - def test_012_double_detach(self): - self.loop.run_until_complete(self.collection.attach(self.assignment)) + def test_013_double_detach(self): + self.attach() self.loop.run_until_complete(self.collection.detach(self.assignment)) + self.detach() - with self.assertRaises(qubes.devices.DeviceNotAttached): + with self.assertRaises(qubes.devices.DeviceNotAssigned): self.loop.run_until_complete( self.collection.detach(self.assignment)) - def test_013_list_attached_persistent(self): + def test_014_double_assign(self): + self.loop.run_until_complete(self.collection.assign(self.assignment)) + + with self.assertRaises(qubes.devices.DeviceAlreadyAssigned): + self.loop.run_until_complete( + self.collection.assign(self.assignment)) + + def test_015_double_unassign(self): + self.loop.run_until_complete(self.collection.assign(self.assignment)) + self.loop.run_until_complete(self.collection.unassign(self.assignment)) + + with self.assertRaises(qubes.devices.DeviceNotAssigned): + self.loop.run_until_complete( + self.collection.unassign(self.assignment)) + + def test_016_list_assigned(self): self.assertEqual(set([]), set(self.collection.get_assigned_devices())) - self.loop.run_until_complete(self.collection.attach(self.assignment)) - self.assertEventFired(self.emitter, 'device-list-attached:testclass') - self.assertEqual({self.device}, set(self.collection.get_assigned_devices())) + self.loop.run_until_complete(self.collection.assign(self.assignment)) + self.assertEqual({self.device}, + set(self.collection.get_assigned_devices())) self.assertEqual(set([]), set(self.collection.get_attached_devices())) + self.assertEqual({self.device}, + set(self.collection.get_dedicated_devices())) - def test_014_list_attached_non_persistent(self): - self.assignment.persistent = False - self.emitter.running = True - self.loop.run_until_complete(self.collection.attach(self.assignment)) - # device-attach event not implemented, so manipulate object manually - self.device.data['test_frontend_domain'] = self.emitter + def test_017_list_attached(self): + self.assignment.required = False + self.attach() self.assertEqual({self.device}, set(self.collection.get_attached_devices())) self.assertEqual(set([]), set(self.collection.get_assigned_devices())) self.assertEqual({self.device}, - set(self.collection.get_attached_devices())) + set(self.collection.get_dedicated_devices())) self.assertEventFired(self.emitter, 'device-list-attached:testclass') - def test_015_list_available(self): + def test_018_list_available(self): self.assertEqual({self.device}, set(self.collection)) self.assertEventFired(self.emitter, 'device-list:testclass') - def test_020_update_persistent_to_false(self): - self.emitter.running = True + def test_020_update_required_to_false(self): self.assertEqual(set([]), set(self.collection.get_assigned_devices())) - self.loop.run_until_complete(self.collection.attach(self.assignment)) - # device-attach event not implemented, so manipulate object manually - self.device.data['test_frontend_domain'] = self.emitter - self.assertEqual({self.device}, set(self.collection.get_assigned_devices())) - self.assertEqual({self.device}, set(self.collection.get_attached_devices())) - self.assertEqual({self.device}, set(self.collection.get_assigned_devices())) - self.assertEqual({self.device}, set(self.collection.get_attached_devices())) - self.collection.update_assignment(self.device, False) - self.assertEqual(set(), set(self.collection.get_assigned_devices())) - self.assertEqual({self.device}, set(self.collection.get_attached_devices())) + self.loop.run_until_complete(self.collection.assign(self.assignment)) + self.attach() + self.assertEqual( + {self.device}, + set(self.collection.get_assigned_devices(required_only=True))) + self.assertEqual( + {self.device}, set(self.collection.get_assigned_devices())) + self.loop.run_until_complete( + self.collection.update_assignment(self.device, False)) + self.assertEqual( + {self.device}, set(self.collection.get_assigned_devices())) + self.assertEqual( + {self.device}, set(self.collection.get_attached_devices())) - def test_021_update_persistent_to_true(self): - self.assignment.persistent = False - self.emitter.running = True + def test_021_update_required_to_none(self): self.assertEqual(set([]), set(self.collection.get_assigned_devices())) - self.loop.run_until_complete(self.collection.attach(self.assignment)) - # device-attach event not implemented, so manipulate object manually - self.device.data['test_frontend_domain'] = self.emitter - self.assertEqual(set(), set(self.collection.get_assigned_devices())) - self.assertEqual({self.device}, set(self.collection.get_attached_devices())) + self.assignment.required = False + self.loop.run_until_complete(self.collection.assign(self.assignment)) + self.attach() + self.assertEqual( + set(), + set(self.collection.get_assigned_devices(required_only=True))) + self.assertEqual( + {self.device}, set(self.collection.get_assigned_devices())) + self.loop.run_until_complete( + self.collection.update_assignment(self.device, None)) + self.assertEqual( + set(), set(self.collection.get_assigned_devices())) + self.assertEqual( + {self.device}, set(self.collection.get_attached_devices())) + + def test_022_update_required_to_true(self): + self.assignment.required = False + self.attach() self.assertEqual(set(), set(self.collection.get_assigned_devices())) - self.assertEqual({self.device}, set(self.collection.get_attached_devices())) - self.collection.update_assignment(self.device, True) - self.assertEqual({self.device}, set(self.collection.get_assigned_devices())) - self.assertEqual({self.device}, set(self.collection.get_attached_devices())) + self.loop.run_until_complete(self.collection.assign(self.assignment)) + self.assertEqual( + set(), + set(self.collection.get_assigned_devices(required_only=True))) + self.assertEqual({self.device}, + set(self.collection.get_attached_devices())) + self.assertEqual({self.device} + , set(self.collection.get_assigned_devices())) + self.assertEqual({self.device}, + set(self.collection.get_attached_devices())) + self.loop.run_until_complete( + self.collection.update_assignment(self.device, True)) + self.assertEqual({self.device}, + set(self.collection.get_assigned_devices())) + self.assertEqual({self.device}, + set(self.collection.get_attached_devices())) - def test_022_update_persistent_reject_not_running(self): + def test_023_update_required_reject_not_running(self): self.assertEqual(set([]), set(self.collection.get_assigned_devices())) - self.loop.run_until_complete(self.collection.attach(self.assignment)) - self.assertEqual({self.device}, set(self.collection.get_assigned_devices())) + self.loop.run_until_complete(self.collection.assign(self.assignment)) + self.assertEqual({self.device}, + set(self.collection.get_assigned_devices())) self.assertEqual(set(), set(self.collection.get_attached_devices())) with self.assertRaises(qubes.exc.QubesVMNotStartedError): - self.collection.update_assignment(self.device, False) + self.loop.run_until_complete( + self.collection.update_assignment(self.device, False)) - def test_023_update_persistent_reject_not_attached(self): + def test_024_update_required_reject_not_attached(self): self.assertEqual(set(), set(self.collection.get_assigned_devices())) self.assertEqual(set(), set(self.collection.get_attached_devices())) self.emitter.running = True with self.assertRaises(qubes.exc.QubesValueError): - self.collection.update_assignment(self.device, True) + self.loop.run_until_complete( + self.collection.update_assignment(self.device, True)) with self.assertRaises(qubes.exc.QubesValueError): - self.collection.update_assignment(self.device, False) + self.loop.run_until_complete( + self.collection.update_assignment(self.device, False)) + + def test_030_assign(self): + self.emitter.running = True + self.assignment.required = False + self.loop.run_until_complete(self.collection.assign(self.assignment)) + self.assertEventFired(self.emitter, 'device-assign:testclass') + self.assertEventNotFired(self.emitter, 'device-unassign:testclass') + + def test_031_assign_to_halted(self): + self.assignment.required = False + self.loop.run_until_complete(self.collection.assign(self.assignment)) + self.assertEventFired(self.emitter, 'device-assign:testclass') + self.assertEventNotFired(self.emitter, 'device-unassign:testclass') + + def test_032_assign_required(self): + self.emitter.running = True + self.loop.run_until_complete(self.collection.assign(self.assignment)) + self.assertEventFired(self.emitter, 'device-assign:testclass') + self.assertEventNotFired(self.emitter, 'device-unassign:testclass') + + def test_033_assign_required_to_halted(self): + self.loop.run_until_complete(self.collection.assign(self.assignment)) + self.assertEventFired(self.emitter, 'device-assign:testclass') + self.assertEventNotFired(self.emitter, 'device-unassign:testclass') + + def test_034_unassign_from_halted(self): + self.assignment.required = False + self.loop.run_until_complete(self.collection.assign(self.assignment)) + self.loop.run_until_complete(self.collection.unassign(self.assignment)) + self.assertEventFired(self.emitter, 'device-assign:testclass') + self.assertEventFired(self.emitter, 'device-unassign:testclass') + + def test_035_unassign(self): + self.emitter.running = True + self.assignment.required = False + self.loop.run_until_complete(self.collection.assign(self.assignment)) + self.loop.run_until_complete(self.collection.unassign(self.assignment)) + self.assertEventFired(self.emitter, 'device-assign:testclass') + self.assertEventFired(self.emitter, 'device-unassign:testclass') + + def test_040_detach_required(self): + self.loop.run_until_complete(self.collection.assign(self.assignment)) + self.attach() + with self.assertRaises(qubes.exc.QubesVMNotHaltedError): + self.loop.run_until_complete( + self.collection.detach(self.assignment)) + + def test_041_detach_required_from_halted(self): + self.loop.run_until_complete(self.collection.assign(self.assignment)) + with self.assertRaises(LookupError): + self.loop.run_until_complete( + self.collection.detach(self.assignment)) + + def test_042_unassign_required(self): + self.loop.run_until_complete(self.collection.assign(self.assignment)) + self.emitter.running = True + with self.assertRaises(qubes.exc.QubesVMNotHaltedError): + self.loop.run_until_complete( + self.collection.unassign(self.assignment)) + + def test_043_detach_assigned(self): + self.assignment.required = False + self.loop.run_until_complete(self.collection.assign(self.assignment)) + self.attach() + self.loop.run_until_complete(self.collection.detach(self.assignment)) + self.assertEventFired(self.emitter, 'device-assign:testclass') + self.assertEventFired(self.emitter, 'device-pre-detach:testclass') + self.assertEventFired(self.emitter, 'device-detach:testclass') class TC_01_DeviceManager(qubes.tests.QubesTestCase): @@ -223,8 +361,9 @@ def test_001_missing(self): ident=device.ident, attach_automatically=True, required=True) self.loop.run_until_complete( - self.manager['testclass'].attach(assignment)) - self.assertEventFired(self.emitter, 'device-attach:testclass') + self.manager['testclass'].assign(assignment)) + self.assertEqual( + len(list(self.manager['testclass'].get_assigned_devices())), 1) class TC_02_DeviceInfo(qubes.tests.QubesTestCase): @@ -233,9 +372,6 @@ def setUp(self): self.app = TestApp() self.vm = TestVM(self.app, 'vm') - def test_001_init(self): - pass # TODO - def test_010_serialize(self): device = DeviceInfo( backend_domain=self.vm, @@ -246,41 +382,77 @@ def test_010_serialize(self): manufacturer="", name="Some untrusted garbage", serial=None, - interfaces=[DeviceCategory.Other, DeviceCategory.USB_HID], - # additional_info="", # TODO - # date="06.12.23", # TODO + interfaces=[qubes.devices.DeviceInterface(" ******"), + qubes.devices.DeviceInterface("u03**01")], + additional_info="", + date="06.12.23", + ) + actual = device.serialize() + expected = ( + b"manufacturer='unknown' self_identity='0000:0000::?******' " + b"serial='unknown' ident='1-1.1.1' product='Qubes' " + b"vendor='ITL' name='Some untrusted garbage' devclass='bus' " + b"backend_domain='vm' interfaces=' ******u03**01' " + b"_additional_info='' _date='06.12.23'") + expected = set(expected.replace(b"Some untrusted garbage", + b"Some_untrusted_garbage").split(b" ")) + actual = set(actual.replace(b"Some untrusted garbage", + b"Some_untrusted_garbage").split(b" ")) + self.assertEqual(actual, expected) + + def test_011_serialize_with_parent(self): + device = DeviceInfo( + backend_domain=self.vm, + ident="1-1.1.1", + devclass="bus", + vendor="ITL", + product="Qubes", + manufacturer="", + name="Some untrusted garbage", + serial=None, + interfaces=[qubes.devices.DeviceInterface(" ******"), + qubes.devices.DeviceInterface("u03**01")], + additional_info="", + date="06.12.23", + parent=qubes.devices.Device(self.vm, '1-1.1', 'pci') ) - actual = sorted(device.serialize().split(b' ')) - expected = [ - b'YmFja2VuZF9kb21haW49dm0=', b'ZGV2Y2xhc3M9YnVz', - b'aW50ZXJmYWNlcz0qKioqKiowMyoqKio=', b'aWRlbnQ9MS0xLjEuMQ==', - b'bWFudWZhY3R1cmVyPXVua25vd24=', - b'bmFtZT1Tb21lIHVudHJ1c3RlZCBnYXJiYWdl', - b'c2VyaWFsPXVua25vd24=', b'cHJvZHVjdD1RdWJlcw==', - b'dmVuZG9yPUlUTA=='] + actual = device.serialize() + expected = ( + b"manufacturer='unknown' self_identity='0000:0000::?******' " + b"serial='unknown' ident='1-1.1.1' product='Qubes' " + b"vendor='ITL' name='Some untrusted garbage' devclass='bus' " + b"backend_domain='vm' interfaces=' ******u03**01' " + b"_additional_info='' _date='06.12.23' " + b"parent_ident='1-1.1' parent_devclass='pci'") + expected = set(expected.replace(b"Some untrusted garbage", + b"Some_untrusted_garbage").split(b" ")) + actual = set(actual.replace(b"Some untrusted garbage", + b"Some_untrusted_garbage").split(b" ")) self.assertEqual(actual, expected) def test_020_deserialize(self): - expected = [ - b'YmFja2VuZF9kb21haW49dm0=', b'ZGV2Y2xhc3M9YnVz', - b'aW50ZXJmYWNlcz0qKioqKiowMyoqKio=', b'aWRlbnQ9MS0xLjEuMQ==', - b'bWFudWZhY3R1cmVyPXVua25vd24=', - b'bmFtZT1Tb21lIHVudHJ1c3RlZCBnYXJiYWdl', - b'c2VyaWFsPXVua25vd24=', b'cHJvZHVjdD1RdWJlcw==', - b'dmVuZG9yPUlUTA=='] - actual = DeviceInfo.deserialize(b' '.join(expected), self.vm) + serialized = ( + b"1-1.1.1 " + b"manufacturer='unknown' self_identity='0000:0000::?******' " + b"serial='unknown' ident='1-1.1.1' product='Qubes' " + b"vendor='ITL' name='Some untrusted garbage' devclass='bus' " + b"backend_domain='vm' interfaces=' ******u03**01' " + b"_additional_info='' _date='06.12.23' " + b"parent_ident='1-1.1' parent_devclass='None'") + actual = DeviceInfo.deserialize(serialized, self.vm) expected = DeviceInfo( backend_domain=self.vm, ident="1-1.1.1", devclass="bus", vendor="ITL", product="Qubes", - manufacturer="", + manufacturer="unknown", name="Some untrusted garbage", serial=None, - interfaces=[DeviceCategory.Other, DeviceCategory.USB_HID], - # additional_info="", # TODO - # date="06.12.23", # TODO + interfaces=[qubes.devices.DeviceInterface(" ******"), + qubes.devices.DeviceInterface("u03**01")], + additional_info="", + date="06.12.23", ) self.assertEqual(actual.backend_domain, expected.backend_domain) @@ -291,6 +463,6 @@ def test_020_deserialize(self): self.assertEqual(actual.manufacturer, expected.manufacturer) self.assertEqual(actual.name, expected.name) self.assertEqual(actual.serial, expected.serial) - self.assertEqual(actual.interfaces, expected.interfaces) - # self.assertEqual(actual.data, expected.data) # TODO - + self.assertEqual(repr(actual.interfaces), repr(expected.interfaces)) + self.assertEqual(actual.self_identity, expected.self_identity) + self.assertEqual(actual.data, expected.data) diff --git a/qubes/tests/devices_block.py b/qubes/tests/devices_block.py index 1c495fb0c..b0fd06c57 100644 --- a/qubes/tests/devices_block.py +++ b/qubes/tests/devices_block.py @@ -115,8 +115,11 @@ def __init__(self): self.domains = {} -class TestVM(object): - def __init__(self, qdb, domain_xml=None, running=True, name='test-vm'): +class TestVM(qubes.tests.TestEmitter): + def __init__( + self, qdb, domain_xml=None, running=True, name='test-vm', + *args, **kwargs): + super(TestVM, self).__init__(*args, **kwargs) self.name = name self.untrusted_qdb = TestQubesDB(qdb) self.libvirt_domain = mock.Mock() @@ -132,6 +135,9 @@ def __init__(self, qdb, domain_xml=None, running=True, name='test-vm'): self.libvirt_domain.configure_mock(**{ 'XMLDesc.return_value': domain_xml }) + self.devices = { + 'testclass': qubes.devices.DeviceCollection(self, 'testclass') + } def __eq__(self, other): if isinstance(other, TestVM): @@ -147,7 +153,7 @@ def setUp(self): def test_000_device_get(self): vm = TestVM({ '/qubes-block-devices/sda': b'', - '/qubes-block-devices/sda/desc': b'Test device', + '/qubes-block-devices/sda/desc': b'Test_ (device)', '/qubes-block-devices/sda/size': b'1024000', '/qubes-block-devices/sda/mode': b'w', }) @@ -155,8 +161,10 @@ def test_000_device_get(self): self.assertIsInstance(device_info, qubes.ext.block.BlockDevice) self.assertEqual(device_info.backend_domain, vm) self.assertEqual(device_info.ident, 'sda') - self.assertEqual(device_info.name, 'Test device') - self.assertEqual(device_info._name, 'Test device') + self.assertEqual(device_info.name, 'device') + self.assertEqual(device_info._name, 'device') + self.assertEqual(device_info.serial, 'Test') + self.assertEqual(device_info._serial, 'Test') self.assertEqual(device_info.size, 1024000) self.assertEqual(device_info.mode, 'w') self.assertEqual( @@ -166,7 +174,7 @@ def test_000_device_get(self): def test_001_device_get_other_node(self): vm = TestVM({ '/qubes-block-devices/mapper_dmroot': b'', - '/qubes-block-devices/mapper_dmroot/desc': b'Test device', + '/qubes-block-devices/mapper_dmroot/desc': b'Test_device', '/qubes-block-devices/mapper_dmroot/size': b'1024000', '/qubes-block-devices/mapper_dmroot/mode': b'w', }) @@ -174,8 +182,10 @@ def test_001_device_get_other_node(self): self.assertIsInstance(device_info, qubes.ext.block.BlockDevice) self.assertEqual(device_info.backend_domain, vm) self.assertEqual(device_info.ident, 'mapper_dmroot') - self.assertEqual(device_info.name, 'Test device') - self.assertEqual(device_info._name, 'Test device') + self.assertEqual(device_info._name, None) + self.assertEqual(device_info.name, 'unknown') + self.assertEqual(device_info.serial, 'Test device') + self.assertEqual(device_info._serial, 'Test device') self.assertEqual(device_info.size, 1024000) self.assertEqual(device_info.mode, 'w') self.assertEqual( @@ -185,12 +195,13 @@ def test_001_device_get_other_node(self): def test_002_device_get_invalid_desc(self): vm = TestVM({ '/qubes-block-devices/sda': b'', - '/qubes-block-devices/sda/desc': b'Test device<>za\xc4\x87abc', + '/qubes-block-devices/sda/desc': b'Test (device<>za\xc4\x87abc)', '/qubes-block-devices/sda/size': b'1024000', '/qubes-block-devices/sda/mode': b'w', }) device_info = self.ext.device_get(vm, 'sda') - self.assertEqual(device_info.name, 'Test device__za__abc') + self.assertEqual(device_info.serial, 'Test') + self.assertEqual(device_info.name, 'device zaabc') def test_003_device_get_invalid_size(self): vm = TestVM({ @@ -227,11 +238,11 @@ def test_005_device_get_none(self): def test_010_devices_list(self): vm = TestVM({ '/qubes-block-devices/sda': b'', - '/qubes-block-devices/sda/desc': b'Test device', + '/qubes-block-devices/sda/desc': b'Test_device', '/qubes-block-devices/sda/size': b'1024000', '/qubes-block-devices/sda/mode': b'w', '/qubes-block-devices/sdb': b'', - '/qubes-block-devices/sdb/desc': b'Test device2', + '/qubes-block-devices/sdb/desc': b'Test_device (2)', '/qubes-block-devices/sdb/size': b'2048000', '/qubes-block-devices/sdb/mode': b'r', }) @@ -239,12 +250,14 @@ def test_010_devices_list(self): self.assertEqual(len(devices), 2) self.assertEqual(devices[0].backend_domain, vm) self.assertEqual(devices[0].ident, 'sda') - self.assertEqual(devices[0].description, 'Test device') + self.assertEqual(devices[0].serial, 'Test device') + self.assertEqual(devices[0].name, 'unknown') self.assertEqual(devices[0].size, 1024000) self.assertEqual(devices[0].mode, 'w') self.assertEqual(devices[1].backend_domain, vm) self.assertEqual(devices[1].ident, 'sdb') - self.assertEqual(devices[1].description, 'Test device2') + self.assertEqual(devices[1].serial, 'Test device') + self.assertEqual(devices[1].name, '2') self.assertEqual(devices[1].size, 2048000) self.assertEqual(devices[1].mode, 'r') @@ -585,27 +598,3 @@ def test_051_detach_not_attached(self): dev = qubes.ext.block.BlockDevice(back_vm, 'sda') self.ext.on_device_pre_detached_block(vm, '', dev) self.assertFalse(vm.libvirt_domain.detachDevice.called) - - def test_060_devices_added(self): - vm = TestVM({ - '/qubes-block-devices/sda': b'', - '/qubes-block-devices/sda/desc': b'Test device', - '/qubes-block-devices/sda/size': b'1024000', - '/qubes-block-devices/sda/mode': b'w', - '/qubes-block-devices/sdb': b'', - '/qubes-block-devices/sdb/desc': b'Test device2', - '/qubes-block-devices/sdb/size': b'2048000', - '/qubes-block-devices/sdb/mode': b'r', - }) - devices = sorted(list(self.ext.on_qdb_change(vm, ''))) - self.assertEqual(len(devices), 2) - self.assertEqual(devices[0].backend_domain, vm) - self.assertEqual(devices[0].ident, 'sda') - self.assertEqual(devices[0].description, 'Test device') - self.assertEqual(devices[0].size, 1024000) - self.assertEqual(devices[0].mode, 'w') - self.assertEqual(devices[1].backend_domain, vm) - self.assertEqual(devices[1].ident, 'sdb') - self.assertEqual(devices[1].description, 'Test device2') - self.assertEqual(devices[1].size, 2048000) - self.assertEqual(devices[1].mode, 'r') From 39e9408bf08d344abd1523c2333a7a8e955a65a5 Mon Sep 17 00:00:00 2001 From: Piotr Bartman-Szwarc Date: Fri, 15 Mar 2024 18:03:17 +0100 Subject: [PATCH 18/49] q-dev: typo in doc/qubes-devices.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Marek Marczykowski-Górecki --- doc/qubes-devices.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/qubes-devices.rst b/doc/qubes-devices.rst index 5879a077d..7e71ab244 100644 --- a/doc/qubes-devices.rst +++ b/doc/qubes-devices.rst @@ -73,7 +73,7 @@ and could be manually detach any time (see 4.), but in the future will be auto-attached again. #. `(True, True, True)` -> domain is running, device is attached and couldn't be detached. -#. `(False, Ture, False)` -> device is assigned to domain, but not attached +#. `(False, True, False)` -> device is assigned to domain, but not attached because either (i) domain is halted, device (ii) manually detached or (iii) attach to different domain. #. `(False, True, True)` -> domain is halted, device assigned to domain From 6e2faf0bf591dd0780dc460886aa21ad32bd664e Mon Sep 17 00:00:00 2001 From: Piotr Bartman-Szwarc Date: Sun, 17 Mar 2024 13:07:16 +0100 Subject: [PATCH 19/49] q-dev: fix path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Marek Marczykowski-Górecki --- qubes/ext/pci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubes/ext/pci.py b/qubes/ext/pci.py index 787bc9096..ce5cb0dba 100644 --- a/qubes/ext/pci.py +++ b/qubes/ext/pci.py @@ -189,7 +189,7 @@ def __init__(self, backend_domain, ident, libvirt_name=None): @property def vendor(self) -> str: """ - Device vendor from local database `/usr/share/hwdata/usb.ids` + Device vendor from local database `/usr/share/hwdata/pci.ids` Could be empty string or "unknown". From 28f178900a4a60f4656264c7b4f82a15303e482f Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Thu, 22 Feb 2024 00:55:09 +0100 Subject: [PATCH 20/49] q-dev: fix block device events To properly report attachments or detachments of block devices, the logic has been moved to the extension. As a bonus, the following issue is fix: https://github.com/QubesOS/qubes-issues/issues/4692 --- qubes/ext/block.py | 227 ++++++++++++++++++++++++++++++-------- templates/libvirt/xen.xml | 5 - 2 files changed, 184 insertions(+), 48 deletions(-) diff --git a/qubes/ext/block.py b/qubes/ext/block.py index 318d2a920..8622fb248 100644 --- a/qubes/ext/block.py +++ b/qubes/ext/block.py @@ -4,6 +4,7 @@ # # Copyright (C) 2017 Marek Marczykowski-Górecki # +# Copyright (C) 2024 Piotr Bartman-Szwarc # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -189,10 +190,41 @@ def parent_device(self) -> Optional[qubes.devices.Device]: self._interface_num = interface_num return self._parent + @property + def attachment(self) -> Optional['qubes.vm.BaseVM']: + """ + Warning: this property is time-consuming, do not run in loop! + """ + if not self.backend_domain.is_running(): + return None + for vm in self.backend_domain.app.domains: + if not vm.is_running(): + continue + if self._is_attached_to(vm): + return vm + + return None + + def _is_attached_to(self, vm): + xml_desc = lxml.etree.fromstring(vm.libvirt_domain.XMLDesc()) + + for disk in xml_desc.findall('devices/disk'): + info = _try_get_block_device_info(vm.app, disk) + if not info: + continue + backend_domain, ident = info + + if backend_domain.name != self.backend_domain.name: + continue + + if self.ident == ident: + return True + return False + @property def self_identity(self) -> str: """ - Get identification of device not related to port. + Get identification of a device not related to port. """ parent_identity = '' p = self.parent_device @@ -231,6 +263,32 @@ def _sanitize( ) +def _try_get_block_device_info(app, disk): + if disk.get('type') != 'block': + return None + dev_path_node = disk.find('source') + if dev_path_node is None: + return None + + backend_domain_node = disk.find('backenddomain') + if backend_domain_node is not None: + backend_domain = app.domains[ + backend_domain_node.get('name')] + else: + backend_domain = app.domains[0] + + dev_path = dev_path_node.get('dev') + + if dev_path.startswith('/dev/'): + ident = dev_path[len('/dev/'):] + else: + ident = dev_path + + ident = ident.replace('/', '_') + + return backend_domain, ident + + class BlockDeviceExtension(qubes.ext.Extension): def __init__(self): super().__init__() @@ -245,28 +303,62 @@ def on_domain_init_load(self, vm, event): # avoid building a cache on domain-init, as it isn't fully set yet, # and definitely isn't running yet current_devices = { - dev.ident: dev.attachment + dev.ident: dev.attachment # TODO load all attachments at once for dev in self.on_device_list_block(vm, None) } - self.devices_cache[vm.name] = current_devices + self.devices_cache[vm.name] = current_devices.copy() else: - self.devices_cache[vm.name] = {} + self.devices_cache[vm.name] = {}.copy() @qubes.ext.handler('domain-qdb-change:/qubes-block-devices') def on_qdb_change(self, vm, event, path): - """A change in QubesDB means a change in device list.""" + """A change in QubesDB means a change in a device list.""" # pylint: disable=unused-argument - vm.fire_event('device-list-change:block') - current_devices = dict((dev.ident, dev.attachment) - for dev in self.on_device_list_block(vm, None)) + if path is not None: + vm.fire_event('device-list-change:block') + device_attachments = self.get_device_attachments(vm) + current_devices = dict( + (dev.ident, device_attachments.get(dev.ident, None)) + for dev in self.on_device_list_block(vm, None)) + + added, attached, detached, removed = ( + self._compare_cache(vm, current_devices)) # send events about devices detached/attached outside by themselves - # (like device pulled out or manual qubes.USB qrexec call) + for dev_id, front_vm in detached.items(): + dev = BlockDevice(vm, dev_id) + asyncio.ensure_future(front_vm.fire_event_async( + 'device-detach:block', device=dev)) + for dev_id in removed: + device = BlockDevice(vm, dev_id) + vm.fire_event('device-removed:block', device=device) + for dev_id in added: + device = BlockDevice(vm, dev_id) + vm.fire_event('device-added:block', device=device) + for dev_ident, front_vm in attached.items(): + dev = BlockDevice(vm, dev_ident) + asyncio.ensure_future(front_vm.fire_event_async( + 'device-attach:block', device=dev, options={})) + + self.devices_cache[vm.name] = current_devices.copy() + + for front_vm in vm.app.domains: + if not front_vm.is_running(): + continue + for assignment in front_vm.devices['block'].get_assigned_devices(): + if (assignment.backend_domain == vm + and assignment.ident in added + and assignment.ident not in attached + ): + asyncio.ensure_future(self._attach_and_notify( + front_vm, assignment.device, assignment.options)) + + def _compare_cache(self, vm, current_devices): # compare cached devices and current devices, collect: # - newly appeared devices (ident) # - devices attached from a vm to frontend vm (ident: frontend_vm) # - devices detached from frontend vm (ident: frontend_vm) - # - disappeared devices, e.g. plugged out (ident) + # - disappeared devices, e.g., plugged out (ident) added = set() attached = dict() detached = dict() @@ -284,8 +376,8 @@ def on_qdb_change(self, vm, event, path): elif cached_front is None: attached[dev_id] = front_vm else: - # front changed from one to another, so we signal it as: - # detach from first one and attach to the second one. + # a front changed from one to another, so we signal it as: + # detach from the first one and attach to the second one. detached[dev_id] = cached_front attached[dev_id] = front_vm @@ -294,41 +386,33 @@ def on_qdb_change(self, vm, event, path): removed.add(dev_id) if cached_front is not None: detached[dev_id] = cached_front + return added, attached, detached, removed - for dev_id, front_vm in detached.items(): - dev = BlockDevice(vm, dev_id) - asyncio.ensure_future(front_vm.fire_event_async( - 'device-detach:block', device=dev)) - for dev_id in removed: - device = BlockDevice(vm, dev_id) - vm.fire_event('device-removed:block', device=device) - for dev_id in added: - device = BlockDevice(vm, dev_id) - vm.fire_event('device-added:block', device=device) - for dev_ident, front_vm in attached.items(): - dev = BlockDevice(vm, dev_ident) - asyncio.ensure_future(front_vm.fire_event_async( - 'device-attach:block', device=dev, options={})) + def get_device_attachments(self, vm_): + result = {} + for vm in vm_.app.domains: + if not vm.is_running(): + continue - self.devices_cache[vm.name] = current_devices + xml_desc = lxml.etree.fromstring(vm.libvirt_domain.XMLDesc()) - for front_vm in vm.app.domains: - if not front_vm.is_running(): - continue - for assignment in front_vm.devices['block'].get_assigned_devices(): - if (assignment.backend_domain == vm - and assignment.ident in added - and assignment.ident not in attached - ): - asyncio.ensure_future(self._attach_and_notify( - front_vm, assignment.device, assignment.options)) + for disk in xml_desc.findall('devices/disk'): + info = _try_get_block_device_info(vm.app, disk) + if not info: + continue + _backend_domain, ident = info + + result[ident] = vm + return result def device_get(self, vm, ident): - '''Read information about device from QubesDB + """ + Read information about a device from QubesDB :param vm: backend VM object :param ident: device identifier - :returns BlockDevice''' + :returns BlockDevice + """ untrusted_qubes_device_attrs = vm.untrusted_qdb.list( '/qubes-block-devices/{}/'.format(ident)) @@ -446,7 +530,7 @@ def on_device_pre_attached_block(self, vm, event, device, options): if isinstance(device, qubes.devices.UnknownDevice): print(f'{device.devclass.capitalize()} device {device} ' 'not available, skipping.', file=sys.stderr) - return + raise qubes.devices.UnrecognizedDevice() # validate options for option, value in options.items(): @@ -482,17 +566,34 @@ def on_device_pre_attached_block(self, vm, event, device, options): 'This device can be attached only read-only') if not vm.is_running(): + print("Not attach, not running", file=sys.stderr) + return + + if not isinstance(device, BlockDevice): + print("The device is not recognized as block device, " + f"skipping attachment of {device}", + file=sys.stderr) return + if device.attachment: + raise qubes.devices.DeviceAlreadyAttached( + 'Device {!s} already attached to {!s}'.format( + device, device.attachment) + ) + if not device.backend_domain.is_running(): - raise qubes.exc.QubesVMNotRunningError(device.backend_domain, - 'Domain {} needs to be running to attach device from ' - 'it'.format(device.backend_domain.name)) + raise qubes.exc.QubesVMNotRunningError( + device.backend_domain, + f'Domain {device.backend_domain.name} needs to be running ' + f'to attach device from it') + + self.devices_cache[device.backend_domain.name][device.ident] = vm if 'frontend-dev' not in options: options['frontend-dev'] = self.find_unused_frontend( vm, options.get('devtype', 'disk')) + print(f'attaching {device} exposed by {device.backend_domain.name}') vm.libvirt_domain.attachDevice( vm.app.env.get_template('libvirt/devices/block.xml').render( device=device, vm=vm, options=options)) @@ -514,6 +615,44 @@ async def _attach_and_notify(self, vm, device, options): await vm.fire_event_async( 'device-attach:block', device=device, options=options) + @qubes.ext.handler('domain-shutdown') + async def on_domain_shutdown(self, vm, event, **_kwargs): + """ + Remove from cache devices attached to or exposed by the vm. + """ + # pylint: disable=unused-argument + + new_cache = {} + for domain in vm.app.domains.values(): + new_cache[domain.name] = {} + if domain == vm: + for dev_id, front_vm in self.devices_cache[domain.name].items(): + if front_vm is None: + continue + dev = BlockDevice(vm, dev_id) + await self._detach_and_notify(vm, dev, options=None) + continue + for dev_id, front_vm in self.devices_cache[domain.name].items(): + if front_vm == vm: + dev = BlockDevice(vm, dev_id) + asyncio.ensure_future(front_vm.fire_event_async( + 'device-detach:block', device=dev)) + else: + new_cache[domain.name][dev_id] = front_vm + self.devices_cache = new_cache.copy() + + async def _detach_and_notify(self, vm, device, options): + # bypass DeviceCollection logic preventing double attach + self.on_device_pre_detached_block( + vm, 'device-pre-detach:block', device) + await vm.fire_event_async( + 'device-detach:block', device=device, options=options) + + @qubes.ext.handler('qubes-close', system=True) + def on_qubes_close(self, app, event): + # pylint: disable=unused-argument + self.devices_cache.clear() + @qubes.ext.handler('device-pre-detach:block') def on_device_pre_detached_block(self, vm, event, device): # pylint: disable=unused-argument @@ -524,6 +663,8 @@ def on_device_pre_detached_block(self, vm, event, device): # least) for attached_device, options in self.on_device_list_attached(vm, event): if attached_device == device: + self.devices_cache[device.backend_domain.name][ + device.ident] = None vm.libvirt_domain.detachDevice( vm.app.env.get_template('libvirt/devices/block.xml').render( device=device, vm=vm, options=options)) diff --git a/templates/libvirt/xen.xml b/templates/libvirt/xen.xml index 95b59a0b6..155f82e46 100644 --- a/templates/libvirt/xen.xml +++ b/templates/libvirt/xen.xml @@ -156,11 +156,6 @@ {# start external devices from xvdi #} {% set counter = {'i': 4} %} - {% for assignment in vm.devices.block.get_assigned_devices(True) %} - {% set device = assignment.device %} - {% set options = assignment.options %} - {% include 'libvirt/devices/block.xml' %} - {% endfor %} {% if vm.netvm %} {% include 'libvirt/devices/net.xml' with context %} From 9796c578071c97f44712f6a00ed02288d4ae33bd Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Tue, 19 Mar 2024 11:04:50 +0100 Subject: [PATCH 21/49] q-dev: extract ext/utils --- qubes/api/admin.py | 4 +- qubes/devices.py | 180 +++++++++++++++++++------------------ qubes/ext/block.py | 97 ++++---------------- qubes/ext/utils.py | 101 +++++++++++++++++++++ rpm_spec/core-dom0.spec.in | 1 + 5 files changed, 215 insertions(+), 168 deletions(-) create mode 100644 qubes/ext/utils.py diff --git a/qubes/api/admin.py b/qubes/api/admin.py index c796dbd4a..0a79264bb 100644 --- a/qubes/api/admin.py +++ b/qubes/api/admin.py @@ -1381,10 +1381,10 @@ async def vm_device_detach(self, endpoint): scope='local', write=True) async def vm_device_set_assignment(self, endpoint, untrusted_payload): """ - Update assignment of already attached device. + Update assignment of an already attached device. Payload: - `None` -> unassign device from qube + `None` -> unassign device from a qube `False` -> device will be auto-attached to qube `True` -> device is required to start qube """ diff --git a/qubes/devices.py b/qubes/devices.py index 6b269950c..3d4960ab6 100644 --- a/qubes/devices.py +++ b/qubes/devices.py @@ -535,9 +535,11 @@ def deserialize( expected_backend_domain: 'qubes.vm.BaseVM', expected_devclass: Optional[str] = None, ) -> 'DeviceInfo': + ident, _, rest = serialization.partition(b' ') + ident = ident.decode('ascii', errors='ignore') try: result = DeviceInfo._deserialize( - cls, serialization, expected_backend_domain, expected_devclass) + cls, rest, expected_backend_domain, ident, expected_devclass) except Exception as exc: print(exc, file=sys.stderr) ident = serialization.split(b' ')[0].decode( @@ -552,37 +554,25 @@ def deserialize( @staticmethod def _deserialize( cls: Type, - serialization: bytes, + untrusted_serialization: bytes, expected_backend_domain: 'qubes.vm.BaseVM', + expected_ident: str, expected_devclass: Optional[str] = None, ) -> 'DeviceInfo': - decoded = serialization.decode('ascii', errors='ignore') - ident, _, rest = decoded.partition(' ') - keys = [] - values = [] - key, _, rest = rest.partition("='") - keys.append(key) - while "='" in rest: - value_key, _, rest = rest.partition("='") - value, _, key = value_key.rpartition("' ") - values.append(deserialize_str(value)) - keys.append(key) - value = rest[:-1] # ending ' - values.append(deserialize_str(value)) - - properties = dict() - for key, value in zip(keys, values): - if key.startswith("_"): - # it's handled in cls.__init__ - properties[key[1:]] = value - else: - properties[key] = value - - if properties['backend_domain'] != expected_backend_domain.name: - raise UnexpectedDeviceProperty( - f"Got device exposed by {properties['backend_domain']}" - f"when expected devices from {expected_backend_domain.name}.") - properties['backend_domain'] = expected_backend_domain + allowed_chars_key = string.digits + string.ascii_letters + '-_.' + allowed_chars_value = ( + allowed_chars_key + ',+:' + string.punctuation + ' ') + + properties, options = unpack_properties( + untrusted_serialization, allowed_chars_key, allowed_chars_value) + properties.update(options) + + check_device_properties( + expected_backend_domain, + expected_ident, + expected_devclass, + properties + ) if 'attachment' not in properties or not properties['attachment']: properties['attachment'] = None @@ -596,11 +586,6 @@ def _deserialize( f"Got {properties['devclass']} device " f"when expected {expected_devclass}.") - if properties["ident"] != ident: - raise UnexpectedDeviceProperty( - f"Got device with id: {properties['ident']} " - f"when expected id: {ident}.") - interfaces = properties['interfaces'] interfaces = [ DeviceInterface(interfaces[i:i + 7]) @@ -672,6 +657,71 @@ def sanitize_str( return result +def unpack_properties( + untrusted_serialization: bytes, + allowed_chars_key: str, + allowed_chars_value: str +): + ut_decoded = untrusted_serialization.decode( + 'ascii', errors='strict').strip() + + options = {} + keys = [] + values = [] + ut_key, _, ut_rest = ut_decoded.partition("='") + + key = sanitize_str( + ut_key, allowed_chars_key, + error_message='Invalid chars in property name') + keys.append(key) + while "='" in ut_rest: + ut_value_key, _, ut_rest = ut_rest.partition("='") + ut_value, _, ut_key = ut_value_key.rpartition("' ") + value = sanitize_str( + deserialize_str(ut_value), allowed_chars_value, + error_message='Invalid chars in property value') + values.append(value) + key = sanitize_str( + ut_key, allowed_chars_key, + error_message='Invalid chars in property name') + keys.append(key) + ut_value = ut_rest[:-1] # ending ' + value = sanitize_str( + deserialize_str(ut_value), allowed_chars_value, + error_message='Invalid chars in property value') + values.append(value) + + properties = dict() + for key, value in zip(keys, values): + if key.startswith("_"): + # it's handled in cls.__init__ + options[key[1:]] = value + else: + properties[key] = value + + return properties, options + + +def check_device_properties( + expected_backend_domain, expected_ident, expected_devclass, properties +): + if properties['backend_domain'] != expected_backend_domain.name: + raise UnexpectedDeviceProperty( + f"Got device exposed by {properties['backend_domain']}" + f"when expected devices from {expected_backend_domain.name}.") + properties['backend_domain'] = expected_backend_domain + + if properties['ident'] != expected_ident: + raise UnexpectedDeviceProperty( + f"Got device with id: {properties['ident']} " + f"when expected id: {expected_ident}.") + + if expected_devclass and properties['devclass'] != expected_devclass: + raise UnexpectedDeviceProperty( + f"Got {properties['devclass']} device " + f"when expected {expected_devclass}.") + + class UnknownDevice(DeviceInfo): # pylint: disable=too-few-public-methods """Unknown device - for example exposed by domain not running currently""" @@ -741,13 +791,13 @@ def device(self) -> DeviceInfo: return self.backend_domain.devices[self.devclass][self.ident] @property - def frontend_domain(self) -> Optional['qubes.vm.qubesvm.QubesVM']: + def frontend_domain(self) -> Optional['qubes.vm.BaseVM']: """ Which domain the device is attached/assigned to. """ return self.__frontend_domain @frontend_domain.setter def frontend_domain( - self, frontend_domain: Optional[Union[str, 'qubes.vm.qubesvm.QubesVM']] + self, frontend_domain: Optional[Union[str, 'qubes.vm.BaseVM']] ): """ Which domain the device is attached/assigned to. """ if isinstance(frontend_domain, str): @@ -852,61 +902,19 @@ def _deserialize( expected_ident: str, expected_devclass: Optional[str] = None, ) -> 'DeviceAssignment': - options = {} allowed_chars_key = string.digits + string.ascii_letters + '-_.' allowed_chars_value = allowed_chars_key + ',+:' - untrusted_decoded = untrusted_serialization.decode( - 'ascii', 'strict').strip() - keys = [] - values = [] - untrusted_key, _, untrusted_rest = untrusted_decoded.partition("='") - - key = sanitize_str( - untrusted_key, allowed_chars_key, - error_message='Invalid chars in property name') - keys.append(key) - while "='" in untrusted_rest: - ut_value_key, _, untrusted_rest = untrusted_rest.partition("='") - untrusted_value, _, untrusted_key = ut_value_key.rpartition("' ") - value = sanitize_str( - deserialize_str(untrusted_value), allowed_chars_value, - error_message='Invalid chars in property value') - values.append(value) - key = sanitize_str( - untrusted_key, allowed_chars_key, - error_message='Invalid chars in property name') - keys.append(key) - untrusted_value = untrusted_rest[:-1] # ending ' - value = sanitize_str( - deserialize_str(untrusted_value), allowed_chars_value, - error_message='Invalid chars in property value') - values.append(value) - - properties = dict() - for key, value in zip(keys, values): - if key.startswith("_"): - options[key[1:]] = value - else: - properties[key] = value - + properties, options = unpack_properties( + untrusted_serialization, allowed_chars_key, allowed_chars_value) properties['options'] = options - if properties['backend_domain'] != expected_backend_domain.name: - raise UnexpectedDeviceProperty( - f"Got device exposed by {properties['backend_domain']} " - f"when expected devices from {expected_backend_domain.name}.") - properties['backend_domain'] = expected_backend_domain - - if properties["ident"] != expected_ident: - raise UnexpectedDeviceProperty( - f"Got device with id: {properties['ident']} " - f"when expected id: {expected_ident}.") - - if expected_devclass and properties['devclass'] != expected_devclass: - raise UnexpectedDeviceProperty( - f"Got {properties['devclass']} device " - f"when expected {expected_devclass}.") + check_device_properties( + expected_backend_domain, + expected_ident, + expected_devclass, + properties + ) properties['attach_automatically'] = qubes.property.bool( None, None, properties['attach_automatically']) diff --git a/qubes/ext/block.py b/qubes/ext/block.py index 8622fb248..2e65837c2 100644 --- a/qubes/ext/block.py +++ b/qubes/ext/block.py @@ -31,6 +31,7 @@ import qubes.devices import qubes.ext +from qubes.ext.utils import device_list_change name_re = re.compile(r"\A[a-z0-9-]{1,12}\Z") device_re = re.compile(r"\A[a-z0-9/-]{1,64}\Z") @@ -302,10 +303,10 @@ def on_domain_init_load(self, vm, event): if event == 'domain-load': # avoid building a cache on domain-init, as it isn't fully set yet, # and definitely isn't running yet - current_devices = { - dev.ident: dev.attachment # TODO load all attachments at once - for dev in self.on_device_list_block(vm, None) - } + device_attachments = self.get_device_attachments(vm) + current_devices = dict( + (dev.ident, device_attachments.get(dev.ident, None)) + for dev in self.on_device_list_block(vm, None)) self.devices_cache[vm.name] = current_devices.copy() else: self.devices_cache[vm.name] = {}.copy() @@ -314,81 +315,14 @@ def on_domain_init_load(self, vm, event): def on_qdb_change(self, vm, event, path): """A change in QubesDB means a change in a device list.""" # pylint: disable=unused-argument - if path is not None: - vm.fire_event('device-list-change:block') device_attachments = self.get_device_attachments(vm) current_devices = dict( (dev.ident, device_attachments.get(dev.ident, None)) for dev in self.on_device_list_block(vm, None)) + device_list_change(self, current_devices, vm, path, BlockDevice) - added, attached, detached, removed = ( - self._compare_cache(vm, current_devices)) - - # send events about devices detached/attached outside by themselves - for dev_id, front_vm in detached.items(): - dev = BlockDevice(vm, dev_id) - asyncio.ensure_future(front_vm.fire_event_async( - 'device-detach:block', device=dev)) - for dev_id in removed: - device = BlockDevice(vm, dev_id) - vm.fire_event('device-removed:block', device=device) - for dev_id in added: - device = BlockDevice(vm, dev_id) - vm.fire_event('device-added:block', device=device) - for dev_ident, front_vm in attached.items(): - dev = BlockDevice(vm, dev_ident) - asyncio.ensure_future(front_vm.fire_event_async( - 'device-attach:block', device=dev, options={})) - - self.devices_cache[vm.name] = current_devices.copy() - - for front_vm in vm.app.domains: - if not front_vm.is_running(): - continue - for assignment in front_vm.devices['block'].get_assigned_devices(): - if (assignment.backend_domain == vm - and assignment.ident in added - and assignment.ident not in attached - ): - asyncio.ensure_future(self._attach_and_notify( - front_vm, assignment.device, assignment.options)) - - def _compare_cache(self, vm, current_devices): - # compare cached devices and current devices, collect: - # - newly appeared devices (ident) - # - devices attached from a vm to frontend vm (ident: frontend_vm) - # - devices detached from frontend vm (ident: frontend_vm) - # - disappeared devices, e.g., plugged out (ident) - added = set() - attached = dict() - detached = dict() - removed = set() - cache = self.devices_cache[vm.name] - for dev_id, front_vm in current_devices.items(): - if dev_id not in cache: - added.add(dev_id) - if front_vm is not None: - attached[dev_id] = front_vm - elif cache[dev_id] != front_vm: - cached_front = cache[dev_id] - if front_vm is None: - detached[dev_id] = cached_front - elif cached_front is None: - attached[dev_id] = front_vm - else: - # a front changed from one to another, so we signal it as: - # detach from the first one and attach to the second one. - detached[dev_id] = cached_front - attached[dev_id] = front_vm - - for dev_id, cached_front in cache.items(): - if dev_id not in current_devices: - removed.add(dev_id) - if cached_front is not None: - detached[dev_id] = cached_front - return added, attached, detached, removed - - def get_device_attachments(self, vm_): + @staticmethod + def get_device_attachments(vm_): result = {} for vm in vm_.app.domains: if not vm.is_running(): @@ -405,7 +339,8 @@ def get_device_attachments(self, vm_): result[ident] = vm return result - def device_get(self, vm, ident): + @staticmethod + def device_get(vm, ident): """ Read information about a device from QubesDB @@ -506,9 +441,11 @@ def on_device_list_attached(self, vm, event, **kwargs): yield (BlockDevice(backend_domain, ident), options) - def find_unused_frontend(self, vm, devtype='disk'): - '''Find unused block frontend device node for - parameter''' + @staticmethod + def find_unused_frontend(vm, devtype='disk'): + """ + Find unused block frontend device node for parameter + """ assert vm.is_running() xml = vm.libvirt_domain.XMLDesc() @@ -602,10 +539,10 @@ def on_device_pre_attached_block(self, vm, event, device, options): async def on_domain_start(self, vm, _event, **_kwargs): # pylint: disable=unused-argument for assignment in vm.devices['block'].get_assigned_devices(): - asyncio.ensure_future(self._attach_and_notify( + asyncio.ensure_future(self.attach_and_notify( vm, assignment.device, assignment.options)) - async def _attach_and_notify(self, vm, device, options): + async def attach_and_notify(self, vm, device, options): # bypass DeviceCollection logic preventing double attach try: self.on_device_pre_attached_block( diff --git a/qubes/ext/utils.py b/qubes/ext/utils.py new file mode 100644 index 000000000..592d2d9ff --- /dev/null +++ b/qubes/ext/utils.py @@ -0,0 +1,101 @@ +# coding=utf-8 +# +# The Qubes OS Project, https://www.qubes-os.org +# +# Copyright (C) 2023 Piotr Bartman-Szwarc +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. +import asyncio + +import qubes + + +def device_list_change( + ext: qubes.ext.Extension, current_devices, + vm, path, device_class: qubes.devices.DeviceInfo +): + devclass = device_class.__name__[:-len('Device')].lower() + + if path is not None: + vm.fire_event(f'device-list-change:{devclass}') + + added, attached, detached, removed = ( + compare_device_cache(vm, ext.devices_cache, current_devices)) + + # send events about devices detached/attached outside by themselves + for dev_id, front_vm in detached.items(): + dev = device_class(vm, dev_id) + asyncio.ensure_future(front_vm.fire_event_async( + f'device-detach:{devclass}', device=dev)) + for dev_id in removed: + device = device_class(vm, dev_id) + vm.fire_event(f'device-removed:{devclass}', device=device) + for dev_id in added: + device = device_class(vm, dev_id) + vm.fire_event(f'device-added:{devclass}', device=device) + for dev_ident, front_vm in attached.items(): + dev = device_class(vm, dev_ident) + asyncio.ensure_future(front_vm.fire_event_async( + f'device-attach:{devclass}', device=dev, options={})) + + ext.devices_cache[vm.name] = current_devices + + for front_vm in vm.app.domains: + if not front_vm.is_running(): + continue + for assignment in front_vm.devices[devclass].get_assigned_devices(): + if (assignment.backend_domain == vm + and assignment.ident in added + and assignment.ident not in attached + ): + asyncio.ensure_future(ext.attach_and_notify( + front_vm, assignment.device, assignment.options)) + + +def compare_device_cache(vm, devices_cache, current_devices): + # compare cached devices and current devices, collect: + # - newly appeared devices (ident) + # - devices attached from a vm to frontend vm (ident: frontend_vm) + # - devices detached from frontend vm (ident: frontend_vm) + # - disappeared devices, e.g., plugged out (ident) + added = set() + attached = dict() + detached = dict() + removed = set() + cache = devices_cache[vm.name] + for dev_id, front_vm in current_devices.items(): + if dev_id not in cache: + added.add(dev_id) + if front_vm is not None: + attached[dev_id] = front_vm + elif cache[dev_id] != front_vm: + cached_front = cache[dev_id] + if front_vm is None: + detached[dev_id] = cached_front + elif cached_front is None: + attached[dev_id] = front_vm + else: + # a front changed from one to another, so we signal it as: + # detach from the first one and attach to the second one. + detached[dev_id] = cached_front + attached[dev_id] = front_vm + + for dev_id, cached_front in cache.items(): + if dev_id not in current_devices: + removed.add(dev_id) + if cached_front is not None: + detached[dev_id] = cached_front + return added, attached, detached, removed diff --git a/rpm_spec/core-dom0.spec.in b/rpm_spec/core-dom0.spec.in index 0ba6008cd..a38baa9e2 100644 --- a/rpm_spec/core-dom0.spec.in +++ b/rpm_spec/core-dom0.spec.in @@ -444,6 +444,7 @@ done %{python3_sitelib}/qubes/ext/r3compatibility.py %{python3_sitelib}/qubes/ext/services.py %{python3_sitelib}/qubes/ext/supported_features.py +%{python3_sitelib}/qubes/ext/utils.py %{python3_sitelib}/qubes/ext/windows.py %{python3_sitelib}/qubes/ext/vm_config.py From ad35d13955be9ef8ae431b916c754d8f2dcfd232 Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Tue, 19 Mar 2024 17:16:16 +0100 Subject: [PATCH 22/49] q-dev: device protocol --- qubes/device_protocol.py | 896 +++++++++++++++++++++++++++++++++++++ qubes/devices.py | 736 +----------------------------- qubes/ext/block.py | 13 +- qubes/ext/pci.py | 12 +- rpm_spec/core-dom0.spec.in | 1 + 5 files changed, 913 insertions(+), 745 deletions(-) create mode 100644 qubes/device_protocol.py diff --git a/qubes/device_protocol.py b/qubes/device_protocol.py new file mode 100644 index 000000000..e5c3e1578 --- /dev/null +++ b/qubes/device_protocol.py @@ -0,0 +1,896 @@ +# -*- encoding: utf8 -*- +# +# The Qubes OS Project, http://www.qubes-os.org +# +# Copyright (C) 2010-2016 Joanna Rutkowska +# Copyright (C) 2015-2016 Wojtek Porczyk +# Copyright (C) 2016 Bahtiar `kalkin-` Gadimov +# Copyright (C) 2017 Marek Marczykowski-Górecki +# +# Copyright (C) 2024 Piotr Bartman-Szwarc +# +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, see . + +""" +Common part of device API. + +The same in `qubes-core-admin` and `qubes-core-admin-client`, +should be moved to one place. +""" +import string +import sys +from enum import Enum +from typing import Optional, Dict, Any, List, Type, Union + +import qubes.utils + +from qubes.api import ProtocolError + + +QubesVM = 'qubes.vm.BaseVM' + + +class UnexpectedDeviceProperty(qubes.exc.QubesException, ValueError): + """ + Device has unexpected property such as backend_domain, devclass etc. + """ + + +def qbool(value): + return qubes.property.bool(None, None, value) + + +class Device: + def __init__(self, backend_domain, ident, devclass=None): + self.__backend_domain = backend_domain + self.__ident = ident + self.__bus = devclass + + def __hash__(self): + return hash((str(self.backend_domain), self.ident)) + + def __eq__(self, other): + return ( + self.backend_domain == other.backend_domain and + self.ident == other.ident + ) + + def __lt__(self, other): + if isinstance(other, Device): + return (self.backend_domain.name, self.ident) < \ + (other.backend_domain.name, other.ident) + return NotImplemented() + + def __repr__(self): + return "[%s]:%s" % (self.backend_domain, self.ident) + + def __str__(self): + return '{!s}:{!s}'.format(self.backend_domain, self.ident) + + @property + def ident(self) -> str: + """ + Immutable device identifier. + + Unique for given domain and device type. + """ + return self.__ident + + @property + def backend_domain(self) -> QubesVM: + """ Which domain provides this device. (immutable)""" + return self.__backend_domain + + @property + def devclass(self) -> str: + """ Immutable* Device class such like: 'usb', 'pci' etc. + + For unknown devices "peripheral" is returned. + + *see `@devclass.setter` + """ + if self.__bus: + return self.__bus + else: + return "peripheral" + + @property + def devclass_is_set(self) -> bool: + """ + Returns true if devclass is already initialised. + """ + return bool(self.__bus) + + @devclass.setter + def devclass(self, devclass: str): + """ Once a value is set, it should not be overridden. + + However, if it has not been set, i.e., the value is `None`, + we can override it.""" + if self.__bus != None: + raise TypeError("Attribute devclass is immutable") + self.__bus = devclass + + +class DeviceCategory(Enum): + """ + Category of peripheral device. + + Arbitrarily selected interfaces that are important to users, + thus deserving special recognition such as a custom icon, etc. + """ + Other = "*******" + + Communication = ("u02****", "p07****") # eg. modems + Input = ("u03****", "p09****") # HID etc. + Keyboard = ("u03**01", "p0900**") + Mouse = ("u03**02", "p0902**") + Printer = ("u07****",) + Scanner = ("p0903**",) + # Multimedia = Audio, Video, Displays etc. + Microphone = ("m******",) + Multimedia = ("u01****", "u0e****", "u06****", "u10****", "p03****", + "p04****") + Wireless = ("ue0****", "p0d****") + Bluetooth = ("ue00101", "p0d11**") + Mass_Data = ("b******", "u08****", "p01****") + Network = ("p02****",) + Memory = ("p05****",) + PCI_Bridge = ("p06****",) + Docking_Station = ("p0a****",) + Processor = ("p0b****", "p40****") + PCI_Serial_Bus = ("p0c****",) + PCI_USB = ("p0c03**",) + + @staticmethod + def from_str(interface_encoding: str) -> 'DeviceCategory': + result = DeviceCategory.Other + if len(interface_encoding) != len(DeviceCategory.Other.value): + return result + best_score = 0 + + for interface in DeviceCategory: + for pattern in interface.value: + score = 0 + for t, p in zip(interface_encoding, pattern): + if t == p: + score += 1 + elif p != "*": + score = -1 # inconsistent with pattern + break + + if score > best_score: + best_score = score + result = interface + + return result + + +class DeviceInterface: + """ + Peripheral device interface wrapper. + """ + + def __init__(self, interface_encoding: str, devclass: Optional[str] = None): + ifc_padded = interface_encoding.ljust(6, '*') + if devclass: + if len(ifc_padded) > 6: + print( + f"{interface_encoding=} is too long " + f"(is {len(interface_encoding)}, expected max. 6) " + f"for given {devclass=}", + file=sys.stderr + ) + ifc_full = devclass[0] + ifc_padded + else: + known_devclasses = { + 'p': 'pci', 'u': 'usb', 'b': 'block', 'm': 'mic'} + devclass = known_devclasses.get(interface_encoding[0], None) + if len(ifc_padded) > 7: + print( + f"{interface_encoding=} is too long " + f"(is {len(interface_encoding)}, expected max. 7)", + file=sys.stderr + ) + ifc_full = ifc_padded + elif len(ifc_padded) == 6: + ifc_full = '?' + ifc_padded + else: + ifc_full = ifc_padded + + self._devclass = devclass + self._interface_encoding = ifc_full + self._category = DeviceCategory.from_str(self._interface_encoding) + + @property + def devclass(self) -> Optional[str]: + """ Immutable Device class such like: 'usb', 'pci' etc. """ + return self._devclass + + @property + def category(self) -> DeviceCategory: + """ Immutable Device category such like: 'Mouse', 'Mass_Data' etc. """ + return self._category + + @classmethod + def unknown(cls) -> 'DeviceInterface': + """ Value for unknown device interface. """ + return cls("?******") + + def __repr__(self): + return self._interface_encoding + + def __str__(self): + if self.devclass == "block": + return "Block device" + if self.devclass in ("usb", "pci"): + result = self._load_classes(self.devclass).get( + self._interface_encoding[1:], None) + if result is None: + result = self._load_classes(self.devclass).get( + self._interface_encoding[1:-2] + '**', None) + if result is None: + result = self._load_classes(self.devclass).get( + self._interface_encoding[1:-4] + '****', None) + if result is None: + result = f"Unclassified {self.devclass} device" + return result + if self.devclass == 'mic': + return "Microphone" + return repr(self) + + @staticmethod + def _load_classes(bus: str): + """ + List of known device classes, subclasses and programming interfaces. + """ + # Syntax: + # C class class_name + # subclass subclass_name <-- single tab + # prog-if prog-if_name <-- two tabs + result = {} + with open(f'/usr/share/hwdata/{bus}.ids', + encoding='utf-8', errors='ignore') as pciids: + class_id = None + subclass_id = None + for line in pciids.readlines(): + line = line.rstrip() + if line.startswith('\t\t') and class_id and subclass_id: + (progif_id, _, progif_name) = line[2:].split(' ', 2) + result[class_id + subclass_id + progif_id] = \ + f"{class_name}: {subclass_name} ({progif_name})" + elif line.startswith('\t') and class_id: + (subclass_id, _, subclass_name) = line[1:].split(' ', 2) + # store both prog-if specific entry and generic one + result[class_id + subclass_id + '**'] = \ + f"{class_name}: {subclass_name}" + elif line.startswith('C '): + (_, class_id, _, class_name) = line.split(' ', 3) + result[class_id + '****'] = class_name + subclass_id = None + + return result + + +class DeviceInfo(Device): + """ Holds all information about a device """ + + def __init__( + self, + backend_domain: QubesVM, + ident: str, + devclass: Optional[str] = None, + vendor: Optional[str] = None, + product: Optional[str] = None, + manufacturer: Optional[str] = None, + name: Optional[str] = None, + serial: Optional[str] = None, + interfaces: Optional[List[DeviceInterface]] = None, + parent: Optional[Device] = None, + attachment: Optional[QubesVM] = None, + self_identity: Optional[str] = None, + **kwargs + ): + super().__init__(backend_domain, ident, devclass) + + self._vendor = vendor + self._product = product + self._manufacturer = manufacturer + self._name = name + self._serial = serial + self._interfaces = interfaces + self._parent = parent + self._attachment = attachment + self._self_identity = self_identity + + self.data = kwargs + + @property + def vendor(self) -> str: + """ + Device vendor name from local database. + + Could be empty string or "unknown". + + Override this method to return proper name from `/usr/share/hwdata/*`. + """ + if not self._vendor: + return "unknown" + return self._vendor + + @property + def product(self) -> str: + """ + Device name from local database. + + Could be empty string or "unknown". + + Override this method to return proper name from `/usr/share/hwdata/*`. + """ + if not self._product: + return "unknown" + return self._product + + @property + def manufacturer(self) -> str: + """ + The name of the manufacturer of the device introduced by device itself. + + Could be empty string or "unknown". + + Override this method to return proper name directly from device itself. + """ + if not self._manufacturer: + return "unknown" + return self._manufacturer + + @property + def name(self) -> str: + """ + The name of the device it introduced itself with. + + Could be empty string or "unknown". + + Override this method to return proper name directly from device itself. + """ + if not self._name: + return "unknown" + return self._name + + @property + def serial(self) -> str: + """ + The serial number of the device it introduced itself with. + + Could be empty string or "unknown". + + Override this method to return proper name directly from device itself. + """ + if not self._serial: + return "unknown" + return self._serial + + @property + def description(self) -> str: + """ + Short human-readable description. + + For unknown device returns `unknown device (unknown vendor)`. + For unknown USB device returns `unknown usb device (unknown vendor)`. + For unknown USB device with known serial number returns + ` (unknown vendor)`. + """ + if self.product and self.product != "unknown": + prod = self.product + elif self.name and self.name != "unknown": + prod = self.name + elif self.serial and self.serial != "unknown": + prod = self.serial + elif self.parent_device is not None: + return f"sub-device of {self.parent_device}" + else: + prod = f"unknown {self.devclass if self.devclass else ''} device" + + if self.vendor and self.vendor != "unknown": + vendor = self.vendor + elif self.manufacturer and self.manufacturer != "unknown": + vendor = self.manufacturer + else: + vendor = "unknown vendor" + + return f"{prod} ({vendor})" + + @property + def interfaces(self) -> List[DeviceInterface]: + """ + Non-empty list of device interfaces. + + Every device should have at least one interface. + """ + if not self._interfaces: + return [DeviceInterface.unknown()] + return self._interfaces + + @property + def parent_device(self) -> Optional[Device]: + """ + The parent device if any. + + If the device is part of another device (e.g. it's a single + partition of an usb stick), the parent device id should be here. + """ + return self._parent + + @property + def subdevices(self) -> List['DeviceInfo']: + """ + The list of children devices if any. + + If the device has subdevices (e.g. partitions of an usb stick), + the subdevices id should be here. + """ + return [dev for dev in self.backend_domain.devices[self.devclass] + if dev.parent_device.ident == self.ident] + + @property + def attachment(self) -> Optional[QubesVM]: + """ + VM to which device is attached (frontend domain). + """ + return self._attachment + + def serialize(self) -> bytes: + """ + Serialize object to be transmitted via Qubes API. + """ + # 'backend_domain', 'attachment', 'interfaces', 'data', 'parent_device' + # are not string, so they need special treatment + default_attrs = { + 'ident', 'devclass', 'vendor', 'product', 'manufacturer', 'name', + 'serial', 'self_identity'} + properties = b' '.join( + f'{prop}={serialize_str(value)}'.encode('ascii') + for prop, value in ( + (key, getattr(self, key)) for key in default_attrs) + ) + + qname = serialize_str(self.backend_domain.name) + backend_prop = (b"backend_domain=" + qname.encode('ascii')) + properties += b' ' + backend_prop + + if self.attachment: + qname = serialize_str(self.attachment.name) + attachment_prop = (b"attachment=" + qname.encode('ascii')) + properties += b' ' + attachment_prop + + interfaces = serialize_str( + ''.join(repr(ifc) for ifc in self.interfaces)) + interfaces_prop = (b'interfaces=' + interfaces.encode('ascii')) + properties += b' ' + interfaces_prop + + if self.parent_device is not None: + ident = serialize_str(self.parent_device.ident) + ident_prop = (b'parent_ident=' + ident.encode('ascii')) + properties += b' ' + ident_prop + devclass = serialize_str(self.parent_device.devclass) + devclass_prop = (b'parent_devclass=' + devclass.encode('ascii')) + properties += b' ' + devclass_prop + + data = b' '.join( + f'_{prop}={serialize_str(value)}'.encode('ascii') + for prop, value in ((key, self.data[key]) for key in self.data) + ) + if data: + properties += b' ' + data + + return properties + + @classmethod + def deserialize( + cls, + serialization: bytes, + expected_backend_domain: QubesVM, + expected_devclass: Optional[str] = None, + ) -> 'DeviceInfo': + ident, _, rest = serialization.partition(b' ') + ident = ident.decode('ascii', errors='ignore') + try: + result = DeviceInfo._deserialize( + cls, rest, expected_backend_domain, ident, expected_devclass) + except Exception as exc: + print(exc, file=sys.stderr) + ident = serialization.split(b' ')[0].decode( + 'ascii', errors='ignore') + result = UnknownDevice( + backend_domain=expected_backend_domain, + ident=ident, + devclass=expected_devclass, + ) + return result + + @staticmethod + def _deserialize( + cls: Type, + untrusted_serialization: bytes, + expected_backend_domain: QubesVM, + expected_ident: str, + expected_devclass: Optional[str] = None, + ) -> 'DeviceInfo': + allowed_chars_key = string.digits + string.ascii_letters + '-_.' + allowed_chars_value = ( + allowed_chars_key + ',+:' + string.punctuation + ' ') + + properties, options = unpack_properties( + untrusted_serialization, allowed_chars_key, allowed_chars_value) + properties.update(options) + + check_device_properties( + expected_backend_domain, + expected_ident, + expected_devclass, + properties + ) + + if 'attachment' not in properties or not properties['attachment']: + properties['attachment'] = None + else: + app = expected_backend_domain.app + properties['attachment'] = app.domains.get_blind( + properties['attachment']) + + if expected_devclass and properties['devclass'] != expected_devclass: + raise UnexpectedDeviceProperty( + f"Got {properties['devclass']} device " + f"when expected {expected_devclass}.") + + interfaces = properties['interfaces'] + interfaces = [ + DeviceInterface(interfaces[i:i + 7]) + for i in range(0, len(interfaces), 7)] + properties['interfaces'] = interfaces + + if 'parent_ident' in properties: + properties['parent'] = Device( + backend_domain=expected_backend_domain, + ident=properties['parent_ident'], + devclass=properties['parent_devclass'], + ) + del properties['parent_ident'] + del properties['parent_devclass'] + + return cls(**properties) + + @property + def self_identity(self) -> str: + """ + Get additional identification of device presented by device itself. + + For pci/usb we expect: + ::: + For block devices: + : + + In addition to the description returns presented interfaces. + It is used to auto-attach usb devices, so an attacking device needs to + mimic not only a name, but also interfaces of trusted device (and have + to be plugged to the same port). For a common user it is all the data + she uses to recognize the device. + """ + if not self._self_identity: + return "0000:0000::?******" + return self._self_identity + + +def serialize_str(value: str): + return repr(str(value)) + + +def deserialize_str(value: str): + return value.replace("\\\'", "'") + + +def sanitize_str( + untrusted_value: str, + allowed_chars: str, + replace_char: str = None, + error_message: str = "" +) -> str: + """ + Sanitize given untrusted string. + + If `replace_char` is not None, ignore `error_message` and replace invalid + characters with the string. + """ + if replace_char is None: + if any(x not in allowed_chars for x in untrusted_value): + raise ProtocolError(error_message) + return untrusted_value + result = "" + for char in untrusted_value: + if char in allowed_chars: + result += char + else: + result += replace_char + return result + + +def unpack_properties( + untrusted_serialization: bytes, + allowed_chars_key: str, + allowed_chars_value: str +): + ut_decoded = untrusted_serialization.decode( + 'ascii', errors='strict').strip() + + options = {} + keys = [] + values = [] + ut_key, _, ut_rest = ut_decoded.partition("='") + + key = sanitize_str( + ut_key, allowed_chars_key, + error_message='Invalid chars in property name') + keys.append(key) + while "='" in ut_rest: + ut_value_key, _, ut_rest = ut_rest.partition("='") + ut_value, _, ut_key = ut_value_key.rpartition("' ") + value = sanitize_str( + deserialize_str(ut_value), allowed_chars_value, + error_message='Invalid chars in property value') + values.append(value) + key = sanitize_str( + ut_key, allowed_chars_key, + error_message='Invalid chars in property name') + keys.append(key) + ut_value = ut_rest[:-1] # ending ' + value = sanitize_str( + deserialize_str(ut_value), allowed_chars_value, + error_message='Invalid chars in property value') + values.append(value) + + properties = dict() + for key, value in zip(keys, values): + if key.startswith("_"): + # it's handled in cls.__init__ + options[key[1:]] = value + else: + properties[key] = value + + return properties, options + + +def check_device_properties( + expected_backend_domain, expected_ident, expected_devclass, properties +): + if properties['backend_domain'] != expected_backend_domain.name: + raise UnexpectedDeviceProperty( + f"Got device exposed by {properties['backend_domain']}" + f"when expected devices from {expected_backend_domain.name}.") + properties['backend_domain'] = expected_backend_domain + + if properties['ident'] != expected_ident: + raise UnexpectedDeviceProperty( + f"Got device with id: {properties['ident']} " + f"when expected id: {expected_ident}.") + + if expected_devclass and properties['devclass'] != expected_devclass: + raise UnexpectedDeviceProperty( + f"Got {properties['devclass']} device " + f"when expected {expected_devclass}.") + + +class UnknownDevice(DeviceInfo): + # pylint: disable=too-few-public-methods + """Unknown device - for example exposed by domain not running currently""" + + def __init__(self, backend_domain, ident, *, devclass, **kwargs): + super().__init__(backend_domain, ident, devclass=devclass, **kwargs) + + +class DeviceAssignment(Device): + """ Maps a device to a frontend_domain. + + There are 3 flags `attached`, `automatically_attached` and `required`. + The meaning of valid combinations is as follows: + 1. (True, False, False) -> domain is running, device is manually attached + and could be manually detach any time. + 2. (True, True, False) -> domain is running, device is attached + and could be manually detach any time (see 4.), + but in the future will be auto-attached again. + 3. (True, True, True) -> domain is running, device is attached + and couldn't be detached. + 4. (False, Ture, False) -> device is assigned to domain, but not attached + because either (i) domain is halted, + device (ii) manually detached or + (iii) attach to different domain. + 5. (False, True, True) -> domain is halted, device assigned to domain + and required to start domain. + """ + + def __init__(self, backend_domain, ident, options=None, + frontend_domain=None, devclass=None, + required=False, attach_automatically=False): + super().__init__(backend_domain, ident, devclass) + self.__options = options or {} + if required: + assert attach_automatically + self.__required = required + self.__attach_automatically = attach_automatically + self.frontend_domain = frontend_domain + + def clone(self, **kwargs): + """ + Clone object and substitute attributes with explicitly given. + """ + attr = { + "backend_domain": self.backend_domain, + "ident": self.ident, + "options": self.options, + "required": self.required, + "attach_automatically": self.attach_automatically, + "frontend_domain": self.frontend_domain, + "devclass": self.devclass, + } + attr.update(kwargs) + return self.__class__(**attr) + + @classmethod + def from_device(cls, device: Device, **kwargs) -> 'DeviceAssignment': + """ + Get assignment of the device. + """ + return cls( + backend_domain=device.backend_domain, + ident=device.ident, + devclass=device.devclass, + **kwargs + ) + + @property + def device(self) -> DeviceInfo: + """Get DeviceInfo object corresponding to this DeviceAssignment""" + return self.backend_domain.devices[self.devclass][self.ident] + + @property + def frontend_domain(self) -> Optional[QubesVM]: + """ Which domain the device is attached/assigned to. """ + return self.__frontend_domain + + @frontend_domain.setter + def frontend_domain( + self, frontend_domain: Optional[Union[str, QubesVM]] + ): + """ Which domain the device is attached/assigned to. """ + if isinstance(frontend_domain, str): + frontend_domain = self.backend_domain.app.domains[frontend_domain] + self.__frontend_domain = frontend_domain + + @property + def attached(self) -> bool: + """ + Is the device attached to the fronted domain? + + Returns False if device is attached to different domain + """ + return self.device.attachment == self.frontend_domain + + @property + def required(self) -> bool: + """ + Is the presence of this device required for the domain to start? If yes, + it will be attached automatically. + """ + return self.__required + + @required.setter + def required(self, required: bool): + self.__required = required + + @property + def attach_automatically(self) -> bool: + """ + Should this device automatically connect to the frontend domain when + available and not connected to other qubes? + """ + return self.__attach_automatically + + @attach_automatically.setter + def attach_automatically(self, attach_automatically: bool): + self.__attach_automatically = attach_automatically + + @property + def options(self) -> Dict[str, Any]: + """ Device options (same as in the legacy API). """ + return self.__options + + @options.setter + def options(self, options: Optional[Dict[str, Any]]): + """ Device options (same as in the legacy API). """ + self.__options = options or {} + + def serialize(self) -> bytes: + properties = b' '.join( + f'{prop}={serialize_str(value)}'.encode('ascii') + for prop, value in ( + ('required', 'yes' if self.required else 'no'), + ('attach_automatically', + 'yes' if self.attach_automatically else 'no'), + ('ident', self.ident), + ('devclass', self.devclass) + ) + ) + + back_name = serialize_str(self.backend_domain.name) + backend_domain_prop = (b"backend_domain=" + back_name.encode('ascii')) + properties += b' ' + backend_domain_prop + + if self.frontend_domain is not None: + front_name = serialize_str(self.frontend_domain.name) + frontend_domain_prop = ( + b"frontend_domain=" + front_name.encode('ascii')) + properties += b' ' + frontend_domain_prop + + if self.options: + properties += b' ' + b' '.join( + f'_{prop}={serialize_str(value)}'.encode('ascii') + for prop, value in self.options.items() + ) + + return properties + + @classmethod + def deserialize( + cls, + serialization: bytes, + expected_backend_domain: QubesVM, + expected_ident: str, + expected_devclass: Optional[str] = None, + ) -> 'DeviceAssignment': + try: + result = DeviceAssignment._deserialize( + cls, serialization, + expected_backend_domain, expected_ident, expected_devclass + ) + except Exception as exc: + raise ProtocolError() from exc + return result + + @staticmethod + def _deserialize( + cls: Type, + untrusted_serialization: bytes, + expected_backend_domain: QubesVM, + expected_ident: str, + expected_devclass: Optional[str] = None, + ) -> 'DeviceAssignment': + allowed_chars_key = string.digits + string.ascii_letters + '-_.' + allowed_chars_value = allowed_chars_key + ',+:' + + properties, options = unpack_properties( + untrusted_serialization, allowed_chars_key, allowed_chars_value) + properties['options'] = options + + check_device_properties( + expected_backend_domain, + expected_ident, + expected_devclass, + properties + ) + + properties['attach_automatically'] = qbool( + properties['attach_automatically']) + properties['required'] = qbool(properties['required']) + + return cls(**properties) diff --git a/qubes/devices.py b/qubes/devices.py index 3d4960ab6..c1f04e623 100644 --- a/qubes/devices.py +++ b/qubes/devices.py @@ -59,13 +59,12 @@ `device-list-change:class` event. """ import itertools -import string -import sys -from enum import Enum -from typing import Optional, List, Type, Dict, Any, Iterable, Union +from typing import Optional, Iterable import qubes.utils from qubes.api import ProtocolError +from qubes.device_protocol import (Device, DeviceInfo, UnknownDevice, + DeviceAssignment) class DeviceNotAssigned(qubes.exc.QubesException, KeyError): @@ -97,533 +96,6 @@ class UnrecognizedDevice(qubes.exc.QubesException, ValueError): """ -class Device: - def __init__(self, backend_domain, ident, devclass=None): - self.__backend_domain = backend_domain - self.__ident = ident - self.__bus = devclass - - def __hash__(self): - return hash((str(self.backend_domain), self.ident)) - - def __eq__(self, other): - return ( - self.backend_domain == other.backend_domain and - self.ident == other.ident - ) - - def __lt__(self, other): - if isinstance(other, Device): - return (self.backend_domain.name, self.ident) < \ - (other.backend_domain.name, other.ident) - return NotImplemented - - def __repr__(self): - return "[%s]:%s" % (self.backend_domain, self.ident) - - def __str__(self): - return '{!s}:{!s}'.format(self.backend_domain, self.ident) - - @property - def ident(self) -> str: - """ - Immutable device identifier. - - Unique for given domain and device type. - """ - return self.__ident - - @property - def backend_domain(self) -> 'qubes.vm.BaseVM': - """ Which domain provides this device. (immutable)""" - return self.__backend_domain - - @property - def devclass(self) -> Optional[str]: - """ Immutable* Device class such like: 'usb', 'pci' etc. - - * see `@devclass.setter` - """ - return self.__bus - - @devclass.setter - def devclass(self, devclass: str): - """ Once a value is set, it should not be overridden. - - However, if it has not been set, i.e., the value is `None`, - we can override it.""" - if self.__bus != None: - raise TypeError("Attribute devclass is immutable") - self.__bus = devclass - - -class DeviceCategory(Enum): - """ - Category of peripheral device. - - Arbitrarily selected interfaces that are important to users, - thus deserving special recognition such as a custom icon, etc. - """ - Other = "*******" - - Communication = ("u02****", "p07****") # eg. modems - Input = ("u03****", "p09****") # HID etc. - Keyboard = ("u03**01", "p0900**") - Mouse = ("u03**02", "p0902**") - Printer = ("u07****",) - Scanner = ("p0903**",) - # Multimedia = Audio, Video, Displays etc. - Microphone = ("m******",) - Multimedia = ("u01****", "u0e****", "u06****", "u10****", "p03****", - "p04****") - Wireless = ("ue0****", "p0d****") - Bluetooth = ("ue00101", "p0d11**") - Mass_Data = ("b******", "u08****", "p01****") - Network = ("p02****",) - Memory = ("p05****",) - PCI_Bridge = ("p06****",) - Docking_Station = ("p0a****",) - Processor = ("p0b****", "p40****") - PCI_Serial_Bus = ("p0c****",) - PCI_USB = ("p0c03**",) - - @staticmethod - def from_str(interface_encoding: str) -> 'DeviceCategory': - result = DeviceCategory.Other - if len(interface_encoding) != len(DeviceCategory.Other.value): - return result - best_score = 0 - - for interface in DeviceCategory: - for pattern in interface.value: - score = 0 - for t, p in zip(interface_encoding, pattern): - if t == p: - score += 1 - elif p != "*": - score = -1 # inconsistent with pattern - break - - if score > best_score: - best_score = score - result = interface - - return result - - -class DeviceInterface: - """ - Peripheral device interface wrapper. - """ - - def __init__(self, interface_encoding: str, devclass: Optional[str] = None): - ifc_padded = interface_encoding.ljust(6, '*') - if devclass: - if len(ifc_padded) > 6: - print( - f"{interface_encoding=} is too long " - f"(is {len(interface_encoding)}, expected max. 6) " - f"for given {devclass=}", - file=sys.stderr - ) - ifc_full = devclass[0] + ifc_padded - else: - known_devclasses = { - 'p': 'pci', 'u': 'usb', 'b': 'block', 'm': 'mic'} - devclass = known_devclasses.get(interface_encoding[0], None) - if len(ifc_padded) > 7: - print( - f"{interface_encoding=} is too long " - f"(is {len(interface_encoding)}, expected max. 7)", - file=sys.stderr - ) - ifc_full = ifc_padded - elif len(ifc_padded) == 6: - ifc_full = '?' + ifc_padded - else: - ifc_full = ifc_padded - - self._devclass = devclass - self._interface_encoding = ifc_full - self._category = DeviceCategory.from_str(self._interface_encoding) - - @property - def devclass(self) -> Optional[str]: - """ Immutable Device class such like: 'usb', 'pci' etc. """ - return self._devclass - - @property - def category(self) -> DeviceCategory: - """ Immutable Device category such like: 'Mouse', 'Mass_Data' etc. """ - return self._category - - @classmethod - def unknown(cls) -> 'DeviceInterface': - """ Value for unknown device interface. """ - return cls("?******") - - def __repr__(self): - return self._interface_encoding - - def __str__(self): - if self.devclass == "block": - return "Block device" - if self.devclass in ("usb", "pci"): - result = self._load_classes(self.devclass).get( - self._interface_encoding[1:], None) - if result is None: - result = self._load_classes(self.devclass).get( - self._interface_encoding[1:-2] + '**', None) - if result is None: - result = self._load_classes(self.devclass).get( - self._interface_encoding[1:-4] + '****', None) - if result is None: - result = f"Unclassified {self.devclass} device" - return result - if self.devclass == 'mic': - return "Microphone" - return repr(self) - - @staticmethod - def _load_classes(bus: str): - """ - List of known device classes, subclasses and programming interfaces. - """ - # Syntax: - # C class class_name - # subclass subclass_name <-- single tab - # prog-if prog-if_name <-- two tabs - result = {} - with open(f'/usr/share/hwdata/{bus}.ids', - encoding='utf-8', errors='ignore') as pciids: - class_id = None - subclass_id = None - for line in pciids.readlines(): - line = line.rstrip() - if line.startswith('\t\t') and class_id and subclass_id: - (progif_id, _, progif_name) = line[2:].split(' ', 2) - result[class_id + subclass_id + progif_id] = \ - f"{class_name}: {subclass_name} ({progif_name})" - elif line.startswith('\t') and class_id: - (subclass_id, _, subclass_name) = line[1:].split(' ', 2) - # store both prog-if specific entry and generic one - result[class_id + subclass_id + '**'] = \ - f"{class_name}: {subclass_name}" - elif line.startswith('C '): - (_, class_id, _, class_name) = line.split(' ', 3) - result[class_id + '****'] = class_name - subclass_id = None - - return result - - -class DeviceInfo(Device): - """ Holds all information about a device """ - - def __init__( - self, - backend_domain: 'qubes.vm.BaseVM', - ident: str, - devclass: Optional[str] = None, - vendor: Optional[str] = None, - product: Optional[str] = None, - manufacturer: Optional[str] = None, - name: Optional[str] = None, - serial: Optional[str] = None, - interfaces: Optional[List[DeviceInterface]] = None, - parent: Optional[Device] = None, - attachment: Optional['qubes.vm.BaseVM'] = None, - self_identity: Optional[str] = None, - **kwargs - ): - super().__init__(backend_domain, ident, devclass) - - self._vendor = vendor - self._product = product - self._manufacturer = manufacturer - self._name = name - self._serial = serial - self._interfaces = interfaces - self._parent = parent - self._attachment = attachment - self._self_identity = self_identity - - self.data = kwargs - - @property - def vendor(self) -> str: - """ - Device vendor name from local database. - - Could be empty string or "unknown". - - Override this method to return proper name from `/usr/share/hwdata/*`. - """ - if not self._vendor: - return "unknown" - return self._vendor - - @property - def product(self) -> str: - """ - Device name from local database. - - Could be empty string or "unknown". - - Override this method to return proper name from `/usr/share/hwdata/*`. - """ - if not self._product: - return "unknown" - return self._product - - @property - def manufacturer(self) -> str: - """ - The name of the manufacturer of the device introduced by device itself. - - Could be empty string or "unknown". - - Override this method to return proper name directly from device itself. - """ - if not self._manufacturer: - return "unknown" - return self._manufacturer - - @property - def name(self) -> str: - """ - The name of the device it introduced itself with. - - Could be empty string or "unknown". - - Override this method to return proper name directly from device itself. - """ - if not self._name: - return "unknown" - return self._name - - @property - def serial(self) -> str: - """ - The serial number of the device it introduced itself with. - - Could be empty string or "unknown". - - Override this method to return proper name directly from device itself. - """ - if not self._serial: - return "unknown" - return self._serial - - @property - def description(self) -> str: - """ - Short human-readable description. - - For unknown device returns `unknown device (unknown vendor)`. - For unknown USB device returns `unknown usb device (unknown vendor)`. - For unknown USB device with known serial number returns - ` (unknown vendor)`. - """ - if self.product and self.product != "unknown": - prod = self.product - elif self.name and self.name != "unknown": - prod = self.name - elif self.serial and self.serial != "unknown": - prod = self.serial - else: - prod = f"unknown {self.devclass if self.devclass else ''} device" - - if self.vendor and self.vendor != "unknown": - vendor = self.vendor - elif self.manufacturer and self.manufacturer != "unknown": - vendor = self.manufacturer - else: - vendor = "unknown vendor" - - return f"{prod} ({vendor})" - - @property - def interfaces(self) -> List[DeviceInterface]: - """ - Non-empty list of device interfaces. - - Every device should have at least one interface. - """ - if not self._interfaces: - return [DeviceInterface.unknown()] - return self._interfaces - - @property - def parent_device(self) -> Optional[Device]: - """ - The parent device if any. - - If the device is part of another device (e.g. it's a single - partition of an usb stick), the parent device id should be here. - """ - return self._parent - - @property - def subdevices(self) -> List['DeviceInfo']: - """ - The list of children devices if any. - - If the device has subdevices (e.g. partitions of an usb stick), - the subdevices id should be here. - """ - return [dev for dev in self.backend_domain.devices[self.devclass] - if dev.parent_device.ident == self.ident] - - @property - def attachment(self) -> Optional['qubes.vm.BaseVM']: - """ - VM to which device is attached (frontend domain). - """ - return self._attachment - - def serialize(self) -> bytes: - """ - Serialize object to be transmitted via Qubes API. - """ - # 'backend_domain', 'attachment', 'interfaces', 'data', 'parent_device' - # are not string, so they need special treatment - default_attrs = { - 'ident', 'devclass', 'vendor', 'product', 'manufacturer', 'name', - 'serial', 'self_identity'} - properties = b' '.join( - f'{prop}={serialize_str(value)}'.encode('ascii') - for prop, value in ( - (key, getattr(self, key)) for key in default_attrs) - ) - - qname = serialize_str(self.backend_domain.name) - backend_prop = (b"backend_domain=" + qname.encode('ascii')) - properties += b' ' + backend_prop - - if self.attachment: - qname = serialize_str(self.attachment.name) - attachment_prop = (b"attachment=" + qname.encode('ascii')) - properties += b' ' + attachment_prop - - interfaces = serialize_str( - ''.join(repr(ifc) for ifc in self.interfaces)) - interfaces_prop = (b'interfaces=' + interfaces.encode('ascii')) - properties += b' ' + interfaces_prop - - if self.parent_device is not None: - ident = serialize_str(self.parent_device.ident) - ident_prop = (b'parent_ident=' + ident.encode('ascii')) - properties += b' ' + ident_prop - devclass = serialize_str(self.parent_device.devclass) - devclass_prop = (b'parent_devclass=' + devclass.encode('ascii')) - properties += b' ' + devclass_prop - - data = b' '.join( - f'_{prop}={serialize_str(value)}'.encode('ascii') - for prop, value in ((key, self.data[key]) for key in self.data) - ) - if data: - properties += b' ' + data - - return properties - - @classmethod - def deserialize( - cls, - serialization: bytes, - expected_backend_domain: 'qubes.vm.BaseVM', - expected_devclass: Optional[str] = None, - ) -> 'DeviceInfo': - ident, _, rest = serialization.partition(b' ') - ident = ident.decode('ascii', errors='ignore') - try: - result = DeviceInfo._deserialize( - cls, rest, expected_backend_domain, ident, expected_devclass) - except Exception as exc: - print(exc, file=sys.stderr) - ident = serialization.split(b' ')[0].decode( - 'ascii', errors='ignore') - result = UnknownDevice( - backend_domain=expected_backend_domain, - ident=ident, - devclass=expected_devclass, - ) - return result - - @staticmethod - def _deserialize( - cls: Type, - untrusted_serialization: bytes, - expected_backend_domain: 'qubes.vm.BaseVM', - expected_ident: str, - expected_devclass: Optional[str] = None, - ) -> 'DeviceInfo': - allowed_chars_key = string.digits + string.ascii_letters + '-_.' - allowed_chars_value = ( - allowed_chars_key + ',+:' + string.punctuation + ' ') - - properties, options = unpack_properties( - untrusted_serialization, allowed_chars_key, allowed_chars_value) - properties.update(options) - - check_device_properties( - expected_backend_domain, - expected_ident, - expected_devclass, - properties - ) - - if 'attachment' not in properties or not properties['attachment']: - properties['attachment'] = None - else: - app = expected_backend_domain.app - properties['attachment'] = app.domains.get_blind( - properties['attachment']) - - if expected_devclass and properties['devclass'] != expected_devclass: - raise UnexpectedDeviceProperty( - f"Got {properties['devclass']} device " - f"when expected {expected_devclass}.") - - interfaces = properties['interfaces'] - interfaces = [ - DeviceInterface(interfaces[i:i + 7]) - for i in range(0, len(interfaces), 7)] - properties['interfaces'] = interfaces - - if 'parent_ident' in properties: - properties['parent'] = Device( - backend_domain=expected_backend_domain, - ident=properties['parent_ident'], - devclass=properties['parent_devclass'], - ) - del properties['parent_ident'] - del properties['parent_devclass'] - - return cls(**properties) - - @property - def self_identity(self) -> str: - """ - Get additional identification of device presented by device itself. - - For pci/usb we expect: - ::: - For block devices: - : - - In addition to the description returns presented interfaces. - It is used to auto-attach usb devices, so an attacking device needs to - mimic not only a name, but also interfaces of trusted device (and have - to be plugged to the same port). For a common user it is all the data - she uses to recognize the device. - """ - if not self._self_identity: - return "0000:0000::?******" - return self._self_identity - - def serialize_str(value: str): return repr(str(value)) @@ -722,208 +194,6 @@ def check_device_properties( f"when expected {expected_devclass}.") -class UnknownDevice(DeviceInfo): - # pylint: disable=too-few-public-methods - """Unknown device - for example exposed by domain not running currently""" - - def __init__(self, backend_domain, ident, *, devclass, **kwargs): - super().__init__(backend_domain, ident, devclass=devclass, **kwargs) - - -class DeviceAssignment(Device): - """ Maps a device to a frontend_domain. - - There are 3 flags `attached`, `automatically_attached` and `required`. - The meaning of valid combinations is as follows: - 1. (True, False, False) -> domain is running, device is manually attached - and could be manually detach any time. - 2. (True, True, False) -> domain is running, device is attached - and could be manually detach any time (see 4.), - but in the future will be auto-attached again. - 3. (True, True, True) -> domain is running, device is attached - and couldn't be detached. - 4. (False, Ture, False) -> device is assigned to domain, but not attached - because either (i) domain is halted, - device (ii) manually detached or - (iii) attach to different domain. - 5. (False, True, True) -> domain is halted, device assigned to domain - and required to start domain. - """ - - def __init__(self, backend_domain, ident, options=None, - frontend_domain=None, devclass=None, - required=False, attach_automatically=False): - super().__init__(backend_domain, ident, devclass) - self.__options = options or {} - if required: - assert attach_automatically - self.__required = required - self.__attach_automatically = attach_automatically - self.frontend_domain = frontend_domain - - def clone(self): - """Clone object instance""" - return self.__class__( - backend_domain=self.backend_domain, - ident=self.ident, - options=self.options, - required=self.required, - attach_automatically=self.attach_automatically, - frontend_domain=self.frontend_domain, - devclass=self.devclass, - ) - - @classmethod - def from_device(cls, device: Device, **kwargs) -> 'DeviceAssignment': - """ - Get assignment of the device. - """ - return cls( - backend_domain=device.backend_domain, - ident=device.ident, - devclass=device.devclass, - **kwargs - ) - - @property - def device(self) -> DeviceInfo: - """Get DeviceInfo object corresponding to this DeviceAssignment""" - return self.backend_domain.devices[self.devclass][self.ident] - - @property - def frontend_domain(self) -> Optional['qubes.vm.BaseVM']: - """ Which domain the device is attached/assigned to. """ - return self.__frontend_domain - - @frontend_domain.setter - def frontend_domain( - self, frontend_domain: Optional[Union[str, 'qubes.vm.BaseVM']] - ): - """ Which domain the device is attached/assigned to. """ - if isinstance(frontend_domain, str): - frontend_domain = self.backend_domain.app.domains[frontend_domain] - self.__frontend_domain = frontend_domain - - @property - def attached(self) -> bool: - """ - Is the device attached to the fronted domain? - - Returns False if device is attached to different domain - """ - return self.device.attachment == self.frontend_domain - - @property - def required(self) -> bool: - """ - Is the presence of this device required for the domain to start? If yes, - it will be attached automatically. - """ - return self.__required - - @required.setter - def required(self, required: bool): - self.__required = required - - @property - def attach_automatically(self) -> bool: - """ - Should this device automatically connect to the frontend domain when - available and not connected to other qubes? - """ - return self.__attach_automatically - - @attach_automatically.setter - def attach_automatically(self, attach_automatically: bool): - self.__attach_automatically = attach_automatically - - @property - def options(self) -> Dict[str, Any]: - """ Device options (same as in the legacy API). """ - return self.__options - - @options.setter - def options(self, options: Optional[Dict[str, Any]]): - """ Device options (same as in the legacy API). """ - self.__options = options or {} - - def serialize(self) -> bytes: - properties = b' '.join( - f'{prop}={serialize_str(value)}'.encode('ascii') - for prop, value in ( - ('required', 'yes' if self.required else 'no'), - ('attach_automatically', - 'yes' if self.attach_automatically else 'no'), - ('ident', self.ident), - ('devclass', self.devclass) - ) - ) - - back_name = serialize_str(self.backend_domain.name) - backend_domain_prop = (b"backend_domain=" + back_name.encode('ascii')) - properties += b' ' + backend_domain_prop - - if self.frontend_domain is not None: - front_name = serialize_str(self.frontend_domain.name) - frontend_domain_prop = ( - b"frontend_domain=" + front_name.encode('ascii')) - properties += b' ' + frontend_domain_prop - - if self.options: - properties += b' ' + b' '.join( - f'_{prop}={serialize_str(value)}'.encode('ascii') - for prop, value in self.options.items() - ) - - return properties - - @classmethod - def deserialize( - cls, - serialization: bytes, - expected_backend_domain: 'qubes.vm.BaseVM', - expected_ident: str, - expected_devclass: Optional[str] = None, - ) -> 'DeviceAssignment': - try: - result = DeviceAssignment._deserialize( - cls, serialization, - expected_backend_domain, expected_ident, expected_devclass - ) - except Exception as exc: - raise ProtocolError() from exc - return result - - @staticmethod - def _deserialize( - cls: Type, - untrusted_serialization: bytes, - expected_backend_domain: 'qubes.vm.BaseVM', - expected_ident: str, - expected_devclass: Optional[str] = None, - ) -> 'DeviceAssignment': - allowed_chars_key = string.digits + string.ascii_letters + '-_.' - allowed_chars_value = allowed_chars_key + ',+:' - - properties, options = unpack_properties( - untrusted_serialization, allowed_chars_key, allowed_chars_value) - properties['options'] = options - - check_device_properties( - expected_backend_domain, - expected_ident, - expected_devclass, - properties - ) - - properties['attach_automatically'] = qubes.property.bool( - None, None, properties['attach_automatically']) - properties['required'] = qubes.property.bool( - None, None, properties['required']) - - return cls(**properties) - - class DeviceCollection: """Bag for devices. diff --git a/qubes/ext/block.py b/qubes/ext/block.py index 2e65837c2..7cb172a4d 100644 --- a/qubes/ext/block.py +++ b/qubes/ext/block.py @@ -29,6 +29,7 @@ import lxml.etree +import qubes.device_protocol import qubes.devices import qubes.ext from qubes.ext.utils import device_list_change @@ -49,7 +50,7 @@ SYSTEM_DISKS_DOM0_KERNEL = SYSTEM_DISKS + ('xvdd',) -class BlockDevice(qubes.devices.DeviceInfo): +class BlockDevice(qubes.device_protocol.DeviceInfo): def __init__(self, backend_domain, ident): super().__init__( backend_domain=backend_domain, ident=ident, devclass="block") @@ -155,16 +156,16 @@ def device_node(self): return '/dev/' + self.ident.replace('_', '/') @property - def interfaces(self) -> List[qubes.devices.DeviceInterface]: + def interfaces(self) -> List[qubes.device_protocol.DeviceInterface]: """ List of device interfaces. Every device should have at least one interface. """ - return [qubes.devices.DeviceInterface("******", "block")] + return [qubes.device_protocol.DeviceInterface("******", "block")] @property - def parent_device(self) -> Optional[qubes.devices.Device]: + def parent_device(self) -> Optional[qubes.device_protocol.Device]: """ The parent device if any. @@ -186,7 +187,7 @@ def parent_device(self) -> Optional[qubes.devices.Device]: devclass = 'usb' if sep == ':' else 'block' if not parent_ident: return None - self._parent = qubes.devices.Device( + self._parent = qubes.device_protocol.Device( self.backend_domain, parent_ident, devclass=devclass) self._interface_num = interface_num return self._parent @@ -464,7 +465,7 @@ def find_unused_frontend(vm, devtype='disk'): @qubes.ext.handler('device-pre-attach:block') def on_device_pre_attached_block(self, vm, event, device, options): # pylint: disable=unused-argument - if isinstance(device, qubes.devices.UnknownDevice): + if isinstance(device, qubes.device_protocol.UnknownDevice): print(f'{device.devclass.capitalize()} device {device} ' 'not available, skipping.', file=sys.stderr) raise qubes.devices.UnrecognizedDevice() diff --git a/qubes/ext/pci.py b/qubes/ext/pci.py index ce5cb0dba..3fd56c46b 100644 --- a/qubes/ext/pci.py +++ b/qubes/ext/pci.py @@ -25,13 +25,13 @@ import re import string import subprocess -import sys -from typing import Optional, List, Dict, Tuple +from typing import Optional, List, Dict import libvirt import lxml import lxml.etree +import qubes.device_protocol import qubes.devices import qubes.ext @@ -152,7 +152,7 @@ def _device_desc(hostdev_xml): ) -class PCIDevice(qubes.devices.DeviceInfo): +class PCIDevice(qubes.device_protocol.DeviceInfo): # pylint: disable=too-few-public-methods regex = re.compile( r'\A(?P[0-9a-f]+)_(?P[0-9a-f]+)\.' @@ -217,7 +217,7 @@ def product(self) -> str: return result @property - def interfaces(self) -> List[qubes.devices.DeviceInterface]: + def interfaces(self) -> List[qubes.device_protocol.DeviceInterface]: """ List of device interfaces. @@ -230,12 +230,12 @@ def interfaces(self) -> List[qubes.devices.DeviceInterface]: ) interface_encoding = pcidev_interface(lxml.etree.fromstring( hostdev_details.XMLDesc())) - self._interfaces = [qubes.devices.DeviceInterface( + self._interfaces = [qubes.device_protocol.DeviceInterface( interface_encoding, devclass='pci')] return self._interfaces @property - def parent_device(self) -> Optional[qubes.devices.DeviceInfo]: + def parent_device(self) -> Optional[qubes.device_protocol.DeviceInfo]: """ The parent device if any. diff --git a/rpm_spec/core-dom0.spec.in b/rpm_spec/core-dom0.spec.in index a38baa9e2..1ea276b1d 100644 --- a/rpm_spec/core-dom0.spec.in +++ b/rpm_spec/core-dom0.spec.in @@ -372,6 +372,7 @@ done %{python3_sitelib}/qubes/app.py %{python3_sitelib}/qubes/backup.py %{python3_sitelib}/qubes/config.py +%{python3_sitelib}/qubes/device_protocol.py %{python3_sitelib}/qubes/devices.py %{python3_sitelib}/qubes/dochelpers.py %{python3_sitelib}/qubes/events.py From cd2515f23920db9d75a17550ae5b0af368ea086e Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Wed, 20 Mar 2024 10:50:40 +0100 Subject: [PATCH 23/49] q-dev: @marmarek comments --- qubes/api/__init__.py | 14 ++---- qubes/api/admin.py | 23 +++++----- qubes/device_protocol.py | 3 +- qubes/devices.py | 5 ++- qubes/exc.py | 8 ++++ qubes/ext/admin.py | 3 +- qubes/ext/pci.py | 14 +++--- qubes/tests/api.py | 3 +- qubes/tests/api_admin.py | 97 ++++++++++++++++++++-------------------- qubes/tests/api_misc.py | 12 ++--- 10 files changed, 94 insertions(+), 88 deletions(-) diff --git a/qubes/api/__init__.py b/qubes/api/__init__.py index ca548d3b4..28b0d99d4 100644 --- a/qubes/api/__init__.py +++ b/qubes/api/__init__.py @@ -30,13 +30,7 @@ import traceback import qubes.exc - -class ProtocolError(AssertionError): - '''Raised when something is wrong with data received''' - - -class PermissionDenied(Exception): - '''Raised deliberately by handlers when we decide not to cooperate''' +from qubes.exc import ProtocolError, PermissionDenied def method(name, *, no_payload=False, endpoints=None, **classifiers): @@ -213,11 +207,11 @@ def enforce(predicate): def validate_size(self, untrusted_size: bytes) -> int: self.enforce(isinstance(untrusted_size, bytes)) if not untrusted_size.isdigit(): - raise qubes.api.ProtocolError('Size must be ASCII digits (only)') + raise qubes.exc.ProtocolError('Size must be ASCII digits (only)') if len(untrusted_size) >= 20: - raise qubes.api.ProtocolError('Sizes limited to 19 decimal digits') + raise qubes.exc.ProtocolError('Sizes limited to 19 decimal digits') if untrusted_size[0] == 48 and untrusted_size != b'0': - raise qubes.api.ProtocolError('Spurious leading zeros not allowed') + raise qubes.exc.ProtocolError('Spurious leading zeros not allowed') return int(untrusted_size) class QubesDaemonProtocol(asyncio.Protocol): diff --git a/qubes/api/admin.py b/qubes/api/admin.py index 0a79264bb..b06526fc3 100644 --- a/qubes/api/admin.py +++ b/qubes/api/admin.py @@ -37,6 +37,7 @@ import qubes.backup import qubes.config import qubes.devices +import qubes.exc import qubes.ext import qubes.firewall import qubes.storage @@ -501,7 +502,7 @@ async def vm_volume_set_rw(self, untrusted_payload): newvalue = qubes.property.bool(None, None, untrusted_payload.decode('ascii')) except (UnicodeDecodeError, ValueError): - raise qubes.api.ProtocolError('Invalid value') + raise qubes.exc.ProtocolError('Invalid value') del untrusted_payload self.fire_event_for_permission(newvalue=newvalue) @@ -520,7 +521,7 @@ async def vm_volume_set_ephemeral(self, untrusted_payload): newvalue = qubes.property.bool(None, None, untrusted_payload.decode('ascii')) except (UnicodeDecodeError, ValueError): - raise qubes.api.ProtocolError('Invalid value') + raise qubes.exc.ProtocolError('Invalid value') del untrusted_payload self.fire_event_for_permission(newvalue=newvalue) @@ -751,7 +752,7 @@ async def pool_set_ephemeral(self, untrusted_payload): newvalue = qubes.property.bool(None, None, untrusted_payload.decode('ascii', 'strict')) except (UnicodeDecodeError, ValueError): - raise qubes.api.ProtocolError('Invalid value') + raise qubes.exc.ProtocolError('Invalid value') del untrusted_payload self.fire_event_for_permission(newvalue=newvalue) @@ -1076,7 +1077,7 @@ async def _vm_create(self, vm_type, allow_pool=False, errors='strict').split(' '): untrusted_key, untrusted_value = untrusted_param.split('=', 1) if untrusted_key in kwargs: - raise qubes.api.ProtocolError('duplicated parameters') + raise qubes.exc.ProtocolError('duplicated parameters') if untrusted_key == 'name': qubes.vm.validate_name(None, None, untrusted_value) @@ -1094,7 +1095,7 @@ async def _vm_create(self, vm_type, allow_pool=False, elif untrusted_key == 'pool' and allow_pool: if pool is not None: - raise qubes.api.ProtocolError('duplicated pool parameter') + raise qubes.exc.ProtocolError('duplicated pool parameter') pool = self.app.get_pool(untrusted_value) elif untrusted_key.startswith('pool:') and allow_pool: untrusted_volume = untrusted_key.split(':', 1)[1] @@ -1104,19 +1105,19 @@ async def _vm_create(self, vm_type, allow_pool=False, 'root', 'private', 'volatile', 'kernel']) volume = untrusted_volume if volume in pools: - raise qubes.api.ProtocolError( + raise qubes.exc.ProtocolError( 'duplicated pool:{} parameter'.format(volume)) pools[volume] = self.app.get_pool(untrusted_value) else: - raise qubes.api.ProtocolError('Invalid param name') + raise qubes.exc.ProtocolError('Invalid param name') del untrusted_payload if 'name' not in kwargs or 'label' not in kwargs: - raise qubes.api.ProtocolError('Missing name or label') + raise qubes.exc.ProtocolError('Missing name or label') if pool and pools: - raise qubes.api.ProtocolError( + raise qubes.exc.ProtocolError( 'Only one of \'pool=\' and \'pool:volume=\' can be used') if kwargs['name'] in self.app.domains: @@ -1550,7 +1551,7 @@ async def backup_execute(self): profile_path = os.path.join(qubes.config.backup_profile_dir, self.arg + '.conf') if not os.path.exists(profile_path): - raise qubes.api.PermissionDenied( + raise qubes.exc.PermissionDenied( 'Backup profile {} does not exist'.format(self.arg)) if not hasattr(self.app, 'api_admin_running_backups'): @@ -1602,7 +1603,7 @@ async def backup_info(self): profile_path = os.path.join(qubes.config.backup_profile_dir, self.arg + '.conf') if not os.path.exists(profile_path): - raise qubes.api.PermissionDenied( + raise qubes.exc.PermissionDenied( 'Backup profile {} does not exist'.format(self.arg)) backup = await self._load_backup_profile(self.arg, diff --git a/qubes/device_protocol.py b/qubes/device_protocol.py index e5c3e1578..cf7dc6081 100644 --- a/qubes/device_protocol.py +++ b/qubes/device_protocol.py @@ -36,8 +36,7 @@ import qubes.utils -from qubes.api import ProtocolError - +from qubes.exc import ProtocolError QubesVM = 'qubes.vm.BaseVM' diff --git a/qubes/devices.py b/qubes/devices.py index c1f04e623..801a13903 100644 --- a/qubes/devices.py +++ b/qubes/devices.py @@ -61,8 +61,9 @@ import itertools from typing import Optional, Iterable +import qubes.exc import qubes.utils -from qubes.api import ProtocolError +from qubes.exc import ProtocolError from qubes.device_protocol import (Device, DeviceInfo, UnknownDevice, DeviceAssignment) @@ -118,7 +119,7 @@ def sanitize_str( """ if replace_char is None: if any(x not in allowed_chars for x in untrusted_value): - raise qubes.api.ProtocolError(error_message) + raise qubes.exc.ProtocolError(error_message) return untrusted_value result = "" for char in untrusted_value: diff --git a/qubes/exc.py b/qubes/exc.py index 5139a5724..45a67fab0 100644 --- a/qubes/exc.py +++ b/qubes/exc.py @@ -219,3 +219,11 @@ def __init__(self, label): def __str__(self): # KeyError overrides __str__ method return QubesException.__str__(self) + + +class ProtocolError(AssertionError): + '''Raised when something is wrong with data received''' + + +class PermissionDenied(Exception): + '''Raised deliberately by handlers when we decide not to cooperate''' diff --git a/qubes/ext/admin.py b/qubes/ext/admin.py index 39a6e34a6..4abc04418 100644 --- a/qubes/ext/admin.py +++ b/qubes/ext/admin.py @@ -19,6 +19,7 @@ import qubes.api import qubes.api.internal +import qubes.exc import qubes.ext import qubes.vm.adminvm from qrexec.policy import utils, parser @@ -52,7 +53,7 @@ def on_tag_set_or_remove(self, vm, event, arg, **kwargs): # pylint: disable=unused-argument if arg.startswith('created-by-') and \ not isinstance(vm, qubes.vm.adminvm.AdminVM): - raise qubes.api.PermissionDenied( + raise qubes.exc.PermissionDenied( 'changing this tag is prohibited by {}.{}'.format( __name__, type(self).__name__)) diff --git a/qubes/ext/pci.py b/qubes/ext/pci.py index 3fd56c46b..1d76bdc79 100644 --- a/qubes/ext/pci.py +++ b/qubes/ext/pci.py @@ -171,15 +171,13 @@ def __init__(self, backend_domain, ident, libvirt_name=None): super().__init__( backend_domain=backend_domain, ident=ident, devclass="pci") - if hasattr(self, 'regex'): - # pylint: disable=no-member - dev_match = self.regex.match(ident) - if not dev_match: - raise ValueError('Invalid device identifier: {!r}'.format( - ident)) + dev_match = self.regex.match(ident) + if not dev_match: + raise ValueError('Invalid device identifier: {!r}'.format( + ident)) - for group in self.regex.groupindex: - setattr(self, group, dev_match.group(group)) + for group in self.regex.groupindex: + setattr(self, group, dev_match.group(group)) # lazy loading self._description = None diff --git a/qubes/tests/api.py b/qubes/tests/api.py index 54ca186d9..c900307c1 100644 --- a/qubes/tests/api.py +++ b/qubes/tests/api.py @@ -23,6 +23,7 @@ import unittest.mock import qubes.api +import qubes.exc import qubes.tests @@ -43,7 +44,7 @@ def __init__(self, app, src, method, dest, arg, send_event=None): 'mgmt.event': self.event, }[self.method.decode()] except KeyError: - raise qubes.api.ProtocolError('Invalid method') + raise qubes.exc.ProtocolError('Invalid method') def execute(self, untrusted_payload): self.task = asyncio.Task(self.function( diff --git a/qubes/tests/api_admin.py b/qubes/tests/api_admin.py index d4781a5c2..a8b40b2e5 100644 --- a/qubes/tests/api_admin.py +++ b/qubes/tests/api_admin.py @@ -34,6 +34,7 @@ import qubes import qubes.devices +import qubes.exc import qubes.firewall import qubes.api.admin import qubes.api.internal @@ -483,7 +484,7 @@ def test_080_vm_volume_info_invalid_volume(self): 'keys.return_value': ['root', 'private', 'volatile', 'kernel'] } self.vm.volumes.configure_mock(**volumes_conf) - with self.assertRaises(qubes.api.PermissionDenied): + with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func(b'admin.vm.volume.Info', b'test-vm1', b'no-such-volume') self.assertEqual(self.vm.volumes.mock_calls, @@ -511,7 +512,7 @@ def test_090_vm_volume_listsnapshots_invalid_volume(self): 'keys.return_value': ['root', 'private', 'volatile', 'kernel'] } self.vm.volumes.configure_mock(**volumes_conf) - with self.assertRaises(qubes.api.PermissionDenied): + with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func(b'admin.vm.volume.ListSnapshots', b'test-vm1', b'no-such-volume') self.assertEqual(self.vm.volumes.mock_calls, @@ -530,7 +531,7 @@ def test_100_vm_volume_snapshot_invalid_volume(self): {'rev2': '2018-02-22T22:22:22', 'rev1': '2018-01-11T11:11:11'}, } self.vm.volumes.configure_mock(**volumes_conf) - with self.assertRaises(qubes.api.PermissionDenied): + with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func(b'admin.vm.volume.Snapshots', b'test-vm1', b'no-such-volume') self.assertEqual(self.vm.volumes.mock_calls, @@ -543,7 +544,7 @@ def test_100_vm_volume_snapshot_invalid_revision(self): 'keys.return_value': ['root', 'private', 'volatile', 'kernel'] } self.vm.volumes.configure_mock(**volumes_conf) - with self.assertRaises(qubes.api.PermissionDenied): + with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func(b'admin.vm.volume.Snapshots', b'test-vm1', b'private', b'no-such-rev') self.assertEqual(self.vm.volumes.mock_calls, @@ -582,7 +583,7 @@ def test_110_vm_volume_revert_invalid_rev(self): } self.vm.volumes.configure_mock(**volumes_conf) self.vm.storage = unittest.mock.Mock() - with self.assertRaises(qubes.api.PermissionDenied): + with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func(b'admin.vm.volume.Revert', b'test-vm1', b'private', b'no-such-rev') self.assertEqual(self.vm.volumes.mock_calls, @@ -630,7 +631,7 @@ def test_120_vm_volume_resize_invalid_size1(self): self.vm.volumes.configure_mock(**volumes_conf) self.vm.storage = unittest.mock.Mock() self.vm.storage.resize.side_effect = self.dummy_coro - with self.assertRaises(qubes.api.ProtocolError): + with self.assertRaises(qubes.exc.ProtocolError): self.call_mgmt_func(b'admin.vm.volume.Resize', b'test-vm1', b'private', b'no-int-size') self.assertEqual(self.vm.volumes.mock_calls, @@ -645,7 +646,7 @@ def test_120_vm_volume_resize_invalid_size2(self): self.vm.volumes.configure_mock(**volumes_conf) self.vm.storage = unittest.mock.Mock() self.vm.storage.resize.side_effect = self.dummy_coro - with self.assertRaises(qubes.api.ProtocolError): + with self.assertRaises(qubes.exc.ProtocolError): self.call_mgmt_func(b'admin.vm.volume.Resize', b'test-vm1', b'private', b'-1') self.assertEqual(self.vm.volumes.mock_calls, @@ -660,7 +661,7 @@ def test_120_vm_volume_resize_invalid_size3(self): self.vm.volumes.configure_mock(**volumes_conf) self.vm.storage = unittest.mock.Mock() self.vm.storage.resize.side_effect = self.dummy_coro - with self.assertRaises(qubes.api.ProtocolError): + with self.assertRaises(qubes.exc.ProtocolError): self.call_mgmt_func(b'admin.vm.volume.Resize', b'test-vm1', b'private', b'10000000000000000000') self.assertEqual(self.vm.volumes.mock_calls, @@ -675,7 +676,7 @@ def test_120_vm_volume_resize_invalid_size4(self): self.vm.volumes.configure_mock(**volumes_conf) self.vm.storage = unittest.mock.Mock() self.vm.storage.resize.side_effect = self.dummy_coro - with self.assertRaises(qubes.api.ProtocolError): + with self.assertRaises(qubes.exc.ProtocolError): self.call_mgmt_func(b'admin.vm.volume.Resize', b'test-vm1', b'private', b'01') self.assertEqual(self.vm.volumes.mock_calls, @@ -867,7 +868,7 @@ def test_160_pool_add_missing_name(self, mock_parameters, mock_drivers): add_pool_mock, self.app.add_pool = self.coroutine_mock() - with self.assertRaises(qubes.api.PermissionDenied): + with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func(b'admin.pool.Add', b'dom0', b'driver1', b'param1=value\nparam2=some-value\n') self.assertEqual(mock_drivers.mock_calls, [unittest.mock.call()]) @@ -892,7 +893,7 @@ def test_160_pool_add_existing_pool(self, mock_parameters, mock_drivers): add_pool_mock, self.app.add_pool = self.coroutine_mock() - with self.assertRaises(qubes.api.PermissionDenied): + with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func(b'admin.pool.Add', b'dom0', b'driver1', b'name=file\nparam1=value\nparam2=some-value\n') self.assertEqual(mock_drivers.mock_calls, [unittest.mock.call()]) @@ -918,7 +919,7 @@ def test_160_pool_add_invalid_config_format(self, mock_parameters, add_pool_mock, self.app.add_pool = self.coroutine_mock() - with self.assertRaises(qubes.api.PermissionDenied): + with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func(b'admin.pool.Add', b'dom0', b'driver1', b'name=test-pool\nparam 1=value\n_param2\n') self.assertEqual(mock_drivers.mock_calls, [unittest.mock.call()]) @@ -946,7 +947,7 @@ def test_170_pool_remove_invalid_pool(self): 'test-pool': unittest.mock.Mock(), } remove_pool_mock, self.app.remove_pool = self.coroutine_mock() - with self.assertRaises(qubes.api.PermissionDenied): + with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func(b'admin.pool.Remove', b'dom0', b'no-such-pool') self.assertEqual(remove_pool_mock.mock_calls, []) @@ -1003,7 +1004,7 @@ def test_200_label_create_invalid_color(self): 'keys.return_value': range(1, 9), } self.app.labels.configure_mock(**labels_config) - with self.assertRaises(qubes.api.PermissionDenied): + with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func(b'admin.label.Create', b'dom0', b'cyan', b'abcd') self.assertEqual(self.app.get_label.mock_calls, @@ -1019,13 +1020,13 @@ def test_200_label_create_invalid_name(self): 'keys.return_value': range(1, 9), } self.app.labels.configure_mock(**labels_config) - with self.assertRaises(qubes.api.PermissionDenied): + with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func(b'admin.label.Create', b'dom0', b'01', b'0xff0000') - with self.assertRaises(qubes.api.PermissionDenied): + with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func(b'admin.label.Create', b'dom0', b'../xxx', b'0xff0000') - with self.assertRaises(qubes.api.PermissionDenied): + with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func(b'admin.label.Create', b'dom0', b'strange-name!@#$', b'0xff0000') @@ -1067,7 +1068,7 @@ def test_210_label_remove_default_label(self): self.app.labels = unittest.mock.MagicMock(wraps=self.app.labels) self.app.get_label = unittest.mock.Mock( **{'return_value.index': 6}) - with self.assertRaises(qubes.api.PermissionDenied): + with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func(b'admin.label.Remove', b'dom0', b'blue') self.assertEqual(self.app.labels.mock_calls, []) @@ -1077,7 +1078,7 @@ def test_210_label_remove_in_use(self): self.app.labels = unittest.mock.MagicMock(wraps=self.app.labels) self.app.get_label = unittest.mock.Mock( **{'return_value.index': 1}) - with self.assertRaises(qubes.api.PermissionDenied): + with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func(b'admin.label.Remove', b'dom0', b'red') self.assertEqual(self.app.labels.mock_calls, []) @@ -1149,7 +1150,7 @@ def test_234_shutdown_force_wait_invalid(self): async def coroutine_mock(*args, **kwargs): return func_mock(*args, **kwargs) self.vm.shutdown = coroutine_mock - with self.assertRaises(qubes.api.PermissionDenied): + with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func(b'admin.vm.Shutdown', b'test-vm1', b'forcewait') func_mock.assert_not_called() @@ -1496,7 +1497,7 @@ def test_334_vm_create_invalid_name(self, storage_mock): @unittest.mock.patch('qubes.storage.Storage.create') def test_335_vm_create_missing_name(self, storage_mock): storage_mock.side_effect = self.dummy_coro - with self.assertRaises(qubes.api.ProtocolError): + with self.assertRaises(qubes.exc.ProtocolError): self.call_mgmt_func(b'admin.vm.Create.AppVM', b'dom0', b'test-template', b'label=red') @@ -1505,7 +1506,7 @@ def test_335_vm_create_missing_name(self, storage_mock): @unittest.mock.patch('qubes.storage.Storage.create') def test_336_vm_create_spurious_pool(self, storage_mock): storage_mock.side_effect = self.dummy_coro - with self.assertRaises(qubes.api.ProtocolError): + with self.assertRaises(qubes.exc.ProtocolError): self.call_mgmt_func(b'admin.vm.Create.AppVM', b'dom0', b'test-template', b'name=test-vm2 label=red pool=default') @@ -1526,7 +1527,7 @@ def test_337_vm_create_duplicate_name(self, storage_mock): @unittest.mock.patch('qubes.storage.Storage.create') def test_338_vm_create_name_twice(self, storage_mock): storage_mock.side_effect = self.dummy_coro - with self.assertRaises(qubes.api.ProtocolError): + with self.assertRaises(qubes.exc.ProtocolError): self.call_mgmt_func(b'admin.vm.Create.AppVM', b'dom0', b'test-template', b'name=test-vm2 name=test-vm3 label=red') @@ -1608,7 +1609,7 @@ def test_343_vm_create_in_pool_invalid_pool2(self, storage_mock): @unittest.mock.patch('qubes.storage.Storage.create') def test_344_vm_create_in_pool_invalid_volume(self, storage_mock): storage_mock.side_effect = self.dummy_coro - with self.assertRaises(qubes.api.PermissionDenied): + with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func(b'admin.vm.CreateInPool.AppVM', b'dom0', b'test-template', b'name=test-vm2 label=red ' b'pool:invalid=test') @@ -1634,7 +1635,7 @@ def test_346_vm_create_in_pool_duplicate_pool(self, storage_mock): # setting custom pool for 'root' volume of AppVM should not be # allowed - this volume belongs to the template storage_mock.side_effect = self.dummy_coro - with self.assertRaises(qubes.api.ProtocolError): + with self.assertRaises(qubes.exc.ProtocolError): self.call_mgmt_func(b'admin.vm.CreateInPool.AppVM', b'dom0', b'test-template', b'name=test-vm2 label=red ' b'pool=test pool:root=test') @@ -2075,7 +2076,7 @@ def test_520_vm_volume_clone(self): def test_521_vm_volume_clone_invalid_volume(self): self.setup_for_clone() - with self.assertRaises(qubes.api.PermissionDenied): + with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func(b'admin.vm.volume.CloneFrom', b'test-vm1', b'private123', b'') self.assertNotIn('init_volume().import_volume', @@ -2087,7 +2088,7 @@ def test_522_vm_volume_clone_invalid_volume2(self): token = self.call_mgmt_func(b'admin.vm.volume.CloneFrom', b'test-vm1', b'private', b'') - with self.assertRaises(qubes.api.PermissionDenied): + with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func(b'admin.vm.volume.CloneTo', b'test-vm1', b'private123', token.encode()) self.assertNotIn('init_volume().import_volume', @@ -2105,7 +2106,7 @@ def get_volume(vid): else: return unittest.mock.DEFAULT self.pool.get_volume.side_effect = get_volume - with self.assertRaises(qubes.api.PermissionDenied): + with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func(b'admin.vm.volume.CloneTo', b'test-vm1', b'private', token.encode()) self.assertNotIn('init_volume().import_volume', @@ -2115,7 +2116,7 @@ def get_volume(vid): def test_524_vm_volume_clone_invlid_token(self): self.setup_for_clone() - with self.assertRaises(qubes.api.PermissionDenied): + with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func(b'admin.vm.volume.CloneTo', b'test-vm1', b'private', b'no-such-token') self.assertNotIn('init_volume().import_volume', @@ -2707,7 +2708,7 @@ def test_654_vm_device_set_persistent_not_attached(self): self.device_list_testclass) with unittest.mock.patch.object(qubes.vm.qubesvm.QubesVM, 'is_halted', lambda _: False): - with self.assertRaises(qubes.api.PermissionDenied): + with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func( b'admin.vm.device.testclass.Set.persistent', b'test-vm1', b'test-vm1+1234', b'True') @@ -2720,7 +2721,7 @@ def test_655_vm_device_set_persistent_invalid_value(self): self.device_list_testclass) with unittest.mock.patch.object(qubes.vm.qubesvm.QubesVM, 'is_halted', lambda _: False): - with self.assertRaises(qubes.api.PermissionDenied): + with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func( b'admin.vm.device.testclass.Set.persistent', b'test-vm1', b'test-vm1+1234', b'maybe') @@ -2739,7 +2740,7 @@ def test_660_pool_set_revisions_to_keep(self): def test_661_pool_set_revisions_to_keep_negative(self): self.app.pools['test-pool'] = unittest.mock.Mock() - with self.assertRaises(qubes.api.ProtocolError): + with self.assertRaises(qubes.exc.ProtocolError): self.call_mgmt_func(b'admin.pool.Set.revisions_to_keep', b'dom0', b'test-pool', b'-2') self.assertEqual(self.app.pools['test-pool'].mock_calls, []) @@ -2747,7 +2748,7 @@ def test_661_pool_set_revisions_to_keep_negative(self): def test_662_pool_set_revisions_to_keep_not_a_number(self): self.app.pools['test-pool'] = unittest.mock.Mock() - with self.assertRaises(qubes.api.ProtocolError): + with self.assertRaises(qubes.exc.ProtocolError): self.call_mgmt_func(b'admin.pool.Set.revisions_to_keep', b'dom0', b'test-pool', b'abc') self.assertEqual(self.app.pools['test-pool'].mock_calls, []) @@ -2764,7 +2765,7 @@ def test_663_pool_set_ephemeral(self): def test_664_pool_set_ephemeral_not_a_boolean(self): self.app.pools['test-pool'] = unittest.mock.Mock() - with self.assertRaises(qubes.api.ProtocolError): + with self.assertRaises(qubes.exc.ProtocolError): self.call_mgmt_func(b'admin.pool.Set.ephemeral_volatile', b'dom0', b'test-pool', b'abc') self.assertEqual(self.app.pools['test-pool'].mock_calls, []) @@ -2793,7 +2794,7 @@ def test_671_vm_volume_set_revisions_to_keep_negative(self): } self.vm.volumes.configure_mock(**volumes_conf) self.vm.storage = unittest.mock.Mock() - with self.assertRaises(qubes.api.ProtocolError): + with self.assertRaises(qubes.exc.ProtocolError): self.call_mgmt_func(b'admin.vm.volume.Set.revisions_to_keep', b'test-vm1', b'private', b'-2') @@ -2804,7 +2805,7 @@ def test_672_vm_volume_set_revisions_to_keep_not_a_number(self): } self.vm.volumes.configure_mock(**volumes_conf) self.vm.storage = unittest.mock.Mock() - with self.assertRaises(qubes.api.ProtocolError): + with self.assertRaises(qubes.exc.ProtocolError): self.call_mgmt_func(b'admin.vm.volume.Set.revisions_to_keep', b'test-vm1', b'private', b'abc') @@ -2831,7 +2832,7 @@ def test_681_vm_volume_set_rw_invalid(self): } self.vm.volumes.configure_mock(**volumes_conf) self.vm.storage = unittest.mock.Mock() - with self.assertRaises(qubes.api.ProtocolError): + with self.assertRaises(qubes.exc.ProtocolError): self.call_mgmt_func(b'admin.vm.volume.Set.rw', b'test-vm1', b'private', b'abc') self.assertFalse(self.app.save.called) @@ -2859,7 +2860,7 @@ def test_686_vm_volume_set_ephemeral_invalid(self): } self.vm.volumes.configure_mock(**volumes_conf) self.vm.storage = unittest.mock.Mock() - with self.assertRaises(qubes.api.ProtocolError): + with self.assertRaises(qubes.exc.ProtocolError): self.call_mgmt_func(b'admin.vm.volume.Set.ephemeral', b'test-vm1', b'volatile', b'abc') self.assertFalse(self.app.save.called) @@ -3014,14 +3015,14 @@ def test_990_vm_unexpected_payload(self): for method in methods_with_no_payload: # should reject payload regardless of having argument or not with self.subTest(method.decode('ascii')): - with self.assertRaises(qubes.api.ProtocolError): + with self.assertRaises(qubes.exc.ProtocolError): self.call_mgmt_func(method, b'test-vm1', b'', b'unexpected-payload') self.assertFalse(vm_mock.called) self.assertFalse(self.app.save.called) with self.subTest(method.decode('ascii') + '+arg'): - with self.assertRaises(qubes.api.ProtocolError): + with self.assertRaises(qubes.exc.ProtocolError): self.call_mgmt_func(method, b'test-vm1', b'some-arg', b'unexpected-payload') self.assertFalse(vm_mock.called) @@ -3052,11 +3053,11 @@ def test_991_vm_unexpected_argument(self): vm_mock.qid = self.vm.qid vm_mock.__lt__ = (lambda x, y: x.qid < y.qid) self.app.domains._dict[self.vm.qid] = vm_mock - exceptions = (qubes.api.PermissionDenied, qubes.api.ProtocolError) + exceptions = (qubes.exc.PermissionDenied, qubes.exc.ProtocolError) for method in methods_with_no_argument: # should reject argument regardless of having payload or not with self.subTest(method.decode('ascii')): - with self.assertRaises(qubes.api.PermissionDenied): + with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func(method, b'test-vm1', b'some-arg', b'') self.assertFalse(vm_mock.called) @@ -3099,14 +3100,14 @@ def test_992_dom0_unexpected_payload(self): for method in methods_with_no_payload: # should reject payload regardless of having argument or not with self.subTest(method.decode('ascii')): - with self.assertRaises(qubes.api.ProtocolError): + with self.assertRaises(qubes.exc.ProtocolError): self.call_mgmt_func(method, b'dom0', b'', b'unexpected-payload') self.assertFalse(vm_mock.called) self.assertFalse(self.app.save.called) with self.subTest(method.decode('ascii') + '+arg'): - with self.assertRaises(qubes.api.ProtocolError): + with self.assertRaises(qubes.exc.ProtocolError): self.call_mgmt_func(method, b'dom0', b'some-arg', b'unexpected-payload') self.assertFalse(vm_mock.called) @@ -3129,11 +3130,11 @@ def test_993_dom0_unexpected_argument(self): vm_mock.qid = self.vm.qid vm_mock.__lt__ = (lambda x, y: x.qid < y.qid) self.app.domains._dict[self.vm.qid] = vm_mock - exceptions = (qubes.api.PermissionDenied, qubes.api.ProtocolError) + exceptions = (qubes.exc.PermissionDenied, qubes.exc.ProtocolError) for method in methods_with_no_argument: # should reject argument regardless of having payload or not with self.subTest(method.decode('ascii')): - with self.assertRaises(qubes.api.PermissionDenied): + with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func(method, b'dom0', b'some-arg', b'') self.assertFalse(vm_mock.called) @@ -3185,7 +3186,7 @@ def test_994_dom0_only_calls(self): vm_mock.qid = self.vm.qid vm_mock.__lt__ = (lambda x, y: x.qid < y.qid) self.app.domains._dict[self.vm.qid] = vm_mock - exceptions = (qubes.api.PermissionDenied, qubes.api.ProtocolError) + exceptions = (qubes.exc.PermissionDenied, qubes.exc.ProtocolError) for method in methods_for_dom0_only: # should reject call regardless of having payload or not with self.subTest(method.decode('ascii')): @@ -3272,7 +3273,7 @@ def test_995_vm_only_calls(self): vm_mock.qid = self.vm.qid vm_mock.__lt__ = (lambda x, y: x.qid < y.qid) self.app.domains._dict[self.vm.qid] = vm_mock - exceptions = (qubes.api.PermissionDenied, qubes.api.ProtocolError) + exceptions = (qubes.exc.PermissionDenied, qubes.exc.ProtocolError) for method in methods_for_vm_only: # should reject payload regardless of having argument or not # should reject call regardless of having payload or not diff --git a/qubes/tests/api_misc.py b/qubes/tests/api_misc.py index 7c8776dae..3345cea8c 100644 --- a/qubes/tests/api_misc.py +++ b/qubes/tests/api_misc.py @@ -20,6 +20,8 @@ import asyncio from datetime import datetime from unittest import mock + +import qubes.exc import qubes.tests import qubes.api.misc @@ -98,7 +100,7 @@ def test_002_features_request_invalid1(self): '/features-request/feature1': b'test spaces', } self.configure_qdb(qdb_entries) - with self.assertRaises(qubes.api.PermissionDenied): + with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func(b'qubes.FeaturesRequest') self.assertEqual(self.app.mock_calls, []) self.assertEqual(self.src.mock_calls, [ @@ -180,7 +182,7 @@ def test_015_notify_tools_invalid_value_qrexec(self): '/qubes-tools/default-user': b'user', } self.configure_qdb(qdb_entries) - with self.assertRaises(qubes.api.PermissionDenied): + with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func(b'qubes.NotifyTools') self.assertEqual(self.app.mock_calls, []) self.assertEqual(self.src.mock_calls, [ @@ -196,7 +198,7 @@ def test_016_notify_tools_invalid_value_gui(self): '/qubes-tools/default-user': b'user', } self.configure_qdb(qdb_entries) - with self.assertRaises(qubes.api.PermissionDenied): + with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func(b'qubes.NotifyTools') self.assertEqual(self.app.mock_calls, []) self.assertEqual(self.src.mock_calls, [ @@ -242,7 +244,7 @@ def test_021_notify_updates_standalone2(self): def test_022_notify_updates_invalid(self): del self.src.template - with self.assertRaises(qubes.api.PermissionDenied): + with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func(b'qubes.NotifyUpdates', payload=b'') self.assertEqual(self.src.mock_calls, []) self.assertEqual(self.tpl.mock_calls, []) @@ -250,7 +252,7 @@ def test_022_notify_updates_invalid(self): def test_023_notify_updates_invalid2(self): del self.src.template - with self.assertRaises(qubes.api.PermissionDenied): + with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func(b'qubes.NotifyUpdates', payload=b'no updates') self.assertEqual(self.src.mock_calls, []) self.assertEqual(self.tpl.mock_calls, []) From 76f55382efa483e1221b2e0f68bddcce20704fdd Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Wed, 20 Mar 2024 11:35:15 +0100 Subject: [PATCH 24/49] q-dev: device protocol refactor + tests --- doc/qubes-devices.rst | 8 +- qubes/api/admin.py | 43 +-- qubes/device_protocol.py | 272 +++++++------- qubes/devices.py | 146 ++------ qubes/ext/block.py | 23 +- qubes/ext/utils.py | 6 +- qubes/tests/api_admin.py | 565 +++++++++++++++++++++-------- qubes/tests/devices.py | 23 +- qubes/tests/integ/devices_block.py | 2 +- qubes/tests/integ/devices_pci.py | 8 +- qubes/tests/integ/vm_qrexec_gui.py | 2 +- qubes/tests/vm/qubesvm.py | 13 +- qubes/vm/__init__.py | 11 +- qubes/vm/qubesvm.py | 4 +- 14 files changed, 627 insertions(+), 499 deletions(-) diff --git a/doc/qubes-devices.rst b/doc/qubes-devices.rst index 7e71ab244..8a3c7141f 100644 --- a/doc/qubes-devices.rst +++ b/doc/qubes-devices.rst @@ -13,7 +13,7 @@ Devices are identified by pair of (backend domain, `ident`), where `ident` is Device Assignment vs Attachment ------------------------------- -:py:class:`qubes.devices.DeviceAssignment` describes the assignment of a device +:py:class:`qubes.device_protocol.DeviceAssignment` describes the assignment of a device to a frontend VM. For clarity let's us introduce two types of assignments: *potential* and *real* (attachment). Attachment indicates that the device has been attached by the Qubes backend to its frontend VM and is visible @@ -22,9 +22,9 @@ has two additional options: `automatically_attach` and `required`. For detailed descriptions, refer to the `DeviceAssignment` documentation. In general we refer to potential assignment as assignment and real assignment as attachment. To check whether the device is currently -attached, we check :py:meth:`qubes.devices.DeviceAssignment.attached`, +attached, we check :py:meth:`qubes.device_protocol.DeviceAssignment.attached`, while to check whether an (potential) assignment exists, -we check :py:meth:`qubes.devices.DeviceAssignment.attach_automatically`. +we check :py:meth:`qubes.device_protocol.DeviceAssignment.attach_automatically`. Potential and real connections may coexist at the same time, in which case both values will be true. @@ -106,7 +106,7 @@ is connected. Therefore, when assigning a device to a VM, such as `sys-usb:1-1.1`, the port `1-1.1` is actually assigned, and thus *every* devices connected to it will be automatically attached. Similarly, when assigning `vm:sda`, every block device with the name `sda` -will be automatically attached. We can limit this using :py:meth:`qubes.devices.DeviceInfo.self_identity`, which returns a string containing information +will be automatically attached. We can limit this using :py:meth:`qubes.device_protocol.DeviceInfo.self_identity`, which returns a string containing information presented by the device, such as, `vendor_id`, `product_id`, `serial_number`, and encoded interfaces. In the case of block devices, `self_identity` consists of the parent port to which the device is connected (if any), diff --git a/qubes/api/admin.py b/qubes/api/admin.py index b06526fc3..f834be175 100644 --- a/qubes/api/admin.py +++ b/qubes/api/admin.py @@ -45,6 +45,7 @@ import qubes.vm import qubes.vm.adminvm import qubes.vm.qubesvm +from qubes.device_protocol import Device class QubesMgmtEventsDispatcher: @@ -1276,11 +1277,8 @@ async def vm_device_assign(self, endpoint, untrusted_payload): # may raise KeyError, either on domain or ident dev = self.app.domains[backend_domain].devices[devclass][ident] - assignment = qubes.devices.DeviceAssignment.deserialize( - untrusted_payload, - expected_backend_domain=dev.backend_domain, - expected_ident=ident, - expected_devclass=devclass + assignment = qubes.device_protocol.DeviceAssignment.deserialize( + untrusted_payload, expected_device=dev ) self.fire_event_for_permission( @@ -1312,8 +1310,8 @@ async def vm_device_unassign(self, endpoint): self.fire_event_for_permission(device=dev, devclass=devclass) - assignment = qubes.devices.DeviceAssignment( - dev.backend_domain, dev.ident) + assignment = qubes.device_protocol.DeviceAssignment( + dev.backend_domain, dev.ident, devclass) await self.dest.devices[devclass].unassign(assignment) self.app.save() @@ -1333,11 +1331,8 @@ async def vm_device_attach(self, endpoint, untrusted_payload): # may raise KeyError, either on domain or ident dev = self.app.domains[backend_domain].devices[devclass][ident] - assignment = qubes.devices.DeviceAssignment.deserialize( - untrusted_payload, - expected_backend_domain=dev.backend_domain, - expected_ident=ident, - expected_devclass=devclass + assignment = qubes.device_protocol.DeviceAssignment.deserialize( + untrusted_payload, expected_device=dev ) self.fire_event_for_permission( @@ -1348,9 +1343,8 @@ async def vm_device_attach(self, endpoint, untrusted_payload): ) await self.dest.devices[devclass].attach(assignment) - self.app.save() # not needed? - # Attach/Detach action can modify only volatile state of running VM. + # Attach/Detach action can modify only a volatile state of running VM. # For this reason, execute=True @qubes.api.method( 'admin.vm.device.{endpoint}.Detach', @@ -1370,11 +1364,11 @@ async def vm_device_detach(self, endpoint): self.fire_event_for_permission(device=dev, devclass=devclass) - assignment = qubes.devices.DeviceAssignment( - dev.backend_domain, dev.ident) + assignment = qubes.device_protocol.DeviceAssignment( + dev.backend_domain, dev.ident, devclass) await self.dest.devices[devclass].detach(assignment) - # Assign/Unassign action can modify only persistent state of running VM. + # Assign/Unassign action can modify only a persistent state of running VM. # For this reason, write=True @qubes.api.method('admin.vm.device.{endpoint}.Set.assignment', endpoints=(ep.name @@ -1382,7 +1376,7 @@ async def vm_device_detach(self, endpoint): scope='local', write=True) async def vm_device_set_assignment(self, endpoint, untrusted_payload): """ - Update assignment of an already attached device. + Update assignment of an already assigned device. Payload: `None` -> unassign device from a qube @@ -1392,18 +1386,15 @@ async def vm_device_set_assignment(self, endpoint, untrusted_payload): devclass = endpoint self.enforce(untrusted_payload in (b'True', b'False', b'None')) - # now is safe to eval, since value of untrusted_payload is trusted + # now is safe to eval, since the value of untrusted_payload is trusted + # pylint: disable=eval-used assignment = eval(untrusted_payload) del untrusted_payload # qrexec already verified that no strange characters are in self.arg - backend_domain, ident = self.arg.split('+', 1) - # device must be already attached - matching_devices = [dev for dev - in self.dest.devices[devclass].get_attached_devices() - if dev.backend_domain.name == backend_domain and dev.ident == ident] - self.enforce(len(matching_devices) == 1) - dev = matching_devices[0] + backend_domain_name, ident = self.arg.split('+', 1) + backend_domain = self.app.domains[backend_domain_name] + dev = Device(backend_domain, ident, devclass) self.fire_event_for_permission(device=dev, assignment=assignment) diff --git a/qubes/device_protocol.py b/qubes/device_protocol.py index cf7dc6081..bea5e61c3 100644 --- a/qubes/device_protocol.py +++ b/qubes/device_protocol.py @@ -32,7 +32,7 @@ import string import sys from enum import Enum -from typing import Optional, Dict, Any, List, Type, Union +from typing import Optional, Dict, Any, List, Union import qubes.utils @@ -52,6 +52,9 @@ def qbool(value): class Device: + ALLOWED_CHARS_KEY = string.digits + string.ascii_letters + '-_.' + ALLOWED_CHARS_PARAM = ALLOWED_CHARS_KEY + ',+:' + def __init__(self, backend_domain, ident, devclass=None): self.__backend_domain = backend_domain self.__ident = ident @@ -70,7 +73,7 @@ def __lt__(self, other): if isinstance(other, Device): return (self.backend_domain.name, self.ident) < \ (other.backend_domain.name, other.ident) - return NotImplemented() + raise NotImplementedError() def __repr__(self): return "[%s]:%s" % (self.backend_domain, self.ident) @@ -102,8 +105,7 @@ def devclass(self) -> str: """ if self.__bus: return self.__bus - else: - return "peripheral" + return "peripheral" @property def devclass_is_set(self) -> bool: @@ -118,10 +120,80 @@ def devclass(self, devclass: str): However, if it has not been set, i.e., the value is `None`, we can override it.""" - if self.__bus != None: + if self.__bus is not None: raise TypeError("Attribute devclass is immutable") self.__bus = devclass + @classmethod + def unpack_properties(cls, untrusted_serialization: bytes): + ut_decoded = untrusted_serialization.decode( + 'ascii', errors='strict').strip() + + properties = {} + options = {} + + if not ut_decoded: + return properties, options + + keys = [] + values = [] + ut_key, _, ut_rest = ut_decoded.partition("='") + + key = sanitize_str( + ut_key, cls.ALLOWED_CHARS_KEY, + error_message='Invalid chars in property name: ') + keys.append(key) + while "='" in ut_rest: + ut_value_key, _, ut_rest = ut_rest.partition("='") + ut_value, _, ut_key = ut_value_key.rpartition("' ") + value = sanitize_str( + deserialize_str(ut_value), cls.ALLOWED_CHARS_PARAM, + error_message='Invalid chars in property value: ') + values.append(value) + key = sanitize_str( + ut_key, cls.ALLOWED_CHARS_KEY, + error_message='Invalid chars in property name: ') + keys.append(key) + ut_value = ut_rest[:-1] # ending ' + value = sanitize_str( + deserialize_str(ut_value), cls.ALLOWED_CHARS_PARAM, + error_message='Invalid chars in property value: ') + values.append(value) + + for key, value in zip(keys, values): + if key.startswith("_"): + # it's handled in cls.__init__ + options[key[1:]] = value + else: + properties[key] = value + + return properties, options + + @staticmethod + def check_device_properties(expected_device, properties): + expected = expected_device + exp_vm_name = expected.backend_domain.name + if properties.get('backend_domain', exp_vm_name) != exp_vm_name: + raise UnexpectedDeviceProperty( + f"Got device exposed by {properties['backend_domain']}" + f"when expected devices from {exp_vm_name}.") + properties['backend_domain'] = expected.backend_domain + + if properties.get('ident', expected.ident) != expected.ident: + raise UnexpectedDeviceProperty( + f"Got device with id: {properties['ident']} " + f"when expected id: {expected.ident}.") + properties['ident'] = expected.ident + + if expected.devclass_is_set: + if (properties.get('devclass', expected.devclass) + != expected.devclass): + raise UnexpectedDeviceProperty( + f"Got {properties['devclass']} device " + f"when expected {expected.devclass}.") + properties['devclass'] = expected.devclass + + class DeviceCategory(Enum): """ @@ -130,6 +202,7 @@ class DeviceCategory(Enum): Arbitrarily selected interfaces that are important to users, thus deserving special recognition such as a custom icon, etc. """ + # pylint: disable=invalid-name Other = "*******" Communication = ("u02****", "p07****") # eg. modems @@ -163,11 +236,11 @@ def from_str(interface_encoding: str) -> 'DeviceCategory': for interface in DeviceCategory: for pattern in interface.value: score = 0 - for t, p in zip(interface_encoding, pattern): - if t == p: + for itf, pat in zip(interface_encoding, pattern): + if itf == pat: score += 1 - elif p != "*": - score = -1 # inconsistent with pattern + elif pat != "*": + score = -1 # inconsistent with a pattern break if score > best_score: @@ -285,6 +358,7 @@ def _load_classes(bus: str): class DeviceInfo(Device): """ Holds all information about a device """ + ALLOWED_CHARS_PARAM = Device.ALLOWED_CHARS_PARAM + string.punctuation + ' ' def __init__( self, @@ -466,31 +540,30 @@ def serialize(self) -> bytes: ) qname = serialize_str(self.backend_domain.name) - backend_prop = (b"backend_domain=" + qname.encode('ascii')) + backend_prop = b"backend_domain=" + qname.encode('ascii') properties += b' ' + backend_prop if self.attachment: qname = serialize_str(self.attachment.name) - attachment_prop = (b"attachment=" + qname.encode('ascii')) + attachment_prop = b"attachment=" + qname.encode('ascii') properties += b' ' + attachment_prop interfaces = serialize_str( ''.join(repr(ifc) for ifc in self.interfaces)) - interfaces_prop = (b'interfaces=' + interfaces.encode('ascii')) + interfaces_prop = b'interfaces=' + interfaces.encode('ascii') properties += b' ' + interfaces_prop if self.parent_device is not None: ident = serialize_str(self.parent_device.ident) - ident_prop = (b'parent_ident=' + ident.encode('ascii')) + ident_prop = b'parent_ident=' + ident.encode('ascii') properties += b' ' + ident_prop devclass = serialize_str(self.parent_device.devclass) - devclass_prop = (b'parent_devclass=' + devclass.encode('ascii')) + devclass_prop = b'parent_devclass=' + devclass.encode('ascii') properties += b' ' + devclass_prop data = b' '.join( f'_{prop}={serialize_str(value)}'.encode('ascii') - for prop, value in ((key, self.data[key]) for key in self.data) - ) + for prop, value in self.data.items()) if data: properties += b' ' + data @@ -505,64 +578,53 @@ def deserialize( ) -> 'DeviceInfo': ident, _, rest = serialization.partition(b' ') ident = ident.decode('ascii', errors='ignore') + device = UnknownDevice( + backend_domain=expected_backend_domain, + ident=ident, + devclass=expected_devclass, + ) + try: - result = DeviceInfo._deserialize( - cls, rest, expected_backend_domain, ident, expected_devclass) + device = cls._deserialize(rest, device) except Exception as exc: print(exc, file=sys.stderr) - ident = serialization.split(b' ')[0].decode( - 'ascii', errors='ignore') - result = UnknownDevice( - backend_domain=expected_backend_domain, - ident=ident, - devclass=expected_devclass, - ) - return result - @staticmethod + return device + + @classmethod def _deserialize( - cls: Type, + cls, untrusted_serialization: bytes, - expected_backend_domain: QubesVM, - expected_ident: str, - expected_devclass: Optional[str] = None, + expected_device: Device ) -> 'DeviceInfo': - allowed_chars_key = string.digits + string.ascii_letters + '-_.' - allowed_chars_value = ( - allowed_chars_key + ',+:' + string.punctuation + ' ') - - properties, options = unpack_properties( - untrusted_serialization, allowed_chars_key, allowed_chars_value) + properties, options = cls.unpack_properties(untrusted_serialization) properties.update(options) - check_device_properties( - expected_backend_domain, - expected_ident, - expected_devclass, - properties - ) + cls.check_device_properties(expected_device, properties) if 'attachment' not in properties or not properties['attachment']: properties['attachment'] = None else: - app = expected_backend_domain.app + app = expected_device.backend_domain.app properties['attachment'] = app.domains.get_blind( properties['attachment']) - if expected_devclass and properties['devclass'] != expected_devclass: + if (expected_device.devclass_is_set + and properties['devclass'] != expected_device.devclass): raise UnexpectedDeviceProperty( f"Got {properties['devclass']} device " - f"when expected {expected_devclass}.") + f"when expected {expected_device.devclass}.") - interfaces = properties['interfaces'] - interfaces = [ - DeviceInterface(interfaces[i:i + 7]) - for i in range(0, len(interfaces), 7)] - properties['interfaces'] = interfaces + if 'interfaces' in properties: + interfaces = properties['interfaces'] + interfaces = [ + DeviceInterface(interfaces[i:i + 7]) + for i in range(0, len(interfaces), 7)] + properties['interfaces'] = interfaces if 'parent_ident' in properties: properties['parent'] = Device( - backend_domain=expected_backend_domain, + backend_domain=expected_device.backend_domain, ident=properties['parent_ident'], devclass=properties['parent_devclass'], ) @@ -613,8 +675,9 @@ def sanitize_str( characters with the string. """ if replace_char is None: - if any(x not in allowed_chars for x in untrusted_value): - raise ProtocolError(error_message) + not_allowed_chars = set(untrusted_value) - set(allowed_chars) + if not_allowed_chars: + raise ProtocolError(error_message + repr(not_allowed_chars)) return untrusted_value result = "" for char in untrusted_value: @@ -625,71 +688,6 @@ def sanitize_str( return result -def unpack_properties( - untrusted_serialization: bytes, - allowed_chars_key: str, - allowed_chars_value: str -): - ut_decoded = untrusted_serialization.decode( - 'ascii', errors='strict').strip() - - options = {} - keys = [] - values = [] - ut_key, _, ut_rest = ut_decoded.partition("='") - - key = sanitize_str( - ut_key, allowed_chars_key, - error_message='Invalid chars in property name') - keys.append(key) - while "='" in ut_rest: - ut_value_key, _, ut_rest = ut_rest.partition("='") - ut_value, _, ut_key = ut_value_key.rpartition("' ") - value = sanitize_str( - deserialize_str(ut_value), allowed_chars_value, - error_message='Invalid chars in property value') - values.append(value) - key = sanitize_str( - ut_key, allowed_chars_key, - error_message='Invalid chars in property name') - keys.append(key) - ut_value = ut_rest[:-1] # ending ' - value = sanitize_str( - deserialize_str(ut_value), allowed_chars_value, - error_message='Invalid chars in property value') - values.append(value) - - properties = dict() - for key, value in zip(keys, values): - if key.startswith("_"): - # it's handled in cls.__init__ - options[key[1:]] = value - else: - properties[key] = value - - return properties, options - - -def check_device_properties( - expected_backend_domain, expected_ident, expected_devclass, properties -): - if properties['backend_domain'] != expected_backend_domain.name: - raise UnexpectedDeviceProperty( - f"Got device exposed by {properties['backend_domain']}" - f"when expected devices from {expected_backend_domain.name}.") - properties['backend_domain'] = expected_backend_domain - - if properties['ident'] != expected_ident: - raise UnexpectedDeviceProperty( - f"Got device with id: {properties['ident']} " - f"when expected id: {expected_ident}.") - - if expected_devclass and properties['devclass'] != expected_devclass: - raise UnexpectedDeviceProperty( - f"Got {properties['devclass']} device " - f"when expected {expected_devclass}.") - - class UnknownDevice(DeviceInfo): # pylint: disable=too-few-public-methods """Unknown device - for example exposed by domain not running currently""" @@ -832,7 +830,7 @@ def serialize(self) -> bytes: ) back_name = serialize_str(self.backend_domain.name) - backend_domain_prop = (b"backend_domain=" + back_name.encode('ascii')) + backend_domain_prop = b"backend_domain=" + back_name.encode('ascii') properties += b' ' + backend_domain_prop if self.frontend_domain is not None: @@ -853,43 +851,27 @@ def serialize(self) -> bytes: def deserialize( cls, serialization: bytes, - expected_backend_domain: QubesVM, - expected_ident: str, - expected_devclass: Optional[str] = None, + expected_device: Device, ) -> 'DeviceAssignment': try: - result = DeviceAssignment._deserialize( - cls, serialization, - expected_backend_domain, expected_ident, expected_devclass - ) + result = cls._deserialize(serialization, expected_device) except Exception as exc: raise ProtocolError() from exc return result - @staticmethod + @classmethod def _deserialize( - cls: Type, + cls, untrusted_serialization: bytes, - expected_backend_domain: QubesVM, - expected_ident: str, - expected_devclass: Optional[str] = None, + expected_device: Device, ) -> 'DeviceAssignment': - allowed_chars_key = string.digits + string.ascii_letters + '-_.' - allowed_chars_value = allowed_chars_key + ',+:' - - properties, options = unpack_properties( - untrusted_serialization, allowed_chars_key, allowed_chars_value) + properties, options = cls.unpack_properties(untrusted_serialization) properties['options'] = options - check_device_properties( - expected_backend_domain, - expected_ident, - expected_devclass, - properties - ) + cls.check_device_properties(expected_device, properties) properties['attach_automatically'] = qbool( - properties['attach_automatically']) - properties['required'] = qbool(properties['required']) + properties.get('attach_automatically', 'no')) + properties['required'] = qbool(properties.get('required', 'no')) return cls(**properties) diff --git a/qubes/devices.py b/qubes/devices.py index 801a13903..deedf04b3 100644 --- a/qubes/devices.py +++ b/qubes/devices.py @@ -33,8 +33,8 @@ Such extension should: - provide `qubes.devices` endpoint - a class descendant from - :py:class:`qubes.devices.DeviceInfo`, designed to hold device description - (including bus-specific properties) + :py:class:`qubes.device_protocol.DeviceInfo`, designed to hold device + description (including bus-specific properties) - handle `device-attach:bus` and `device-detach:bus` events for performing the attach/detach action; events are fired even when domain isn't running and extension should be prepared for this; handlers for those events @@ -63,7 +63,6 @@ import qubes.exc import qubes.utils -from qubes.exc import ProtocolError from qubes.device_protocol import (Device, DeviceInfo, UnknownDevice, DeviceAssignment) @@ -86,115 +85,12 @@ class DeviceAlreadyAssigned(qubes.exc.QubesException, KeyError): """ -class UnexpectedDeviceProperty(qubes.exc.QubesException, ValueError): - """ - Device has unexpected property such as backend_domain, devclass etc. - """ - class UnrecognizedDevice(qubes.exc.QubesException, ValueError): """ Device identity is not as expected. """ -def serialize_str(value: str): - return repr(str(value)) - - -def deserialize_str(value: str): - return value.replace("\\\'", "'") - - -def sanitize_str( - untrusted_value: str, - allowed_chars: str, - replace_char: str = None, - error_message: str = "" -) -> str: - """ - Sanitize given untrusted string. - - If `replace_char` is not None, ignore `error_message` and replace invalid - characters with the string. - """ - if replace_char is None: - if any(x not in allowed_chars for x in untrusted_value): - raise qubes.exc.ProtocolError(error_message) - return untrusted_value - result = "" - for char in untrusted_value: - if char in allowed_chars: - result += char - else: - result += replace_char - return result - - -def unpack_properties( - untrusted_serialization: bytes, - allowed_chars_key: str, - allowed_chars_value: str -): - ut_decoded = untrusted_serialization.decode( - 'ascii', errors='strict').strip() - - options = {} - keys = [] - values = [] - ut_key, _, ut_rest = ut_decoded.partition("='") - - key = sanitize_str( - ut_key, allowed_chars_key, - error_message='Invalid chars in property name') - keys.append(key) - while "='" in ut_rest: - ut_value_key, _, ut_rest = ut_rest.partition("='") - ut_value, _, ut_key = ut_value_key.rpartition("' ") - value = sanitize_str( - deserialize_str(ut_value), allowed_chars_value, - error_message='Invalid chars in property value') - values.append(value) - key = sanitize_str( - ut_key, allowed_chars_key, - error_message='Invalid chars in property name') - keys.append(key) - ut_value = ut_rest[:-1] # ending ' - value = sanitize_str( - deserialize_str(ut_value), allowed_chars_value, - error_message='Invalid chars in property value') - values.append(value) - - properties = dict() - for key, value in zip(keys, values): - if key.startswith("_"): - # it's handled in cls.__init__ - options[key[1:]] = value - else: - properties[key] = value - - return properties, options - - -def check_device_properties( - expected_backend_domain, expected_ident, expected_devclass, properties -): - if properties['backend_domain'] != expected_backend_domain.name: - raise UnexpectedDeviceProperty( - f"Got device exposed by {properties['backend_domain']}" - f"when expected devices from {expected_backend_domain.name}.") - properties['backend_domain'] = expected_backend_domain - - if properties['ident'] != expected_ident: - raise UnexpectedDeviceProperty( - f"Got device with id: {properties['ident']} " - f"when expected id: {expected_ident}.") - - if expected_devclass and properties['devclass'] != expected_devclass: - raise UnexpectedDeviceProperty( - f"Got {properties['devclass']} device " - f"when expected {expected_devclass}.") - - class DeviceCollection: """Bag for devices. @@ -297,11 +193,12 @@ async def attach(self, assignment: DeviceAssignment): Attach device to domain. """ - if assignment.devclass is None: + if not assignment.devclass_is_set: assignment.devclass = self._bus elif assignment.devclass != self._bus: raise ValueError( - 'Trying to attach DeviceAssignment of a different device class') + f'Trying to attach {assignment.devclass} device ' + f'when {self._bus} device expected.') if self._vm.is_halted(): raise qubes.exc.QubesVMNotRunningError( @@ -326,11 +223,12 @@ async def assign(self, assignment: DeviceAssignment): """ Assign device to domain. """ - if assignment.devclass is None: + if not assignment.devclass_is_set: assignment.devclass = self._bus elif assignment.devclass != self._bus: raise ValueError( - 'Trying to assign DeviceAssignment of a different device class') + f'Trying to attach {assignment.devclass} device ' + f'when {self._bus} device expected.') device = assignment.device if device in self.get_assigned_devices(): @@ -355,12 +253,11 @@ def load_assignment(self, device_assignment: DeviceAssignment): device_assignment.devclass = self._bus self._set.add(device_assignment) - async def update_assignment( - self, device: DeviceInfo, required: Optional[bool]): + async def update_assignment(self, device: Device, required: Optional[bool]): """ - Update assignment of already attached device. + Update assignment of an already attached device. - :param DeviceInfo device: device for which change required flag + :param Device device: device for which change required flag :param bool required: new assignment: `None` -> unassign device from qube `False` -> device will be auto-attached to qube @@ -372,9 +269,10 @@ async def update_assignment( 'VM must be running to modify device assignment' ) assignments = [a for a in self.get_assigned_devices() - if a.device == device] + if a == device] if not assignments: - raise qubes.exc.QubesValueError('Device not assigned') + raise qubes.exc.QubesValueError( + f'Device {device} not assigned to {self._vm.name}') assert len(assignments) == 1 assignment = assignments[0] @@ -388,6 +286,8 @@ async def update_assignment( await self._vm.fire_event_async( 'device-assignment-changed:' + self._bus, device=device) else: + await self._vm.fire_event_async( + 'device-unassign:' + self._bus, device=device) await self.unassign(assignment) async def detach(self, device: Device): @@ -438,7 +338,7 @@ async def unassign(self, device_assignment: DeviceAssignment): "Can not remove an required assignment from " "a non halted qube.") - self._set.discard(device_assignment) + self._set.discard(assignment) device = device_assignment.device await self._vm.fire_event_async( @@ -448,10 +348,8 @@ def get_dedicated_devices(self) -> Iterable[DeviceAssignment]: """ List devices which are attached or assigned to this vm. """ - dedicated = {dev for dev in itertools.chain( - self.get_attached_devices(), self.get_assigned_devices())} - for dev in dedicated: - yield dev + yield from itertools.chain( + self.get_attached_devices(), self.get_assigned_devices()) def get_attached_devices(self) -> Iterable[DeviceAssignment]: """ @@ -491,9 +389,7 @@ def get_exposed_devices(self) -> Iterable[DeviceInfo]: """ List devices exposed by this vm. """ - devices = self._vm.fire_event('device-list:' + self._bus) - for device in devices: - yield device + yield from self._vm.fire_event('device-list:' + self._bus) __iter__ = get_exposed_devices @@ -554,7 +450,7 @@ def add(self, assignment: DeviceAssignment): def discard(self, assignment: DeviceAssignment): """ - Discard assignment from collection. + Discard assignment from a collection. """ assert assignment.attach_automatically vm = assignment.backend_domain diff --git a/qubes/ext/block.py b/qubes/ext/block.py index 7cb172a4d..7a1c46a91 100644 --- a/qubes/ext/block.py +++ b/qubes/ext/block.py @@ -179,17 +179,16 @@ def parent_device(self) -> Optional[qubes.device_protocol.Device]: f'/qubes-block-devices/{self.ident}/parent') if untrusted_parent_info is None: return None - else: - # '4-4.1:1.0' -> parent_ident='4-4.1', interface_num='1.0' - # 'sda' -> parent_ident='sda', interface_num='' - parent_ident, sep, interface_num = self._sanitize( - untrusted_parent_info).partition(":") - devclass = 'usb' if sep == ':' else 'block' - if not parent_ident: - return None - self._parent = qubes.device_protocol.Device( - self.backend_domain, parent_ident, devclass=devclass) - self._interface_num = interface_num + # '4-4.1:1.0' -> parent_ident='4-4.1', interface_num='1.0' + # 'sda' -> parent_ident='sda', interface_num='' + parent_ident, sep, interface_num = self._sanitize( + untrusted_parent_info).partition(":") + devclass = 'usb' if sep == ':' else 'block' + if not parent_ident: + return None + self._parent = qubes.device_protocol.Device( + self.backend_domain, parent_ident, devclass=devclass) + self._interface_num = interface_num return self._parent @property @@ -486,7 +485,7 @@ def on_device_pre_attached_block(self, vm, event, device, options): '\'disk\' or \'cdrom\' value') elif option == 'identity': identity = value - if identity != 'any' and device.self_identity != identity: + if identity not in ('any', device.self_identity): print("Unrecognized identity, skipping attachment of" f" {device}", file=sys.stderr) raise qubes.devices.UnrecognizedDevice( diff --git a/qubes/ext/utils.py b/qubes/ext/utils.py index 592d2d9ff..60216d243 100644 --- a/qubes/ext/utils.py +++ b/qubes/ext/utils.py @@ -25,7 +25,7 @@ def device_list_change( ext: qubes.ext.Extension, current_devices, - vm, path, device_class: qubes.devices.DeviceInfo + vm, path, device_class: qubes.device_protocol.DeviceInfo ): devclass = device_class.__name__[:-len('Device')].lower() @@ -72,8 +72,8 @@ def compare_device_cache(vm, devices_cache, current_devices): # - devices detached from frontend vm (ident: frontend_vm) # - disappeared devices, e.g., plugged out (ident) added = set() - attached = dict() - detached = dict() + attached = {} + detached = {} removed = set() cache = devices_cache[vm.name] for dev_id, front_vm in current_devices.items(): diff --git a/qubes/tests/api_admin.py b/qubes/tests/api_admin.py index a8b40b2e5..a2e39b1ca 100644 --- a/qubes/tests/api_admin.py +++ b/qubes/tests/api_admin.py @@ -125,6 +125,7 @@ def call_internal_mgmt_func(self, method, dest, arg=b'', payload=b''): return response +# noinspection PyUnresolvedReferences class TC_00_VMs(AdminAPITestCase): def test_000_vm_list(self): value = self.call_mgmt_func(b'admin.vm.List', b'dom0') @@ -1268,7 +1269,7 @@ async def fire_event(): mgmt_obj.execute(untrusted_payload=b'')) event_task = asyncio.ensure_future(fire_event()) loop.run_until_complete(execute_task) - vm2 = event_task.result() + event_task.result() self.assertIsNone(execute_task.result()) self.assertEqual(send_event.mock_calls, [ @@ -1688,30 +1689,52 @@ def test_450_property_reset(self): def device_list_testclass(self, vm, event): if vm is not self.vm: return - dev = qubes.devices.DeviceInfo(self.vm, '1234') - dev.description = 'Some device' + dev = qubes.device_protocol.DeviceInfo( + self.vm, '1234', product='Some device') dev.extra_prop = 'xx' yield dev - dev = qubes.devices.DeviceInfo(self.vm, '4321') - dev.description = 'Some other device' + dev = qubes.device_protocol.DeviceInfo( + self.vm, '4321', product='Some other device') yield dev + def assertSerializedEqual(self, actual, expected): + """ + Works only for props without spaces! + """ + actual = sorted(actual.split(' ')) + expected = sorted(expected.split(' ')) + self.assertEqual(actual, expected) + def test_460_vm_device_available(self): self.vm.add_handler('device-list:testclass', self.device_list_testclass) value = self.call_mgmt_func(b'admin.vm.device.testclass.Available', b'test-vm1') - self.assertEqual(value, - '1234 extra_prop=xx description=Some ' - 'device\n' - '4321 description=Some other device\n') + # the only props with sapces + value = value.replace("'Some device'", "'Some_device'") + value = value.replace("'Some other device'", "'Some_other_device'") + self.assertSerializedEqual(value, + "1234 serial='unknown' manufacturer='unknown' " + "self_identity='0000:0000::?******' vendor='unknown' " + "devclass='peripheral' product='Some_device' ident='1234' " + "name='unknown' backend_domain='test-vm1' interfaces='?******'\n" + "4321 serial='unknown' manufacturer='unknown' " + "self_identity='0000:0000::?******' vendor='unknown' " + "devclass='peripheral' product='Some_other_device' " + "ident='4321' name='unknown' backend_domain='test-vm1' " + "interfaces='?******'\n") self.assertFalse(self.app.save.called) def test_461_vm_device_available_specific(self): self.vm.add_handler('device-list:testclass', self.device_list_testclass) value = self.call_mgmt_func(b'admin.vm.device.testclass.Available', b'test-vm1', b'4321') - self.assertEqual(value, - '4321 description=Some other device\n') + value = value.replace("'Some other device'", "'Some_other_device'") + self.assertSerializedEqual(value, + "4321 serial='unknown' manufacturer='unknown' " + "self_identity='0000:0000::?******' vendor='unknown' " + "devclass='peripheral' product='Some_other_device' " + "ident='4321' name='unknown' backend_domain='test-vm1' " + "interfaces='?******'\n") self.assertFalse(self.app.save.called) def test_462_vm_device_available_invalid(self): @@ -1721,156 +1744,205 @@ def test_462_vm_device_available_invalid(self): self.assertEqual(value, '') self.assertFalse(self.app.save.called) - def test_470_vm_device_list_persistent(self): - assignment = qubes.devices.DeviceAssignment(self.vm, '1234', + def test_470_vm_device_list_assigned(self): + assignment = qubes.device_protocol.DeviceAssignment(self.vm, '1234', attach_automatically=True, required=True) self.loop.run_until_complete( - self.vm.devices['testclass'].attach(assignment)) - value = self.call_mgmt_func(b'admin.vm.device.testclass.List', + self.vm.devices['testclass'].assign(assignment)) + value = self.call_mgmt_func(b'admin.vm.device.testclass.Assigned', b'test-vm1') self.assertEqual(value, - 'test-vm1+1234 persistent=yes\n') + "test-vm1+1234 required='yes' attach_automatically='yes' " + "ident='1234' devclass='testclass' backend_domain='test-vm1'\n") self.assertFalse(self.app.save.called) - def test_471_vm_device_list_persistent_options(self): - assignment = qubes.devices.DeviceAssignment(self.vm, '1234', + def test_471_vm_device_list_assigned_options(self): + assignment = qubes.device_protocol.DeviceAssignment(self.vm, '1234', attach_automatically=True, required=True, options={'opt1': 'value'}) self.loop.run_until_complete( - self.vm.devices['testclass'].attach(assignment)) - assignment = qubes.devices.DeviceAssignment(self.vm, '4321', + self.vm.devices['testclass'].assign(assignment)) + assignment = qubes.device_protocol.DeviceAssignment(self.vm, '4321', attach_automatically=True, required=True) self.loop.run_until_complete( - self.vm.devices['testclass'].attach(assignment)) - value = self.call_mgmt_func(b'admin.vm.device.testclass.List', + self.vm.devices['testclass'].assign(assignment)) + value = self.call_mgmt_func(b'admin.vm.device.testclass.Assigned', b'test-vm1') self.assertEqual(value, - 'test-vm1+1234 opt1=value persistent=yes\n' - 'test-vm1+4321 persistent=yes\n') + "test-vm1+1234 required='yes' attach_automatically='yes' " + "ident='1234' devclass='testclass' backend_domain='test-vm1' " + "_opt1='value'\n" + "test-vm1+4321 required='yes' attach_automatically='yes' " + "ident='4321' devclass='testclass' backend_domain='test-vm1'\n") self.assertFalse(self.app.save.called) - def device_list_attached_testclass(self, vm, event, **kwargs): + def device_list_single_attached_testclass(self, vm, event, **kwargs): if vm is not self.vm: return - dev = qubes.devices.DeviceInfo(self.vm, '1234') + dev = qubes.device_protocol.DeviceInfo(self.vm, '1234', 'testclass') yield (dev, {'attach_opt': 'value'}) - def test_472_vm_device_list_temporary(self): + def test_472_vm_device_list_attached(self): self.vm.add_handler('device-list-attached:testclass', - self.device_list_attached_testclass) - value = self.call_mgmt_func(b'admin.vm.device.testclass.List', + self.device_list_single_attached_testclass) + value = self.call_mgmt_func(b'admin.vm.device.testclass.Attached', b'test-vm1') self.assertEqual(value, - 'test-vm1+1234 attach_opt=value persistent=no\n') + "test-vm1+1234 required='no' attach_automatically='no' " + "ident='1234' devclass='testclass' backend_domain='test-vm1' " + "frontend_domain='test-vm1' _attach_opt='value'\n") self.assertFalse(self.app.save.called) - def test_473_vm_device_list_mixed(self): - self.vm.add_handler('device-list-attached:testclass', - self.device_list_attached_testclass) - assignment = qubes.devices.DeviceAssignment(self.vm, '4321', - attach_automatically=True, required=True) + def test_473_vm_device_list_assigned_specific(self): + assignment = qubes.device_protocol.DeviceAssignment( + self.vm, '1234', attach_automatically=True, required=True) self.loop.run_until_complete( - self.vm.devices['testclass'].attach(assignment)) - value = self.call_mgmt_func(b'admin.vm.device.testclass.List', - b'test-vm1') + self.vm.devices['testclass'].assign(assignment)) + assignment = qubes.device_protocol.DeviceAssignment( + self.vm, '4321', attach_automatically=True, required=True) + self.loop.run_until_complete( + self.vm.devices['testclass'].assign(assignment)) + value = self.call_mgmt_func(b'admin.vm.device.testclass.Assigned', + b'test-vm1', b'test-vm1+1234') self.assertEqual(value, - 'test-vm1+1234 attach_opt=value persistent=no\n' - 'test-vm1+4321 persistent=yes\n') + "test-vm1+1234 required='yes' attach_automatically='yes' " + "ident='1234' devclass='testclass' backend_domain='test-vm1'\n") self.assertFalse(self.app.save.called) - def test_474_vm_device_list_specific(self): + def device_list_multiple_attached_testclass(self, vm, event, **kwargs): + if vm is not self.vm: + return + dev = qubes.device_protocol.DeviceInfo(self.vm, '1234', 'testclass') + yield (dev, {'attach_opt': 'value'}) + dev = qubes.device_protocol.DeviceInfo(self.vm, '4321', 'testclass') + yield (dev, {'attach_opt': 'value'}) + + def test_474_vm_device_list_attached_specific(self): self.vm.add_handler('device-list-attached:testclass', - self.device_list_attached_testclass) - assignment = qubes.devices.DeviceAssignment(self.vm, '4321', - attach_automatically=True, required=True) - self.loop.run_until_complete( - self.vm.devices['testclass'].attach(assignment)) - value = self.call_mgmt_func(b'admin.vm.device.testclass.List', + self.device_list_multiple_attached_testclass) + value = self.call_mgmt_func(b'admin.vm.device.testclass.Attached', b'test-vm1', b'test-vm1+1234') self.assertEqual(value, - 'test-vm1+1234 attach_opt=value persistent=no\n') + "test-vm1+1234 required='no' attach_automatically='no' " + "ident='1234' devclass='testclass' backend_domain='test-vm1' " + "frontend_domain='test-vm1' _attach_opt='value'\n") self.assertFalse(self.app.save.called) def test_480_vm_device_attach(self): self.vm.add_handler('device-list:testclass', self.device_list_testclass) - mock_attach = unittest.mock.Mock() - mock_attach.return_value = None - del mock_attach._is_coroutine - self.vm.add_handler('device-attach:testclass', mock_attach) + mock_action = unittest.mock.Mock() + mock_action.return_value = None + del mock_action._is_coroutine + self.vm.add_handler(f'device-attach:testclass', mock_action) with unittest.mock.patch.object(qubes.vm.qubesvm.QubesVM, 'is_halted', lambda _: False): value = self.call_mgmt_func(b'admin.vm.device.testclass.Attach', - b'test-vm1', b'test-vm1+1234') + b'test-vm1', b'test-vm1+1234') + self.assertIsNone(value) + mock_action.assert_called_once_with( + self.vm, f'device-attach:testclass', + device=self.vm.devices['testclass']['1234'], + options={}) + self.assertEqual( + len(list(self.vm.devices['testclass'].get_assigned_devices())), + 0) + self.assertFalse(self.app.save.called) + + def test_481_vm_device_assign(self): + self.vm.add_handler('device-list:testclass', self.device_list_testclass) + mock_action = unittest.mock.Mock() + mock_action.return_value = None + del mock_action._is_coroutine + self.vm.add_handler(f'device-assign:testclass', mock_action) + with unittest.mock.patch.object(qubes.vm.qubesvm.QubesVM, + 'is_halted', lambda _: False): + value = self.call_mgmt_func( + b'admin.vm.device.testclass.Assign', + b'test-vm1', b'test-vm1+1234', + b"attach_automatically='yes'") self.assertIsNone(value) - mock_attach.assert_called_once_with(self.vm, 'device-attach:testclass', + mock_action.assert_called_once_with( + self.vm, f'device-assign:testclass', device=self.vm.devices['testclass']['1234'], options={}) - self.assertEqual(len(self.vm.devices['testclass'].get_assigned_devices()), 0) + self.assertEqual( + len(list(self.vm.devices['testclass'].get_assigned_devices())), + 1) self.app.save.assert_called_once_with() - def test_481_vm_device_attach(self): + def test_483_vm_device_assign_required(self): self.vm.add_handler('device-list:testclass', self.device_list_testclass) - mock_attach = unittest.mock.Mock() - mock_attach.return_value = None - del mock_attach._is_coroutine - self.vm.add_handler('device-attach:testclass', mock_attach) + mock_action = unittest.mock.Mock() + mock_action.return_value = None + del mock_action._is_coroutine + self.vm.add_handler(f'device-assign:testclass', mock_action) with unittest.mock.patch.object(qubes.vm.qubesvm.QubesVM, 'is_halted', lambda _: False): - value = self.call_mgmt_func(b'admin.vm.device.testclass.Attach', - b'test-vm1', b'test-vm1+1234', b'persistent=no') + value = self.call_mgmt_func( + b'admin.vm.device.testclass.Assign', + b'test-vm1', b'test-vm1+1234', + b"attach_automatically='yes' required='yes'") self.assertIsNone(value) - mock_attach.assert_called_once_with(self.vm, 'device-attach:testclass', + mock_action.assert_called_once_with( + self.vm, f'device-assign:testclass', device=self.vm.devices['testclass']['1234'], options={}) - self.assertEqual(len(self.vm.devices['testclass'].get_assigned_devices()), 0) + self.assertEqual( + len(list(self.vm.devices['testclass'].get_assigned_devices())), + 1) self.app.save.assert_called_once_with() - def test_482_vm_device_attach_not_running(self): + def test_484_vm_device_attach_not_running(self): self.vm.add_handler('device-list:testclass', self.device_list_testclass) - mock_attach = unittest.mock.Mock() - del mock_attach._is_coroutine - self.vm.add_handler('device-attach:testclass', mock_attach) + mock_action = unittest.mock.Mock() + del mock_action._is_coroutine + self.vm.add_handler('device-attach:testclass', mock_action) with self.assertRaises(qubes.exc.QubesVMNotRunningError): self.call_mgmt_func(b'admin.vm.device.testclass.Attach', b'test-vm1', b'test-vm1+1234') - self.assertFalse(mock_attach.called) - self.assertEqual(len(self.vm.devices['testclass'].get_assigned_devices()), 0) + self.assertFalse(mock_action.called) + self.assertEqual( + len(list(self.vm.devices['testclass'].get_assigned_devices())), 0) self.assertFalse(self.app.save.called) - def test_483_vm_device_attach_persistent(self): + def test_485_vm_device_assign_not_running(self): self.vm.add_handler('device-list:testclass', self.device_list_testclass) - mock_attach = unittest.mock.Mock() - mock_attach.return_value = None - del mock_attach._is_coroutine - self.vm.add_handler('device-attach:testclass', mock_attach) + mock_action = unittest.mock.Mock() + mock_action.return_value = None + del mock_action._is_coroutine + self.vm.add_handler('device-assign:testclass', mock_action) with unittest.mock.patch.object(qubes.vm.qubesvm.QubesVM, - 'is_halted', lambda _: False): - value = self.call_mgmt_func(b'admin.vm.device.testclass.Attach', - b'test-vm1', b'test-vm1+1234', b'persistent=yes') - self.assertIsNone(value) - dev = self.vm.devices['testclass']['1234'] - mock_attach.assert_called_once_with(self.vm, 'device-attach:testclass', - device=dev, + 'is_halted', lambda _: False): + self.call_mgmt_func(b'admin.vm.device.testclass.Assign', + b'test-vm1', b'test-vm1+1234', + b"attach_automatically='yes'") + mock_action.assert_called_once_with( + self.vm, f'device-assign:testclass', + device=self.vm.devices['testclass']['1234'], options={}) - self.assertIn(dev, self.vm.devices['testclass'].get_assigned_devices()) + self.assertEqual( + len(list(self.vm.devices['testclass'].get_assigned_devices())), 1) self.app.save.assert_called_once_with() - def test_484_vm_device_attach_persistent_not_running(self): + def test_486_vm_device_assign_required_not_running(self): self.vm.add_handler('device-list:testclass', self.device_list_testclass) - mock_attach = unittest.mock.Mock() - mock_attach.return_value = None - del mock_attach._is_coroutine - self.vm.add_handler('device-attach:testclass', mock_attach) - value = self.call_mgmt_func(b'admin.vm.device.testclass.Attach', - b'test-vm1', b'test-vm1+1234', b'persistent=yes') - self.assertIsNone(value) - dev = self.vm.devices['testclass']['1234'] - mock_attach.assert_called_once_with(self.vm, 'device-attach:testclass', - device=dev, + mock_action = unittest.mock.Mock() + mock_action.return_value = None + del mock_action._is_coroutine + self.vm.add_handler('device-assign:testclass', mock_action) + with unittest.mock.patch.object(qubes.vm.qubesvm.QubesVM, + 'is_halted', lambda _: False): + self.call_mgmt_func(b'admin.vm.device.testclass.Assign', + b'test-vm1', b'test-vm1+1234', + b"attach_automatically='yes' required='yes'") + mock_action.assert_called_once_with( + self.vm, f'device-assign:testclass', + device=self.vm.devices['testclass']['1234'], options={}) - self.assertIn(dev, self.vm.devices['testclass'].get_assigned_devices()) + self.assertEqual( + len(list(self.vm.devices['testclass'].get_assigned_devices())), 1) self.app.save.assert_called_once_with() - def test_485_vm_device_attach_options(self): + def test_487_vm_device_attach_options(self): self.vm.add_handler('device-list:testclass', self.device_list_testclass) mock_attach = unittest.mock.Mock() mock_attach.return_value = None @@ -1879,18 +1951,145 @@ def test_485_vm_device_attach_options(self): with unittest.mock.patch.object(qubes.vm.qubesvm.QubesVM, 'is_halted', lambda _: False): value = self.call_mgmt_func(b'admin.vm.device.testclass.Attach', - b'test-vm1', b'test-vm1+1234', b'option1=value2') + b'test-vm1', b'test-vm1+1234', b"_option1='value2'") + self.assertIsNone(value) + dev = self.vm.devices['testclass']['1234'] + mock_attach.assert_called_once_with( + self.vm, 'device-attach:testclass', device=dev, + options={'option1': 'value2'}) + self.assertFalse(self.app.save.called) + + def test_488_vm_device_assign_options(self): + self.vm.add_handler('device-list:testclass', + self.device_list_testclass) + mock_attach = unittest.mock.Mock() + mock_attach.return_value = None + del mock_attach._is_coroutine + self.vm.add_handler('device-assign:testclass', mock_attach) + with unittest.mock.patch.object(qubes.vm.qubesvm.QubesVM, + 'is_halted', lambda _: False): + value = self.call_mgmt_func( + b'admin.vm.device.testclass.Assign', + b'test-vm1', b'test-vm1+1234', + b"attach_automatically='yes' _option1='value2'") self.assertIsNone(value) dev = self.vm.devices['testclass']['1234'] - mock_attach.assert_called_once_with(self.vm, 'device-attach:testclass', - device=dev, + mock_attach.assert_called_once_with( + self.vm, 'device-assign:testclass', device=dev, options={'option1': 'value2'}) self.app.save.assert_called_once_with() - def test_490_vm_device_detach(self): + def test_490_vm_device_unassign_from_running(self): + assignment = qubes.device_protocol.DeviceAssignment( + self.vm, '1234', attach_automatically=True, required=False, + options={'opt1': 'value'}) + self.loop.run_until_complete( + self.vm.devices['testclass'].assign(assignment)) + mock_action = unittest.mock.Mock() + mock_action.return_value = None + del mock_action._is_coroutine + self.vm.add_handler('device-unassign:testclass', mock_action) + with unittest.mock.patch.object( + qubes.vm.qubesvm.QubesVM, 'is_halted', lambda _: False): + value = self.call_mgmt_func(b'admin.vm.device.testclass.Unassign', + b'test-vm1', b'test-vm1+1234') + self.assertIsNone(value) + mock_action.assert_called_once_with(self.vm, 'device-unassign:testclass', + device=self.vm.devices['testclass']['1234']) + self.app.save.assert_called_once_with() + + def test_491_vm_device_unassign_required_from_running(self): + assignment = qubes.device_protocol.DeviceAssignment( + self.vm, '1234', attach_automatically=True, required=True, + options={'opt1': 'value'}) + self.loop.run_until_complete( + self.vm.devices['testclass'].assign(assignment)) + mock_action = unittest.mock.Mock() + mock_action.return_value = None + del mock_action._is_coroutine + self.vm.add_handler('device-unassign:testclass', mock_action) + with unittest.mock.patch.object( + qubes.vm.qubesvm.QubesVM, 'is_halted', lambda _: False): + with self.assertRaises(qubes.exc.QubesVMNotHaltedError): + self.call_mgmt_func( + b'admin.vm.device.testclass.Unassign', + b'test-vm1', b'test-vm1+1234') + self.assertFalse(mock_action.called) + self.assertFalse(self.app.save.called) + def test_492_vm_device_unassign_from_halted(self): + assignment = qubes.device_protocol.DeviceAssignment( + self.vm, '1234', attach_automatically=True, required=False, + options={'opt1': 'value'}) + self.loop.run_until_complete( + self.vm.devices['testclass'].assign(assignment)) + mock_action = unittest.mock.Mock() + mock_action.return_value = None + del mock_action._is_coroutine + self.vm.add_handler('device-unassign:testclass', mock_action) + self.call_mgmt_func( + b'admin.vm.device.testclass.Unassign', + b'test-vm1', b'test-vm1+1234') + mock_action.assert_called_once_with( + self.vm, 'device-unassign:testclass', + device=self.vm.devices['testclass']['1234']) + self.app.save.assert_called_once_with() + + def test_493_vm_device_unassign_required_from_halted(self): + assignment = qubes.device_protocol.DeviceAssignment( + self.vm, '1234', attach_automatically=True, required=True, + options={'opt1': 'value'}) + self.loop.run_until_complete( + self.vm.devices['testclass'].assign(assignment)) + mock_action = unittest.mock.Mock() + mock_action.return_value = None + del mock_action._is_coroutine + self.vm.add_handler('device-unassign:testclass', mock_action) + self.call_mgmt_func( + b'admin.vm.device.testclass.Unassign', + b'test-vm1', b'test-vm1+1234') + mock_action.assert_called_once_with( + self.vm, 'device-unassign:testclass', + device=self.vm.devices['testclass']['1234']) + self.app.save.assert_called_once_with() + + def test_494_vm_device_unassign_attached(self): + self.vm.add_handler('device-list:testclass', self.device_list_testclass) + self.vm.add_handler('device-list-attached:testclass', + self.device_list_single_attached_testclass) + assignment = qubes.device_protocol.DeviceAssignment( + self.vm, '1234', attach_automatically=True, required=False, + options={'opt1': 'value'}) + self.loop.run_until_complete( + self.vm.devices['testclass'].assign(assignment)) + mock_action = unittest.mock.Mock() + mock_action.return_value = None + del mock_action._is_coroutine + self.vm.add_handler('device-unassign:testclass', mock_action) + self.call_mgmt_func( + b'admin.vm.device.testclass.Unassign', + b'test-vm1', b'test-vm1+1234') + mock_action.assert_called_once_with( + self.vm, 'device-unassign:testclass', + device=self.vm.devices['testclass']['1234']) + self.app.save.assert_called_once_with() + + def test_495_vm_device_unassign_not_assigned(self): + mock_detach = unittest.mock.Mock() + mock_detach.return_value = None + del mock_detach._is_coroutine + self.vm.add_handler('device-detach:testclass', mock_detach) + with unittest.mock.patch.object(qubes.vm.qubesvm.QubesVM, + 'is_halted', lambda _: False): + with self.assertRaises(qubes.devices.DeviceNotAssigned): + self.call_mgmt_func(b'admin.vm.device.testclass.Detach', + b'test-vm1', b'test-vm1+1234') + self.assertFalse(mock_detach.called) + self.assertFalse(self.app.save.called) + + def test_496_vm_device_detach(self): self.vm.add_handler('device-list:testclass', self.device_list_testclass) self.vm.add_handler('device-list-attached:testclass', - self.device_list_attached_testclass) + self.device_list_single_attached_testclass) mock_detach = unittest.mock.Mock() mock_detach.return_value = None del mock_detach._is_coroutine @@ -1902,9 +2101,9 @@ def test_490_vm_device_detach(self): self.assertIsNone(value) mock_detach.assert_called_once_with(self.vm, 'device-detach:testclass', device=self.vm.devices['testclass']['1234']) - self.app.save.assert_called_once_with() + self.assertFalse(self.app.save.called) - def test_491_vm_device_detach_not_attached(self): + def test_497_vm_device_detach_not_attached(self): mock_detach = unittest.mock.Mock() mock_detach.return_value = None del mock_detach._is_coroutine @@ -1944,10 +2143,10 @@ def test_501_vm_remove_running(self, mock_rmtree, mock_remove): @unittest.mock.patch('shutil.rmtree') def test_502_vm_remove_attached(self, mock_rmtree, mock_remove): self.setup_for_clone() - assignment = qubes.devices.DeviceAssignment( + assignment = qubes.device_protocol.DeviceAssignment( self.vm, '1234', attach_automatically=True, required=True) self.loop.run_until_complete( - self.vm2.devices['testclass'].attach(assignment)) + self.vm2.devices['testclass'].assign(assignment)) mock_remove.side_effect = self.dummy_coro with self.assertRaises(qubes.exc.QubesVMInUseError): @@ -2632,100 +2831,156 @@ def test_642_vm_create_disposable_not_allowed(self, storage_mock): b'test-vm1') self.assertFalse(self.app.save.called) - def test_650_vm_device_set_persistent_true(self): - self.vm.add_handler('device-list:testclass', - self.device_list_testclass) - self.vm.add_handler('device-list-attached:testclass', - self.device_list_attached_testclass) + def test_650_vm_device_set_assignment_true(self): + assignment = qubes.device_protocol.DeviceAssignment( + self.vm, '1234', attach_automatically=True, required=False, + options={'opt1': 'value'}) + self.loop.run_until_complete( + self.vm.devices['testclass'].assign(assignment)) + mock_action = unittest.mock.Mock() + mock_action.return_value = None + del mock_action._is_coroutine + self.vm.add_handler(f'device-assignment-changed:testclass', mock_action) + with unittest.mock.patch.object(qubes.vm.qubesvm.QubesVM, 'is_halted', lambda _: False): value = self.call_mgmt_func( - b'admin.vm.device.testclass.Set.persistent', + b'admin.vm.device.testclass.Set.assignment', b'test-vm1', b'test-vm1+1234', b'True') + self.assertIsNone(value) - dev = qubes.devices.DeviceInfo(self.vm, '1234') - self.assertIn(dev, self.vm.devices['testclass'].get_assigned_devices()) + dev = qubes.device_protocol.DeviceInfo(self.vm, '1234') + required = self.vm.devices['testclass'].get_assigned_devices( + required_only=True) + self.assertIn(dev, required) + self.assertEventFired( + self.emitter, + 'admin-permission:admin.vm.device.testclass.Set.assignment') + mock_action.assert_called_once_with( + self.vm, f'device-assignment-changed:testclass', + device=self.vm.devices['testclass']['1234']) self.app.save.assert_called_once_with() - def test_651_vm_device_set_persistent_false_unchanged(self): - self.vm.add_handler('device-list:testclass', - self.device_list_testclass) - self.vm.add_handler('device-list-attached:testclass', - self.device_list_attached_testclass) + def test_651_vm_device_set_assignment_false(self): + assignment = qubes.device_protocol.DeviceAssignment( + self.vm, '1234', attach_automatically=True, required=True, + options={'opt1': 'value'}) + self.loop.run_until_complete( + self.vm.devices['testclass'].assign(assignment)) + mock_action = unittest.mock.Mock() + mock_action.return_value = None + del mock_action._is_coroutine + self.vm.add_handler(f'device-assignment-changed:testclass', mock_action) + with unittest.mock.patch.object(qubes.vm.qubesvm.QubesVM, 'is_halted', lambda _: False): value = self.call_mgmt_func( - b'admin.vm.device.testclass.Set.persistent', + b'admin.vm.device.testclass.Set.assignment', b'test-vm1', b'test-vm1+1234', b'False') + self.assertIsNone(value) - dev = qubes.devices.DeviceInfo(self.vm, '1234') - self.assertNotIn(dev, self.vm.devices['testclass'].get_assigned_devices()) + dev = qubes.device_protocol.DeviceInfo(self.vm, '1234') + required = self.vm.devices['testclass'].get_assigned_devices( + required_only=True) + self.assertNotIn(dev, required) + self.assertEventFired( + self.emitter, + 'admin-permission:admin.vm.device.testclass.Set.assignment') + mock_action.assert_called_once_with( + self.vm, f'device-assignment-changed:testclass', + device=self.vm.devices['testclass']['1234']) self.app.save.assert_called_once_with() - def test_652_vm_device_set_persistent_false(self): - self.vm.add_handler('device-list:testclass', - self.device_list_testclass) - assignment = qubes.devices.DeviceAssignment(self.vm, '1234', {}, - attach_automatically=True, required=True) + def test_652_vm_device_set_assignment_none(self): + assignment = qubes.device_protocol.DeviceAssignment( + self.vm, '1234', attach_automatically=True, required=False, + options={'opt1': 'value'}) self.loop.run_until_complete( - self.vm.devices['testclass'].attach(assignment)) - self.vm.add_handler('device-list-attached:testclass', - self.device_list_attached_testclass) - dev = qubes.devices.DeviceInfo(self.vm, '1234') - self.assertIn(dev, self.vm.devices['testclass'].get_assigned_devices()) + self.vm.devices['testclass'].assign(assignment)) + mock_action = unittest.mock.Mock() + mock_action.return_value = None + del mock_action._is_coroutine + self.vm.add_handler(f'device-unassign:testclass', mock_action) + with unittest.mock.patch.object(qubes.vm.qubesvm.QubesVM, 'is_halted', lambda _: False): value = self.call_mgmt_func( - b'admin.vm.device.testclass.Set.persistent', # TODO - b'test-vm1', b'test-vm1+1234', b'False') + b'admin.vm.device.testclass.Set.assignment', + b'test-vm1', b'test-vm1+1234', b'None') + self.assertIsNone(value) - self.assertNotIn(dev, self.vm.devices['testclass'].get_assigned_devices()) - self.assertIn(dev, self.vm.devices['testclass'].get_attached_devices()) + dev = qubes.device_protocol.DeviceInfo(self.vm, '1234') + required = self.vm.devices['testclass'].get_assigned_devices( + required_only=False) + self.assertNotIn(dev, required) + self.assertEventFired( + self.emitter, + 'admin-permission:admin.vm.device.testclass.Set.assignment') + mock_action.assert_called_once_with( + self.vm, f'device-unassign:testclass', + device=self.vm.devices['testclass']['1234']) self.app.save.assert_called_once_with() - def test_653_vm_device_set_persistent_true_unchanged(self): - self.vm.add_handler('device-list:testclass', - self.device_list_testclass) - assignment = qubes.devices.DeviceAssignment(self.vm, '1234', {}, - attach_automatically=True, required=True) + def test_653_vm_device_set_assignment_true_unchanged(self): + assignment = qubes.device_protocol.DeviceAssignment( + self.vm, '1234', attach_automatically=True, required=True, + options={'opt1': 'value'}) self.loop.run_until_complete( - self.vm.devices['testclass'].attach(assignment)) - self.vm.add_handler('device-list-attached:testclass', - self.device_list_attached_testclass) + self.vm.devices['testclass'].assign(assignment)) with unittest.mock.patch.object(qubes.vm.qubesvm.QubesVM, 'is_halted', lambda _: False): value = self.call_mgmt_func( - b'admin.vm.device.testclass.Set.persistent', # TODO + b'admin.vm.device.testclass.Set.assignment', b'test-vm1', b'test-vm1+1234', b'True') self.assertIsNone(value) - dev = qubes.devices.DeviceInfo(self.vm, '1234') - self.assertIn(dev, self.vm.devices['testclass'].get_assigned_devices()) - self.assertIn(dev, self.vm.devices['testclass'].get_attached_devices()) + dev = qubes.device_protocol.DeviceInfo(self.vm, '1234') + required = self.vm.devices['testclass'].get_assigned_devices( + required_only=True) + self.assertIn(dev, required) self.app.save.assert_called_once_with() - def test_654_vm_device_set_persistent_not_attached(self): + def test_654_vm_device_set_assignment_false_unchanged(self): + assignment = qubes.device_protocol.DeviceAssignment( + self.vm, '1234', attach_automatically=True, required=False, + options={'opt1': 'value'}) + self.loop.run_until_complete( + self.vm.devices['testclass'].assign(assignment)) + with unittest.mock.patch.object(qubes.vm.qubesvm.QubesVM, + 'is_halted', lambda _: False): + value = self.call_mgmt_func( + b'admin.vm.device.testclass.Set.assignment', + b'test-vm1', b'test-vm1+1234', b'False') + self.assertIsNone(value) + dev = qubes.device_protocol.DeviceInfo(self.vm, '1234') + required = self.vm.devices['testclass'].get_assigned_devices( + required_only=True) + self.assertNotIn(dev, required) + self.app.save.assert_called_once_with() + + def test_655_vm_device_set_persistent_not_assigned(self): self.vm.add_handler('device-list:testclass', self.device_list_testclass) with unittest.mock.patch.object(qubes.vm.qubesvm.QubesVM, 'is_halted', lambda _: False): - with self.assertRaises(qubes.exc.PermissionDenied): + with self.assertRaises(qubes.exc.QubesValueError): self.call_mgmt_func( - b'admin.vm.device.testclass.Set.persistent', + b'admin.vm.device.testclass.Set.assignment', b'test-vm1', b'test-vm1+1234', b'True') - dev = qubes.devices.DeviceInfo(self.vm, '1234') - self.assertNotIn(dev, self.vm.devices['testclass'].get_assigned_devices()) + dev = qubes.device_protocol.DeviceInfo(self.vm, '1234') + self.assertNotIn( + dev, self.vm.devices['testclass'].get_assigned_devices()) self.assertFalse(self.app.save.called) - def test_655_vm_device_set_persistent_invalid_value(self): + def test_656_vm_device_set_persistent_invalid_value(self): self.vm.add_handler('device-list:testclass', self.device_list_testclass) with unittest.mock.patch.object(qubes.vm.qubesvm.QubesVM, 'is_halted', lambda _: False): with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func( - b'admin.vm.device.testclass.Set.persistent', + b'admin.vm.device.testclass.Set.assignment', b'test-vm1', b'test-vm1+1234', b'maybe') - dev = qubes.devices.DeviceInfo(self.vm, '1234') + dev = qubes.device_protocol.DeviceInfo(self.vm, '1234') self.assertNotIn(dev, self.vm.devices['testclass'].get_assigned_devices()) self.assertFalse(self.app.save.called) diff --git a/qubes/tests/devices.py b/qubes/tests/devices.py index 7d9534007..a1ac3e2f3 100644 --- a/qubes/tests/devices.py +++ b/qubes/tests/devices.py @@ -21,12 +21,13 @@ # import qubes.devices -from qubes.devices import DeviceInfo +from qubes.device_protocol import (Device, DeviceInfo, DeviceAssignment, + DeviceInterface) import qubes.tests -class TestDevice(qubes.devices.DeviceInfo): +class TestDevice(DeviceInfo): # pylint: disable=too-few-public-methods pass @@ -89,7 +90,7 @@ def setUp(self): self.app.domains['vm'] = self.emitter self.device = self.emitter.device self.collection = self.emitter.devices['testclass'] - self.assignment = qubes.devices.DeviceAssignment( + self.assignment = DeviceAssignment( backend_domain=self.device.backend_domain, ident=self.device.ident, attach_automatically=True, @@ -356,7 +357,7 @@ def test_000_init(self): def test_001_missing(self): device = TestDevice(self.emitter.app.domains['vm'], 'testdev') - assignment = qubes.devices.DeviceAssignment( + assignment = DeviceAssignment( backend_domain=device.backend_domain, ident=device.ident, attach_automatically=True, required=True) @@ -382,8 +383,8 @@ def test_010_serialize(self): manufacturer="", name="Some untrusted garbage", serial=None, - interfaces=[qubes.devices.DeviceInterface(" ******"), - qubes.devices.DeviceInterface("u03**01")], + interfaces=[DeviceInterface(" ******"), + DeviceInterface("u03**01")], additional_info="", date="06.12.23", ) @@ -410,11 +411,11 @@ def test_011_serialize_with_parent(self): manufacturer="", name="Some untrusted garbage", serial=None, - interfaces=[qubes.devices.DeviceInterface(" ******"), - qubes.devices.DeviceInterface("u03**01")], + interfaces=[DeviceInterface(" ******"), + DeviceInterface("u03**01")], additional_info="", date="06.12.23", - parent=qubes.devices.Device(self.vm, '1-1.1', 'pci') + parent=Device(self.vm, '1-1.1', 'pci') ) actual = device.serialize() expected = ( @@ -449,8 +450,8 @@ def test_020_deserialize(self): manufacturer="unknown", name="Some untrusted garbage", serial=None, - interfaces=[qubes.devices.DeviceInterface(" ******"), - qubes.devices.DeviceInterface("u03**01")], + interfaces=[DeviceInterface(" ******"), + DeviceInterface("u03**01")], additional_info="", date="06.12.23", ) diff --git a/qubes/tests/integ/devices_block.py b/qubes/tests/integ/devices_block.py index c8d39ffbf..c39bc0e42 100644 --- a/qubes/tests/integ/devices_block.py +++ b/qubes/tests/integ/devices_block.py @@ -327,7 +327,7 @@ def setUp(self): self.img_path, self.backend.name)) def test_000_attach_reattach(self): - ass = qubes.devices.DeviceAssignment(self.backend, self.device_ident) + ass = qubes.device_protocol.DeviceAssignment(self.backend, self.device_ident) with self.subTest('attach'): self.loop.run_until_complete( self.frontend.devices['block'].attach(ass)) diff --git a/qubes/tests/integ/devices_pci.py b/qubes/tests/integ/devices_pci.py index 96c6eea3b..7b1ea5579 100644 --- a/qubes/tests/integ/devices_pci.py +++ b/qubes/tests/integ/devices_pci.py @@ -37,18 +37,18 @@ def setUp(self): if self._testMethodName not in ['test_000_list']: pcidev = os.environ['QUBES_TEST_PCIDEV'] self.dev = self.app.domains[0].devices['pci'][pcidev] - self.assignment = qubes.devices.DeviceAssignment( + self.assignment = qubes.device_protocol.DeviceAssignment( backend_domain=self.dev.backend_domain, ident=self.dev.ident, attach_automatically=True, ) - self.required_assignment = qubes.devices.DeviceAssignment( + self.required_assignment = qubes.device_protocol.DeviceAssignment( backend_domain=self.dev.backend_domain, ident=self.dev.ident, attach_automatically=True, required=True, ) - if isinstance(self.dev, qubes.devices.UnknownDevice): + if isinstance(self.dev, qubes.device_protocol.UnknownDevice): self.skipTest('Specified device {} does not exists'.format(pcidev)) self.init_default_template() self.vm = self.app.add_new_vm(qubes.vm.appvm.AppVM, @@ -118,7 +118,7 @@ def test_011_attach_offline_temp_fail(self): dev_col = self.vm.devices['pci'] self.assertDeviceIs( self.dev, attached=False, assigned=False, required=False) - self.assignment.persistent = False + self.assignment.required = False with self.assertRaises(qubes.exc.QubesVMNotRunningError): self.loop.run_until_complete( dev_col.attach(self.assignment)) diff --git a/qubes/tests/integ/vm_qrexec_gui.py b/qubes/tests/integ/vm_qrexec_gui.py index e55671963..011ca9cff 100644 --- a/qubes/tests/integ/vm_qrexec_gui.py +++ b/qubes/tests/integ/vm_qrexec_gui.py @@ -308,7 +308,7 @@ def common_audio_record_muted(self): self.fail('VM recorded something, even though mic disabled') def common_audio_record_unmuted(self): - deva = qubes.devices.DeviceAssignment(self.app.domains[0], 'mic') + deva = qubes.device_protocol.DeviceAssignment(self.app.domains[0], 'mic') self.loop.run_until_complete( self.testvm1.devices['mic'].attach(deva)) # connect VM's recording source output monitor (instead of mic) diff --git a/qubes/tests/vm/qubesvm.py b/qubes/tests/vm/qubesvm.py index c818afbc2..c1c50b5cb 100644 --- a/qubes/tests/vm/qubesvm.py +++ b/qubes/tests/vm/qubesvm.py @@ -79,9 +79,10 @@ class TestDeviceCollection(object): def __init__(self): self._list = [] - def persistent(self): + def get_assigned_devices(self, required_only=False): return self._list + class TestQubesDB(object): def __init__(self, data=None): self.data = {} @@ -214,7 +215,7 @@ def test_010_default_virt_mode(self): self.vm.template = None self.assertEqual(qubes.vm.qubesvm._default_virt_mode(self.vm), 'pvh') - self.vm.devices['pci'].persistent().append('some-dev') + self.vm.devices['pci'].get_assigned_devices().append('some-dev') self.assertEqual(qubes.vm.qubesvm._default_virt_mode(self.vm), 'hvm') @@ -246,7 +247,7 @@ def test_021_default_maxmem_with_pcidevs(self): self.vm.is_memory_balancing_possible = \ lambda: qubes.vm.qubesvm.QubesVM.is_memory_balancing_possible( self.vm) - self.vm.devices['pci'].persistent().append('00_00.0') + self.vm.devices['pci'].get_assigned_devices().append('00_00.0') self.assertEqual(qubes.vm.qubesvm._default_maxmem(self.vm), 0) def test_022_default_maxmem_linux(self): @@ -1391,7 +1392,7 @@ def test_600_libvirt_xml_hvm_pcidev_s0ix(self): vm.kernel = None # even with meminfo-writer enabled, should have memory==maxmem vm.features['service.meminfo-writer'] = True - assignment = qubes.devices.DeviceAssignment( + assignment = qubes.device_protocol.DeviceAssignment( vm, # this is violation of API, but for PCI the argument # is unused '00_00.0', @@ -1477,7 +1478,7 @@ def test_600_libvirt_xml_hvm_cdrom_boot(self): vm.kernel = None dom0.events_enabled = True self.app.vmm.offline_mode = False - dev = qubes.devices.DeviceAssignment( + dev = qubes.device_protocol.DeviceAssignment( dom0, 'sda', {'devtype': 'cdrom', 'read-only': 'yes'}, attach_automatically=True, required=True) @@ -1582,7 +1583,7 @@ def test_600_libvirt_xml_hvm_cdrom_dom0_kernel_boot(self): }) dom0.events_enabled = True self.app.vmm.offline_mode = False - dev = qubes.devices.DeviceAssignment( + dev = qubes.device_protocol.DeviceAssignment( dom0, 'sda', {'devtype': 'cdrom', 'read-only': 'yes'}, attach_automatically=True, required=True) diff --git a/qubes/vm/__init__.py b/qubes/vm/__init__.py index 4d5ed777a..1b0772b23 100644 --- a/qubes/vm/__init__.py +++ b/qubes/vm/__init__.py @@ -276,7 +276,7 @@ def load_extras(self): options[option.get('name')] = str(option.text) try: - device_assignment = qubes.devices.DeviceAssignment( + device_assignment = qubes.device_protocol.DeviceAssignment( self.app.domains[node.get('backend-domain')], node.get('id'), options, @@ -299,10 +299,10 @@ def load_extras(self): # SEE:1815 firewall, policy. - def get_provided_assignments(self, required_only: bool)\ - -> List['qubes.devices.DeviceAssignment']: + def get_provided_assignments(self, required_only: bool = False)\ + -> List['qubes.device_protocol.DeviceAssignment']: """ - List device assignments from this VM. + List device assignments from this VM. """ assignments = [] @@ -310,7 +310,8 @@ def get_provided_assignments(self, required_only: bool)\ if domain == self: continue for device_collection in domain.devices.values(): - for assignment in device_collection.get_assigned_devices(): + for assignment in device_collection.get_assigned_devices( + required_only): if assignment.backend_domain == self: assignments.append(assignment) return assignments diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 71ecfe12c..7993e7baa 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -1171,7 +1171,9 @@ async def start(self, start_guid=True, notify_function=None, try: for devclass in self.devices: for ass in self.devices[devclass].get_assigned_devices(): - if isinstance(ass.device, qubes.devices.UnknownDevice) \ + if isinstance( + ass.device, + qubes.device_protocol.UnknownDevice) \ and ass.required: raise qubes.exc.QubesException( f'{devclass.capitalize()} device {ass} ' From da079f2408db8a73bbc90c057dc49c1c9c48979d Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Sun, 24 Mar 2024 21:00:28 +0100 Subject: [PATCH 25/49] q-dev: update docs --- doc/qubes-devices.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/qubes-devices.rst b/doc/qubes-devices.rst index 8a3c7141f..e64811a3e 100644 --- a/doc/qubes-devices.rst +++ b/doc/qubes-devices.rst @@ -100,7 +100,7 @@ The microphone cannot be assigned (potentially) to any VM (attempting to attach Understanding Device Self Identity ---------------------------------- -It is important to understand that :py:class:`qubes.devices.Device` does not +It is important to understand that :py:class:`qubes.device_protocol.Device` does not correspond to the device itself, but rather to the *port* to which the device is connected. Therefore, when assigning a device to a VM, such as `sys-usb:1-1.1`, the port `1-1.1` is actually assigned, and thus From b89a89c851a69527b5824763d2d927a1833c755e Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Mon, 25 Mar 2024 06:58:36 +0100 Subject: [PATCH 26/49] q-dev: broder allowance of names --- qubes/device_protocol.py | 8 ++++++-- qubes/ext/pci.py | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/qubes/device_protocol.py b/qubes/device_protocol.py index bea5e61c3..368267a3b 100644 --- a/qubes/device_protocol.py +++ b/qubes/device_protocol.py @@ -358,7 +358,8 @@ def _load_classes(bus: str): class DeviceInfo(Device): """ Holds all information about a device """ - ALLOWED_CHARS_PARAM = Device.ALLOWED_CHARS_PARAM + string.punctuation + ' ' + ALLOWED_CHARS_KEY = Device.ALLOWED_CHARS_KEY + string.punctuation + ' ' + ALLOWED_CHARS_PARAM = Device.ALLOWED_CHARS_PARAM + string.punctuation + ' ' def __init__( self, @@ -655,7 +656,10 @@ def self_identity(self) -> str: def serialize_str(value: str): - return repr(str(value)) + result = repr(str(value)) + if result.startswith('"'): + result = "'" + result[1:-1] + "'" + return result def deserialize_str(value: str): diff --git a/qubes/ext/pci.py b/qubes/ext/pci.py index 1d76bdc79..9cba325f5 100644 --- a/qubes/ext/pci.py +++ b/qubes/ext/pci.py @@ -299,11 +299,11 @@ def _load_desc(self) -> Dict[str, str]: self._vendor = result["vendor"] = hostdev_xml.findtext( 'capability/vendor') self._vendor_id = result["vendor ID"] = hostdev_xml.xpath( - "//vendor/@id") + "//vendor/@id")[0] self._product = result["product"] = hostdev_xml.findtext( 'capability/product') self._product_id = result["product ID"] = hostdev_xml.xpath( - "//product/@id") + "//product/@id")[0] return result @staticmethod From 0193c0ad78ac8a55c5e8032c73f13ed2b098dd5d Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Mon, 25 Mar 2024 11:01:21 +0100 Subject: [PATCH 27/49] q-dev: attaching block dev refactor --- qubes/ext/block.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/qubes/ext/block.py b/qubes/ext/block.py index 7a1c46a91..495489451 100644 --- a/qubes/ext/block.py +++ b/qubes/ext/block.py @@ -464,6 +464,13 @@ def find_unused_frontend(vm, devtype='disk'): @qubes.ext.handler('device-pre-attach:block') def on_device_pre_attached_block(self, vm, event, device, options): # pylint: disable=unused-argument + self.pre_attachment_internal(vm, device, options) + + vm.libvirt_domain.attachDevice( + vm.app.env.get_template('libvirt/devices/block.xml').render( + device=device, vm=vm, options=options)) + + def pre_attachment_internal(self, vm, device, options): if isinstance(device, qubes.device_protocol.UnknownDevice): print(f'{device.devclass.capitalize()} device {device} ' 'not available, skipping.', file=sys.stderr) @@ -530,27 +537,21 @@ def on_device_pre_attached_block(self, vm, event, device, options): options['frontend-dev'] = self.find_unused_frontend( vm, options.get('devtype', 'disk')) - print(f'attaching {device} exposed by {device.backend_domain.name}') - vm.libvirt_domain.attachDevice( - vm.app.env.get_template('libvirt/devices/block.xml').render( - device=device, vm=vm, options=options)) - @qubes.ext.handler('domain-start') async def on_domain_start(self, vm, _event, **_kwargs): # pylint: disable=unused-argument for assignment in vm.devices['block'].get_assigned_devices(): - asyncio.ensure_future(self.attach_and_notify( - vm, assignment.device, assignment.options)) + self.notify_auto_attached( + vm, assignment.device, assignment.options) - async def attach_and_notify(self, vm, device, options): + def notify_auto_attached(self, vm, device, options): + pass # bypass DeviceCollection logic preventing double attach - try: - self.on_device_pre_attached_block( - vm, 'device-pre-attach:block', device, options) - except qubes.devices.UnrecognizedDevice: - return - await vm.fire_event_async( - 'device-attach:block', device=device, options=options) + self.pre_attachment_internal(vm, device, options) + + vm.libvirt_domain.attachDevice( + vm.app.env.get_template('libvirt/devices/block.xml').render( + device=device, vm=vm, options=options)) @qubes.ext.handler('domain-shutdown') async def on_domain_shutdown(self, vm, event, **_kwargs): From af8996f24830dd8d7f8deed9096928ade997b14d Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Tue, 26 Mar 2024 18:38:53 +0100 Subject: [PATCH 28/49] q-dev: set.assignment -> set.required --- Makefile | 10 ++-- .../90-admin-default.policy.header | 4 +- qubes/api/admin.py | 11 ++-- qubes/devices.py | 22 +++---- qubes/ext/block.py | 1 - qubes/tests/api_admin.py | 58 +++++-------------- qubes/tests/devices.py | 33 +++-------- rpm_spec/core-dom0.spec.in | 4 +- 8 files changed, 44 insertions(+), 99 deletions(-) diff --git a/Makefile b/Makefile index a52eb2e29..7a410ade1 100644 --- a/Makefile +++ b/Makefile @@ -66,7 +66,7 @@ ADMIN_API_METHODS_SIMPLE = \ admin.vm.device.pci.Attached \ admin.vm.device.pci.Available \ admin.vm.device.pci.Detach \ - admin.vm.device.pci.Set.assignment \ + admin.vm.device.pci.Set.required \ admin.vm.device.pci.Unassign \ admin.vm.device.block.Assign \ admin.vm.device.block.Assigned \ @@ -74,7 +74,7 @@ ADMIN_API_METHODS_SIMPLE = \ admin.vm.device.block.Attached \ admin.vm.device.block.Available \ admin.vm.device.block.Detach \ - admin.vm.device.block.Set.assignment \ + admin.vm.device.block.Set.required \ admin.vm.device.block.Unassign \ admin.vm.device.usb.Assign \ admin.vm.device.usb.Assigned \ @@ -82,7 +82,7 @@ ADMIN_API_METHODS_SIMPLE = \ admin.vm.device.usb.Attached \ admin.vm.device.usb.Available \ admin.vm.device.usb.Detach \ - admin.vm.device.usb.Set.assignment \ + admin.vm.device.usb.Set.required \ admin.vm.device.usb.Unassign \ admin.vm.device.mic.Assign \ admin.vm.device.mic.Assigned \ @@ -90,7 +90,7 @@ ADMIN_API_METHODS_SIMPLE = \ admin.vm.device.mic.Attached \ admin.vm.device.mic.Available \ admin.vm.device.mic.Detach \ - admin.vm.device.mic.Set.assignment \ + admin.vm.device.mic.Set.required \ admin.vm.device.mic.Unassign \ admin.vm.feature.CheckWithNetvm \ admin.vm.feature.CheckWithTemplate \ @@ -227,7 +227,7 @@ endif admin.vm.device.testclass.Unassign \ admin.vm.device.testclass.Attached \ admin.vm.device.testclass.Assigned \ - admin.vm.device.testclass.Set.assignment \ + admin.vm.device.testclass.Set.required \ admin.vm.device.testclass.Available install -d $(DESTDIR)/etc/qubes/policy.d/include install -m 0644 qubes-rpc-policy/admin-local-ro \ diff --git a/qubes-rpc-policy/90-admin-default.policy.header b/qubes-rpc-policy/90-admin-default.policy.header index bdd5c7065..864872ba4 100644 --- a/qubes-rpc-policy/90-admin-default.policy.header +++ b/qubes-rpc-policy/90-admin-default.policy.header @@ -26,7 +26,7 @@ !include-service admin.vm.device.mic.Attached * include/admin-local-ro !include-service admin.vm.device.mic.Available * include/admin-local-ro !include-service admin.vm.device.mic.Detach * include/admin-local-rwx -!include-service admin.vm.device.mic.Set.assignment * include/admin-local-rwx +!include-service admin.vm.device.mic.Set.required * include/admin-local-rwx !include-service admin.vm.device.mic.Unassign * include/admin-local-rwx !include-service admin.vm.device.usb.Assign * include/admin-local-rwx !include-service admin.vm.device.usb.Assigned * include/admin-local-ro @@ -34,6 +34,6 @@ !include-service admin.vm.device.usb.Attached * include/admin-local-ro !include-service admin.vm.device.usb.Available * include/admin-local-ro !include-service admin.vm.device.usb.Detach * include/admin-local-rwx -!include-service admin.vm.device.usb.Set.assignment * include/admin-local-rwx +!include-service admin.vm.device.usb.Set.required * include/admin-local-rwx !include-service admin.vm.device.usb.Unassign * include/admin-local-rwx diff --git a/qubes/api/admin.py b/qubes/api/admin.py index f834be175..75b708fd5 100644 --- a/qubes/api/admin.py +++ b/qubes/api/admin.py @@ -1370,22 +1370,21 @@ async def vm_device_detach(self, endpoint): # Assign/Unassign action can modify only a persistent state of running VM. # For this reason, write=True - @qubes.api.method('admin.vm.device.{endpoint}.Set.assignment', + @qubes.api.method('admin.vm.device.{endpoint}.Set.required', endpoints=(ep.name for ep in importlib.metadata.entry_points(group='qubes.devices')), scope='local', write=True) - async def vm_device_set_assignment(self, endpoint, untrusted_payload): + async def vm_device_set_required(self, endpoint, untrusted_payload): """ - Update assignment of an already assigned device. + Update `required` flag of an already assigned device. Payload: - `None` -> unassign device from a qube `False` -> device will be auto-attached to qube `True` -> device is required to start qube """ devclass = endpoint - self.enforce(untrusted_payload in (b'True', b'False', b'None')) + self.enforce(untrusted_payload in (b'True', b'False')) # now is safe to eval, since the value of untrusted_payload is trusted # pylint: disable=eval-used assignment = eval(untrusted_payload) @@ -1398,7 +1397,7 @@ async def vm_device_set_assignment(self, endpoint, untrusted_payload): self.fire_event_for_permission(device=dev, assignment=assignment) - await self.dest.devices[devclass].update_assignment(dev, assignment) + await self.dest.devices[devclass].update_required(dev, assignment) self.app.save() @qubes.api.method('admin.vm.firewall.Get', no_payload=True, diff --git a/qubes/devices.py b/qubes/devices.py index deedf04b3..e06bee101 100644 --- a/qubes/devices.py +++ b/qubes/devices.py @@ -59,7 +59,7 @@ `device-list-change:class` event. """ import itertools -from typing import Optional, Iterable +from typing import Iterable import qubes.exc import qubes.utils @@ -253,13 +253,12 @@ def load_assignment(self, device_assignment: DeviceAssignment): device_assignment.devclass = self._bus self._set.add(device_assignment) - async def update_assignment(self, device: Device, required: Optional[bool]): + async def update_required(self, device: Device, required: bool): """ - Update assignment of an already attached device. + Update `required` flag of an already attached device. :param Device device: device for which change required flag :param bool required: new assignment: - `None` -> unassign device from qube `False` -> device will be auto-attached to qube `True` -> device is required to start qube """ @@ -278,17 +277,12 @@ async def update_assignment(self, device: Device, required: Optional[bool]): # be careful to use already present assignment, not the provided one # - to not change options as a side effect - if required is not None: - if assignment.required == required: - return + if assignment.required == required: + return - assignment.required = required - await self._vm.fire_event_async( - 'device-assignment-changed:' + self._bus, device=device) - else: - await self._vm.fire_event_async( - 'device-unassign:' + self._bus, device=device) - await self.unassign(assignment) + assignment.required = required + await self._vm.fire_event_async( + 'device-assignment-changed:' + self._bus, device=device) async def detach(self, device: Device): """ diff --git a/qubes/ext/block.py b/qubes/ext/block.py index 495489451..0bee71f8a 100644 --- a/qubes/ext/block.py +++ b/qubes/ext/block.py @@ -545,7 +545,6 @@ async def on_domain_start(self, vm, _event, **_kwargs): vm, assignment.device, assignment.options) def notify_auto_attached(self, vm, device, options): - pass # bypass DeviceCollection logic preventing double attach self.pre_attachment_internal(vm, device, options) diff --git a/qubes/tests/api_admin.py b/qubes/tests/api_admin.py index a2e39b1ca..0c88e0f6d 100644 --- a/qubes/tests/api_admin.py +++ b/qubes/tests/api_admin.py @@ -2831,7 +2831,7 @@ def test_642_vm_create_disposable_not_allowed(self, storage_mock): b'test-vm1') self.assertFalse(self.app.save.called) - def test_650_vm_device_set_assignment_true(self): + def test_650_vm_device_set_required_true(self): assignment = qubes.device_protocol.DeviceAssignment( self.vm, '1234', attach_automatically=True, required=False, options={'opt1': 'value'}) @@ -2845,7 +2845,7 @@ def test_650_vm_device_set_assignment_true(self): with unittest.mock.patch.object(qubes.vm.qubesvm.QubesVM, 'is_halted', lambda _: False): value = self.call_mgmt_func( - b'admin.vm.device.testclass.Set.assignment', + b'admin.vm.device.testclass.Set.required', b'test-vm1', b'test-vm1+1234', b'True') self.assertIsNone(value) @@ -2855,13 +2855,13 @@ def test_650_vm_device_set_assignment_true(self): self.assertIn(dev, required) self.assertEventFired( self.emitter, - 'admin-permission:admin.vm.device.testclass.Set.assignment') + 'admin-permission:admin.vm.device.testclass.Set.required') mock_action.assert_called_once_with( self.vm, f'device-assignment-changed:testclass', device=self.vm.devices['testclass']['1234']) self.app.save.assert_called_once_with() - def test_651_vm_device_set_assignment_false(self): + def test_651_vm_device_set_required_false(self): assignment = qubes.device_protocol.DeviceAssignment( self.vm, '1234', attach_automatically=True, required=True, options={'opt1': 'value'}) @@ -2875,7 +2875,7 @@ def test_651_vm_device_set_assignment_false(self): with unittest.mock.patch.object(qubes.vm.qubesvm.QubesVM, 'is_halted', lambda _: False): value = self.call_mgmt_func( - b'admin.vm.device.testclass.Set.assignment', + b'admin.vm.device.testclass.Set.required', b'test-vm1', b'test-vm1+1234', b'False') self.assertIsNone(value) @@ -2885,43 +2885,13 @@ def test_651_vm_device_set_assignment_false(self): self.assertNotIn(dev, required) self.assertEventFired( self.emitter, - 'admin-permission:admin.vm.device.testclass.Set.assignment') + 'admin-permission:admin.vm.device.testclass.Set.required') mock_action.assert_called_once_with( self.vm, f'device-assignment-changed:testclass', device=self.vm.devices['testclass']['1234']) self.app.save.assert_called_once_with() - def test_652_vm_device_set_assignment_none(self): - assignment = qubes.device_protocol.DeviceAssignment( - self.vm, '1234', attach_automatically=True, required=False, - options={'opt1': 'value'}) - self.loop.run_until_complete( - self.vm.devices['testclass'].assign(assignment)) - mock_action = unittest.mock.Mock() - mock_action.return_value = None - del mock_action._is_coroutine - self.vm.add_handler(f'device-unassign:testclass', mock_action) - - with unittest.mock.patch.object(qubes.vm.qubesvm.QubesVM, - 'is_halted', lambda _: False): - value = self.call_mgmt_func( - b'admin.vm.device.testclass.Set.assignment', - b'test-vm1', b'test-vm1+1234', b'None') - - self.assertIsNone(value) - dev = qubes.device_protocol.DeviceInfo(self.vm, '1234') - required = self.vm.devices['testclass'].get_assigned_devices( - required_only=False) - self.assertNotIn(dev, required) - self.assertEventFired( - self.emitter, - 'admin-permission:admin.vm.device.testclass.Set.assignment') - mock_action.assert_called_once_with( - self.vm, f'device-unassign:testclass', - device=self.vm.devices['testclass']['1234']) - self.app.save.assert_called_once_with() - - def test_653_vm_device_set_assignment_true_unchanged(self): + def test_652_vm_device_set_required_true_unchanged(self): assignment = qubes.device_protocol.DeviceAssignment( self.vm, '1234', attach_automatically=True, required=True, options={'opt1': 'value'}) @@ -2930,7 +2900,7 @@ def test_653_vm_device_set_assignment_true_unchanged(self): with unittest.mock.patch.object(qubes.vm.qubesvm.QubesVM, 'is_halted', lambda _: False): value = self.call_mgmt_func( - b'admin.vm.device.testclass.Set.assignment', + b'admin.vm.device.testclass.Set.required', b'test-vm1', b'test-vm1+1234', b'True') self.assertIsNone(value) dev = qubes.device_protocol.DeviceInfo(self.vm, '1234') @@ -2939,7 +2909,7 @@ def test_653_vm_device_set_assignment_true_unchanged(self): self.assertIn(dev, required) self.app.save.assert_called_once_with() - def test_654_vm_device_set_assignment_false_unchanged(self): + def test_653_vm_device_set_required_false_unchanged(self): assignment = qubes.device_protocol.DeviceAssignment( self.vm, '1234', attach_automatically=True, required=False, options={'opt1': 'value'}) @@ -2948,7 +2918,7 @@ def test_654_vm_device_set_assignment_false_unchanged(self): with unittest.mock.patch.object(qubes.vm.qubesvm.QubesVM, 'is_halted', lambda _: False): value = self.call_mgmt_func( - b'admin.vm.device.testclass.Set.assignment', + b'admin.vm.device.testclass.Set.required', b'test-vm1', b'test-vm1+1234', b'False') self.assertIsNone(value) dev = qubes.device_protocol.DeviceInfo(self.vm, '1234') @@ -2957,28 +2927,28 @@ def test_654_vm_device_set_assignment_false_unchanged(self): self.assertNotIn(dev, required) self.app.save.assert_called_once_with() - def test_655_vm_device_set_persistent_not_assigned(self): + def test_654_vm_device_set_persistent_not_assigned(self): self.vm.add_handler('device-list:testclass', self.device_list_testclass) with unittest.mock.patch.object(qubes.vm.qubesvm.QubesVM, 'is_halted', lambda _: False): with self.assertRaises(qubes.exc.QubesValueError): self.call_mgmt_func( - b'admin.vm.device.testclass.Set.assignment', + b'admin.vm.device.testclass.Set.required', b'test-vm1', b'test-vm1+1234', b'True') dev = qubes.device_protocol.DeviceInfo(self.vm, '1234') self.assertNotIn( dev, self.vm.devices['testclass'].get_assigned_devices()) self.assertFalse(self.app.save.called) - def test_656_vm_device_set_persistent_invalid_value(self): + def test_655_vm_device_set_persistent_invalid_value(self): self.vm.add_handler('device-list:testclass', self.device_list_testclass) with unittest.mock.patch.object(qubes.vm.qubesvm.QubesVM, 'is_halted', lambda _: False): with self.assertRaises(qubes.exc.PermissionDenied): self.call_mgmt_func( - b'admin.vm.device.testclass.Set.assignment', + b'admin.vm.device.testclass.Set.required', b'test-vm1', b'test-vm1+1234', b'maybe') dev = qubes.device_protocol.DeviceInfo(self.vm, '1234') self.assertNotIn(dev, self.vm.devices['testclass'].get_assigned_devices()) diff --git a/qubes/tests/devices.py b/qubes/tests/devices.py index a1ac3e2f3..6821292c9 100644 --- a/qubes/tests/devices.py +++ b/qubes/tests/devices.py @@ -211,30 +211,13 @@ def test_020_update_required_to_false(self): self.assertEqual( {self.device}, set(self.collection.get_assigned_devices())) self.loop.run_until_complete( - self.collection.update_assignment(self.device, False)) + self.collection.update_required(self.device, False)) self.assertEqual( {self.device}, set(self.collection.get_assigned_devices())) self.assertEqual( {self.device}, set(self.collection.get_attached_devices())) - def test_021_update_required_to_none(self): - self.assertEqual(set([]), set(self.collection.get_assigned_devices())) - self.assignment.required = False - self.loop.run_until_complete(self.collection.assign(self.assignment)) - self.attach() - self.assertEqual( - set(), - set(self.collection.get_assigned_devices(required_only=True))) - self.assertEqual( - {self.device}, set(self.collection.get_assigned_devices())) - self.loop.run_until_complete( - self.collection.update_assignment(self.device, None)) - self.assertEqual( - set(), set(self.collection.get_assigned_devices())) - self.assertEqual( - {self.device}, set(self.collection.get_attached_devices())) - - def test_022_update_required_to_true(self): + def test_021_update_required_to_true(self): self.assignment.required = False self.attach() self.assertEqual(set(), set(self.collection.get_assigned_devices())) @@ -249,13 +232,13 @@ def test_022_update_required_to_true(self): self.assertEqual({self.device}, set(self.collection.get_attached_devices())) self.loop.run_until_complete( - self.collection.update_assignment(self.device, True)) + self.collection.update_required(self.device, True)) self.assertEqual({self.device}, set(self.collection.get_assigned_devices())) self.assertEqual({self.device}, set(self.collection.get_attached_devices())) - def test_023_update_required_reject_not_running(self): + def test_022_update_required_reject_not_running(self): self.assertEqual(set([]), set(self.collection.get_assigned_devices())) self.loop.run_until_complete(self.collection.assign(self.assignment)) self.assertEqual({self.device}, @@ -263,18 +246,18 @@ def test_023_update_required_reject_not_running(self): self.assertEqual(set(), set(self.collection.get_attached_devices())) with self.assertRaises(qubes.exc.QubesVMNotStartedError): self.loop.run_until_complete( - self.collection.update_assignment(self.device, False)) + self.collection.update_required(self.device, False)) - def test_024_update_required_reject_not_attached(self): + def test_023_update_required_reject_not_attached(self): self.assertEqual(set(), set(self.collection.get_assigned_devices())) self.assertEqual(set(), set(self.collection.get_attached_devices())) self.emitter.running = True with self.assertRaises(qubes.exc.QubesValueError): self.loop.run_until_complete( - self.collection.update_assignment(self.device, True)) + self.collection.update_required(self.device, True)) with self.assertRaises(qubes.exc.QubesValueError): self.loop.run_until_complete( - self.collection.update_assignment(self.device, False)) + self.collection.update_required(self.device, False)) def test_030_assign(self): self.emitter.running = True diff --git a/rpm_spec/core-dom0.spec.in b/rpm_spec/core-dom0.spec.in index 1ea276b1d..3e092681b 100644 --- a/rpm_spec/core-dom0.spec.in +++ b/rpm_spec/core-dom0.spec.in @@ -260,7 +260,7 @@ admin.vm.device.block.Attach admin.vm.device.block.Attached admin.vm.device.block.Available admin.vm.device.block.Detach -admin.vm.device.block.Set.assignment +admin.vm.device.block.Set.required admin.vm.device.block.Unassign admin.vm.device.pci.Assign admin.vm.device.pci.Assigned @@ -268,7 +268,7 @@ admin.vm.device.pci.Attach admin.vm.device.pci.Attached admin.vm.device.pci.Available admin.vm.device.pci.Detach -admin.vm.device.pci.Set.assignment +admin.vm.device.pci.Set.required admin.vm.device.pci.Unassign admin.vm.feature.CheckWithAdminVM admin.vm.feature.CheckWithNetvm From 9dec436bb6aed6859814d8009ae595945dcbfb0f Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Tue, 9 Apr 2024 16:04:07 +0200 Subject: [PATCH 29/49] q-dev: fix auto-attachment of block devices --- qubes/ext/block.py | 13 ++++++------- templates/libvirt/xen.xml | 5 +++++ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/qubes/ext/block.py b/qubes/ext/block.py index 0bee71f8a..f1924340c 100644 --- a/qubes/ext/block.py +++ b/qubes/ext/block.py @@ -470,7 +470,8 @@ def on_device_pre_attached_block(self, vm, event, device, options): vm.app.env.get_template('libvirt/devices/block.xml').render( device=device, vm=vm, options=options)) - def pre_attachment_internal(self, vm, device, options): + def pre_attachment_internal( + self, vm, device, options, expected_attachment=None): if isinstance(device, qubes.device_protocol.UnknownDevice): print(f'{device.devclass.capitalize()} device {device} ' 'not available, skipping.', file=sys.stderr) @@ -519,7 +520,7 @@ def pre_attachment_internal(self, vm, device, options): file=sys.stderr) return - if device.attachment: + if device.attachment != expected_attachment: raise qubes.devices.DeviceAlreadyAttached( 'Device {!s} already attached to {!s}'.format( device, device.attachment) @@ -546,11 +547,9 @@ async def on_domain_start(self, vm, _event, **_kwargs): def notify_auto_attached(self, vm, device, options): # bypass DeviceCollection logic preventing double attach - self.pre_attachment_internal(vm, device, options) - - vm.libvirt_domain.attachDevice( - vm.app.env.get_template('libvirt/devices/block.xml').render( - device=device, vm=vm, options=options)) + # we expected that these devices are already attached to this vm + self.pre_attachment_internal( + vm, device, options, expected_attachment=vm) @qubes.ext.handler('domain-shutdown') async def on_domain_shutdown(self, vm, event, **_kwargs): diff --git a/templates/libvirt/xen.xml b/templates/libvirt/xen.xml index 155f82e46..a9ce7c1db 100644 --- a/templates/libvirt/xen.xml +++ b/templates/libvirt/xen.xml @@ -156,6 +156,11 @@ {# start external devices from xvdi #} {% set counter = {'i': 4} %} + {% for assignment in vm.devices.block.get_assigned_devices(False) %} + {% set device = assignment.device %} + {% set options = assignment.options %} + {% include 'libvirt/devices/block.xml' %} + {% endfor %} {% if vm.netvm %} {% include 'libvirt/devices/net.xml' with context %} From 00256e48e9237855fbd88119474a0a3057d3e4dd Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Fri, 19 Apr 2024 06:39:24 +0200 Subject: [PATCH 30/49] q-dev: typos --- qubes/ext/block.py | 3 ++- qubes/tests/vm/qubesvm.py | 2 +- qubes/vm/qubesvm.py | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/qubes/ext/block.py b/qubes/ext/block.py index f1924340c..0e21424fb 100644 --- a/qubes/ext/block.py +++ b/qubes/ext/block.py @@ -511,7 +511,8 @@ def pre_attachment_internal( 'This device can be attached only read-only') if not vm.is_running(): - print("Not attach, not running", file=sys.stderr) + print(f"Can not attach device, qube {vm.name} is not running." + , file=sys.stderr) return if not isinstance(device, BlockDevice): diff --git a/qubes/tests/vm/qubesvm.py b/qubes/tests/vm/qubesvm.py index c1c50b5cb..04c2f7fd3 100644 --- a/qubes/tests/vm/qubesvm.py +++ b/qubes/tests/vm/qubesvm.py @@ -1393,7 +1393,7 @@ def test_600_libvirt_xml_hvm_pcidev_s0ix(self): # even with meminfo-writer enabled, should have memory==maxmem vm.features['service.meminfo-writer'] = True assignment = qubes.device_protocol.DeviceAssignment( - vm, # this is violation of API, but for PCI the argument + vm, # this is a violation of API, but for PCI the argument # is unused '00_00.0', devclass='pci', diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 7993e7baa..f604eb0ff 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -1629,9 +1629,9 @@ def is_memory_balancing_possible(self): - have PCI devices - balloon driver not present - We don't have reliable way to detect the second point, but good + We don't have a reliable way to detect the second point, but good heuristic is HVM virt_mode (PV and PVH require OS support, and it does - include balloon driver) and lack of qrexec/meminfo-writer service + include the balloon driver) and lack of qrexec/meminfo-writer service support (no qubes tools installed). """ if list(self.devices['pci'].get_assigned_devices()): From 8da324688a9cde207cf2e68d112ef1b52972ff02 Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Mon, 22 Apr 2024 05:42:14 +0200 Subject: [PATCH 31/49] q-dev: fix memory balancing --- qubes/api/admin.py | 3 ++- qubes/vm/qubesvm.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/qubes/api/admin.py b/qubes/api/admin.py index 75b708fd5..5bcb60769 100644 --- a/qubes/api/admin.py +++ b/qubes/api/admin.py @@ -1216,7 +1216,8 @@ async def vm_device_available(self, endpoint): scope='local', read=True) async def vm_device_list(self, endpoint): devclass = endpoint - device_assignments = self.dest.devices[devclass].get_assigned_devices() + device_assignments = list( + self.dest.devices[devclass].get_assigned_devices()) if self.arg: select_backend, select_ident = self.arg.split('+', 1) device_assignments = [dev for dev in device_assignments diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index f604eb0ff..a38d46432 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -169,7 +169,7 @@ def _setter_kbd_layout(self, prop, value): def _default_virt_mode(self): - if self.devices['pci'].get_assigned_devices(): + if list(self.devices['pci'].get_assigned_devices()): return 'hvm' try: return self.template.virt_mode From 54b9f9b3200604b0b1bdb3dd17a64340fbac5ec0 Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Tue, 23 Apr 2024 04:27:38 +0200 Subject: [PATCH 32/49] q-dev: update device_protocol --- qubes/device_protocol.py | 90 +++++++++++++++++++++++++++++++++------- 1 file changed, 76 insertions(+), 14 deletions(-) diff --git a/qubes/device_protocol.py b/qubes/device_protocol.py index 368267a3b..31de9b3d6 100644 --- a/qubes/device_protocol.py +++ b/qubes/device_protocol.py @@ -32,7 +32,7 @@ import string import sys from enum import Enum -from typing import Optional, Dict, Any, List, Union +from typing import Optional, Dict, Any, List, Union, Tuple import qubes.utils @@ -52,8 +52,19 @@ def qbool(value): class Device: - ALLOWED_CHARS_KEY = string.digits + string.ascii_letters + '-_.' - ALLOWED_CHARS_PARAM = ALLOWED_CHARS_KEY + ',+:' + """ + Basic class of a *bus* device with *ident* exposed by a *backend domain*. + + Attributes: + backend_domain (QubesVM): The domain which exposes devices, + e.g.`sys-usb`. + ident (str): A unique identifier for the device within + the backend domain. + devclass (str, optional): The class of the device (e.g., 'usb', 'pci'). + """ + ALLOWED_CHARS_KEY = ( + string.digits + string.ascii_letters + string.punctuation + ' ') + ALLOWED_CHARS_PARAM = ALLOWED_CHARS_KEY + string.punctuation + ' ' def __init__(self, backend_domain, ident, devclass=None): self.__backend_domain = backend_domain @@ -64,16 +75,20 @@ def __hash__(self): return hash((str(self.backend_domain), self.ident)) def __eq__(self, other): - return ( - self.backend_domain == other.backend_domain and - self.ident == other.ident - ) + if isinstance(other, Device): + return ( + self.backend_domain == other.backend_domain and + self.ident == other.ident + ) + raise TypeError(f"Comparing instances of 'Device' and '{type(other)}' " + "is not supported") def __lt__(self, other): if isinstance(other, Device): return (self.backend_domain.name, self.ident) < \ (other.backend_domain.name, other.ident) - raise NotImplementedError() + raise TypeError(f"Comparing instances of 'Device' and '{type(other)}' " + "is not supported") def __repr__(self): return "[%s]:%s" % (self.backend_domain, self.ident) @@ -125,7 +140,20 @@ def devclass(self, devclass: str): self.__bus = devclass @classmethod - def unpack_properties(cls, untrusted_serialization: bytes): + def unpack_properties( + cls, untrusted_serialization: bytes + ) -> Tuple[Dict, Dict]: + """ + Unpacks basic device properties from a serialized encoded string. + + Returns: + tuple: A tuple containing two dictionaries, properties and options, + extracted from the serialization. + + Raises: + ValueError: If unexpected characters are found in property + names or values. + """ ut_decoded = untrusted_serialization.decode( 'ascii', errors='strict').strip() @@ -170,7 +198,17 @@ def unpack_properties(cls, untrusted_serialization: bytes): return properties, options @staticmethod - def check_device_properties(expected_device, properties): + def check_device_properties( + expected_device: 'Device', properties: Dict[str, Any]): + """ + Validates properties against an expected device configuration. + + Modifies `properties`. + + Raises: + UnexpectedDeviceProperty: If any property does not match + the expected values. + """ expected = expected_device exp_vm_name = expected.backend_domain.name if properties.get('backend_domain', exp_vm_name) != exp_vm_name: @@ -194,10 +232,9 @@ def check_device_properties(expected_device, properties): properties['devclass'] = expected.devclass - class DeviceCategory(Enum): """ - Category of peripheral device. + Category of a peripheral device. Arbitrarily selected interfaces that are important to users, thus deserving special recognition such as a custom icon, etc. @@ -228,6 +265,9 @@ class DeviceCategory(Enum): @staticmethod def from_str(interface_encoding: str) -> 'DeviceCategory': + """ + Returns `DeviceCategory` from data encoded in string. + """ result = DeviceCategory.Other if len(interface_encoding) != len(DeviceCategory.Other.value): return result @@ -527,7 +567,7 @@ def attachment(self) -> Optional[QubesVM]: def serialize(self) -> bytes: """ - Serialize object to be transmitted via Qubes API. + Serialize an object to be transmitted via Qubes API. """ # 'backend_domain', 'attachment', 'interfaces', 'data', 'parent_device' # are not string, so they need special treatment @@ -577,6 +617,9 @@ def deserialize( expected_backend_domain: QubesVM, expected_devclass: Optional[str] = None, ) -> 'DeviceInfo': + """ + Recovers a serialized object, see: :py:meth:`serialize`. + """ ident, _, rest = serialization.partition(b' ') ident = ident.decode('ascii', errors='ignore') device = UnknownDevice( @@ -587,6 +630,7 @@ def deserialize( try: device = cls._deserialize(rest, device) + # pylint: disable=broad-exception-caught except Exception as exc: print(exc, file=sys.stderr) @@ -598,6 +642,9 @@ def _deserialize( untrusted_serialization: bytes, expected_device: Device ) -> 'DeviceInfo': + """ + Actually deserializes the object. + """ properties, options = cls.unpack_properties(untrusted_serialization) properties.update(options) @@ -656,6 +703,9 @@ def self_identity(self) -> str: def serialize_str(value: str): + """ + Serialize python string to ensure consistency. + """ result = repr(str(value)) if result.startswith('"'): result = "'" + result[1:-1] + "'" @@ -663,6 +713,9 @@ def serialize_str(value: str): def deserialize_str(value: str): + """ + Deserialize python string to ensure consistency. + """ return value.replace("\\\'", "'") @@ -694,7 +747,7 @@ def sanitize_str( class UnknownDevice(DeviceInfo): # pylint: disable=too-few-public-methods - """Unknown device - for example exposed by domain not running currently""" + """Unknown device - for example, exposed by domain not running currently""" def __init__(self, backend_domain, ident, *, devclass, **kwargs): super().__init__(backend_domain, ident, devclass=devclass, **kwargs) @@ -822,6 +875,9 @@ def options(self, options: Optional[Dict[str, Any]]): self.__options = options or {} def serialize(self) -> bytes: + """ + Serialize an object to be transmitted via Qubes API. + """ properties = b' '.join( f'{prop}={serialize_str(value)}'.encode('ascii') for prop, value in ( @@ -857,6 +913,9 @@ def deserialize( serialization: bytes, expected_device: Device, ) -> 'DeviceAssignment': + """ + Recovers a serialized object, see: :py:meth:`serialize`. + """ try: result = cls._deserialize(serialization, expected_device) except Exception as exc: @@ -869,6 +928,9 @@ def _deserialize( untrusted_serialization: bytes, expected_device: Device, ) -> 'DeviceAssignment': + """ + Actually deserializes the object. + """ properties, options = cls.unpack_properties(untrusted_serialization) properties['options'] = options From 42e062ad24906d290b7c2a719520e3f7802a980f Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Tue, 23 Apr 2024 06:22:28 +0200 Subject: [PATCH 33/49] q-dev: update integration tests --- qubes/tests/integ/devices_block.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/qubes/tests/integ/devices_block.py b/qubes/tests/integ/devices_block.py index c39bc0e42..74e60b4ae 100644 --- a/qubes/tests/integ/devices_block.py +++ b/qubes/tests/integ/devices_block.py @@ -87,7 +87,7 @@ def test_000_list_loop(self): dev_list = list(self.vm.devices['block']) found = False for dev in dev_list: - if dev.name == self.img_path: + if dev.serial == self.img_path: self.assertTrue(dev.ident.startswith('loop')) self.assertEqual(dev.mode, 'w') self.assertEqual(dev.size, 1024 * 1024 * 128) @@ -113,7 +113,7 @@ def test_001_list_loop_mounted(self): dev_list = list(self.vm.devices['block']) for dev in dev_list: - if dev.name == self.img_path: + if dev.serial == self.img_path: self.fail( 'Device {} ({}) should not be listed because is mounted' .format(dev, self.img_path)) @@ -132,11 +132,11 @@ def test_010_list_dm(self): found = False for dev in dev_list: if dev.ident.startswith('loop'): - self.assertNotEquals(dev.name, self.img_path, + self.assertNotEquals(dev.serial, self.img_path, "Device {} ({}) should not be listed as it is used in " "device-mapper".format(dev, self.img_path) ) - elif dev.name == 'test-dm': + elif dev.serial == 'test-dm': self.assertEqual(dev.mode, 'w') self.assertEqual(dev.size, 1024 * 1024 * 128) found = True @@ -163,12 +163,12 @@ def test_011_list_dm_mounted(self): dev_list = list(self.vm.devices['block']) for dev in dev_list: if dev.ident.startswith('loop'): - self.assertNotEquals(dev.name, self.img_path, + self.assertNotEquals(dev.serial, self.img_path, "Device {} ({}) should not be listed as it is used in " "device-mapper".format(dev, self.img_path) ) else: - self.assertNotEquals(dev.name, 'test-dm', + self.assertNotEquals(dev.serial, 'test-dm', "Device {} ({}) should not be listed as it is " "mounted".format(dev, 'test-dm') ) @@ -188,11 +188,11 @@ def test_012_list_dm_delayed(self): found = False for dev in dev_list: if dev.ident.startswith('loop'): - self.assertNotEquals(dev.name, self.img_path, + self.assertNotEquals(dev.serial, self.img_path, "Device {} ({}) should not be listed as it is used in " "device-mapper".format(dev, self.img_path) ) - elif dev.name == 'test-dm': + elif dev.serial == 'test-dm': self.assertEqual(dev.mode, 'w') self.assertEqual(dev.size, 1024 * 1024 * 128) found = True @@ -218,7 +218,7 @@ def test_013_list_dm_removed(self): dev_list = list(self.vm.devices['block']) found = False for dev in dev_list: - if dev.name == self.img_path: + if dev.serial == self.img_path: self.assertTrue(dev.ident.startswith('loop')) self.assertEqual(dev.mode, 'w') self.assertEqual(dev.size, 1024 * 1024 * 128) @@ -242,7 +242,7 @@ def test_020_list_loop_partition(self): dev_list = list(self.vm.devices['block']) found = False for dev in dev_list: - if dev.name == self.img_path: + if dev.serial == self.img_path: self.assertTrue(dev.ident.startswith('loop')) self.assertEqual(dev.mode, 'w') self.assertEqual(dev.size, 1024 * 1024 * 128) @@ -271,7 +271,7 @@ def test_021_list_loop_partition_mounted(self): dev_list = list(self.vm.devices['block']) for dev in dev_list: - if dev.name == self.img_path: + if dev.serial == self.img_path: self.fail( 'Device {} ({}) should not be listed because its ' 'partition is mounted' @@ -318,13 +318,13 @@ def setUp(self): "udevadm settle".format(path=self.img_path), user="root")) dev_list = list(self.backend.devices['block']) for dev in dev_list: - if dev.name == self.img_path: + if dev.serial == self.img_path: self.device = dev self.device_ident = dev.ident break else: self.fail('Device for {} in {} not found'.format( - self.img_path, self.backend.name)) + self.img_path, self.backend.serial)) def test_000_attach_reattach(self): ass = qubes.device_protocol.DeviceAssignment(self.backend, self.device_ident) From 4a13df1cef6304b75ba365f4ed7b75033665745b Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Tue, 23 Apr 2024 16:44:09 +0200 Subject: [PATCH 34/49] q-dev: update integration tests fo block devices --- qubes/ext/block.py | 7 +++++-- qubes/tests/vm/qubesvm.py | 4 ++-- relaxng/qubes.rng | 12 +++++++++++- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/qubes/ext/block.py b/qubes/ext/block.py index 0e21424fb..cca7307e7 100644 --- a/qubes/ext/block.py +++ b/qubes/ext/block.py @@ -307,9 +307,9 @@ def on_domain_init_load(self, vm, event): current_devices = dict( (dev.ident, device_attachments.get(dev.ident, None)) for dev in self.on_device_list_block(vm, None)) - self.devices_cache[vm.name] = current_devices.copy() + self.devices_cache[vm.name] = current_devices else: - self.devices_cache[vm.name] = {}.copy() + self.devices_cache[vm.name] = {} @qubes.ext.handler('domain-qdb-change:/qubes-block-devices') def on_qdb_change(self, vm, event, path): @@ -328,6 +328,9 @@ def get_device_attachments(vm_): if not vm.is_running(): continue + if vm.app.vmm.offline_mode: + return result + xml_desc = lxml.etree.fromstring(vm.libvirt_domain.XMLDesc()) for disk in xml_desc.findall('devices/disk'): diff --git a/qubes/tests/vm/qubesvm.py b/qubes/tests/vm/qubesvm.py index 04c2f7fd3..e73f29968 100644 --- a/qubes/tests/vm/qubesvm.py +++ b/qubes/tests/vm/qubesvm.py @@ -1482,7 +1482,7 @@ def test_600_libvirt_xml_hvm_cdrom_boot(self): dom0, 'sda', {'devtype': 'cdrom', 'read-only': 'yes'}, attach_automatically=True, required=True) - self.loop.run_until_complete(vm.devices['block'].attach(dev)) + self.loop.run_until_complete(vm.devices['block'].assign(dev)) libvirt_xml = vm.create_config_file() self.assertXMLEqual(lxml.etree.XML(libvirt_xml), lxml.etree.XML(expected)) @@ -1587,7 +1587,7 @@ def test_600_libvirt_xml_hvm_cdrom_dom0_kernel_boot(self): dom0, 'sda', {'devtype': 'cdrom', 'read-only': 'yes'}, attach_automatically=True, required=True) - self.loop.run_until_complete(vm.devices['block'].attach(dev)) + self.loop.run_until_complete(vm.devices['block'].assign(dev)) libvirt_xml = vm.create_config_file() self.assertXMLEqual(lxml.etree.XML(libvirt_xml), lxml.etree.XML(expected)) diff --git a/relaxng/qubes.rng b/relaxng/qubes.rng index 39065e288..b286db21b 100644 --- a/relaxng/qubes.rng +++ b/relaxng/qubes.rng @@ -218,11 +218,21 @@ the parser will complain about missing combine= attribute on the second . - [0-9a-f]{2}_[0-9a-f]{2}.[0-9a-f]{2} + + + + Flag: is the device required to start domain? + If not present assume 'yes'. + + + [yesno]+ + + + From fe36156d1c52f82ef84e7fc78cbe52d50603eae0 Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Sun, 28 Apr 2024 05:59:09 +0200 Subject: [PATCH 35/49] q-dev: fix serialization --- qubes/device_protocol.py | 117 ++++++++++++----------- qubes/tests/devices.py | 200 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 260 insertions(+), 57 deletions(-) diff --git a/qubes/device_protocol.py b/qubes/device_protocol.py index 31de9b3d6..6b918317e 100644 --- a/qubes/device_protocol.py +++ b/qubes/device_protocol.py @@ -62,9 +62,10 @@ class Device: the backend domain. devclass (str, optional): The class of the device (e.g., 'usb', 'pci'). """ - ALLOWED_CHARS_KEY = ( - string.digits + string.ascii_letters + string.punctuation + ' ') - ALLOWED_CHARS_PARAM = ALLOWED_CHARS_KEY + string.punctuation + ' ' + ALLOWED_CHARS_KEY = set( + string.digits + string.ascii_letters + + r"!#$%&()*+,-./:;<>?@[\]^_{|}~" + ' ') + ALLOWED_CHARS_PARAM = ALLOWED_CHARS_KEY.union(set(string.punctuation + ' ')) def __init__(self, backend_domain, ident, devclass=None): self.__backend_domain = backend_domain @@ -197,6 +198,20 @@ def unpack_properties( return properties, options + @classmethod + def pack_property(cls, serialization: bytes, key: str, value: str): + """ + Add property `key=value` to serialization. + """ + key = sanitize_str( + key, cls.ALLOWED_CHARS_KEY, + error_message='Invalid chars in property name: ') + value = sanitize_str( + serialize_str(value), cls.ALLOWED_CHARS_PARAM, + error_message='Invalid chars in property value: ') + return (serialization + b' ' + + key.encode('ascii') + b'=' + value.encode('ascii')) + @staticmethod def check_device_properties( expected_device: 'Device', properties: Dict[str, Any]): @@ -344,6 +359,14 @@ def unknown(cls) -> 'DeviceInterface': def __repr__(self): return self._interface_encoding + def __hash__(self): + return hash(repr(self)) + + def __eq__(self, other): + if not isinstance(other, DeviceInterface): + return False + return repr(self) == repr(other) + def __str__(self): if self.devclass == "block": return "Block device" @@ -398,8 +421,6 @@ def _load_classes(bus: str): class DeviceInfo(Device): """ Holds all information about a device """ - ALLOWED_CHARS_KEY = Device.ALLOWED_CHARS_KEY + string.punctuation + ' ' - ALLOWED_CHARS_PARAM = Device.ALLOWED_CHARS_PARAM + string.punctuation + ' ' def __init__( self, @@ -574,39 +595,31 @@ def serialize(self) -> bytes: default_attrs = { 'ident', 'devclass', 'vendor', 'product', 'manufacturer', 'name', 'serial', 'self_identity'} - properties = b' '.join( - f'{prop}={serialize_str(value)}'.encode('ascii') - for prop, value in ( - (key, getattr(self, key)) for key in default_attrs) - ) + properties = b'' + for key, value in ((key, getattr(self, key)) for key in default_attrs): + properties = self.pack_property(properties, key, value) + # remove unnecessary space in the beginning + properties = properties[1:] - qname = serialize_str(self.backend_domain.name) - backend_prop = b"backend_domain=" + qname.encode('ascii') - properties += b' ' + backend_prop + properties = self.pack_property( + properties, 'backend_domain', self.backend_domain.name) if self.attachment: - qname = serialize_str(self.attachment.name) - attachment_prop = b"attachment=" + qname.encode('ascii') - properties += b' ' + attachment_prop + properties = self.pack_property( + properties, 'attachment', self.attachment.name) - interfaces = serialize_str( + properties = self.pack_property( + properties, 'interfaces', ''.join(repr(ifc) for ifc in self.interfaces)) - interfaces_prop = b'interfaces=' + interfaces.encode('ascii') - properties += b' ' + interfaces_prop if self.parent_device is not None: - ident = serialize_str(self.parent_device.ident) - ident_prop = b'parent_ident=' + ident.encode('ascii') - properties += b' ' + ident_prop - devclass = serialize_str(self.parent_device.devclass) - devclass_prop = b'parent_devclass=' + devclass.encode('ascii') - properties += b' ' + devclass_prop - - data = b' '.join( - f'_{prop}={serialize_str(value)}'.encode('ascii') - for prop, value in self.data.items()) - if data: - properties += b' ' + data + properties = self.pack_property( + properties, 'parent_ident', self.parent_device.ident) + properties = self.pack_property( + properties, 'parent_devclass', self.parent_device.devclass) + + for key, value in self.data.items(): + properties = self.pack_property(properties, "_" + key, value) return properties @@ -706,22 +719,19 @@ def serialize_str(value: str): """ Serialize python string to ensure consistency. """ - result = repr(str(value)) - if result.startswith('"'): - result = "'" + result[1:-1] + "'" - return result + return "'" + str(value).replace("'", r"\'") + "'" def deserialize_str(value: str): """ Deserialize python string to ensure consistency. """ - return value.replace("\\\'", "'") + return value.replace(r"\'", "'") def sanitize_str( untrusted_value: str, - allowed_chars: str, + allowed_chars: set, replace_char: str = None, error_message: str = "" ) -> str: @@ -732,7 +742,7 @@ def sanitize_str( characters with the string. """ if replace_char is None: - not_allowed_chars = set(untrusted_value) - set(allowed_chars) + not_allowed_chars = set(untrusted_value) - allowed_chars if not_allowed_chars: raise ProtocolError(error_message + repr(not_allowed_chars)) return untrusted_value @@ -878,32 +888,27 @@ def serialize(self) -> bytes: """ Serialize an object to be transmitted via Qubes API. """ - properties = b' '.join( - f'{prop}={serialize_str(value)}'.encode('ascii') - for prop, value in ( + properties = b'' + for key, value in ( ('required', 'yes' if self.required else 'no'), ('attach_automatically', 'yes' if self.attach_automatically else 'no'), ('ident', self.ident), ('devclass', self.devclass) - ) - ) + ): + properties = self.pack_property(properties, key, value) + # remove unnecessary space in the beginning + properties = properties[1:] - back_name = serialize_str(self.backend_domain.name) - backend_domain_prop = b"backend_domain=" + back_name.encode('ascii') - properties += b' ' + backend_domain_prop + properties = self.pack_property( + properties, 'backend_domain', self.backend_domain.name) if self.frontend_domain is not None: - front_name = serialize_str(self.frontend_domain.name) - frontend_domain_prop = ( - b"frontend_domain=" + front_name.encode('ascii')) - properties += b' ' + frontend_domain_prop - - if self.options: - properties += b' ' + b' '.join( - f'_{prop}={serialize_str(value)}'.encode('ascii') - for prop, value in self.options.items() - ) + properties = self.pack_property( + properties, 'frontend_domain', self.frontend_domain.name) + + for key, value in self.options.items(): + properties = self.pack_property(properties, "_" + key, value) return properties diff --git a/qubes/tests/devices.py b/qubes/tests/devices.py index 6821292c9..f8d954ee6 100644 --- a/qubes/tests/devices.py +++ b/qubes/tests/devices.py @@ -22,7 +22,7 @@ import qubes.devices from qubes.device_protocol import (Device, DeviceInfo, DeviceAssignment, - DeviceInterface) + DeviceInterface, UnknownDevice) import qubes.tests @@ -414,6 +414,19 @@ def test_011_serialize_with_parent(self): b"Some_untrusted_garbage").split(b" ")) self.assertEqual(actual, expected) + def test_012_invalid_serialize(self): + device = DeviceInfo( + backend_domain=self.vm, + ident="1-1.1.1", + devclass="bus?", + vendor="malicious", + product="suspicious", + manufacturer="", + name="""Some='untrusted' garbage="5%\n" ?""", + ) + with self.assertRaises(qubes.exc.ProtocolError): + _ = device.serialize() + def test_020_deserialize(self): serialized = ( b"1-1.1.1 " @@ -450,3 +463,188 @@ def test_020_deserialize(self): self.assertEqual(repr(actual.interfaces), repr(expected.interfaces)) self.assertEqual(actual.self_identity, expected.self_identity) self.assertEqual(actual.data, expected.data) + + def test_021_invalid_deserialize(self): + serialized = ( + b"1-1.1.1 " + b"manufacturer='unknown' self_identity='0000:0000::?******' " + b"serial='unknown' ident='1-1.1.1' product='Qubes' " + b"vendor='ITL' name='Some untrusted garbage' devclass='bus' " + b"backend_domain='vm' interfaces=' ******u03**01' " + b"_additional_info='' _date='06.12.23' " + b"parent_ident='1-1.1' parent_devclass='None' add'tional='info'") + actual = DeviceInfo.deserialize(serialized, self.vm) + self.assertIsInstance(actual, UnknownDevice) + self.assertEqual(actual.backend_domain, self.vm) + self.assertEqual(actual.ident, '1-1.1.1') + self.assertEqual(actual.devclass, 'peripheral') + + def test_030_serialize_and_deserialize(self): + device = DeviceInfo( + backend_domain=self.vm, + ident="1-1.1.1", + devclass="bus?", + vendor="malicious", + product="suspicious", + manufacturer="", + name=r"""Some='untrusted' garbage="5%\n" ?""", + serial=None, + interfaces=[DeviceInterface(" 112233"), + DeviceInterface("&012345")], + **{"additional info": "and='more'", "date": "06.12.23"} + ) + serialized = device.serialize() + deserialized = DeviceInfo.deserialize(b'1-1.1.1 ' + serialized, self.vm) + self.assertEqual(deserialized.backend_domain, device.backend_domain) + self.assertEqual(deserialized.ident, device.ident) + self.assertEqual(deserialized.devclass, device.devclass) + self.assertEqual(deserialized.vendor, device.vendor) + self.assertEqual(deserialized.product, device.product) + self.assertEqual(deserialized.manufacturer, device.manufacturer) + self.assertEqual(deserialized.name, device.name) + self.assertEqual(deserialized.serial, device.serial) + self.assertEqual(deserialized.data, device.data) + self.assertEqual(deserialized.interfaces[0], device.interfaces[0]) + self.assertEqual(deserialized.interfaces[1], device.interfaces[1]) + + +class TC_03_DeviceAssignment(qubes.tests.QubesTestCase): + def setUp(self): + super().setUp() + self.app = TestApp() + self.vm = TestVM(self.app, 'vm') + + def test_010_serialize(self): + assignment = DeviceAssignment( + backend_domain=self.vm, + ident="1-1.1.1", + devclass="bus", + ) + actual = assignment.serialize() + expected = ( + b"ident='1-1.1.1' devclass='bus' " + b"backend_domain='vm' required='no' attach_automatically='no'") + expected = set(expected.split(b" ")) + actual = set(actual.split(b" ")) + self.assertEqual(actual, expected) + + def test_011_serialize_required(self): + assignment = DeviceAssignment( + backend_domain=self.vm, + ident="1-1.1.1", + devclass="bus", + attach_automatically=True, + required=True, + ) + actual = assignment.serialize() + expected = ( + b"ident='1-1.1.1' devclass='bus' " + b"backend_domain='vm' required='yes' attach_automatically='yes'") + expected = set(expected.split(b" ")) + actual = set(actual.split(b" ")) + self.assertEqual(actual, expected) + + def test_012_serialize_fronted(self): + assignment = DeviceAssignment( + backend_domain=self.vm, + ident="1-1.1.1", + devclass="bus", + frontend_domain=self.vm, + ) + actual = assignment.serialize() + expected = ( + b"ident='1-1.1.1' frontend_domain='vm' devclass='bus' " + b"backend_domain='vm' required='no' attach_automatically='no'") + expected = set(expected.split(b" ")) + actual = set(actual.split(b" ")) + self.assertEqual(actual, expected) + + def test_013_serialize_options(self): + assignment = DeviceAssignment( + backend_domain=self.vm, + ident="1-1.1.1", + devclass="bus", + options={'read-only': 'yes'}, + ) + actual = assignment.serialize() + expected = ( + b"ident='1-1.1.1' _read-only='yes' devclass='bus' " + b"backend_domain='vm' required='no' attach_automatically='no'") + expected = set(expected.split(b" ")) + actual = set(actual.split(b" ")) + self.assertEqual(actual, expected) + + def test_014_invalid_serialize(self): + assignment = DeviceAssignment( + backend_domain=self.vm, + ident="1-1.1.1", + devclass="bus", + options={"read'only": 'yes'}, + ) + with self.assertRaises(qubes.exc.ProtocolError): + _ = assignment.serialize() + + def test_020_deserialize(self): + serialized = ( + b"ident='1-1.1.1' frontend_domain='vm' devclass='bus' " + b"backend_domain='vm' required='no' attach_automatically='yes' " + b"_read-only='yes'") + expected_device = Device(self.vm, '1-1.1.1', 'bus') + actual = DeviceAssignment.deserialize(serialized, expected_device) + expected = DeviceAssignment( + backend_domain=self.vm, + ident="1-1.1.1", + devclass="bus", + frontend_domain=self.vm, + attach_automatically=True, + required=False, + options={'read-only': 'yes'}, + ) + + self.assertEqual(actual.backend_domain, expected.backend_domain) + self.assertEqual(actual.ident, expected.ident) + self.assertEqual(actual.devclass, expected.devclass) + self.assertEqual(actual.frontend_domain, expected.frontend_domain) + self.assertEqual(actual.attach_automatically, expected.attach_automatically) + self.assertEqual(actual.required, expected.required) + self.assertEqual(actual.options, expected.options) + + def test_021_invalid_deserialize(self): + serialized = ( + b"ident='1-1.1.1' frontend_domain='vm' devclass='bus' " + b"backend_domain='vm' required='no' attach_automatically='yes' " + b"_read'only='yes'") + expected_device = Device(self.vm, '1-1.1.1', 'bus') + with self.assertRaises(qubes.exc.ProtocolError): + _ = DeviceAssignment.deserialize(serialized, expected_device) + + def test_022_invalid_deserialize_2(self): + serialized = ( + b"ident='1-1.1.1' frontend_domain='vm' devclass='bus' " + b"backend_domain='vm' required='no' attach_automatically='yes' " + b"read-only='yes'") + expected_device = Device(self.vm, '1-1.1.1', 'bus') + with self.assertRaises(qubes.exc.ProtocolError): + _ = DeviceAssignment.deserialize(serialized, expected_device) + + def test_030_serialize_and_deserialize(self): + expected = DeviceAssignment( + backend_domain=self.vm, + ident="1-1.1.1", + devclass="bus", + frontend_domain=self.vm, + attach_automatically=True, + required=False, + options={'read-only': 'yes'}, + ) + serialized = expected.serialize() + expected_device = Device(self.vm, '1-1.1.1', 'bus') + actual = DeviceAssignment.deserialize(serialized, expected_device) + self.assertEqual(actual.backend_domain, expected.backend_domain) + self.assertEqual(actual.ident, expected.ident) + self.assertEqual(actual.devclass, expected.devclass) + self.assertEqual(actual.frontend_domain, expected.frontend_domain) + self.assertEqual(actual.attach_automatically, + expected.attach_automatically) + self.assertEqual(actual.required, expected.required) + self.assertEqual(actual.options, expected.options) From 3527f7b8d04a596ce8b327179da702973b4fd042 Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Tue, 30 Apr 2024 08:01:34 +0200 Subject: [PATCH 36/49] q-dev: simplify pack properties --- qubes/device_protocol.py | 53 ++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 29 deletions(-) diff --git a/qubes/device_protocol.py b/qubes/device_protocol.py index 6b918317e..afe971e85 100644 --- a/qubes/device_protocol.py +++ b/qubes/device_protocol.py @@ -199,7 +199,7 @@ def unpack_properties( return properties, options @classmethod - def pack_property(cls, serialization: bytes, key: str, value: str): + def pack_property(cls, key: str, value: str): """ Add property `key=value` to serialization. """ @@ -209,8 +209,7 @@ def pack_property(cls, serialization: bytes, key: str, value: str): value = sanitize_str( serialize_str(value), cls.ALLOWED_CHARS_PARAM, error_message='Invalid chars in property value: ') - return (serialization + b' ' - + key.encode('ascii') + b'=' + value.encode('ascii')) + return key.encode('ascii') + b'=' + value.encode('ascii') @staticmethod def check_device_properties( @@ -595,31 +594,30 @@ def serialize(self) -> bytes: default_attrs = { 'ident', 'devclass', 'vendor', 'product', 'manufacturer', 'name', 'serial', 'self_identity'} - properties = b'' - for key, value in ((key, getattr(self, key)) for key in default_attrs): - properties = self.pack_property(properties, key, value) - # remove unnecessary space in the beginning - properties = properties[1:] + properties = b' '.join( + self.pack_property(key, value) + for key, value in ((key, getattr(self, key)) + for key in default_attrs)) - properties = self.pack_property( - properties, 'backend_domain', self.backend_domain.name) + properties += b' ' + self.pack_property( + 'backend_domain', self.backend_domain.name) if self.attachment: properties = self.pack_property( properties, 'attachment', self.attachment.name) - properties = self.pack_property( - properties, 'interfaces', + properties += b' ' + self.pack_property( + 'interfaces', ''.join(repr(ifc) for ifc in self.interfaces)) if self.parent_device is not None: - properties = self.pack_property( - properties, 'parent_ident', self.parent_device.ident) - properties = self.pack_property( - properties, 'parent_devclass', self.parent_device.devclass) + properties += b' ' + self.pack_property( + 'parent_ident', self.parent_device.ident) + properties += b' ' + self.pack_property( + 'parent_devclass', self.parent_device.devclass) for key, value in self.data.items(): - properties = self.pack_property(properties, "_" + key, value) + properties += b' ' + self.pack_property("_" + key, value) return properties @@ -888,27 +886,24 @@ def serialize(self) -> bytes: """ Serialize an object to be transmitted via Qubes API. """ - properties = b'' - for key, value in ( + properties = b' '.join( + self.pack_property(key, value) + for key, value in ( ('required', 'yes' if self.required else 'no'), ('attach_automatically', 'yes' if self.attach_automatically else 'no'), ('ident', self.ident), - ('devclass', self.devclass) - ): - properties = self.pack_property(properties, key, value) - # remove unnecessary space in the beginning - properties = properties[1:] + ('devclass', self.devclass))) - properties = self.pack_property( - properties, 'backend_domain', self.backend_domain.name) + properties += b' ' + self.pack_property( + 'backend_domain', self.backend_domain.name) if self.frontend_domain is not None: - properties = self.pack_property( - properties, 'frontend_domain', self.frontend_domain.name) + properties += b' ' + self.pack_property( + 'frontend_domain', self.frontend_domain.name) for key, value in self.options.items(): - properties = self.pack_property(properties, "_" + key, value) + properties += b' ' + self.pack_property("_" + key, value) return properties From 320f246e85274da20d7a52fc2c364bddb113fa00 Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Tue, 30 Apr 2024 08:03:42 +0200 Subject: [PATCH 37/49] q-dev: more restrictive key allowance there is no good reason to allow a space in the option's key --- qubes/device_protocol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubes/device_protocol.py b/qubes/device_protocol.py index afe971e85..51d5ef1b1 100644 --- a/qubes/device_protocol.py +++ b/qubes/device_protocol.py @@ -64,7 +64,7 @@ class Device: """ ALLOWED_CHARS_KEY = set( string.digits + string.ascii_letters - + r"!#$%&()*+,-./:;<>?@[\]^_{|}~" + ' ') + + r"!#$%&()*+,-./:;<>?@[\]^_{|}~") ALLOWED_CHARS_PARAM = ALLOWED_CHARS_KEY.union(set(string.punctuation + ' ')) def __init__(self, backend_domain, ident, devclass=None): From c347bfe8a256c8b2a9f882a5abe1d96d6263a6a0 Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Wed, 8 May 2024 11:42:03 +0200 Subject: [PATCH 38/49] q-dev: do not load block devices in offline mode --- qubes/ext/block.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qubes/ext/block.py b/qubes/ext/block.py index cca7307e7..0fbeb687b 100644 --- a/qubes/ext/block.py +++ b/qubes/ext/block.py @@ -300,6 +300,9 @@ def on_domain_init_load(self, vm, event): """Initialize watching for changes""" # pylint: disable=unused-argument vm.watch_qdb_path('/qubes-block-devices') + if vm.app.vmm.offline_mode: + self.devices_cache[vm.name] = {} + return if event == 'domain-load': # avoid building a cache on domain-init, as it isn't fully set yet, # and definitely isn't running yet From 2969817177e3e2fa293a0214926f841740cdf154 Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Wed, 8 May 2024 15:00:26 +0200 Subject: [PATCH 39/49] q-dev: update tests and minor fixes --- qubes/device_protocol.py | 11 ++++++----- qubes/tests/devices.py | 2 +- qubes/tests/vm/qubesvm.py | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/qubes/device_protocol.py b/qubes/device_protocol.py index 51d5ef1b1..e04340a77 100644 --- a/qubes/device_protocol.py +++ b/qubes/device_protocol.py @@ -560,10 +560,10 @@ def interfaces(self) -> List[DeviceInterface]: @property def parent_device(self) -> Optional[Device]: """ - The parent device if any. + The parent device, if any. - If the device is part of another device (e.g. it's a single - partition of an usb stick), the parent device id should be here. + If the device is part of another device (e.g., it's a single + partition of a USB stick), the parent device id should be here. """ return self._parent @@ -572,7 +572,7 @@ def subdevices(self) -> List['DeviceInfo']: """ The list of children devices if any. - If the device has subdevices (e.g. partitions of an usb stick), + If the device has subdevices (e.g., partitions of a USB stick), the subdevices id should be here. """ return [dev for dev in self.backend_domain.devices[self.devclass] @@ -604,7 +604,7 @@ def serialize(self) -> bytes: if self.attachment: properties = self.pack_property( - properties, 'attachment', self.attachment.name) + 'attachment', self.attachment.name) properties += b' ' + self.pack_property( 'interfaces', @@ -742,6 +742,7 @@ def sanitize_str( if replace_char is None: not_allowed_chars = set(untrusted_value) - allowed_chars if not_allowed_chars: + print(untrusted_value) raise ProtocolError(error_message + repr(not_allowed_chars)) return untrusted_value result = "" diff --git a/qubes/tests/devices.py b/qubes/tests/devices.py index f8d954ee6..b0bc4d4a8 100644 --- a/qubes/tests/devices.py +++ b/qubes/tests/devices.py @@ -491,7 +491,7 @@ def test_030_serialize_and_deserialize(self): serial=None, interfaces=[DeviceInterface(" 112233"), DeviceInterface("&012345")], - **{"additional info": "and='more'", "date": "06.12.23"} + **{"additional_info": "and='more'", "date": "06.12.23"} ) serialized = device.serialize() deserialized = DeviceInfo.deserialize(b'1-1.1.1 ' + serialized, self.vm) diff --git a/qubes/tests/vm/qubesvm.py b/qubes/tests/vm/qubesvm.py index e73f29968..3de6d48da 100644 --- a/qubes/tests/vm/qubesvm.py +++ b/qubes/tests/vm/qubesvm.py @@ -1312,7 +1312,7 @@ def test_600_libvirt_xml_hvm_pcidev(self): vm, # this is violation of API, but for PCI the argument # is unused '00_00.0', - bus='pci', + devclass='pci', persistent=True) vm.devices['pci']._set.add( assignment) From 8fba2be1c4f7e6d4fe29805b0db643665a63f89a Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Wed, 8 May 2024 16:03:06 +0200 Subject: [PATCH 40/49] q-dev: allow an unassignment of required device + typos The unassignment of a required device does not break anything, and it's required to detach the required device. Detaching required device can break things but should be possible. --- qubes/api/admin.py | 6 +++--- qubes/devices.py | 8 +------- qubes/tests/api_admin.py | 14 +++++++++----- qubes/tests/devices.py | 8 ++++---- 4 files changed, 17 insertions(+), 19 deletions(-) diff --git a/qubes/api/admin.py b/qubes/api/admin.py index 5bcb60769..b113e5b07 100644 --- a/qubes/api/admin.py +++ b/qubes/api/admin.py @@ -1304,9 +1304,9 @@ async def vm_device_unassign(self, endpoint): # qrexec already verified that no strange characters are in self.arg backend_domain, ident = self.arg.split('+', 1) - # may raise KeyError; if device isn't found, it will be UnknownDevice - # instance - but allow it, otherwise it will be impossible to detach - # already removed device + # may raise KeyError; if a device isn't found, it will be UnknownDevice + # instance - but allow it, otherwise it will be impossible to unassign + # an already removed device dev = self.app.domains[backend_domain].devices[devclass][ident] self.fire_event_for_permission(device=dev, devclass=devclass) diff --git a/qubes/devices.py b/qubes/devices.py index e06bee101..c073ef759 100644 --- a/qubes/devices.py +++ b/qubes/devices.py @@ -304,7 +304,7 @@ async def detach(self, device: Device): "Can not detach a required device from a non halted qube. " "You need to unassign device first.") - # use local object + # use the local object device = assignment.device await self._vm.fire_event_async( 'device-pre-detach:' + self._bus, pre_event=True, device=device) @@ -326,12 +326,6 @@ async def unassign(self, device_assignment: DeviceAssignment): f'device {device_assignment.ident!s} of class {self._bus} not ' f'assigned to {self._vm!s}') - if not self._vm.is_halted() and assignment.required: - raise qubes.exc.QubesVMNotHaltedError( - self._vm, - "Can not remove an required assignment from " - "a non halted qube.") - self._set.discard(assignment) device = device_assignment.device diff --git a/qubes/tests/api_admin.py b/qubes/tests/api_admin.py index 0c88e0f6d..6b752b392 100644 --- a/qubes/tests/api_admin.py +++ b/qubes/tests/api_admin.py @@ -1994,7 +1994,8 @@ def test_490_vm_device_unassign_from_running(self): value = self.call_mgmt_func(b'admin.vm.device.testclass.Unassign', b'test-vm1', b'test-vm1+1234') self.assertIsNone(value) - mock_action.assert_called_once_with(self.vm, 'device-unassign:testclass', + mock_action.assert_called_once_with( + self.vm, 'device-unassign:testclass', device=self.vm.devices['testclass']['1234']) self.app.save.assert_called_once_with() @@ -2010,12 +2011,15 @@ def test_491_vm_device_unassign_required_from_running(self): self.vm.add_handler('device-unassign:testclass', mock_action) with unittest.mock.patch.object( qubes.vm.qubesvm.QubesVM, 'is_halted', lambda _: False): - with self.assertRaises(qubes.exc.QubesVMNotHaltedError): - self.call_mgmt_func( + value = self.call_mgmt_func( b'admin.vm.device.testclass.Unassign', b'test-vm1', b'test-vm1+1234') - self.assertFalse(mock_action.called) - self.assertFalse(self.app.save.called) + self.assertIsNone(value) + mock_action.assert_called_once_with( + self.vm, 'device-unassign:testclass', + device=self.vm.devices['testclass']['1234']) + self.app.save.assert_called_once_with() + def test_492_vm_device_unassign_from_halted(self): assignment = qubes.device_protocol.DeviceAssignment( self.vm, '1234', attach_automatically=True, required=False, diff --git a/qubes/tests/devices.py b/qubes/tests/devices.py index b0bc4d4a8..22db41c35 100644 --- a/qubes/tests/devices.py +++ b/qubes/tests/devices.py @@ -312,11 +312,11 @@ def test_041_detach_required_from_halted(self): self.collection.detach(self.assignment)) def test_042_unassign_required(self): - self.loop.run_until_complete(self.collection.assign(self.assignment)) self.emitter.running = True - with self.assertRaises(qubes.exc.QubesVMNotHaltedError): - self.loop.run_until_complete( - self.collection.unassign(self.assignment)) + self.loop.run_until_complete(self.collection.assign(self.assignment)) + self.loop.run_until_complete(self.collection.unassign(self.assignment)) + self.assertEventFired(self.emitter, 'device-assign:testclass') + self.assertEventFired(self.emitter, 'device-unassign:testclass') def test_043_detach_assigned(self): self.assignment.required = False From cd2cc2f88c219ecaf8a80d02a9570475842aaf67 Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Wed, 8 May 2024 22:32:37 +0200 Subject: [PATCH 41/49] q-dev: update test --- qubes/tests/vm/qubesvm.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/qubes/tests/vm/qubesvm.py b/qubes/tests/vm/qubesvm.py index 3de6d48da..6821942f7 100644 --- a/qubes/tests/vm/qubesvm.py +++ b/qubes/tests/vm/qubesvm.py @@ -1313,7 +1313,9 @@ def test_600_libvirt_xml_hvm_pcidev(self): # is unused '00_00.0', devclass='pci', - persistent=True) + attach_automatically=True, + required=True, + ) vm.devices['pci']._set.add( assignment) libvirt_xml = vm.create_config_file() From ba66cba75a20e76addb56a7fd78add9159fc39d4 Mon Sep 17 00:00:00 2001 From: Piotr Bartman Date: Thu, 9 May 2024 09:08:27 +0200 Subject: [PATCH 42/49] q-dev: remove leftover debug print --- qubes/device_protocol.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qubes/device_protocol.py b/qubes/device_protocol.py index e04340a77..1589848d7 100644 --- a/qubes/device_protocol.py +++ b/qubes/device_protocol.py @@ -742,7 +742,6 @@ def sanitize_str( if replace_char is None: not_allowed_chars = set(untrusted_value) - allowed_chars if not_allowed_chars: - print(untrusted_value) raise ProtocolError(error_message + repr(not_allowed_chars)) return untrusted_value result = "" From 93f60d54f165b27e63e45c07afd1e3f88fff412d Mon Sep 17 00:00:00 2001 From: Piotr Bartman-Szwarc Date: Mon, 13 May 2024 14:31:26 +0200 Subject: [PATCH 43/49] q-dev: add block and pci tests to improve testcov + typos update pci description format --- qubes/ext/block.py | 6 +- qubes/ext/pci.py | 19 +---- qubes/tests/devices_block.py | 74 ++++++++++++++++--- qubes/tests/devices_pci.py | 136 +++++++++++++++++++++++++---------- 4 files changed, 168 insertions(+), 67 deletions(-) diff --git a/qubes/ext/block.py b/qubes/ext/block.py index 0fbeb687b..7f7b56df5 100644 --- a/qubes/ext/block.py +++ b/qubes/ext/block.py @@ -167,9 +167,9 @@ def interfaces(self) -> List[qubes.device_protocol.DeviceInterface]: @property def parent_device(self) -> Optional[qubes.device_protocol.Device]: """ - The parent device if any. + The parent device, if any. - If the device is part of another device (e.g. it's a single + If the device is part of another device (e.g., it's a single partition of an usb stick), the parent device id should be here. """ if self._parent is None: @@ -602,7 +602,7 @@ def on_device_pre_detached_block(self, vm, event, device): if not vm.is_running(): return - # need to enumerate attached device to find frontend_dev option (at + # need to enumerate attached devices to find frontend_dev option (at # least) for attached_device, options in self.on_device_list_attached(vm, event): if attached_device == device: diff --git a/qubes/ext/pci.py b/qubes/ext/pci.py index 9cba325f5..9915ffd1a 100644 --- a/qubes/ext/pci.py +++ b/qubes/ext/pci.py @@ -145,7 +145,7 @@ def attached_devices(app): def _device_desc(hostdev_xml): - return '{devclass}: {vendor} {product}'.format( + return '{devclass}: {product} ({vendor})'.format( devclass=pcidev_class(hostdev_xml), vendor=hostdev_xml.findtext('capability/vendor'), product=hostdev_xml.findtext('capability/product'), @@ -235,7 +235,7 @@ def interfaces(self) -> List[qubes.device_protocol.DeviceInterface]: @property def parent_device(self) -> Optional[qubes.device_protocol.DeviceInfo]: """ - The parent device if any. + The parent device, if any. PCI device has no parents. """ @@ -261,7 +261,7 @@ def description(self): @property def self_identity(self) -> str: """ - Get identification of device not related to port. + Get identification of the device not related to port. """ allowed_chars = string.digits + string.ascii_letters + '-_.' if self._vendor_id is None: @@ -306,19 +306,6 @@ def _load_desc(self) -> Dict[str, str]: "//product/@id")[0] return result - @staticmethod - def _sanitize( - untrusted_device_desc: bytes, - safe_chars: str = - string.ascii_letters + string.digits + string.punctuation + ' ' - ) -> str: - # b'USB\\x202.0\\x20Camera' -> 'USB 2.0 Camera' - untrusted_device_desc = untrusted_device_desc.decode( - 'unicode_escape', errors='ignore') - return ''.join( - c if c in set(safe_chars) else '_' for c in untrusted_device_desc - ) - @property def frontend_domain(self): # TODO: cache this diff --git a/qubes/tests/devices_block.py b/qubes/tests/devices_block.py index b0fd06c57..8929d3286 100644 --- a/qubes/tests/devices_block.py +++ b/qubes/tests/devices_block.py @@ -24,6 +24,7 @@ import qubes.tests import qubes.ext.block +from qubes.device_protocol import DeviceInterface, Device, DeviceInfo modules_disk = ''' @@ -102,6 +103,13 @@ def list(self, prefix): class TestApp(object): + class Domains(dict): + def __init__(self): + super().__init__() + + def __iter__(self): + return iter(self.values()) + def __init__(self): #: jinja2 environment for libvirt XML templates self.env = jinja2.Environment( @@ -112,7 +120,19 @@ def __init__(self): ]), undefined=jinja2.StrictUndefined, autoescape=True) - self.domains = {} + self.domains = TestApp.Domains() + + +class TestDeviceCollection(object): + def __init__(self, backend_vm, devclass): + self._exposed = [] + self.backend_vm = backend_vm + self.devclass = devclass + + def __getitem__(self, ident): + for dev in self._exposed: + if dev.ident == ident: + return dev class TestVM(qubes.tests.TestEmitter): @@ -136,13 +156,16 @@ def __init__( 'XMLDesc.return_value': domain_xml }) self.devices = { - 'testclass': qubes.devices.DeviceCollection(self, 'testclass') + 'testclass': TestDeviceCollection(self, 'testclass') } def __eq__(self, other): if isinstance(other, TestVM): return self.name == other.name + def __str__(self): + return self.name + class TC_00_Block(qubes.tests.QubesTestCase): @@ -156,7 +179,33 @@ def test_000_device_get(self): '/qubes-block-devices/sda/desc': b'Test_ (device)', '/qubes-block-devices/sda/size': b'1024000', '/qubes-block-devices/sda/mode': b'w', - }) + '/qubes-block-devices/sda/parent': b'1-1.1:1.0', + }, domain_xml=domain_xml_template.format("")) + parent = DeviceInfo(vm, '1-1.1', devclass='usb') + vm.devices['usb'] = TestDeviceCollection(backend_vm=vm, devclass='usb') + vm.devices['usb']._exposed.append(parent) + vm.is_running = lambda: True + + dom0 = TestVM({}, name='dom0', + domain_xml=domain_xml_template.format("")) + + disk = ''' + + + + + + + + ''' + front = TestVM({}, domain_xml=domain_xml_template.format(disk), name='front-vm') + + vm.app.domains[0] = dom0 + vm.app.domains['test-vm'] = vm + vm.app.domains['front-vm'] = front + front.app.domains = vm.app.domains + dom0.app.domains = vm.app.domains + device_info = self.ext.device_get(vm, 'sda') self.assertIsInstance(device_info, qubes.ext.block.BlockDevice) self.assertEqual(device_info.backend_domain, vm) @@ -167,6 +216,16 @@ def test_000_device_get(self): self.assertEqual(device_info._serial, 'Test') self.assertEqual(device_info.size, 1024000) self.assertEqual(device_info.mode, 'w') + self.assertEqual(device_info.manufacturer, + 'sub-device of test-vm:1-1.1') + self.assertEqual(device_info.device_node, '/dev/sda') + self.assertEqual(device_info.interfaces, + [DeviceInterface("b******")]) + self.assertEqual(device_info.parent_device, + Device(vm, '1-1.1', devclass='usb')) + self.assertEqual(device_info.attachment, front) + self.assertEqual(device_info.self_identity, + '1-1.1:0000:0000::?******:1.0') self.assertEqual( device_info.data.get('test_frontend_domain', None), None) self.assertEqual(device_info.device_node, '/dev/sda') @@ -325,6 +384,7 @@ def test_031_list_attached(self): options = devices[0][1] self.assertEqual(dev.backend_domain, vm.app.domains['sys-usb']) self.assertEqual(dev.ident, 'sda') + self.assertEqual(dev.attachment, None) self.assertEqual(options['frontend-dev'], 'xvdi') self.assertEqual(options['read-only'], 'yes') @@ -584,14 +644,6 @@ def test_051_detach_not_attached(self): '/qubes-block-devices/sda/size': b'1024000', '/qubes-block-devices/sda/mode': b'r', }) - device_xml = ( - '\n' - ' \n' - ' \n' - ' \n' - ' \n\n' - ' \n' - '') vm = TestVM({}, domain_xml=domain_xml_template.format('')) vm.app.domains['test-vm'] = vm vm.app.domains['sys-usb'] = TestVM({}, name='sys-usb') diff --git a/qubes/tests/devices_pci.py b/qubes/tests/devices_pci.py index b653337e2..072877221 100644 --- a/qubes/tests/devices_pci.py +++ b/qubes/tests/devices_pci.py @@ -17,13 +17,11 @@ # # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, see . - from unittest import mock -import jinja2 - import qubes.tests import qubes.ext.pci +from qubes.device_protocol import DeviceInterface class TestVM(object): @@ -38,38 +36,10 @@ def __eq__(self, other): if isinstance(other, TestVM): return self.name == other.name -class TC_00_Block(qubes.tests.QubesTestCase): - def setUp(self): - super().setUp() - self.ext = qubes.ext.pci.PCIDeviceExtension() - def test_000_unsupported_device(self): - vm = TestVM() - vm.app.configure_mock(**{ - 'vmm.libvirt_conn.listAllDevices.return_value': - [mock.Mock(**{"XMLDesc.return_value": """ - pci_0000_00_14_0 - /sys/devices/pci0000:00/0000:00:14.0 - computer - - pciback - - - 0x0c0330 - 0 - 0 - 20 - 0 - 9 Series Chipset Family USB xHCI Controller - Intel Corporation - - -""", - "listCaps.return_value": ["pci"] - }), - mock.Mock(**{"XMLDesc.return_value": """ - pci_1000_00_14_0 - /sys/devices/pci1000:00/1000:00:14.0 +PCI_XML = """ + pci_{}_00_14_0 + /sys/devices/pci{}:00/{}:00:14.0 computer pciback @@ -84,11 +54,103 @@ def test_000_unsupported_device(self): Intel Corporation -""", - "listCaps.return_value": ["pci"] +""" + + +def mock_file_open(filename: str, *_args, **_kwargs): + if filename == "/usr/share/hwdata/pci.ids": + # short version of pci.ids + content = """ +# +# List of PCI ID's +# +# (...) +# +0001 SafeNet (wrong ID) +0010 Allied Telesis, Inc (Wrong ID) +# This is a relabelled RTL-8139 + 8139 AT-2500TX V3 Ethernet +# List of known device classes, subclasses and programming interfaces + +# Syntax: +# C class class_name +# subclass subclass_name <-- single tab +# prog-if prog-if_name <-- two tabs + +C 00 Unclassified device +\t00 Non-VGA unclassified device +C 01 Mass storage controller +\t01 IDE interface +\t\t00 ISA Compatibility mode-only controller +C 0c Serial bus controller +\t00 FireWire (IEEE 1394) +\t\t00 Generic +\t\t10 OHCI +\t01 ACCESS Bus +\t02 SSA +\t03 USB controller +\t\t00 UHCI +\t\t10 OHCI +\t\t20 EHCI +\t\t30 XHCI +\t\t40 USB4 Host Interface +\t\t80 Unspecified +\t\tfe USB Device +\t04 Fibre Channel +\t05 SMBus +\t06 InfiniBand +\t07 IPMI Interface +\t\t00 SMIC +\t\t01 KCS +\t\t02 BT (Block Transfer) +\t08 SERCOS interface +\t09 CANBUS +\t80 Serial bus controller +""" + elif filename.startswith("/sys/devices/pci"): + content = "0x0c0330" + else: + raise OSError() + + file_object = mock.mock_open(read_data=content).return_value + file_object.__iter__.return_value = content + return file_object + + +class TC_00_Block(qubes.tests.QubesTestCase): + def setUp(self): + super().setUp() + self.ext = qubes.ext.pci.PCIDeviceExtension() + + @mock.patch('builtins.open', new=mock_file_open) + def test_000_unsupported_device(self): + vm = TestVM() + vm.app.configure_mock(**{ + 'vmm.libvirt_conn.nodeDeviceLookupByName.return_value': + mock.Mock(**{"XMLDesc.return_value": + PCI_XML.format(*["0000"] * 3) }), - ] + 'vmm.libvirt_conn.listAllDevices.return_value': + [mock.Mock(**{"XMLDesc.return_value": + PCI_XML.format(*["0000"] * 3), + "listCaps.return_value": ["pci"] + }), + mock.Mock(**{"XMLDesc.return_value": + PCI_XML.format(*["1000"] * 3), + "listCaps.return_value": ["pci"] + }), + ] }) devices = list(self.ext.on_device_list_pci(vm, 'device-list:pci')) self.assertEqual(len(devices), 1) self.assertEqual(devices[0].ident, "00_14.0") + self.assertEqual(devices[0].vendor, "Intel Corporation") + self.assertEqual(devices[0].product, + "9 Series Chipset Family USB xHCI Controller") + self.assertEqual(devices[0].interfaces, [DeviceInterface("p0c0330")]) + self.assertEqual(devices[0].parent_device, None) + self.assertEqual(devices[0].libvirt_name, "pci_0000_00_14_0") + self.assertEqual(devices[0].description, + "USB controller: 9 Series Chipset Family " + "USB xHCI Controller (Intel Corporation)") + self.assertEqual(devices[0].self_identity, "0x8086:0x8cb1::p0c0330") From fd84f65a22e1012cdf9ac55bd872d549d376e8fb Mon Sep 17 00:00:00 2001 From: Piotr Bartman-Szwarc Date: Tue, 14 May 2024 00:20:29 +0200 Subject: [PATCH 44/49] q-dev: add more block tests + fixes --- qubes/ext/block.py | 4 +- qubes/ext/utils.py | 5 +- qubes/tests/devices_block.py | 239 ++++++++++++++++++++++++++++++++++- 3 files changed, 243 insertions(+), 5 deletions(-) diff --git a/qubes/ext/block.py b/qubes/ext/block.py index 7f7b56df5..96e0a83e4 100644 --- a/qubes/ext/block.py +++ b/qubes/ext/block.py @@ -527,7 +527,7 @@ def pre_attachment_internal( file=sys.stderr) return - if device.attachment != expected_attachment: + if device.attachment and device.attachment != expected_attachment: raise qubes.devices.DeviceAlreadyAttached( 'Device {!s} already attached to {!s}'.format( device, device.attachment) @@ -557,6 +557,8 @@ def notify_auto_attached(self, vm, device, options): # we expected that these devices are already attached to this vm self.pre_attachment_internal( vm, device, options, expected_attachment=vm) + asyncio.ensure_future(vm.fire_event_async( + 'device-attach:block', device=device, options=options)) @qubes.ext.handler('domain-shutdown') async def on_domain_shutdown(self, vm, event, **_kwargs): diff --git a/qubes/ext/utils.py b/qubes/ext/utils.py index 60216d243..fed52b4fb 100644 --- a/qubes/ext/utils.py +++ b/qubes/ext/utils.py @@ -48,6 +48,7 @@ def device_list_change( vm.fire_event(f'device-added:{devclass}', device=device) for dev_ident, front_vm in attached.items(): dev = device_class(vm, dev_ident) + # options are unknown, device already attached asyncio.ensure_future(front_vm.fire_event_async( f'device-attach:{devclass}', device=dev, options={})) @@ -61,8 +62,8 @@ def device_list_change( and assignment.ident in added and assignment.ident not in attached ): - asyncio.ensure_future(ext.attach_and_notify( - front_vm, assignment.device, assignment.options)) + ext.notify_auto_attached( + front_vm, assignment.device, assignment.options) def compare_device_cache(vm, devices_cache, current_devices): diff --git a/qubes/tests/devices_block.py b/qubes/tests/devices_block.py index 8929d3286..f9b19044a 100644 --- a/qubes/tests/devices_block.py +++ b/qubes/tests/devices_block.py @@ -17,14 +17,15 @@ # # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, see . - +import asyncio from unittest import mock import jinja2 import qubes.tests import qubes.ext.block -from qubes.device_protocol import DeviceInterface, Device, DeviceInfo +from qubes.device_protocol import DeviceInterface, Device, DeviceInfo, \ + DeviceAssignment modules_disk = ''' @@ -121,14 +122,19 @@ def __init__(self): undefined=jinja2.StrictUndefined, autoescape=True) self.domains = TestApp.Domains() + self.vmm = mock.Mock() class TestDeviceCollection(object): def __init__(self, backend_vm, devclass): self._exposed = [] + self._assigned = [] self.backend_vm = backend_vm self.devclass = devclass + def get_assigned_devices(self): + return self._assigned + def __getitem__(self, ident): for dev in self._exposed: if dev.ident == ident: @@ -650,3 +656,232 @@ def test_051_detach_not_attached(self): dev = qubes.ext.block.BlockDevice(back_vm, 'sda') self.ext.on_device_pre_detached_block(vm, '', dev) self.assertFalse(vm.libvirt_domain.detachDevice.called) + + def test_060_on_qdb_change_added(self): + back_vm = TestVM(name='sys-usb', qdb={ + '/qubes-block-devices/sda': b'', + '/qubes-block-devices/sda/desc': b'Test device', + '/qubes-block-devices/sda/size': b'1024000', + '/qubes-block-devices/sda/mode': b'r', + }, domain_xml=domain_xml_template.format("")) + exp_dev = Device(back_vm, 'sda', 'block') + + self.ext.on_qdb_change(back_vm, None, None) + + self.assertEqual(self.ext.devices_cache, {'sys-usb': {'sda': None}}) + self.assertEqual( + back_vm.fired_events[ + ('device-added:block', frozenset({('device', exp_dev)}))],1) + + def test_061_on_qdb_change_auto_attached(self): + back_vm = TestVM(name='sys-usb', qdb={ + '/qubes-block-devices/sda': b'', + '/qubes-block-devices/sda/desc': b'Test device', + '/qubes-block-devices/sda/size': b'1024000', + '/qubes-block-devices/sda/mode': b'r', + }, domain_xml=domain_xml_template.format("")) + exp_dev = Device(back_vm, 'sda', 'block') + front = TestVM({}, domain_xml=domain_xml_template.format(""), + name='front-vm') + dom0 = TestVM({}, name='dom0', + domain_xml=domain_xml_template.format("")) + back_vm.app.domains['sys-usb'] = back_vm + back_vm.app.domains['front-vm'] = front + back_vm.app.domains[0] = dom0 + front.app = back_vm.app + dom0.app = back_vm.app + + back_vm.app.vmm.configure_mock(**{'offline_mode': False}) + fire_event_async = mock.Mock() + front.fire_event_async = fire_event_async + + back_vm.devices['block'] = TestDeviceCollection( + backend_vm=back_vm, devclass='block') + front.devices['block'] = TestDeviceCollection( + backend_vm=front, devclass='block') + dom0.devices['block'] = TestDeviceCollection( + backend_vm=dom0, devclass='block') + + front.devices['block']._assigned.append( + DeviceAssignment.from_device(exp_dev)) + back_vm.devices['block']._exposed.append( + qubes.ext.block.BlockDevice(back_vm, 'sda')) + + with mock.patch('asyncio.ensure_future'): + self.ext.on_qdb_change(back_vm, None, None) + self.assertEqual(self.ext.devices_cache, {'sys-usb': {'sda': front}}) + fire_event_async.assert_called_once_with( + 'device-attach:block', device=exp_dev, + options={'read-only': 'yes', 'frontend-dev': 'xvdi'}) + + def test_062_on_qdb_change_attached(self): + # added + back_vm = TestVM(name='sys-usb', qdb={ + '/qubes-block-devices/sda': b'', + '/qubes-block-devices/sda/desc': b'Test device', + '/qubes-block-devices/sda/size': b'1024000', + '/qubes-block-devices/sda/mode': b'r', + }, domain_xml=domain_xml_template.format("")) + exp_dev = Device(back_vm, 'sda', 'block') + + self.ext.devices_cache = {'sys-usb': {'sda': None}} + + # then attached + disk = ''' + + + + + + + + ''' + front = TestVM({}, domain_xml=domain_xml_template.format(disk), + name='front-vm') + dom0 = TestVM({}, name='dom0', + domain_xml=domain_xml_template.format("")) + back_vm.app.domains['sys-usb'] = back_vm + back_vm.app.domains['front-vm'] = front + back_vm.app.domains[0] = dom0 + front.app = back_vm.app + dom0.app = back_vm.app + + back_vm.app.vmm.configure_mock(**{'offline_mode': False}) + fire_event_async = mock.Mock() + front.fire_event_async = fire_event_async + + back_vm.devices['block'] = TestDeviceCollection( + backend_vm=back_vm, devclass='block') + front.devices['block'] = TestDeviceCollection( + backend_vm=front, devclass='block') + dom0.devices['block'] = TestDeviceCollection( + backend_vm=dom0, devclass='block') + + with mock.patch('asyncio.ensure_future'): + self.ext.on_qdb_change(back_vm, None, None) + self.assertEqual(self.ext.devices_cache, {'sys-usb': {'sda': front}}) + fire_event_async.assert_called_once_with( + 'device-attach:block', device=exp_dev, options={}) + + def test_063_on_qdb_change_changed(self): + # attached to front-vm + back_vm = TestVM(name='sys-usb', qdb={ + '/qubes-block-devices/sda': b'', + '/qubes-block-devices/sda/desc': b'Test device', + '/qubes-block-devices/sda/size': b'1024000', + '/qubes-block-devices/sda/mode': b'r', + }, domain_xml=domain_xml_template.format("")) + exp_dev = Device(back_vm, 'sda', 'block') + + front = TestVM({}, name='front-vm') + dom0 = TestVM({}, name='dom0', + domain_xml=domain_xml_template.format("")) + + self.ext.devices_cache = {'sys-usb': {'sda': front}} + + disk = ''' + + + + + + + + ''' + front_2 = TestVM({}, domain_xml=domain_xml_template.format(disk), + name='front-2') + + back_vm.app.vmm.configure_mock(**{'offline_mode': False}) + front.libvirt_domain.configure_mock(**{ + 'XMLDesc.return_value': domain_xml_template.format("") + }) + + back_vm.app.domains['sys-usb'] = back_vm + back_vm.app.domains['front-vm'] = front + back_vm.app.domains['front-2'] = front_2 + back_vm.app.domains[0] = dom0 + + front.app = back_vm.app + front_2.app = back_vm.app + dom0.app = back_vm.app + + fire_event_async = mock.Mock() + front.fire_event_async = fire_event_async + fire_event_async_2 = mock.Mock() + front_2.fire_event_async = fire_event_async_2 + + back_vm.devices['block'] = TestDeviceCollection( + backend_vm=back_vm, devclass='block') + front.devices['block'] = TestDeviceCollection( + backend_vm=front, devclass='block') + dom0.devices['block'] = TestDeviceCollection( + backend_vm=dom0, devclass='block') + front_2.devices['block'] = TestDeviceCollection( + backend_vm=front_2, devclass='block') + + with mock.patch('asyncio.ensure_future'): + self.ext.on_qdb_change(back_vm, None, None) + + self.assertEqual(self.ext.devices_cache, {'sys-usb': {'sda': front_2}}) + fire_event_async.assert_called_with( + 'device-detach:block', device=exp_dev) + fire_event_async_2.assert_called_once_with( + 'device-attach:block', device=exp_dev, options={}) + + def test_064_on_qdb_change_removed_attached(self): + # attached to front-vm + back_vm = TestVM(name='sys-usb', qdb={ + '/qubes-block-devices/sda': b'', + '/qubes-block-devices/sda/desc': b'Test device', + '/qubes-block-devices/sda/size': b'1024000', + '/qubes-block-devices/sda/mode': b'r', + }, domain_xml=domain_xml_template.format("")) + dom0 = TestVM({}, name='dom0', + domain_xml=domain_xml_template.format("")) + exp_dev = Device(back_vm, 'sda', 'block') + + disk = ''' + + + + + + + + ''' + front = TestVM({}, domain_xml=domain_xml_template.format(disk), + name='front') + self.ext.devices_cache = {'sys-usb': {'sda': front}} + + back_vm.app.vmm.configure_mock(**{'offline_mode': False}) + front.libvirt_domain.configure_mock(**{ + 'XMLDesc.return_value': domain_xml_template.format("") + }) + + back_vm.app.domains['sys-usb'] = back_vm + back_vm.app.domains['front-vm'] = front + back_vm.app.domains[0] = dom0 + + front.app = back_vm.app + dom0.app = back_vm.app + + fire_event_async = mock.Mock() + front.fire_event_async = fire_event_async + + back_vm.devices['block'] = TestDeviceCollection( + backend_vm=back_vm, devclass='block') + front.devices['block'] = TestDeviceCollection( + backend_vm=front, devclass='block') + dom0.devices['block'] = TestDeviceCollection( + backend_vm=dom0, devclass='block') + + back_vm.untrusted_qdb = TestQubesDB({}) + with mock.patch('asyncio.ensure_future'): + self.ext.on_qdb_change(back_vm, None, None) + self.assertEqual(self.ext.devices_cache, {'sys-usb': {}}) + fire_event_async.assert_called_with( + 'device-detach:block', device=exp_dev) + self.assertEqual( + back_vm.fired_events[ + ('device-removed:block', frozenset({('device', exp_dev)}))], + 1) From f483e60d6117eb835ee9a76f07d5a6969e265630 Mon Sep 17 00:00:00 2001 From: Piotr Bartman-Szwarc Date: Tue, 14 May 2024 20:33:34 +0200 Subject: [PATCH 45/49] q-dev: make pylint happy --- qubes/api/admin.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/qubes/api/admin.py b/qubes/api/admin.py index b113e5b07..39b9311e4 100644 --- a/qubes/api/admin.py +++ b/qubes/api/admin.py @@ -1297,8 +1297,9 @@ async def vm_device_assign(self, endpoint, untrusted_payload): @qubes.api.method( 'admin.vm.device.{endpoint}.Unassign', endpoints=( - ep.name - for ep in importlib.metadata.entry_points(group='qubes.devices')), no_payload=True, scope='local', write=True) + ep.name + for ep in importlib.metadata.entry_points(group='qubes.devices')), + no_payload=True, scope='local', write=True) async def vm_device_unassign(self, endpoint): devclass = endpoint @@ -1321,7 +1322,7 @@ async def vm_device_unassign(self, endpoint): @qubes.api.method( 'admin.vm.device.{endpoint}.Attach', endpoints=( - ep.name + ep.name for ep in importlib.metadata.entry_points(group='qubes.devices')), scope='local', execute=True) async def vm_device_attach(self, endpoint, untrusted_payload): From 33132917f404237a0de154e6bf0cd28bba89e1ce Mon Sep 17 00:00:00 2001 From: Piotr Bartman-Szwarc Date: Tue, 14 May 2024 22:11:09 +0200 Subject: [PATCH 46/49] q-dev: restore attach_and_notify usb device proxy needs this feature --- qubes/ext/block.py | 7 +++++-- qubes/ext/utils.py | 4 ++-- qubes/tests/devices_block.py | 3 +++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/qubes/ext/block.py b/qubes/ext/block.py index 96e0a83e4..3d3805a14 100644 --- a/qubes/ext/block.py +++ b/qubes/ext/block.py @@ -553,13 +553,16 @@ async def on_domain_start(self, vm, _event, **_kwargs): vm, assignment.device, assignment.options) def notify_auto_attached(self, vm, device, options): - # bypass DeviceCollection logic preventing double attach - # we expected that these devices are already attached to this vm self.pre_attachment_internal( vm, device, options, expected_attachment=vm) asyncio.ensure_future(vm.fire_event_async( 'device-attach:block', device=device, options=options)) + async def attach_and_notify(self, vm, device, options): + # bypass DeviceCollection logic preventing double attach + # we expected that these devices are already attached to this vm + self.notify_auto_attached(vm, device, options) + @qubes.ext.handler('domain-shutdown') async def on_domain_shutdown(self, vm, event, **_kwargs): """ diff --git a/qubes/ext/utils.py b/qubes/ext/utils.py index fed52b4fb..1c7831922 100644 --- a/qubes/ext/utils.py +++ b/qubes/ext/utils.py @@ -62,8 +62,8 @@ def device_list_change( and assignment.ident in added and assignment.ident not in attached ): - ext.notify_auto_attached( - front_vm, assignment.device, assignment.options) + asyncio.ensure_future(ext.attach_and_notify( + front_vm, assignment.device, assignment.options)) def compare_device_cache(vm, devices_cache, current_devices): diff --git a/qubes/tests/devices_block.py b/qubes/tests/devices_block.py index f9b19044a..0d19eebb4 100644 --- a/qubes/tests/devices_block.py +++ b/qubes/tests/devices_block.py @@ -707,6 +707,9 @@ def test_061_on_qdb_change_auto_attached(self): back_vm.devices['block']._exposed.append( qubes.ext.block.BlockDevice(back_vm, 'sda')) + # In the case of block devices it is the same, + # but notify_auto_attached is synchronous + self.ext.attach_and_notify = self.ext.notify_auto_attached with mock.patch('asyncio.ensure_future'): self.ext.on_qdb_change(back_vm, None, None) self.assertEqual(self.ext.devices_cache, {'sys-usb': {'sda': front}}) From 9e5255618d5b845bf47af7fcf69947d18e0ff59d Mon Sep 17 00:00:00 2001 From: Piotr Bartman-Szwarc Date: Thu, 16 May 2024 15:49:07 +0200 Subject: [PATCH 47/49] q-dev: get parent device directly from vm --- qubes/ext/block.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/qubes/ext/block.py b/qubes/ext/block.py index 3d3805a14..3a3eb1e58 100644 --- a/qubes/ext/block.py +++ b/qubes/ext/block.py @@ -186,8 +186,12 @@ def parent_device(self) -> Optional[qubes.device_protocol.Device]: devclass = 'usb' if sep == ':' else 'block' if not parent_ident: return None - self._parent = qubes.device_protocol.Device( - self.backend_domain, parent_ident, devclass=devclass) + try: + self._parent = ( + self.backend_domain.devices)[devclass][parent_ident] + except KeyError: + self._parent = qubes.device_protocol.UnknownDevice( + self.backend_domain, parent_ident, devclass=devclass) self._interface_num = interface_num return self._parent From 5d0ea724b145050472543ec1bcd817cb64d3f29a Mon Sep 17 00:00:00 2001 From: Piotr Bartman-Szwarc Date: Thu, 16 May 2024 23:57:19 +0200 Subject: [PATCH 48/49] q-dev: make pylint happy --- qubes/device_protocol.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/qubes/device_protocol.py b/qubes/device_protocol.py index 1589848d7..f9833854b 100644 --- a/qubes/device_protocol.py +++ b/qubes/device_protocol.py @@ -397,11 +397,15 @@ def _load_classes(bus: str): result = {} with open(f'/usr/share/hwdata/{bus}.ids', encoding='utf-8', errors='ignore') as pciids: + # for `class_name` and `subclass_name` + # pylint: disable=used-before-assignment + # pylint: disable=possibly-used-before-assignment class_id = None subclass_id = None for line in pciids.readlines(): line = line.rstrip() - if line.startswith('\t\t') and class_id and subclass_id: + if line.startswith('\t\t') \ + and class_id is not None and subclass_id is not None: (progif_id, _, progif_name) = line[2:].split(' ', 2) result[class_id + subclass_id + progif_id] = \ f"{class_name}: {subclass_name} ({progif_name})" From 58b7872185b471f4bb8c3dad56dabed92b00f8a0 Mon Sep 17 00:00:00 2001 From: Piotr Bartman-Szwarc Date: Sat, 18 May 2024 21:11:26 +0200 Subject: [PATCH 49/49] q-dev: keep description as in `lspci/lsusb` --- qubes/device_protocol.py | 23 ++++++++++++++--------- qubes/ext/pci.py | 2 +- qubes/tests/devices_pci.py | 4 ++-- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/qubes/device_protocol.py b/qubes/device_protocol.py index f9833854b..3d8b9aaa6 100644 --- a/qubes/device_protocol.py +++ b/qubes/device_protocol.py @@ -370,12 +370,17 @@ def __str__(self): if self.devclass == "block": return "Block device" if self.devclass in ("usb", "pci"): + # try subclass first as in `lspci` result = self._load_classes(self.devclass).get( - self._interface_encoding[1:], None) - if result is None: + self._interface_encoding[1:-2] + '**', None) + if (result is None or result.lower() + in ('none', 'no subclass', 'unused', 'undefined')): + # if not, try interface result = self._load_classes(self.devclass).get( - self._interface_encoding[1:-2] + '**', None) - if result is None: + self._interface_encoding[1:], None) + if (result is None or result.lower() + in ('none', 'no subclass', 'unused', 'undefined')): + # if not, try class result = self._load_classes(self.devclass).get( self._interface_encoding[1:-4] + '****', None) if result is None: @@ -399,21 +404,20 @@ def _load_classes(bus: str): encoding='utf-8', errors='ignore') as pciids: # for `class_name` and `subclass_name` # pylint: disable=used-before-assignment - # pylint: disable=possibly-used-before-assignment class_id = None subclass_id = None for line in pciids.readlines(): line = line.rstrip() if line.startswith('\t\t') \ - and class_id is not None and subclass_id is not None: + and class_id is not None and subclass_id is not None: (progif_id, _, progif_name) = line[2:].split(' ', 2) result[class_id + subclass_id + progif_id] = \ - f"{class_name}: {subclass_name} ({progif_name})" + progif_name elif line.startswith('\t') and class_id: (subclass_id, _, subclass_name) = line[1:].split(' ', 2) # store both prog-if specific entry and generic one result[class_id + subclass_id + '**'] = \ - f"{class_name}: {subclass_name}" + subclass_name elif line.startswith('C '): (_, class_id, _, class_name) = line.split(' ', 3) result[class_id + '****'] = class_name @@ -548,7 +552,8 @@ def description(self) -> str: else: vendor = "unknown vendor" - return f"{prod} ({vendor})" + main_interface = str(self.interfaces[0]) + return f"{main_interface}: {vendor} {prod}" @property def interfaces(self) -> List[DeviceInterface]: diff --git a/qubes/ext/pci.py b/qubes/ext/pci.py index 9915ffd1a..1535a7f9d 100644 --- a/qubes/ext/pci.py +++ b/qubes/ext/pci.py @@ -145,7 +145,7 @@ def attached_devices(app): def _device_desc(hostdev_xml): - return '{devclass}: {product} ({vendor})'.format( + return '{devclass}: {vendor} {product}'.format( devclass=pcidev_class(hostdev_xml), vendor=hostdev_xml.findtext('capability/vendor'), product=hostdev_xml.findtext('capability/product'), diff --git a/qubes/tests/devices_pci.py b/qubes/tests/devices_pci.py index 072877221..01011cadd 100644 --- a/qubes/tests/devices_pci.py +++ b/qubes/tests/devices_pci.py @@ -151,6 +151,6 @@ def test_000_unsupported_device(self): self.assertEqual(devices[0].parent_device, None) self.assertEqual(devices[0].libvirt_name, "pci_0000_00_14_0") self.assertEqual(devices[0].description, - "USB controller: 9 Series Chipset Family " - "USB xHCI Controller (Intel Corporation)") + "USB controller: Intel Corporation 9 Series " + "Chipset Family USB xHCI Controller") self.assertEqual(devices[0].self_identity, "0x8086:0x8cb1::p0c0330")