From 21d4f9165979149212295be2f136126d27f84f47 Mon Sep 17 00:00:00 2001 From: Multyxu Date: Mon, 9 Mar 2026 15:03:28 -0400 Subject: [PATCH 1/7] faster gdsam2 segmentation with bf16 and tf32. Also using a smaller model for sam2 --- .../semantic_inference/models/wrappers.py | 23 ++++++++++++------- .../config/instance_segmentation/gdsam2.yaml | 4 ++-- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/semantic_inference/python/semantic_inference/models/wrappers.py b/semantic_inference/python/semantic_inference/models/wrappers.py index bdd6206..11cf953 100644 --- a/semantic_inference/python/semantic_inference/models/wrappers.py +++ b/semantic_inference/python/semantic_inference/models/wrappers.py @@ -425,6 +425,12 @@ def __init__(self, config): self.config = config self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + # Enable TF32 for faster fp32 matrix ops on Ampere+ GPUs (Blackwell: major=10) + if torch.cuda.is_available() and torch.cuda.get_device_properties(0).major >= 8: + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.allow_tf32 = True + self.text_prompt = config.text_prompt self.multimask_output = config.multimask_output self.erosion = config.erosion @@ -530,14 +536,15 @@ def forward(self, img): torch.backends.cudnn.allow_tf32 = True """ - # SAM 2 predicts mask - self.sam2_predictor.set_image(img) - masks, scores, logits = self.sam2_predictor.predict( - point_coords=None, - point_labels=None, - box=input_boxes, - multimask_output=self.multimask_output, - ) + # SAM2 predicts mask — bfloat16 autocast scoped to SAM2 only (leaves GD in fp32) + with torch.autocast(device_type=self.device.type, dtype=torch.bfloat16): + self.sam2_predictor.set_image(img) + masks, scores, logits = self.sam2_predictor.predict( + point_coords=None, + point_labels=None, + box=input_boxes, + multimask_output=self.multimask_output, + ) # Sample best according to scores if multimask output if self.multimask_output: diff --git a/semantic_inference_ros/config/instance_segmentation/gdsam2.yaml b/semantic_inference_ros/config/instance_segmentation/gdsam2.yaml index f943275..6b08789 100644 --- a/semantic_inference_ros/config/instance_segmentation/gdsam2.yaml +++ b/semantic_inference_ros/config/instance_segmentation/gdsam2.yaml @@ -6,8 +6,8 @@ model: text_prompt: unknown. green chair. van. fire hydrant. ground. grass. sand. sidewalk. dock. road. path. ignore. building. shelter. signal. rock. fence. boat. sign. hill. bridge. wall. floor. ceiling. door. stairs. pole. rail. structure. window. surface. flora. flower. bed. box. storage. barrel. bag. basket. ignore. flag. decor. light. appliance. trash. bicycle. food. clothes. # text_prompt: unknown. adirondack chair. van. fire hydrant. - sam2_checkpoint: sam2.1_hiera_large.pt - sam2_model_config: sam2.1_hiera_l.yaml + sam2_checkpoint: sam2.1_hiera_small.pt + sam2_model_config: sam2.1_hiera_s.yaml grounding_dino_config: GroundingDINO_SwinT_OGC.py grounding_dino_checkpoint: groundingdino_swint_ogc.pth box_threshold: 0.45 From 3926617636162b7a01e30ecd0acb0100e790bd0a Mon Sep 17 00:00:00 2001 From: Multyxu Date: Tue, 10 Mar 2026 01:03:34 -0400 Subject: [PATCH 2/7] publish status --- .../app/instance_segmentation_node | 22 +++++++++++++++++++ semantic_inference_ros/package.xml | 1 + 2 files changed, 23 insertions(+) diff --git a/semantic_inference_ros/app/instance_segmentation_node b/semantic_inference_ros/app/instance_segmentation_node index 5d4fba9..253aebd 100755 --- a/semantic_inference_ros/app/instance_segmentation_node +++ b/semantic_inference_ros/app/instance_segmentation_node @@ -9,6 +9,7 @@ import rclpy import spark_config as sc import torch from rclpy.node import Node +from ros_system_monitor_msgs.msg import NodeInfoMsg from sensor_msgs.msg import Image import semantic_inference.models as models @@ -53,6 +54,10 @@ class InstanceSegmentationNode(Node): self._model.eval() # TODO: causing issue with yolo model self.get_logger().info("Finished initializing!") + self._last_inference_time = None + self._status_pub = self.create_publisher(NodeInfoMsg, "~/status", 1) + self._hb_timer = self.create_timer(0.1, self._hb_callback) + self._pub = self.create_publisher(Image, "semantic/image_raw", 1) self._worker = semantic_inference_ros.ImageWorker( self, self.config.worker, "color/image_raw", self._spin_once @@ -76,6 +81,7 @@ class InstanceSegmentationNode(Node): instance_seg_img = instance_seg_img.astype(np.int32) msg = Conversions.to_image_msg(header, instance_seg_img, encoding="32SC1") self._pub.publish(msg) + self._last_inference_time = self.get_clock().now().nanoseconds * 1e-9 self.get_logger().debug("Published instance segmentation image.") if self.config.visualize_semantic_img: @@ -85,6 +91,22 @@ class InstanceSegmentationNode(Node): Conversions.to_image_msg(header, color_img, encoding="rgb8") ) + def _hb_callback(self): + msg = NodeInfoMsg() + msg.nickname = "instance_seg" + msg.node_name = self.get_fully_qualified_name() + now = self.get_clock().now().nanoseconds * 1e-9 + if self._last_inference_time is None: + msg.status = NodeInfoMsg.STARTUP + msg.notes = "Waiting for first image." + elif now - self._last_inference_time > 10.0: + msg.status = NodeInfoMsg.WARNING + msg.notes = f"No inference for {now - self._last_inference_time:.1f}s." + else: + msg.status = NodeInfoMsg.NOMINAL + msg.notes = "Running." + self._status_pub.publish(msg) + def stop(self): """Stop the underlying image worker.""" self._worker.stop() diff --git a/semantic_inference_ros/package.xml b/semantic_inference_ros/package.xml index 2fba768..eab9cba 100644 --- a/semantic_inference_ros/package.xml +++ b/semantic_inference_ros/package.xml @@ -24,6 +24,7 @@ tf2_eigen rclpy image_transport_plugins + ros_system_monitor_msgs pcl_conversions ament_cmake_gtest From d208aaa29b4ef251e93a2f217e22f2b99cd23491 Mon Sep 17 00:00:00 2001 From: Multyxu Date: Tue, 10 Mar 2026 21:19:37 -0400 Subject: [PATCH 3/7] Add yoloe as an instance segmenter. Bugfix: we should still publish img even there's no object been detected. --- .../semantic_inference/models/wrappers.py | 76 ++++++++++++++++--- .../semantic_inference/visualization.py | 4 + .../app/instance_segmentation_node | 9 ++- 3 files changed, 76 insertions(+), 13 deletions(-) diff --git a/semantic_inference/python/semantic_inference/models/wrappers.py b/semantic_inference/python/semantic_inference/models/wrappers.py index 11cf953..5be1c70 100644 --- a/semantic_inference/python/semantic_inference/models/wrappers.py +++ b/semantic_inference/python/semantic_inference/models/wrappers.py @@ -352,14 +352,22 @@ class Yolov11InstanceSegmenterWrapper(nn.Module): def __init__(self, config): """Load Yolov11 model.""" super().__init__() - from ultralytics import YOLO self.config = config + self.confidence_threshold = config.confidence_threshold + self.min_segmentation_size = config.min_segmentation_size + self.overlap_merge_iou = config.overlap_merge_iou + + self.init_model() + + def init_model(self): + """Initialize the model.""" + from ultralytics import YOLO + model_weights = os.path.join( - path_to_dot_semantic_inference(), f"{config.model_name}" + path_to_dot_semantic_inference(), f"{self.config.model_name}" ) self.model = YOLO(model_weights) - self.confidence_threshold = config.confidence_threshold def eval(self): """override eval to avoid issues with yolo model""" @@ -379,21 +387,31 @@ def construct(cls, **kwargs): def forward(self, img): """Segment image.""" - result = self.model(img)[0] # assume batch size 1 + result = self.model( + img, + conf=self.confidence_threshold, + imgsz=(img.shape[0], img.shape[1]), + # Set True to maintain original image size for masks, + # required for hydra, but slow down the model + retina_masks=True, + iou=self.overlap_merge_iou, + )[0] # assume batch size 1 if result.masks is None: return None, None, None, None - categories = result.boxes.cls.cpu() # int8 + categories = result.boxes.cls.to(torch.int).cpu() # int8 masks = result.masks.data.to(torch.bool).cpu() boxes = result.boxes.xyxy.cpu() # float32 confidences = result.boxes.conf.cpu() # float32 - # filter by confidence threshold - mask_indices = confidences > self.confidence_threshold - categories = categories[mask_indices] - masks = masks[mask_indices] - boxes = boxes[mask_indices] - confidences = confidences[mask_indices] + # filter out small masks + if self.min_segmentation_size > 0: + mask_sizes = masks.sum(dim=(1, 2)) # number of pixels in each mask + size_indices = mask_sizes > self.min_segmentation_size + categories = categories[size_indices] + masks = masks[size_indices] + boxes = boxes[size_indices] + confidences = confidences[size_indices] return categories, masks, boxes, confidences @@ -406,6 +424,8 @@ class Yolov11InstanceSegmenterConfig(Config): model_name: str = "yolo11n-seg.pt" confidence_threshold: float = 0.25 + min_segmentation_size: int = 0 # number of pixels to filter out small masks + overlap_merge_iou: float = 0.7 # merging overlaping object detection @classmethod def load(cls, filepath): @@ -413,6 +433,40 @@ def load(cls, filepath): return Config.load(cls, filepath) +class YoloeInstanceSegmenterWrapper(Yolov11InstanceSegmenterWrapper): + """Yoloe instance segmentation wrapper.""" + + def __init__(self, config): + super().__init__(config) + + def init_model(self): + """Initialize the model.""" + from ultralytics import YOLOE + + model_weights = os.path.join( + path_to_dot_semantic_inference(), f"{self.config.model_name}" + ) + self.model = YOLOE(model_weights) + + if not self.config.text_prompt: + raise ValueError( + "text_prompt must be provided for YoloeInstanceSegmenterWrapper" + ) + + self.model.set_classes(self.config.text_prompt) + + +@register_config( + "instance_model", name="yoloe", constructor=YoloeInstanceSegmenterWrapper +) +@dataclasses.dataclass +class YoloeInstanceSegmenterConfig(Yolov11InstanceSegmenterConfig): + """Configuration for Yoloe instance segmenter.""" + + model_name: str = "yoloe-26l-seg.pt" + text_prompt: list[str] = dataclasses.field(default_factory=list) + + class GDSam2InstanceSegmenterWrapper(nn.Module): """Grounded SAM 2 instance segmentation wrapper.""" diff --git a/semantic_inference/python/semantic_inference/visualization.py b/semantic_inference/python/semantic_inference/visualization.py index fa193d4..44cc7ed 100644 --- a/semantic_inference/python/semantic_inference/visualization.py +++ b/semantic_inference/python/semantic_inference/visualization.py @@ -328,6 +328,10 @@ def get_semantic_overlay_img(category_names, ret, img): boxes = ret.boxes confidences = ret.confidences + if masks is None: + # no bounding boxes to draw + return img + # Convert RGB to BGR for OpenCV vis_img_bgr = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) diff --git a/semantic_inference_ros/app/instance_segmentation_node b/semantic_inference_ros/app/instance_segmentation_node index 253aebd..7df84cb 100755 --- a/semantic_inference_ros/app/instance_segmentation_node +++ b/semantic_inference_ros/app/instance_segmentation_node @@ -69,14 +69,19 @@ class InstanceSegmentationNode(Node): ) def _spin_once(self, header, img): + self.get_logger().debug(f"recived image of size: {img.shape}") with torch.no_grad(): ret = self._model.segment(img, is_rgb_order=True).cpu() if ret.masks is None: self.get_logger().debug("No masks detected in the image.") - return + # we should still publish image even when detection is None + instance_seg_img = np.zeros((img.shape[0], img.shape[1])) + else: + instance_seg_img = ret.instance_seg_img + self.get_logger().debug("get image size: " + str(instance_seg_img.shape)) + self.get_logger().debug(f"result category type: {ret.categories.dtype}") - instance_seg_img = ret.instance_seg_img # Convert to int32 to match 32SC1 encoding expected by cv_bridge instance_seg_img = instance_seg_img.astype(np.int32) msg = Conversions.to_image_msg(header, instance_seg_img, encoding="32SC1") From afb85078b12b5eacc1dd2e1a2501456c3f59d095 Mon Sep 17 00:00:00 2001 From: Multyxu Date: Sat, 14 Mar 2026 21:00:35 -0400 Subject: [PATCH 4/7] Use json monitor and remove ros_system_monitor_msgs dependency --- .../app/instance_segmentation_node | 32 ++++++++++++------- semantic_inference_ros/package.xml | 1 - 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/semantic_inference_ros/app/instance_segmentation_node b/semantic_inference_ros/app/instance_segmentation_node index 7df84cb..3b14ead 100755 --- a/semantic_inference_ros/app/instance_segmentation_node +++ b/semantic_inference_ros/app/instance_segmentation_node @@ -1,7 +1,10 @@ #!/usr/bin/env python3 """Node that runs openset segmentation.""" +import collections +import json import pathlib +import time from dataclasses import dataclass, field import numpy as np @@ -9,8 +12,8 @@ import rclpy import spark_config as sc import torch from rclpy.node import Node -from ros_system_monitor_msgs.msg import NodeInfoMsg from sensor_msgs.msg import Image +from std_msgs.msg import String import semantic_inference.models as models import semantic_inference_ros @@ -55,7 +58,8 @@ class InstanceSegmentationNode(Node): self.get_logger().info("Finished initializing!") self._last_inference_time = None - self._status_pub = self.create_publisher(NodeInfoMsg, "~/status", 1) + self._inference_times_ms = collections.deque(maxlen=30) + self._status_pub = self.create_publisher(String, "~/status", 1) self._hb_timer = self.create_timer(0.1, self._hb_callback) self._pub = self.create_publisher(Image, "semantic/image_raw", 1) @@ -70,8 +74,10 @@ class InstanceSegmentationNode(Node): def _spin_once(self, header, img): self.get_logger().debug(f"recived image of size: {img.shape}") + t0 = time.monotonic() with torch.no_grad(): ret = self._model.segment(img, is_rgb_order=True).cpu() + self._inference_times_ms.append((time.monotonic() - t0) * 1e3) if ret.masks is None: self.get_logger().debug("No masks detected in the image.") @@ -97,19 +103,23 @@ class InstanceSegmentationNode(Node): ) def _hb_callback(self): - msg = NodeInfoMsg() - msg.nickname = "instance_seg" - msg.node_name = self.get_fully_qualified_name() + record = { + "nickname": "instance_seg", + "node_name": self.get_fully_qualified_name(), + } now = self.get_clock().now().nanoseconds * 1e-9 if self._last_inference_time is None: - msg.status = NodeInfoMsg.STARTUP - msg.notes = "Waiting for first image." + record["status"] = "STARTUP" + record["note"] = "Waiting for first image." elif now - self._last_inference_time > 10.0: - msg.status = NodeInfoMsg.WARNING - msg.notes = f"No inference for {now - self._last_inference_time:.1f}s." + record["status"] = "WARNING" + record["note"] = f"No inference for {now - self._last_inference_time:.1f}s." else: - msg.status = NodeInfoMsg.NOMINAL - msg.notes = "Running." + record["status"] = "NOMINAL" + avg_ms = sum(self._inference_times_ms) / len(self._inference_times_ms) + record["note"] = f"Average inference: {avg_ms:.1f} ms" + msg = String() + msg.data = json.dumps(record) self._status_pub.publish(msg) def stop(self): diff --git a/semantic_inference_ros/package.xml b/semantic_inference_ros/package.xml index eab9cba..2fba768 100644 --- a/semantic_inference_ros/package.xml +++ b/semantic_inference_ros/package.xml @@ -24,7 +24,6 @@ tf2_eigen rclpy image_transport_plugins - ros_system_monitor_msgs pcl_conversions ament_cmake_gtest From ea49987aa88e11dc35020771ddbfb60622f0cced Mon Sep 17 00:00:00 2001 From: Multyxu Date: Sat, 14 Mar 2026 21:16:06 -0400 Subject: [PATCH 5/7] let result property handle emtpy image creation. --- .../semantic_inference/models/instance_segmenter.py | 12 ++++++++++-- .../python/semantic_inference/models/wrappers.py | 8 ++++---- .../app/instance_segmentation_node | 9 ++++----- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/semantic_inference/python/semantic_inference/models/instance_segmenter.py b/semantic_inference/python/semantic_inference/models/instance_segmenter.py index 946ef7b..8d25eeb 100644 --- a/semantic_inference/python/semantic_inference/models/instance_segmenter.py +++ b/semantic_inference/python/semantic_inference/models/instance_segmenter.py @@ -52,6 +52,7 @@ class Results: boxes: torch.Tensor # (n, 4) xyxy format, torch.float32 categories: torch.Tensor # (n,), torch.float32/int64 (doesn't matter) confidences: torch.Tensor # (n,), torch.float32 + img_shape: tuple[int, int] @property def instance_seg_img(self): @@ -60,6 +61,9 @@ def instance_seg_img(self): Each pixel value encodes both category id and instance id. First 16 bits are category id, last 16 bits are instance id. """ + if self.masks is None: + return np.zeros(self.img_shape) + masks = self.masks.cpu().numpy() category_ids = self.categories.cpu().numpy() img = np.zeros(masks[0].shape, dtype=np.uint32) @@ -152,8 +156,12 @@ def forward(self, rgb_img): Returns: Encoded image """ - categories, masks, boxes, confidences = self.segmenter(rgb_img) + categories, masks, boxes, confidences, img_shape = self.segmenter(rgb_img) return Results( - masks=masks, boxes=boxes, categories=categories, confidences=confidences + masks=masks, + boxes=boxes, + categories=categories, + confidences=confidences, + img_shape=img_shape, ) diff --git a/semantic_inference/python/semantic_inference/models/wrappers.py b/semantic_inference/python/semantic_inference/models/wrappers.py index 5be1c70..ab3a4d8 100644 --- a/semantic_inference/python/semantic_inference/models/wrappers.py +++ b/semantic_inference/python/semantic_inference/models/wrappers.py @@ -397,7 +397,7 @@ def forward(self, img): iou=self.overlap_merge_iou, )[0] # assume batch size 1 if result.masks is None: - return None, None, None, None + return None, None, None, None, (img.shape[0], img.shape[1]) categories = result.boxes.cls.to(torch.int).cpu() # int8 masks = result.masks.data.to(torch.bool).cpu() @@ -412,7 +412,7 @@ def forward(self, img): masks = masks[size_indices] boxes = boxes[size_indices] confidences = confidences[size_indices] - return categories, masks, boxes, confidences + return categories, masks, boxes, confidences, (masks.shape[1], masks.shape[2]) @register_config( @@ -572,7 +572,7 @@ def forward(self, img): # if nothing detected if boxes.shape[0] == 0: - return None, None, None, None + return None, None, None, None, (img.shape[0], img.shape[1]) # process the box prompt for SAM 2 h, w, _ = img.shape @@ -636,7 +636,7 @@ def forward(self, img): # use xyxy boxes boxes = torch.tensor(input_boxes) - return categories, masks, boxes, confidences + return categories, masks, boxes, confidences, (masks.shape[1], masks.shape[2]) @register_config( diff --git a/semantic_inference_ros/app/instance_segmentation_node b/semantic_inference_ros/app/instance_segmentation_node index 3b14ead..82de733 100755 --- a/semantic_inference_ros/app/instance_segmentation_node +++ b/semantic_inference_ros/app/instance_segmentation_node @@ -76,18 +76,17 @@ class InstanceSegmentationNode(Node): self.get_logger().debug(f"recived image of size: {img.shape}") t0 = time.monotonic() with torch.no_grad(): - ret = self._model.segment(img, is_rgb_order=True).cpu() + ret = self._model.segment(img, is_rgb_order=True) self._inference_times_ms.append((time.monotonic() - t0) * 1e3) if ret.masks is None: self.get_logger().debug("No masks detected in the image.") - # we should still publish image even when detection is None - instance_seg_img = np.zeros((img.shape[0], img.shape[1])) else: - instance_seg_img = ret.instance_seg_img - self.get_logger().debug("get image size: " + str(instance_seg_img.shape)) + self.get_logger().debug("get image size: " + str(ret.img_shape)) self.get_logger().debug(f"result category type: {ret.categories.dtype}") + # get 32 bit instance + cateogry img, all 0 if no box detected + instance_seg_img = ret.instance_seg_img # Convert to int32 to match 32SC1 encoding expected by cv_bridge instance_seg_img = instance_seg_img.astype(np.int32) msg = Conversions.to_image_msg(header, instance_seg_img, encoding="32SC1") From 5338e6d309863430f7fe3892506978251099adb6 Mon Sep 17 00:00:00 2001 From: Multyxu Date: Sun, 15 Mar 2026 11:45:43 -0400 Subject: [PATCH 6/7] clean up yolo instance seg family to have proper inherentence from a base class. Change yolov11 name to yolo-seg becuase it now supports multiple segmentation version. --- docs/instance_seg.md | 6 +- .../models/instance_segmenter.py | 2 +- .../semantic_inference/models/wrappers.py | 85 ++++++++++--------- .../{yolov11.yaml => yolo-seg.yaml} | 2 +- .../config/instance_segmentation/yoloe.yaml | 8 ++ 5 files changed, 59 insertions(+), 44 deletions(-) rename semantic_inference_ros/config/instance_segmentation/{yolov11.yaml => yolo-seg.yaml} (82%) create mode 100644 semantic_inference_ros/config/instance_segmentation/yoloe.yaml diff --git a/docs/instance_seg.md b/docs/instance_seg.md index e9fb806..faeed0d 100644 --- a/docs/instance_seg.md +++ b/docs/instance_seg.md @@ -25,7 +25,7 @@ source /bin/activate pip install ./semantic_inference ``` -The above setup allows you to use `yolov11`, in order to use `Grounded Sam 2`, we have to manually install it. +The above setup allows you to use `yolo instance segmentation`, in order to use `Grounded Sam 2`, we have to manually install it. ```shell # cd to your favorite path, we can default to `~/.semantic_inference/` git clone -b more_gpu https://github.com/MultyXu/Grounded-SAM-2.git @@ -35,7 +35,9 @@ And follow the `README.md` in the cloned repo to install gdsam2. ## Setup models For `Grounded Sam 2`, put (or symlink) `GroundingDINO_SwinT_OGC.py` under `~/.semantic_inference/gdsam2_config/`. And, put `sam2.1_hiera_large.pt` and `groundingdino_swint_ogc.pth` under `~/.semantic_inference/` -For `Yolo`, download [yolo11n-seg.pt](https://github.com/ultralytics/assets/releases/download/v8.3.0/yolo11n-seg.pt) under `~/.semantic_inference/` +For `YOLO`, download [yolo11n-seg.pt](https://github.com/ultralytics/assets/releases/download/v8.3.0/yolo11n-seg.pt) under `~/.semantic_inference/`. + +For `YOLOE`, download [yoloe-26m-seg.pt](https://github.com/ultralytics/assets/releases/download/v8.4.0/yoloe-26m-seg.pt) under `~/.semantic_inference/`. ## Trying out close-set instance segmentation nodes diff --git a/semantic_inference/python/semantic_inference/models/instance_segmenter.py b/semantic_inference/python/semantic_inference/models/instance_segmenter.py index 8d25eeb..7b8137e 100644 --- a/semantic_inference/python/semantic_inference/models/instance_segmenter.py +++ b/semantic_inference/python/semantic_inference/models/instance_segmenter.py @@ -93,7 +93,7 @@ class InstanceSegmenterConfig(Config): """Main config for instance segmenter.""" # relevant configs (model path, model weights) for the model - instance_model: Any = config_field("instance_model", default="yolov11") + instance_model: Any = config_field("instance_model", default="yolo-seg") class InstanceSegmenter(nn.Module): diff --git a/semantic_inference/python/semantic_inference/models/wrappers.py b/semantic_inference/python/semantic_inference/models/wrappers.py index ab3a4d8..2a4c237 100644 --- a/semantic_inference/python/semantic_inference/models/wrappers.py +++ b/semantic_inference/python/semantic_inference/models/wrappers.py @@ -346,31 +346,25 @@ def load(cls, filepath): return Config.load(cls, filepath) -class Yolov11InstanceSegmenterWrapper(nn.Module): - """Yolov11 instance segmentation wrapper.""" +class YoloInstanceSegmenterBase(nn.Module): + """Base class for YOLO instance segmentation wrappers. + + Subclasses must assign self.model in their __init__. + """ def __init__(self, config): - """Load Yolov11 model.""" + """Store shared configuration.""" super().__init__() - self.config = config self.confidence_threshold = config.confidence_threshold self.min_segmentation_size = config.min_segmentation_size self.overlap_merge_iou = config.overlap_merge_iou - - self.init_model() - - def init_model(self): - """Initialize the model.""" - from ultralytics import YOLO - - model_weights = os.path.join( + self.model_weights = os.path.join( path_to_dot_semantic_inference(), f"{self.config.model_name}" ) - self.model = YOLO(model_weights) def eval(self): - """override eval to avoid issues with yolo model""" + """Override eval to avoid issues with yolo model.""" self.model.model.eval() @property @@ -378,13 +372,6 @@ def category_names(self): """Get category names.""" return self.model.names - @classmethod - def construct(cls, **kwargs): - """Load model from configuration dictionary.""" - config = Yolov11InstanceSegmenterConfig() - config.update(kwargs) - return cls(config) - def forward(self, img): """Segment image.""" result = self.model( @@ -415,17 +402,14 @@ def forward(self, img): return categories, masks, boxes, confidences, (masks.shape[1], masks.shape[2]) -@register_config( - "instance_model", name="yolov11", constructor=Yolov11InstanceSegmenterWrapper -) @dataclasses.dataclass -class Yolov11InstanceSegmenterConfig(Config): - """Configuration for Yolov11 instance segmenter.""" +class YoloInstanceSegmenterBaseConfig(Config): + """Shared configuration for YOLO instance segmenters.""" - model_name: str = "yolo11n-seg.pt" + model_name: str = "" confidence_threshold: float = 0.25 min_segmentation_size: int = 0 # number of pixels to filter out small masks - overlap_merge_iou: float = 0.7 # merging overlaping object detection + overlap_merge_iou: float = 0.7 # merging overlapping object detection @classmethod def load(cls, filepath): @@ -433,26 +417,47 @@ def load(cls, filepath): return Config.load(cls, filepath) -class YoloeInstanceSegmenterWrapper(Yolov11InstanceSegmenterWrapper): - """Yoloe instance segmentation wrapper.""" +class YolosegInstanceSegmenterWrapper(YoloInstanceSegmenterBase): + """Yolov11 instance segmentation wrapper.""" def __init__(self, config): + """Load Yolov11 model.""" + from ultralytics import YOLO + super().__init__(config) + self.model = YOLO(self.model_weights) - def init_model(self): - """Initialize the model.""" - from ultralytics import YOLOE + @classmethod + def construct(cls, **kwargs): + """Load model from configuration dictionary.""" + config = YolosegInstanceSegmenterConfig() + config.update(kwargs) + return cls(config) - model_weights = os.path.join( - path_to_dot_semantic_inference(), f"{self.config.model_name}" - ) - self.model = YOLOE(model_weights) - if not self.config.text_prompt: +@register_config( + "instance_model", name="yolo-seg", constructor=YolosegInstanceSegmenterWrapper +) +@dataclasses.dataclass +class YolosegInstanceSegmenterConfig(YoloInstanceSegmenterBaseConfig): + """Configuration for YOLO segmentation instance segmenter.""" + + model_name: str = "yolo11n-seg.pt" + + +class YoloeInstanceSegmenterWrapper(YoloInstanceSegmenterBase): + """Yoloe instance segmentation wrapper.""" + + def __init__(self, config): + from ultralytics import YOLOE + + if not config.text_prompt: raise ValueError( "text_prompt must be provided for YoloeInstanceSegmenterWrapper" ) + super().__init__(config) + self.model = YOLOE(self.model_weights) self.model.set_classes(self.config.text_prompt) @@ -460,10 +465,10 @@ def init_model(self): "instance_model", name="yoloe", constructor=YoloeInstanceSegmenterWrapper ) @dataclasses.dataclass -class YoloeInstanceSegmenterConfig(Yolov11InstanceSegmenterConfig): +class YoloeInstanceSegmenterConfig(YoloInstanceSegmenterBaseConfig): """Configuration for Yoloe instance segmenter.""" - model_name: str = "yoloe-26l-seg.pt" + model_name: str = "yoloe-26m-seg.pt" text_prompt: list[str] = dataclasses.field(default_factory=list) diff --git a/semantic_inference_ros/config/instance_segmentation/yolov11.yaml b/semantic_inference_ros/config/instance_segmentation/yolo-seg.yaml similarity index 82% rename from semantic_inference_ros/config/instance_segmentation/yolov11.yaml rename to semantic_inference_ros/config/instance_segmentation/yolo-seg.yaml index cef82eb..c2852e1 100644 --- a/semantic_inference_ros/config/instance_segmentation/yolov11.yaml +++ b/semantic_inference_ros/config/instance_segmentation/yolo-seg.yaml @@ -1,6 +1,6 @@ --- model: instance_model: - type: yolov11 + type: yolo-seg model_name: yolo11n-seg.pt confidence_threshold: 0.25 diff --git a/semantic_inference_ros/config/instance_segmentation/yoloe.yaml b/semantic_inference_ros/config/instance_segmentation/yoloe.yaml new file mode 100644 index 0000000..902b3d8 --- /dev/null +++ b/semantic_inference_ros/config/instance_segmentation/yoloe.yaml @@ -0,0 +1,8 @@ +--- +model: + instance_model: + type: yoloe + model_name: yoloe-26m-seg.pt + text_prompt: [ignore, chair, cone, bag] + confidence_threshold: 0.1 + min_segmentation_size: 0 From 66c81f95481331985000a37150b536f96ff95e9f Mon Sep 17 00:00:00 2001 From: Multyxu Date: Sun, 15 Mar 2026 11:48:10 -0400 Subject: [PATCH 7/7] reference to instance seg --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9a186b2..14b03c1 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,7 @@ echo "build: {cmake-args: [--no-warn-unused-cli, -DCMAKE_BUILD_TYPE=Release]}" > Once you've added this repository to your workspace, follow one (or both) of the following setup-guides as necessary: - [Closed-Set](docs/closed_set.md#setting-up) - [Open-Set](docs/open_set.md#setting-up) +- [Instance-Seg](docs/instance_seg.md) > **Note**
> Some of our other (larger) packages have or will have more accessible guides to getting `semantic_inference` set up for specific applications, such as [Hydra](https://github.com/MIT-SPARK/Hydra), [Khronos](https://github.com/MIT-SPARK/Khronos) or [Clio](https://github.com/MIT-SPARK/Clio).