Feature/resize

semantic_inference/CMakeLists.txt

@@ -36,7 +36,7 @@ configure_file(cmake/config.h.in ${CMAKE_CURRENT_BINARY_DIR}/semantic_inference_
 add_library(
   ${PROJECT_NAME}
   src/image_recolor.cpp src/image_rotator.cpp src/image_utilities.cpp src/logging.cpp
-  src/model_config.cpp src/segmenter.cpp
+  src/model_config.cpp src/segmenter.cpp src/image_resizer.cpp
 )
 target_link_libraries(${PROJECT_NAME} PUBLIC config_utilities::config_utilities ${OpenCV_LIBRARIES})
 target_include_directories(

semantic_inference/include/semantic_inference/image_resizer.h

@@ -0,0 +1,63 @@
+/* -----------------------------------------------------------------------------
+ * BSD 3-Clause License
+ *
+ * Copyright (c) 2021-2024, Massachusetts Institute of Technology.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its
+ *    contributors may be used to endorse or promote products derived from
+ *    this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * * -------------------------------------------------------------------------- */
+
+#pragma once
+
+#include <opencv2/core.hpp>
+
+namespace semantic_inference {
+
+struct ImageResizer {
+  struct Config {
+    int width = -1;
+    int height = -1;
+  } const config;
+
+  ImageResizer();
+  explicit ImageResizer(const Config& config);
+  ImageResizer(const ImageResizer& other);
+  ImageResizer& operator=(const ImageResizer& other);
+
+  cv::Mat resizeForModelInput(const cv::Mat& input) const;
+  // @brief Resize an RGB image to the target width and height prior to passing into a
+  // segmentation model.
+  //
+  // @param original input image.
+
+  cv::Mat restoreToOriginal(const cv::Mat& input, const cv::Size& target) const;
+  // @brief Resize an image of (semantic) labels back to the size of the original image.
+  //
+  // @param original input (label) image.
+};
+
+void declare_config(ImageResizer::Config& config);
+
+}  // namespace semantic_inference

semantic_inference/src/image_resizer.cpp

@@ -0,0 +1,81 @@
+/* -----------------------------------------------------------------------------
+ * BSD 3-Clause License
+ *
+ * Copyright (c) 2021-2024, Massachusetts Institute of Technology.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its
+ *    contributors may be used to endorse or promote products derived from
+ *    this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * * -------------------------------------------------------------------------- */
+
+#include "semantic_inference/image_resizer.h"
+
+#include <config_utilities/config.h>
+#include <config_utilities/types/enum.h>
+#include <config_utilities/validation.h>
+
+#include <opencv2/opencv.hpp>
+
+#include "semantic_inference/logging.h"
+
+namespace semantic_inference {
+
+void declare_config(ImageResizer::Config& config) {
+  using namespace config;
+  name("ImageResizer::Config");
+  config::field(config.width, "width", "pixels");
+  config::field(config.height, "height", "pixels");
+}
+
+ImageResizer::ImageResizer() : ImageResizer(Config{}) {}
+
+ImageResizer::ImageResizer(const Config& config) : config(config::checkValid(config)) {}
+
+ImageResizer::ImageResizer(const ImageResizer& other) : config(other.config) {}
+
+ImageResizer& ImageResizer::operator=(const ImageResizer& other) {
+  const_cast<Config&>(config) = other.config;
+  return *this;
+}
+
+cv::Mat ImageResizer::resizeForModelInput(const cv::Mat& input) const {
+  // if either width or height is negative, then no op. Note defaults are both -1
+  if (config.width < 0 || config.height < 0) return input;
+
+  cv::Mat scaled;
+  cv::resize(
+      input, scaled, cv::Size(config.width, config.height), 0.0, 0.0, cv::INTER_AREA);
+  return scaled;
+}
+
+cv::Mat ImageResizer::restoreToOriginal(const cv::Mat& input,
+                                        const cv::Size& target) const {
+  if (config.width < 0 || config.height < 0) return input;
+
+  cv::Mat scaled;
+  cv::resize(input, scaled, target, 0, 0, cv::INTER_NEAREST);
+  return scaled;
+}
+
+}  // namespace semantic_inference

semantic_inference/tests/CMakeLists.txt

@@ -2,6 +2,6 @@ find_package(GTest REQUIRED)
 include(GoogleTest)
 enable_testing()
 
-add_executable(test_${PROJECT_NAME} utest_image_utilities.cpp utest_main.cpp)
+add_executable(test_${PROJECT_NAME} utest_image_utilities.cpp utest_image_resizer.cpp utest_main.cpp)
 target_link_libraries(test_${PROJECT_NAME} PRIVATE ${PROJECT_NAME} GTest::gtest_main)
 gtest_add_tests(TARGET test_${PROJECT_NAME})

semantic_inference/tests/utest_image_resizer.cpp

@@ -0,0 +1,117 @@
+/* -----------------------------------------------------------------------------
+ * BSD 3-Clause License
+ *
+ * Copyright (c) 2021-2024, Massachusetts Institute of Technology.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its
+ *    contributors may be used to endorse or promote products derived from
+ *    this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * * -------------------------------------------------------------------------- */
+
+#include <gtest/gtest.h>
+#include <semantic_inference/image_resizer.h>
+
+#include <opencv2/imgproc.hpp>
+#include <opencv2/opencv.hpp>
+
+namespace semantic_inference {
+
+TEST(ImageResizer, ResizeForModelInput) {
+  cv::Mat input(cv::Size(500, 500), CV_8UC3);
+  cv::randu(input, 0, 255);
+
+  {  // Expect output is unchanged if height is -1
+    ImageResizer::Config config;
+    config.width = 200;
+    config.height = -1;
+    const ImageResizer resizer(config);
+
+    cv::Mat output = resizer.resizeForModelInput(input);
+    EXPECT_EQ(input.size(), output.size());
+    EXPECT_EQ(input.type(), output.type());
+
+    cv::Mat diff;
+    cv::compare(input, output, diff, cv::CmpTypes::CMP_NE);
+    EXPECT_TRUE(cv::countNonZero(diff.reshape(1)) == 0);
+  }
+
+  {  // Expect output is unchanged if width is -1
+    ImageResizer::Config config;
+    config.width = -1;
+    config.height = 200;
+    const ImageResizer resizer(config);
+
+    cv::Mat output = resizer.resizeForModelInput(input);
+    EXPECT_EQ(input.size(), output.size());
+    EXPECT_EQ(input.type(), output.type());
+
+    cv::Mat diff;
+    cv::compare(input, output, diff, cv::CmpTypes::CMP_NE);
+    EXPECT_TRUE(cv::countNonZero(diff.reshape(1)) == 0);
+  }
+
+  {  // Expect output is unchanged with default configuration
+    ImageResizer::Config config;
+    const ImageResizer resizer(config);
+
+    cv::Mat output = resizer.resizeForModelInput(input);
+    EXPECT_EQ(input.size(), output.size());
+    EXPECT_EQ(input.type(), output.type());
+
+    cv::Mat diff;
+    cv::compare(input, output, diff, cv::CmpTypes::CMP_NE);
+    EXPECT_TRUE(cv::countNonZero(diff.reshape(1)) == 0);
+  }
+
+  {  // Expect output size is changed to configuration
+    ImageResizer::Config config;
+    config.width = 300;
+    config.height = 200;
+    const ImageResizer resizer(config);
+
+    cv::Mat output = resizer.resizeForModelInput(input);
+    EXPECT_EQ(output.size(), cv::Size(config.width, config.height));
+  }
+}
+
+TEST(ImageResizer, RestoreToOriginal) {
+  cv::Mat input(cv::Size(300, 200), CV_16SC1);
+  cv::randu(input, 0, 32);
+
+  ImageResizer::Config config;
+  config.width = 300;
+  config.height = 200;
+  const ImageResizer resizer(config);
+
+  int original_width = 600;
+  int original_height = 400;
+
+  cv::Mat output =
+      resizer.restoreToOriginal(input, cv::Size(original_width, original_height));
+
+  EXPECT_EQ(output.size(), cv::Size(original_width, original_height));
+  EXPECT_EQ(input.type(), output.type());
+}
+
+}  // namespace semantic_inference

semantic_inference_ros/src/segmentation_nodelet.cpp

@@ -32,6 +32,7 @@
 #include <config_utilities/config_utilities.h>
 #include <config_utilities/parsing/commandline.h>
 #include <ianvs/image_subscription.h>
+#include <semantic_inference/image_resizer.h>
 #include <semantic_inference/image_rotator.h>
 #include <semantic_inference/model_config.h>
 #include <semantic_inference/segmenter.h>
@@ -63,6 +64,7 @@ class SegmentationNode : public rclcpp::Node {
   struct Config {
     Segmenter::Config segmenter;
     WorkerConfig worker;
+    ImageResizer::Config image_resizer;
     ImageRotator::Config image_rotator;
     bool show_config = true;
     bool show_output_config = false;
@@ -99,6 +101,7 @@ class SegmentationNode : public rclcpp::Node {
 
   OutputPublisher output_pub_;
   ImageRotator image_rotator_;
+  ImageResizer image_resizer_;
   ianvs::ImageSubscription sub_;
 };
 
@@ -121,6 +124,7 @@ void declare_config(SegmentationNode::Config& config) {
   name("SegmentationNode::Config");
   field(config.segmenter, "segmenter");
   field(config.worker, "worker");
+  field(config.image_resizer, "image_resizer");
   field(config.image_rotator, "image_rotator");
   field(config.show_config, "show_config");
   field(config.show_output_config, "show_output_config");
@@ -134,6 +138,7 @@ SegmentationNode::SegmentationNode(const rclcpp::NodeOptions& options)
           config::fromCLI<OutputPublisher::Config>(options.arguments(), "output")),
       output_pub_(output_config, *this),
       image_rotator_(config.image_rotator),
+      image_resizer_(config.image_resizer),
       sub_(*this) {
   logging::Logger::addSink("ros", std::make_shared<RosLogSink>(get_logger()));
   logging::setConfigUtilitiesLogger();
@@ -192,7 +197,9 @@ void SegmentationNode::runSegmentation(const Image::ConstSharedPtr& msg) {
               << (img_ptr->image.type() == CV_8UC3 ? "yes" : "no");
 
   const auto start = std::chrono::steady_clock::now();
-  const auto rotated = image_rotator_.rotate(img_ptr->image);
+  cv::Mat scaled_image = image_resizer_.resizeForModelInput(img_ptr->image);
+  const auto rotated = image_rotator_.rotate(scaled_image);
+
   const auto result = segmenter_->infer(rotated);
   if (!result) {
     SLOG(ERROR) << "failed to run inference!";
@@ -201,7 +208,9 @@ void SegmentationNode::runSegmentation(const Image::ConstSharedPtr& msg) {
   }
 
   const auto derotated = image_rotator_.derotate(result.labels);
-  output_pub_.publish(img_ptr->header, derotated, img_ptr->image);
+  const auto descaled =
+      image_resizer_.restoreToOriginal(derotated, img_ptr->image.size());
+  output_pub_.publish(img_ptr->header, descaled, img_ptr->image);
   const auto end = std::chrono::steady_clock::now();
 
   const auto elapsed =