diff --git a/semantic_inference/CMakeLists.txt b/semantic_inference/CMakeLists.txt index ffa619d..878e0fa 100644 --- a/semantic_inference/CMakeLists.txt +++ b/semantic_inference/CMakeLists.txt @@ -36,7 +36,7 @@ configure_file(cmake/config.h.in ${CMAKE_CURRENT_BINARY_DIR}/semantic_inference_ add_library( ${PROJECT_NAME} src/image_recolor.cpp src/image_rotator.cpp src/image_utilities.cpp src/logging.cpp - src/model_config.cpp src/segmenter.cpp + src/model_config.cpp src/segmenter.cpp src/image_resizer.cpp ) target_link_libraries(${PROJECT_NAME} PUBLIC config_utilities::config_utilities ${OpenCV_LIBRARIES}) target_include_directories( diff --git a/semantic_inference/include/semantic_inference/image_resizer.h b/semantic_inference/include/semantic_inference/image_resizer.h new file mode 100644 index 0000000..ad5f7ac --- /dev/null +++ b/semantic_inference/include/semantic_inference/image_resizer.h @@ -0,0 +1,63 @@ +/* ----------------------------------------------------------------------------- + * BSD 3-Clause License + * + * Copyright (c) 2021-2024, Massachusetts Institute of Technology. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * * -------------------------------------------------------------------------- */ + +#pragma once + +#include + +namespace semantic_inference { + +struct ImageResizer { + struct Config { + int width = -1; + int height = -1; + } const config; + + ImageResizer(); + explicit ImageResizer(const Config& config); + ImageResizer(const ImageResizer& other); + ImageResizer& operator=(const ImageResizer& other); + + cv::Mat resizeForModelInput(const cv::Mat& input) const; + // @brief Resize an RGB image to the target width and height prior to passing into a + // segmentation model. + // + // @param original input image. + + cv::Mat restoreToOriginal(const cv::Mat& input, const cv::Size& target) const; + // @brief Resize an image of (semantic) labels back to the size of the original image. + // + // @param original input (label) image. +}; + +void declare_config(ImageResizer::Config& config); + +} // namespace semantic_inference diff --git a/semantic_inference/src/image_resizer.cpp b/semantic_inference/src/image_resizer.cpp new file mode 100644 index 0000000..aa54f66 --- /dev/null +++ b/semantic_inference/src/image_resizer.cpp @@ -0,0 +1,81 @@ +/* ----------------------------------------------------------------------------- + * BSD 3-Clause License + * + * Copyright (c) 2021-2024, Massachusetts Institute of Technology. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * * -------------------------------------------------------------------------- */ + +#include "semantic_inference/image_resizer.h" + +#include +#include +#include + +#include + +#include "semantic_inference/logging.h" + +namespace semantic_inference { + +void declare_config(ImageResizer::Config& config) { + using namespace config; + name("ImageResizer::Config"); + config::field(config.width, "width", "pixels"); + config::field(config.height, "height", "pixels"); +} + +ImageResizer::ImageResizer() : ImageResizer(Config{}) {} + +ImageResizer::ImageResizer(const Config& config) : config(config::checkValid(config)) {} + +ImageResizer::ImageResizer(const ImageResizer& other) : config(other.config) {} + +ImageResizer& ImageResizer::operator=(const ImageResizer& other) { + const_cast(config) = other.config; + return *this; +} + +cv::Mat ImageResizer::resizeForModelInput(const cv::Mat& input) const { + // if either width or height is negative, then no op. Note defaults are both -1 + if (config.width < 0 || config.height < 0) return input; + + cv::Mat scaled; + cv::resize( + input, scaled, cv::Size(config.width, config.height), 0.0, 0.0, cv::INTER_AREA); + return scaled; +} + +cv::Mat ImageResizer::restoreToOriginal(const cv::Mat& input, + const cv::Size& target) const { + if (config.width < 0 || config.height < 0) return input; + + cv::Mat scaled; + cv::resize(input, scaled, target, 0, 0, cv::INTER_NEAREST); + return scaled; +} + +} // namespace semantic_inference diff --git a/semantic_inference/tests/CMakeLists.txt b/semantic_inference/tests/CMakeLists.txt index 635c905..3ea29b5 100644 --- a/semantic_inference/tests/CMakeLists.txt +++ b/semantic_inference/tests/CMakeLists.txt @@ -2,6 +2,6 @@ find_package(GTest REQUIRED) include(GoogleTest) enable_testing() -add_executable(test_${PROJECT_NAME} utest_image_utilities.cpp utest_main.cpp) +add_executable(test_${PROJECT_NAME} utest_image_utilities.cpp utest_image_resizer.cpp utest_main.cpp) target_link_libraries(test_${PROJECT_NAME} PRIVATE ${PROJECT_NAME} GTest::gtest_main) gtest_add_tests(TARGET test_${PROJECT_NAME}) diff --git a/semantic_inference/tests/utest_image_resizer.cpp b/semantic_inference/tests/utest_image_resizer.cpp new file mode 100644 index 0000000..e805f43 --- /dev/null +++ b/semantic_inference/tests/utest_image_resizer.cpp @@ -0,0 +1,117 @@ +/* ----------------------------------------------------------------------------- + * BSD 3-Clause License + * + * Copyright (c) 2021-2024, Massachusetts Institute of Technology. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * * -------------------------------------------------------------------------- */ + +#include +#include + +#include +#include + +namespace semantic_inference { + +TEST(ImageResizer, ResizeForModelInput) { + cv::Mat input(cv::Size(500, 500), CV_8UC3); + cv::randu(input, 0, 255); + + { // Expect output is unchanged if height is -1 + ImageResizer::Config config; + config.width = 200; + config.height = -1; + const ImageResizer resizer(config); + + cv::Mat output = resizer.resizeForModelInput(input); + EXPECT_EQ(input.size(), output.size()); + EXPECT_EQ(input.type(), output.type()); + + cv::Mat diff; + cv::compare(input, output, diff, cv::CmpTypes::CMP_NE); + EXPECT_TRUE(cv::countNonZero(diff.reshape(1)) == 0); + } + + { // Expect output is unchanged if width is -1 + ImageResizer::Config config; + config.width = -1; + config.height = 200; + const ImageResizer resizer(config); + + cv::Mat output = resizer.resizeForModelInput(input); + EXPECT_EQ(input.size(), output.size()); + EXPECT_EQ(input.type(), output.type()); + + cv::Mat diff; + cv::compare(input, output, diff, cv::CmpTypes::CMP_NE); + EXPECT_TRUE(cv::countNonZero(diff.reshape(1)) == 0); + } + + { // Expect output is unchanged with default configuration + ImageResizer::Config config; + const ImageResizer resizer(config); + + cv::Mat output = resizer.resizeForModelInput(input); + EXPECT_EQ(input.size(), output.size()); + EXPECT_EQ(input.type(), output.type()); + + cv::Mat diff; + cv::compare(input, output, diff, cv::CmpTypes::CMP_NE); + EXPECT_TRUE(cv::countNonZero(diff.reshape(1)) == 0); + } + + { // Expect output size is changed to configuration + ImageResizer::Config config; + config.width = 300; + config.height = 200; + const ImageResizer resizer(config); + + cv::Mat output = resizer.resizeForModelInput(input); + EXPECT_EQ(output.size(), cv::Size(config.width, config.height)); + } +} + +TEST(ImageResizer, RestoreToOriginal) { + cv::Mat input(cv::Size(300, 200), CV_16SC1); + cv::randu(input, 0, 32); + + ImageResizer::Config config; + config.width = 300; + config.height = 200; + const ImageResizer resizer(config); + + int original_width = 600; + int original_height = 400; + + cv::Mat output = + resizer.restoreToOriginal(input, cv::Size(original_width, original_height)); + + EXPECT_EQ(output.size(), cv::Size(original_width, original_height)); + EXPECT_EQ(input.type(), output.type()); +} + +} // namespace semantic_inference diff --git a/semantic_inference_ros/src/segmentation_nodelet.cpp b/semantic_inference_ros/src/segmentation_nodelet.cpp index 32539b6..44df48b 100644 --- a/semantic_inference_ros/src/segmentation_nodelet.cpp +++ b/semantic_inference_ros/src/segmentation_nodelet.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -63,6 +64,7 @@ class SegmentationNode : public rclcpp::Node { struct Config { Segmenter::Config segmenter; WorkerConfig worker; + ImageResizer::Config image_resizer; ImageRotator::Config image_rotator; bool show_config = true; bool show_output_config = false; @@ -99,6 +101,7 @@ class SegmentationNode : public rclcpp::Node { OutputPublisher output_pub_; ImageRotator image_rotator_; + ImageResizer image_resizer_; ianvs::ImageSubscription sub_; }; @@ -121,6 +124,7 @@ void declare_config(SegmentationNode::Config& config) { name("SegmentationNode::Config"); field(config.segmenter, "segmenter"); field(config.worker, "worker"); + field(config.image_resizer, "image_resizer"); field(config.image_rotator, "image_rotator"); field(config.show_config, "show_config"); field(config.show_output_config, "show_output_config"); @@ -134,6 +138,7 @@ SegmentationNode::SegmentationNode(const rclcpp::NodeOptions& options) config::fromCLI(options.arguments(), "output")), output_pub_(output_config, *this), image_rotator_(config.image_rotator), + image_resizer_(config.image_resizer), sub_(*this) { logging::Logger::addSink("ros", std::make_shared(get_logger())); logging::setConfigUtilitiesLogger(); @@ -192,7 +197,9 @@ void SegmentationNode::runSegmentation(const Image::ConstSharedPtr& msg) { << (img_ptr->image.type() == CV_8UC3 ? "yes" : "no"); const auto start = std::chrono::steady_clock::now(); - const auto rotated = image_rotator_.rotate(img_ptr->image); + cv::Mat scaled_image = image_resizer_.resizeForModelInput(img_ptr->image); + const auto rotated = image_rotator_.rotate(scaled_image); + const auto result = segmenter_->infer(rotated); if (!result) { SLOG(ERROR) << "failed to run inference!"; @@ -201,7 +208,9 @@ void SegmentationNode::runSegmentation(const Image::ConstSharedPtr& msg) { } const auto derotated = image_rotator_.derotate(result.labels); - output_pub_.publish(img_ptr->header, derotated, img_ptr->image); + const auto descaled = + image_resizer_.restoreToOriginal(derotated, img_ptr->image.size()); + output_pub_.publish(img_ptr->header, descaled, img_ptr->image); const auto end = std::chrono::steady_clock::now(); const auto elapsed =