From 7290fe28a6b09c9437568439324abe185e8ad1f6 Mon Sep 17 00:00:00 2001 From: shreyaskommuri Date: Tue, 5 May 2026 11:19:01 -0700 Subject: [PATCH 1/4] Suppress spurious libtiff warnings when decoding GeoTIFF files GeoTIFF files embed geographic metadata in five TIFF tags (ModelPixelScaleTag 33550, ModelTiepointTag 33922, GeoKeyDirectoryTag 34735, GeoDoubleParamsTag 34736, GeoAsciiParamsTag 34737) plus two common GDAL tags (42112, 42113). libtiff does not know these tags natively and emits "Unknown field with tag X encountered" warnings for each one. Install a custom TIFFErrorHandler in the ImageDecoder constructor (guarded by LIBTIFF_ENABLED) that silently drops warnings for these known GeoTIFF/GDAL tags and forwards all other warnings to stderr unchanged. The handler is installed once per process via std::call_once. Also add test_image_decoder_geotiff which builds a minimal GeoTIFF from scratch (no third-party dependency) and decodes it with both CPU and mixed backends, asserting correct pixel values. Fixes: #6114 Signed-off-by: shreyaskommuri --- dali/operators/imgcodec/image_decoder.h | 51 +++++++++++++ dali/test/python/decoder/test_image.py | 99 +++++++++++++++++++++++++ 2 files changed, 150 insertions(+) diff --git a/dali/operators/imgcodec/image_decoder.h b/dali/operators/imgcodec/image_decoder.h index 795422ede6e..d57ccbdfbb9 100644 --- a/dali/operators/imgcodec/image_decoder.h +++ b/dali/operators/imgcodec/image_decoder.h @@ -12,11 +12,18 @@ // limitations under the License. #include +#include +#include +#include #include #include +#include #include #include #include +#if LIBTIFF_ENABLED +#include +#endif #include "dali/core/call_at_exit.h" #include "dali/core/mm/memory.h" #include "dali/operators.h" @@ -46,6 +53,47 @@ nvimgcodecStatus_t get_nvjpeg2k_extension_desc(nvimgcodecExtensionDesc_t *ext_de namespace dali { namespace imgcodec { +#if LIBTIFF_ENABLED +namespace { + +// GeoTIFF-specific TIFF tag IDs that libtiff does not recognize natively. +// When decoding GeoTIFF files, libtiff emits "Unknown field with tag X" warnings for these tags. +// The image data is decoded correctly; the tags only carry geographic metadata. +constexpr uint32_t kGeoTIFFTags[] = { + 33550, // ModelPixelScaleTag + 33922, // ModelTiepointTag + 34264, // ModelTransformationTag + 34735, // GeoKeyDirectoryTag + 34736, // GeoDoubleParamsTag + 34737, // GeoAsciiParamsTag + 42112, // GDAL_METADATA + 42113, // GDAL_NODATA +}; + +void SuppressGeoTIFFTagWarnings(const char *module, const char *fmt, va_list ap) { + if (strstr(fmt, "Unknown field with tag") != nullptr) { + va_list ap_copy; + va_copy(ap_copy, ap); + unsigned int tag = va_arg(ap_copy, unsigned int); + va_end(ap_copy); + for (auto geotiff_tag : kGeoTIFFTags) { + if (tag == geotiff_tag) + return; + } + } + char buf[1024]; + vsnprintf(buf, sizeof(buf), fmt, ap); + std::cerr << module << ": " << buf << "\n"; +} + +void InstallGeoTIFFWarningFilter() { + static std::once_flag flag; + std::call_once(flag, [] { TIFFSetWarningHandler(SuppressGeoTIFFTagWarnings); }); +} + +} // namespace +#endif // LIBTIFF_ENABLED + template struct OutBackend { using type = GPUBackend; @@ -219,6 +267,9 @@ class ImageDecoder : public StatelessOperator { explicit ImageDecoder(const OpSpec &spec) : StatelessOperator(spec) { +#if LIBTIFF_ENABLED + InstallGeoTIFFWarningFilter(); +#endif device_id_ = std::is_same::value ? CPU_ONLY_DEVICE_ID : spec.GetArgument("device_id"); format_ = spec.GetArgument("output_type"); diff --git a/dali/test/python/decoder/test_image.py b/dali/test/python/decoder/test_image.py index 8c4398e1957..8e578425570 100644 --- a/dali/test/python/decoder/test_image.py +++ b/dali/test/python/decoder/test_image.py @@ -491,6 +491,105 @@ def pipe(): assert np.quantile(delta, 0.9) < 0.05, "Original and palette TIFF differ significantly" +def _create_geotiff(path, width=4, height=4): + """Build a minimal GeoTIFF file from scratch with the five standard GeoTIFF metadata tags. + + Returns the expected pixel array (HxW uint8) so callers can compare against decoded output. + The file is constructed with struct so no third-party TIFF library is required. + """ + import struct + + # Image: row-major, values 0..(H*W-1), clipped to uint8 + image_data = bytes(i % 256 for i in range(width * height)) + + # Extra tag payloads (stored after the IFD) + model_pixel_scale = struct.pack("<3d", 1.0, 1.0, 0.0) # ModelPixelScaleTag (3 doubles) + model_tiepoint = struct.pack("<6d", *([0.0] * 6)) # ModelTiepointTag (6 doubles) + geo_key_dir = struct.pack("<4H", 1, 1, 0, 0) # GeoKeyDirectoryTag (4 shorts) + geo_double = struct.pack(" Date: Tue, 5 May 2026 11:35:14 -0700 Subject: [PATCH 2/4] Address review feedback on GeoTIFF warning handler - Guard against null module name in SuppressGeoTIFFTagWarnings (libtiff permits a null module argument in some code paths) - Clarify comment on once_flag scope: each TU gets its own copy because the function lives in an anonymous namespace in a header; behaviour is still correct since all copies install the same handler - Skip the mixed-backend path in test_image_decoder_geotiff on CPU-only runners (guard with get_gpu_num() > 0) - Update copyright year to 2026 Signed-off-by: shreyaskommuri --- dali/operators/imgcodec/image_decoder.h | 11 ++++++++++- dali/test/python/decoder/test_image.py | 11 ++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/dali/operators/imgcodec/image_decoder.h b/dali/operators/imgcodec/image_decoder.h index d57ccbdfbb9..b612d710146 100644 --- a/dali/operators/imgcodec/image_decoder.h +++ b/dali/operators/imgcodec/image_decoder.h @@ -83,9 +83,18 @@ void SuppressGeoTIFFTagWarnings(const char *module, const char *fmt, va_list ap) } char buf[1024]; vsnprintf(buf, sizeof(buf), fmt, ap); - std::cerr << module << ": " << buf << "\n"; + // libtiff permits a null module name in some code paths + if (module) + std::cerr << module << ": " << buf << "\n"; + else + std::cerr << buf << "\n"; } +// Note: because this function lives in an anonymous namespace in a header, +// each translation unit that includes image_decoder.h gets its own copy of +// the once_flag. TIFFSetWarningHandler may therefore be called once per TU +// (in practice at most twice: host_decoder.cc and mixed_decoder.cc). +// All copies install the same handler so the behaviour is correct. void InstallGeoTIFFWarningFilter() { static std::once_flag flag; std::call_once(flag, [] { TIFFSetWarningHandler(SuppressGeoTIFFTagWarnings); }); diff --git a/dali/test/python/decoder/test_image.py b/dali/test/python/decoder/test_image.py index 8e578425570..8d9eecd53b7 100644 --- a/dali/test/python/decoder/test_image.py +++ b/dali/test/python/decoder/test_image.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -26,8 +26,9 @@ from nose_utils import assert_raises, SkipTest from test_utils import compare_pipelines from test_utils import get_dali_extra_path -from test_utils import to_array +from test_utils import get_gpu_num from test_utils import get_nvjpeg_ver +from test_utils import to_array def get_img_files(data_path, subdir="*", ext=None): @@ -574,7 +575,11 @@ def test_image_decoder_geotiff(): geo_path = os.path.join(tmpdir, "geo.tif") expected = _create_geotiff(geo_path) - for device in ["cpu", "mixed"]: + devices = ["cpu"] + if get_gpu_num() > 0: + devices.append("mixed") + + for device in devices: @pipeline_def(batch_size=1, device_id=0, num_threads=1) def geo_pipe(files): From f6575abf2cd93270ef8f560c46c5be44dca52f31 Mon Sep 17 00:00:00 2001 From: shreyaskommuri Date: Wed, 6 May 2026 09:52:52 -0700 Subject: [PATCH 3/4] Address review feedback: lint fixes and test refactor - Remove anonymous namespace from image_decoder.h (cpplint build/namespaces) - Mark SuppressGeoTIFFTagWarnings and InstallGeoTIFFWarningFilter inline, which also makes the once_flag truly process-wide across TUs - Move import struct to top of test_image.py - Apply black formatting to _create_geotiff - Fix double-space in docstring - Refactor test_image_decoder_geotiff to use @params Signed-off-by: shreyaskommuri --- dali/operators/imgcodec/image_decoder.h | 13 ++-- dali/test/python/decoder/test_image.py | 86 ++++++++++++------------- 2 files changed, 47 insertions(+), 52 deletions(-) diff --git a/dali/operators/imgcodec/image_decoder.h b/dali/operators/imgcodec/image_decoder.h index b612d710146..30da68aec7f 100644 --- a/dali/operators/imgcodec/image_decoder.h +++ b/dali/operators/imgcodec/image_decoder.h @@ -54,7 +54,6 @@ namespace dali { namespace imgcodec { #if LIBTIFF_ENABLED -namespace { // GeoTIFF-specific TIFF tag IDs that libtiff does not recognize natively. // When decoding GeoTIFF files, libtiff emits "Unknown field with tag X" warnings for these tags. @@ -70,7 +69,7 @@ constexpr uint32_t kGeoTIFFTags[] = { 42113, // GDAL_NODATA }; -void SuppressGeoTIFFTagWarnings(const char *module, const char *fmt, va_list ap) { +inline void SuppressGeoTIFFTagWarnings(const char *module, const char *fmt, va_list ap) { if (strstr(fmt, "Unknown field with tag") != nullptr) { va_list ap_copy; va_copy(ap_copy, ap); @@ -90,17 +89,13 @@ void SuppressGeoTIFFTagWarnings(const char *module, const char *fmt, va_list ap) std::cerr << buf << "\n"; } -// Note: because this function lives in an anonymous namespace in a header, -// each translation unit that includes image_decoder.h gets its own copy of -// the once_flag. TIFFSetWarningHandler may therefore be called once per TU -// (in practice at most twice: host_decoder.cc and mixed_decoder.cc). -// All copies install the same handler so the behaviour is correct. -void InstallGeoTIFFWarningFilter() { +// Declared inline so the static once_flag is shared across all translation units, +// guaranteeing TIFFSetWarningHandler is called exactly once per process. +inline void InstallGeoTIFFWarningFilter() { static std::once_flag flag; std::call_once(flag, [] { TIFFSetWarningHandler(SuppressGeoTIFFTagWarnings); }); } -} // namespace #endif // LIBTIFF_ENABLED template diff --git a/dali/test/python/decoder/test_image.py b/dali/test/python/decoder/test_image.py index 8d9eecd53b7..6dd816293b3 100644 --- a/dali/test/python/decoder/test_image.py +++ b/dali/test/python/decoder/test_image.py @@ -19,6 +19,7 @@ import nvidia.dali.types as types import os import random +import struct import tempfile from nvidia.dali import pipeline_def @@ -498,17 +499,15 @@ def _create_geotiff(path, width=4, height=4): Returns the expected pixel array (HxW uint8) so callers can compare against decoded output. The file is constructed with struct so no third-party TIFF library is required. """ - import struct - # Image: row-major, values 0..(H*W-1), clipped to uint8 image_data = bytes(i % 256 for i in range(width * height)) # Extra tag payloads (stored after the IFD) model_pixel_scale = struct.pack("<3d", 1.0, 1.0, 0.0) # ModelPixelScaleTag (3 doubles) - model_tiepoint = struct.pack("<6d", *([0.0] * 6)) # ModelTiepointTag (6 doubles) - geo_key_dir = struct.pack("<4H", 1, 1, 0, 0) # GeoKeyDirectoryTag (4 shorts) - geo_double = struct.pack(" 0: - devices.append("mixed") - - for device in devices: - - @pipeline_def(batch_size=1, device_id=0, num_threads=1) - def geo_pipe(files): - encoded, _ = fn.readers.file(files=files) - decoded = fn.decoders.image(encoded, device=device, output_type=types.ANY_DATA) - return decoded + @pipeline_def(batch_size=1, device_id=0, num_threads=1) + def geo_pipe(files): + encoded, _ = fn.readers.file(files=files) + decoded = fn.decoders.image(encoded, device=device, output_type=types.ANY_DATA) + return decoded - p = geo_pipe(files=[geo_path]) - out = p.run()[0] - if device == "mixed": - out = out.as_cpu() - result = np.array(out[0]).squeeze() - np.testing.assert_array_equal(result, expected) + p = geo_pipe(files=[geo_path]) + out = p.run()[0] + if device == "mixed": + out = out.as_cpu() + result = np.array(out[0]).squeeze() + np.testing.assert_array_equal(result, expected) def test_image_decoder_crafted_tiny_files(): From d619ac6f89dbddf4b8eb66dc7d337ea3f2b385f1 Mon Sep 17 00:00:00 2001 From: shreyaskommuri Date: Wed, 6 May 2026 10:11:36 -0700 Subject: [PATCH 4/4] Remove unused get_gpu_num import (fixes flake8 F401) The import was left over after refactoring the geotiff test to use @params instead of a manual GPU check. Signed-off-by: shreyaskommuri --- dali/test/python/decoder/test_image.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dali/test/python/decoder/test_image.py b/dali/test/python/decoder/test_image.py index 6dd816293b3..69c124f6bb5 100644 --- a/dali/test/python/decoder/test_image.py +++ b/dali/test/python/decoder/test_image.py @@ -27,7 +27,6 @@ from nose_utils import assert_raises, SkipTest from test_utils import compare_pipelines from test_utils import get_dali_extra_path -from test_utils import get_gpu_num from test_utils import get_nvjpeg_ver from test_utils import to_array