option to save quantized meshes in gzip compressed forma

CMakeLists.txt

@@ -53,6 +53,8 @@ find_package(Boost REQUIRED COMPONENTS
     system
 )
 
+find_package(ZLIB REQUIRED)
+
 include("download-deps.cmake")
 find_path(TNTN_LIBGLM_SOURCE_DIR NAMES "glm/glm.hpp" HINTS "${CMAKE_SOURCE_DIR}/3rdparty/glm-0.9.9.0/")
 find_path(TNTN_LIBFMT_SOURCE_DIR NAMES "include/fmt/format.h" HINTS "${CMAKE_SOURCE_DIR}/3rdparty/fmt-5.1.0/")
@@ -231,6 +233,7 @@ target_link_libraries(tntn
 
     PRIVATE
     ${GDAL_LIBRARY}
+    ZLIB::ZLIB
 )
 
 add_executable(tin-terrain

download-deps.cmake

@@ -141,19 +141,19 @@ download_verify_unpack(
 if(TNTN_TEST)
     #see also http://oe.oregonexplorer.info/craterlake/
     download_verify_unpack(
-        "http://oe.oregonexplorer.info/craterlake/products/dem/dems_10m.zip"
+        "https://oe.oregonexplorer.info/craterlake//products/dem/dems_10m.zip"
         "${CMAKE_SOURCE_DIR}/3rdparty/craterlake/dems_10m.zip"
         "c677ac64dd3e443edebf0cdbb8e8785eb2d1dee864d1011b00bd0ef1745c72e4305f81d6630db240a31205d3b7ebed2dd0d52f468c0222013d6d79312a830ae0"
     )
 
-    #see also https://data.gov.uk/dataset/lidar-composite-dsm-50cm1
-    #data is in EPSG:27700 aka OSGB 1936 / British National Grid
-    #look at tile tq3389_DSM_50CM.asc - Tottenham for an interesting city DSM
-    download_verify_unpack(
-        "https://environment.data.gov.uk/UserDownloads/interactive/dcbcae4faa49490186edaa3b991025f549808/LIDARCOMP/LIDAR-DSM-50CM-TQ38nw.zip"
-        "${CMAKE_SOURCE_DIR}/3rdparty/uk-lidar-composite/LIDAR-DSM-50CM-TQ38nw.zip"
-        "f0c8f1cfbffba35122f1be6ba77df15a1f0c666cf4d49b44410bc9ea782cde46c1c24d9e0936aaf477d6a717cfcea64ebc8421155664929cb828e9806748c16b"
-    )
+#    #see also https://data.gov.uk/dataset/lidar-composite-dsm-50cm1
+#    #data is in EPSG:27700 aka OSGB 1936 / British National Grid
+#    #look at tile tq3389_DSM_50CM.asc - Tottenham for an interesting city DSM
+#    download_verify_unpack(
+#        "https://environment.data.gov.uk/UserDownloads/interactive/dcbcae4faa49490186edaa3b991025f549808/LIDARCOMP/LIDAR-DSM-50CM-TQ38nw.zip"
+#        "${CMAKE_SOURCE_DIR}/3rdparty/uk-lidar-composite/LIDAR-DSM-50CM-TQ38nw.zip"
+#        "f0c8f1cfbffba35122f1be6ba77df15a1f0c666cf4d49b44410bc9ea782cde46c1c24d9e0936aaf477d6a717cfcea64ebc8421155664929cb828e9806748c16b"
+#    )
 endif()
 
 #https://github.com/boostorg/filesystem/archive/boost-1.68.0.tar.gz

include/tntn/File.h

@@ -144,8 +144,11 @@ class File : public FileLike
     std::unique_ptr<FileImpl> m_impl;
 };
 
+class GZipWriteFile;
+
 class MemoryFile : public FileLike
 {
+    friend class GZipWriteFile;
   public:
     std::string name() const override { return std::string(); }
 
@@ -165,6 +168,29 @@ class MemoryFile : public FileLike
     bool m_is_good = true;
 };
 
+class GZipWriteFile : public FileLike {
+  public:
+    GZipWriteFile(const std::string& filename) : m_filename(filename) {}
+    ~GZipWriteFile() override;
+
+    std::string name() const override { return m_filename; }
+
+    bool is_good() override { return m_memory_file.is_good();}
+    position_type size() override { return m_memory_file.size(); }
+
+    size_t read(position_type from_offset, unsigned char* buffer, size_t size_of_buffer) override { return m_memory_file.read(from_offset, buffer, size_of_buffer); }
+    using FileLike::read; //import convenience overloads
+
+    bool write(position_type to_offset, const unsigned char* data, size_t data_size) override { return m_memory_file.write(to_offset, data, data_size); }
+    using FileLike::write; //import convenience overloads
+
+    void flush() override {} // don't want to write before everything is there. writing in destructor.
+
+  private:
+    std::string m_filename;
+    MemoryFile m_memory_file;
+};
+
 FileLike::position_type getline(FileLike::position_type from_offset,
                                 FileLike& f,
                                 std::string& str);

include/tntn/FileFormat.h

@@ -21,6 +21,7 @@ class FileFormat
         GEOJSON = 7,
         TIFF = 8,
         TIF = 9,
+        TERRAINGZ = 10,
     };
 
     FileFormat() = default;
@@ -56,6 +57,7 @@ class FileFormat
             case OBJ: return "obj";
             case ASC: return "asc";
             case XYZ: return "xyz";
+            case TERRAINGZ: "terrain.gz";
             case TERRAIN: return "terrain";
             case JSON: return "json";
             case GEOJSON: return "geojson";
@@ -81,6 +83,8 @@ class FileFormat
             return XYZ;
         else if(strcasecmp(s, "terrain") == 0)
             return TERRAIN;
+        else if(strcasecmp(s, "terrain.gz") == 0)
+            return TERRAINGZ;
         else if(strcasecmp(s, "json") == 0)
             return JSON;
         else if(strcasecmp(s, "geojson") == 0)
@@ -113,8 +117,9 @@ class FileFormat
     {
         switch(m_value)
         {
-            case OBJ: //fallthrough
-            case OFF: //fallthrough
+            case OBJ: [[fallthrough]];
+            case OFF: [[fallthrough]];
+            case TERRAINGZ: [[fallthrough]];
             case TERRAIN: return MeshMode::decomposed;
             default: return MeshMode::none;
         }

include/tntn/MeshWriter.h

@@ -23,13 +23,18 @@ class ObjMeshWriter : public MeshWriter
     virtual ~ObjMeshWriter(){};
 };
 
+
 class QuantizedMeshWriter : public MeshWriter
 {
+  public:
+    QuantizedMeshWriter(bool gzipped = false) : m_gzipped(gzipped) {}
     virtual bool write_mesh_to_file(const char* filename,
                                     Mesh& mesh,
                                     const BBox3D& bbox) override;
     virtual std::string file_extension() override;
     virtual ~QuantizedMeshWriter(){};
+  private:
+    bool m_gzipped = false;
 };
 
 } // namespace tntn

include/tntn/QuantizedMeshIO.h

@@ -17,11 +17,12 @@ int16_t zig_zag_decode(uint16_t i);
 } //namespace detail
 
 // unsigned int zig_zag_encode(int i);
-bool write_mesh_as_qm(const char* filename, const Mesh& m);
+bool write_mesh_as_qm(const char* filename, const Mesh& m, bool compress = false);
 bool write_mesh_as_qm(const char* filename,
                       const Mesh& m,
                       const BBox3D& bbox,
-                      bool mesh_is_rescaled = false);
+                      bool mesh_is_rescaled = false,
+                      bool compress = false);
 
 bool write_mesh_as_qm(const std::shared_ptr<FileLike>& f, const Mesh& m);
 bool write_mesh_as_qm(const std::shared_ptr<FileLike>& f,

src/File.cpp

@@ -1,9 +1,11 @@
 #include "tntn/File.h"
 #include "tntn/logging.h"
+#include "tntn/tntn_assert.h"
 
 #include <cstdio>
-#include <errno.h>
+#include <cerrno>
 #include <limits>
+#include <zlib.h>
 
 namespace tntn {
 
@@ -402,4 +404,27 @@ FileLike::position_type getline(FileLike::position_type from_offset,
     return getline(from_offset, *f, str);
 }
 
+GZipWriteFile::~GZipWriteFile()
+{
+    auto* file = gzopen(m_filename.c_str(), "w9");
+    {
+        const auto err = errno;
+        if (file == nullptr) {
+            TNTN_LOG_ERROR("gzopen({}, {}) = {} / errno = {}", m_filename.c_str(), "w+9", file != nullptr, err);
+            return;
+        }
+        TNTN_LOG_TRACE("gzopen({}, {}) = {} / errno = {}", m_filename.c_str(), "w+9", file != nullptr, err);
+    }
+
+    TNTN_ASSERT(size_t(m_memory_file.m_data.size()) < size_t(std::numeric_limits<unsigned>::max()));
+    const auto n_uncompressed_bytes_written = gzwrite(file, voidpc(m_memory_file.m_data.data()), unsigned(m_memory_file.size()));
+    TNTN_ASSERT(n_uncompressed_bytes_written == int(m_memory_file.size()));
+
+    const auto err = gzclose(file);
+    if(err != Z_OK)
+    {
+        TNTN_LOG_DEBUG("gzclose on {} failed with errno {}", m_filename, err);
+    }
+}
+
 } // namespace tntn

src/MeshIO.cpp

@@ -25,6 +25,10 @@ bool write_mesh_to_file(const char* filename, const Mesh& m, const FileFormat& f
     {
         return write_mesh_as_qm(filename, m);
     }
+    else if(f == FileFormat::TERRAINGZ)
+    {
+        return write_mesh_as_qm(filename, m, true);
+    }
     else if(f == FileFormat::JSON || f == FileFormat::GEOJSON)
     {
         return write_mesh_as_geojson(filename, m);

src/MeshWriter.cpp

@@ -16,7 +16,7 @@ std::string ObjMeshWriter::file_extension()
 
 bool QuantizedMeshWriter::write_mesh_to_file(const char* filename, Mesh& mesh, const BBox3D& bbox)
 {
-    return write_mesh_as_qm(filename, mesh, bbox, true);
+    return write_mesh_as_qm(filename, mesh, bbox, true, m_gzipped);
 }
 
 std::string QuantizedMeshWriter::file_extension()

src/QuantizedMeshIO.cpp

@@ -249,24 +249,25 @@ static void write_indices(BinaryIO& bio,
     }
 }
 
-bool write_mesh_as_qm(const char* filename, const Mesh& m)
+bool write_mesh_as_qm(const char* filename, const Mesh& m, bool compress)
 {
     BBox3D bbox;
     m.get_bbox(bbox);
-    return write_mesh_as_qm(filename, m, bbox, false);
+    return write_mesh_as_qm(filename, m, bbox, false, compress);
 }
 
 bool write_mesh_as_qm(const char* filename,
                       const Mesh& m,
                       const BBox3D& bbox,
-                      bool mesh_is_rescaled)
+                      bool mesh_is_rescaled, bool comprress)
 {
-    auto f = std::make_shared<File>();
-    if(!f->open(filename, File::OM_RWCF))
-    {
-        TNTN_LOG_ERROR("unable to open quantized mesh file {}", filename);
-        return false;
+    if (comprress) {
+        auto f = std::make_shared<GZipWriteFile>(filename);
+        return write_mesh_as_qm(f, m, bbox, mesh_is_rescaled);
     }
+
+    auto f = std::make_shared<File>();
+    f->open(filename, File::OM_RWCF);
     return write_mesh_as_qm(f, m, bbox, mesh_is_rescaled);
 }
 

src/cmd.cpp

@@ -54,7 +54,7 @@ static int subcommand_dem2tintiles(bool need_help,
         ("min-zoom", po::value<int>()->default_value(-1), "minimum zoom level to generate tiles for will guesstimate from resolution if not provided.")
         ("max-error", po::value<double>(), "max error parameter when using terra or zemlya method")
         ("step", po::value<int>()->default_value(1), "grid spacing in pixels when using dense method")
-        ("output-format", po::value<std::string>()->default_value("terrain"), "output tiles in terrain (quantized mesh) or obj")
+        ("output-format", po::value<std::string>()->default_value("terrain"), "output tiles in terrain (quantized mesh), terraingz (gzipped terrain), or obj")
 #if defined(TNTN_USE_ADDONS) && TNTN_USE_ADDONS
         ("method", po::value<std::string>()->default_value("terra"), "meshing algorithm. one of: terra, zemlya, curvature or dense")
         ("threshold", po::value<double>(), "threshold when using curvature method");
@@ -132,6 +132,10 @@ static int subcommand_dem2tintiles(bool need_help,
     {
         w.reset(new QuantizedMeshWriter());
     }
+    else if(local_varmap["output-format"].as<std::string>() == "terraingz")
+    {
+        w.reset(new QuantizedMeshWriter(true));
+    }
     else
     {
         throw po::error(std::string("unknow ouput-format: ") +
@@ -254,6 +258,7 @@ static FileFormat validate_output_format_for_mesh(const std::string& output_file
         FileFormat::OBJ,
         FileFormat::OFF,
         FileFormat::TERRAIN,
+        FileFormat::TERRAINGZ,
         FileFormat::JSON,
         FileFormat::GEOJSON,
     };