revert to tile expiry

Author: leijurv

flex-config/gen/grouped-linemerge.lua

@@ -22,14 +22,13 @@
 --   * Apply an update:           osm2pgsql -a -O flex -S grouped-linemerge.lua CHANGES.osc.gz
 --   * Update the merged table:   osm2pgsql-gen -a -S grouped-linemerge.lua
 
--- An expire output with an 'endpoint_table' records the exact endpoints (start
--- and end point) of every way added/edited/deleted during an update, as POINT
--- rows. The grouped-linemerge generalization consumes these points: it walks
--- each affected connected component out from the changed endpoints and
--- re-merges only those, matching by exact endpoint equality (no tiles, no
--- area scan). Deletes contribute the old way's endpoints automatically.
+-- An expire output records which tiles changed during an update. The
+-- grouped-linemerge generalization uses it only as a seed for "where did line
+-- geometry change" - it then walks each affected connected component out from
+-- there and re-merges it. Use a high maxzoom so the seed regions are small.
 local exp_roads = osm2pgsql.define_expire_output({
-    endpoint_table = 'exp_roads_endpoints',
+    maxzoom = 18,
+    table = 'exp_roads',
 })
 
 -- The source table with the original road segments (one row per OSM way).
@@ -42,7 +41,7 @@ local roads = osm2pgsql.define_table({
         { column = 'highway', type = 'text' },
         { column = 'layer', type = 'int' },
         -- Attach the expire output to the geometry so that any change to a
-        -- road's geometry (add/modify/delete) records its endpoints.
+        -- road's geometry (add/modify/delete) expires the tiles it covers.
         { column = 'geom', type = 'linestring', not_null = true,
             expire = { { output = exp_roads } } },
     }
@@ -94,10 +93,10 @@ function osm2pgsql.process_gen()
         -- Here we only merge roads that carry a label or a shield.
         where = 'name IS NOT NULL OR ref IS NOT NULL',
 
-        -- In append mode, the table of exact changed-way endpoints to consume
-        -- (written by the expire output's 'endpoint_table' above). Consumption
-        -- is destructive, so each generalizer needs its own endpoint table.
-        endpoint_table = 'exp_roads_endpoints',
+        -- In append mode, where to read the expired tiles from, and the zoom
+        -- level they were captured at (must match the expire output's maxzoom).
+        expire_list = 'exp_roads',
+        zoom = 18,
 
         -- Create functional endpoint indexes on the src/dest tables in create
         -- mode. These make the incremental component walk fast. Set to false

src/expire-output.cpp

@@ -10,12 +10,9 @@
 #include "expire-output.hpp"
 
 #include "format.hpp"
-#include "geom.hpp"
-#include "hex.hpp"
 #include "logging.hpp"
 #include "pgsql.hpp"
 #include "tile.hpp"
-#include "wkb.hpp"
 
 #include <cerrno>
 #include <system_error>
@@ -53,43 +50,10 @@ void expire_output_t::add_tiles(
     m_tiles.insert(dirty_tiles.cbegin(), dirty_tiles.cend());
 }
 
-void expire_output_t::add_endpoints(geom::geometry_t const &geom)
-{
-    auto const add_line = [this](geom::linestring_t const &line) {
-        if (line.empty()) {
-            return;
-        }
-        auto const &first = line.front();
-        auto const &last = line.back();
-        m_endpoints.emplace(first.x(), first.y());
-        m_endpoints.emplace(last.x(), last.y());
-    };
-
-    std::lock_guard<std::mutex> const guard{*m_tiles_mutex};
-
-    m_endpoint_srid = geom.srid();
-    if (geom.is_linestring()) {
-        add_line(geom.get<geom::linestring_t>());
-    } else if (geom.is_multilinestring()) {
-        for (auto const &line : geom.get<geom::multilinestring_t>()) {
-            add_line(line);
-        }
-    }
-}
-
 bool expire_output_t::empty() noexcept
 {
     std::lock_guard<std::mutex> const guard{*m_tiles_mutex};
-    return m_tiles.empty() && m_endpoints.empty();
-}
-
-std::vector<std::pair<double, double>> expire_output_t::get_endpoints()
-{
-    std::lock_guard<std::mutex> const guard{*m_tiles_mutex};
-    std::vector<std::pair<double, double>> endpoints(m_endpoints.cbegin(),
-                                                     m_endpoints.cend());
-    m_endpoints.clear();
-    return endpoints;
+    return m_tiles.empty();
 }
 
 quadkey_list_t expire_output_t::get_tiles()
@@ -115,9 +79,6 @@ expire_output_t::output(connection_params_t const &connection_params)
     if (!m_table.empty()) {
         num = output_tiles_to_table(get_tiles(), connection_params);
     }
-    if (!m_endpoint_table.empty()) {
-        num += output_endpoints_to_table(get_endpoints(), connection_params);
-    }
     return num;
 }
 
@@ -179,58 +140,16 @@ std::size_t expire_output_t::output_tiles_to_table(
     return count;
 }
 
-std::size_t expire_output_t::output_endpoints_to_table(
-    std::vector<std::pair<double, double>> const &endpoints,
-    connection_params_t const &connection_params) const
-{
-    if (endpoints.empty()) {
-        return 0;
-    }
-
-    auto const qn = qualified_name(m_schema, m_endpoint_table);
-
-    pg_conn_t const db_connection{connection_params, "expire"};
-
-    // COPY is significantly faster than individual INSERTs.
-    db_connection.copy_start(fmt::format("COPY {} (geom) FROM STDIN", qn));
-
-    std::string buffer;
-    for (auto const &[x, y] : endpoints) {
-        geom::geometry_t const point{geom::point_t{x, y}, m_endpoint_srid};
-        buffer += util::encode_hex(geom_to_ewkb(point));
-        buffer += '\n';
-        if (buffer.size() >= 65536) {
-            db_connection.copy_send(buffer, qn);
-            buffer.clear();
-        }
-    }
-    if (!buffer.empty()) {
-        db_connection.copy_send(buffer, qn);
-    }
-    db_connection.copy_end(qn);
-
-    return endpoints.size();
-}
-
 void expire_output_t::create_output_table(pg_conn_t const &db_connection) const
 {
-    if (!m_table.empty()) {
-        auto const qn = qualified_name(m_schema, m_table);
-        db_connection.exec(
-            "CREATE TABLE IF NOT EXISTS {} ("
-            " zoom int4 NOT NULL,"
-            " x int4 NOT NULL,"
-            " y int4 NOT NULL,"
-            " first timestamp with time zone DEFAULT CURRENT_TIMESTAMP(0),"
-            " last timestamp with time zone DEFAULT CURRENT_TIMESTAMP(0),"
-            " PRIMARY KEY (zoom, x, y))",
-            qn);
-    }
-
-    if (!m_endpoint_table.empty()) {
-        auto const qn = qualified_name(m_schema, m_endpoint_table);
-        db_connection.exec("CREATE TABLE IF NOT EXISTS {} ("
-                           " geom geometry(Point) NOT NULL)",
-                           qn);
-    }
+    auto const qn = qualified_name(m_schema, m_table);
+    db_connection.exec(
+        "CREATE TABLE IF NOT EXISTS {} ("
+        " zoom int4 NOT NULL,"
+        " x int4 NOT NULL,"
+        " y int4 NOT NULL,"
+        " first timestamp with time zone DEFAULT CURRENT_TIMESTAMP(0),"
+        " last timestamp with time zone DEFAULT CURRENT_TIMESTAMP(0),"
+        " PRIMARY KEY (zoom, x, y))",
+        qn);
 }

src/expire-output.hpp

@@ -17,22 +17,16 @@
 #include <cstdint>
 #include <memory>
 #include <mutex>
-#include <set>
 #include <string>
 #include <unordered_set>
 #include <utility>
-#include <vector>
 
 constexpr std::size_t DEFAULT_MAX_TILES_GEOMETRY = 10'000'000;
 constexpr std::size_t DEFAULT_MAX_TILES_OVERALL = 50'000'000;
 
 class pg_conn_t;
 class connection_params_t;
 
-namespace geom {
-class geometry_t;
-} // namespace geom
-
 /**
  * Output for tile expiry.
  */
@@ -59,35 +53,6 @@ class expire_output_t
         m_table = std::move(table);
     }
 
-    std::string const &endpoint_table() const noexcept
-    {
-        return m_endpoint_table;
-    }
-
-    void set_endpoint_table(std::string table)
-    {
-        m_endpoint_table = std::move(table);
-    }
-
-    /// Does this output write expired tiles (to a file and/or table)?
-    bool has_tile_output() const noexcept
-    {
-        return !m_filename.empty() || !m_table.empty();
-    }
-
-    /// Does this output write the endpoints of changed geometries to a table?
-    bool has_endpoint_output() const noexcept
-    {
-        return !m_endpoint_table.empty();
-    }
-
-    /**
-     * Record the endpoints (first and last point of each linestring) of a
-     * changed geometry. They are written to the endpoint table on output.
-     * Non-(multi)linestring geometries contribute no endpoints. Thread-safe.
-     */
-    void add_endpoints(geom::geometry_t const &geom);
-
     uint32_t minzoom() const noexcept { return m_minzoom; }
     void set_minzoom(uint32_t minzoom) noexcept { m_minzoom = minzoom; }
 
@@ -151,19 +116,6 @@ class expire_output_t
     output_tiles_to_table(quadkey_list_t const &tiles_at_maxzoom,
                           connection_params_t const &connection_params) const;
 
-    /// Take and clear the collected endpoints. Thread-safe.
-    std::vector<std::pair<double, double>> get_endpoints();
-
-    /**
-     * Write the collected endpoints as POINT geometries to the endpoint table.
-     *
-     * \param endpoints The endpoint coordinates to write
-     * \param connection_params Database connection parameters
-     */
-    std::size_t
-    output_endpoints_to_table(std::vector<std::pair<double, double>> const &endpoints,
-                              connection_params_t const &connection_params) const;
-
     /**
      * Access to the m_tiles collection of expired tiles must go through
      * this mutex, because it can happend from several threads at the same
@@ -184,16 +136,6 @@ class expire_output_t
     /// The table (if any) for output
     std::string m_table;
 
-    /// The table (if any) for changed-geometry endpoints
-    std::string m_endpoint_table;
-
-    /// Collected endpoints (deduplicated) of changed geometries. Guarded by
-    /// m_tiles_mutex (same access pattern as m_tiles).
-    std::set<std::pair<double, double>> m_endpoints;
-
-    /// SRID of the collected endpoints (all geometries share the table's SRID).
-    int m_endpoint_srid = 0;
-
     /// Minimum zoom level for output
     uint32_t m_minzoom = 0;
 

src/flex-lua-expire-output.cpp

@@ -39,20 +39,10 @@ create_expire_output(lua_State *lua_state, std::string const &default_schema,
     new_expire_output.set_schema_and_table(schema, table);
     lua_pop(lua_state, 2); // "schema" and "table"
 
-    // optional "endpoint_table" field: write the endpoints of changed
-    // geometries as POINT rows (in the same schema) instead of, or in addition
-    // to, expired tiles.
-    auto const *endpoint_table = luaX_get_table_string(
-        lua_state, "endpoint_table", -1, "The expire output", "");
-    check_identifier(endpoint_table, "endpoint_table field");
-    new_expire_output.set_endpoint_table(endpoint_table);
-    lua_pop(lua_state, 1); // "endpoint_table"
-
     if (new_expire_output.filename().empty() &&
-        new_expire_output.table().empty() &&
-        new_expire_output.endpoint_table().empty()) {
-        throw std::runtime_error{"Must set 'filename', 'table', and/or "
-                                 "'endpoint_table' on expire output."};
+        new_expire_output.table().empty()) {
+        throw std::runtime_error{
+            "Must set 'filename' and/or 'table' on expire output."};
     }
 
     // required "maxzoom" field

src/flex-table-column.cpp

@@ -339,34 +339,17 @@ void flex_table_column_t::do_expire(
 
     for (auto const &expire_config : m_expires) {
         assert(expire_config.expire_output < expire->size());
-        auto &expire_output = expire_outputs->at(expire_config.expire_output);
-
-        if (expire_output.has_tile_output()) {
-            auto &expire_tiles = expire->at(expire_config.expire_output);
-            if (!expire_config.diff_expire || !enable_diff_expire ||
-                geoms_old->empty() || geoms_new->empty()) {
-                separate_expire(*geoms_old, expire_config, expire_tiles,
-                                expire_outputs);
-                separate_expire(*geoms_new, expire_config, expire_tiles,
-                                expire_outputs);
-            } else {
-                diff_expire(geoms_old, geoms_new, expire_config, expire_tiles,
-                            expire_outputs);
-            }
-        }
+        auto &expire_tiles = expire->at(expire_config.expire_output);
 
-        // Record the endpoints of the changed geometry (old and new), so a
-        // consumer (e.g. the grouped-linemerge generalizer) can re-merge only
-        // the exact connected components touched, instead of everything in a
-        // tile. Deletes provide the old geometry via the geometry cache, so
-        // their endpoints are captured here too.
-        if (expire_output.has_endpoint_output()) {
-            for (auto const &geom : *geoms_old) {
-                expire_output.add_endpoints(geom);
-            }
-            for (auto const &geom : *geoms_new) {
-                expire_output.add_endpoints(geom);
-            }
+        if (!expire_config.diff_expire || !enable_diff_expire ||
+            geoms_old->empty() || geoms_new->empty()) {
+            separate_expire(*geoms_old, expire_config, expire_tiles,
+                            expire_outputs);
+            separate_expire(*geoms_new, expire_config, expire_tiles,
+                            expire_outputs);
+        } else {
+            diff_expire(geoms_old, geoms_new, expire_config, expire_tiles,
+                        expire_outputs);
         }
     }
 }