From 0e15b3b7292008e88d6c6f2bf8a34400e11861b2 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 1 Apr 2016 12:55:14 -0700 Subject: [PATCH 1/2] Update GeoWave tar.gz link before submitting pull request! Updated GeoWave to v0.9.1 --- README.md | 22 ++++++---- bootstrap.sh | 8 ++-- geowave-plugin.gyp | 4 +- include/geowave_datasource.hpp | 1 + src/geowave_datasource.cpp | 76 ++++++++++++++-------------------- test/geowave | 6 +-- test/test.cpp | 70 +++++++++++++++++++++---------- 7 files changed, 105 insertions(+), 82 deletions(-) diff --git a/README.md b/README.md index 46d7fee..8f11bc0 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,8 @@ GeoWave Plugin is an open source input plugin for Mapnik. * jvm - >= JDK 7 * geowave - - >= 0.8.7 - - [source](http://s3.amazonaws.com/geowave-rpms/release/TARBALL/geowave-0.8.7-c8ef40c-jace-source.tar.gz) + - >= 0.9.1 + - [source](http://s3.amazonaws.com/geowave-rpms/release/TARBALL/geowave-0.9.1-???-jace-source.tar.gz) - [docs](http://ngageoint.github.io/geowave/documentation.html#generate-proxies-and-build-from-source) ### Test Libraries @@ -40,9 +40,15 @@ GeoWave Plugin is an open source input plugin for Mapnik. * Java Runtime Environment - >= 7 * GeoWave Runtime Jar - - >= 0.8.7 - * GeoWave Ingest Jar - - >= 0.8.7 + - >= 0.9.1 + * GeoWave Example Jar + - >= 0.9.1 + - Required for test + * GeoWave Format Vector Jar + - >= 0.9.1 + - Required for test + * GeoWave Tools Jar + - >= 0.9.1 - Required for test ## Configuration @@ -95,11 +101,13 @@ For more details, or to build directly with GYP, check out the contents of the M First, download the GeoWave runtime and ingest jars contained within the jace-source packages provided by GeoWave [here](http://ngageoint.github.io/geowave/packages.html). -Next, create two environment variables which point to the runtime and ingest jars: +Next, create four environment variables which point to the runtime and ingest jars: ```bash +export GEOWAVE_EXAMPLE_JAR=/path/to/jar +export GEOWAVE_FORMAT_VECTOR_JAR=/path/to/jar export GEOWAVE_RUNTIME_JAR=/path/to/jar -export GEOWAVE_INGEST_JAR=/path/to/jar +export GEOWAVE_TOOLS_JAR=/path/to/jar ``` ### Running Tests diff --git a/bootstrap.sh b/bootstrap.sh index 3ec917c..e754e1a 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -71,7 +71,7 @@ function build_mapnik() { # boost libraries are included through mapnik function install_mason_deps() { - install geowave-jace 0.8.7 & + install geowave-jace 0.9.1 & install gtest 1.7.0 & wait } @@ -81,8 +81,10 @@ MASON_LINKED_REL=./mason_packages/.link export C_INCLUDE_PATH="${MASON_LINKED_ABS}/include" export CPLUS_INCLUDE_PATH="${MASON_LINKED_ABS}/include" export LIBRARY_PATH="${MASON_LINKED_ABS}/lib" -export GEOWAVE_RUNTIME_JAR=${MASON_LINKED_ABS}/bin/geowave-jace.jar -export GEOWAVE_INGEST_JAR=${MASON_LINKED_ABS}/bin/geowave-ingest.jar +export GEOWAVE_EXAMPLE_JAR=${MASON_LINKED_ABS}/bin/geowave-example.jar +export GEOWAVE_FORMAT_VECTOR_JAR=${MASON_LINKED_ABS}/bin/geowave-format-vector.jar +export GEOWAVE_RUNTIME_JAR=${MASON_LINKED_ABS}/bin/geowave-runtime.jar +export GEOWAVE_TOOLS_JAR=${MASON_LINKED_ABS}/bin/geowave-tools.jar function make_config() { if [[ $(uname -s) == 'Darwin' ]]; then diff --git a/geowave-plugin.gyp b/geowave-plugin.gyp index 647b8f3..65583ab 100644 --- a/geowave-plugin.gyp +++ b/geowave-plugin.gyp @@ -20,7 +20,7 @@ # geowave 'geowave_includes%': '<(geowave_home)/include', 'geowave_libs%': '<(geowave_home)/lib', - 'geowave_runtime_jar%': '<(geowave_home)/bin/geowave-jace.jar', + 'geowave_runtime_jar%': '<(geowave_home)/bin/geowave-runtime.jar', # java # For whatever reason, java_home is not in scope for the conditions @@ -80,7 +80,7 @@ 'link_settings': { 'libraries': [ '-l<(mapnik_name)', - '-ljace', + '-lgeowave', '-ljvm', '-lboost_thread', '-lboost_system' diff --git a/include/geowave_datasource.hpp b/include/geowave_datasource.hpp index e7a1fcd..1afede8 100644 --- a/include/geowave_datasource.hpp +++ b/include/geowave_datasource.hpp @@ -49,6 +49,7 @@ class geowave_datasource : public mapnik::datasource bool init_extent(); void init_layer_descriptor(); bool init_geometry_type(); + mapnik::featureset_ptr query(mapnik::box2d const& box) const; mapnik::layer_descriptor desc_; mapnik::context_ptr ctx_; diff --git a/src/geowave_datasource.cpp b/src/geowave_datasource.cpp index 10450f7..5de6a86 100644 --- a/src/geowave_datasource.cpp +++ b/src/geowave_datasource.cpp @@ -118,14 +118,16 @@ using jace::proxy::mil::nga::giat::geowave::core::index::ByteArrayId; using jace::proxy::mil::nga::giat::geowave::core::geotime::GeometryUtils; #include "jace/proxy/mil/nga/giat/geowave/core/store/adapter/DataAdapter.h" using jace::proxy::mil::nga::giat::geowave::core::store::adapter::DataAdapter; -#include "jace/proxy/mil/nga/giat/geowave/core/store/adapter/statistics/StatisticalDataAdapter.h" -using jace::proxy::mil::nga::giat::geowave::core::store::adapter::statistics::StatisticalDataAdapter; #include "jace/proxy/mil/nga/giat/geowave/core/store/index/Index.h" using jace::proxy::mil::nga::giat::geowave::core::store::index::Index; -#include "jace/proxy/mil/nga/giat/geowave/core/geotime/IndexType_JaceIndexType.h" -using jace::proxy::mil::nga::giat::geowave::core::geotime::IndexType_JaceIndexType; +#include "jace/proxy/mil/nga/giat/geowave/core/store/index/PrimaryIndex.h" +using jace::proxy::mil::nga::giat::geowave::core::store::index::PrimaryIndex; +#include "jace/proxy/mil/nga/giat/geowave/core/geotime/ingest/SpatialDimensionalityTypeProvider.h" +using jace::proxy::mil::nga::giat::geowave::core::geotime::ingest::SpatialDimensionalityTypeProvider; #include "jace/proxy/mil/nga/giat/geowave/core/store/query/Query.h" using jace::proxy::mil::nga::giat::geowave::core::store::query::Query; +#include "jace/proxy/mil/nga/giat/geowave/core/store/query/QueryOptions.h" +using jace::proxy::mil::nga::giat::geowave::core::store::query::QueryOptions; #include "jace/proxy/mil/nga/giat/geowave/core/geotime/store/query/SpatialQuery.h" using jace::proxy::mil::nga::giat::geowave::core::geotime::store::query::SpatialQuery; @@ -134,8 +136,10 @@ using jace::proxy::mil::nga::giat::geowave::core::store::adapter::statistics::Da #include "jace/proxy/mil/nga/giat/geowave/core/geotime/store/statistics/BoundingBoxDataStatistics.h" using jace::proxy::mil::nga::giat::geowave::core::geotime::store::statistics::BoundingBoxDataStatistics; -#include "jace/proxy/mil/nga/giat/geowave/adapter/vector/VectorDataStore.h" -using jace::proxy::mil::nga::giat::geowave::adapter::vector::VectorDataStore; +#include "jace/proxy/mil/nga/giat/geowave/adapter/vector/GeotoolsFeatureDataAdapter.h" +using jace::proxy::mil::nga::giat::geowave::adapter::vector::GeotoolsFeatureDataAdapter; +#include "jace/proxy/mil/nga/giat/geowave/adapter/vector/query/cql/CQLQuery.h" +using jace::proxy::mil::nga::giat::geowave::adapter::vector::query::cql::CQLQuery; #include "jace/proxy/mil/nga/giat/geowave/adapter/vector/plugin/ExtractGeometryFilterVisitor.h" using jace::proxy::mil::nga::giat::geowave::adapter::vector::plugin::ExtractGeometryFilterVisitor; #include "jace/proxy/mil/nga/giat/geowave/adapter/vector/stats/FeatureBoundingBoxStatistics.h" @@ -464,37 +468,7 @@ mapnik::featureset_ptr geowave_datasource::features(mapnik::query const& q) cons { // if the query box intersects our world extent then query for features mapnik::box2d const& box = q.get_bbox(); - if (extent_.intersects(box)) - { - GeometryFactory factory = java_new(); - - JDouble lonMin = box.minx(); - JDouble lonMax = box.maxx(); - JDouble latMin = box.miny(); - JDouble latMax = box.maxy(); - - JArray coordArray(5); - coordArray[0] = java_new(lonMin, latMin); - coordArray[1] = java_new(lonMax, latMin); - coordArray[2] = java_new(lonMax, latMax); - coordArray[3] = java_new(lonMin, latMax); - coordArray[4] = java_new(lonMin, latMin); - - VectorDataStore vector_datastore = java_new( - accumulo_operations_); - - return std::make_shared(vector_datastore.query(feature_data_adapter_, - IndexType_JaceIndexType::createSpatialVectorIndex(), - java_new(factory.createPolygon(coordArray)), - filter_, - Integer(0), - JArray(auths_)), - desc_.get_encoding(), - ctx_); - } - - // otherwise return an empty featureset pointer - return mapnik::featureset_ptr(); + return geowave_datasource::query(box); } mapnik::featureset_ptr geowave_datasource::features_at_point(mapnik::coord2d const& pt, double tol) const @@ -502,6 +476,11 @@ mapnik::featureset_ptr geowave_datasource::features_at_point(mapnik::coord2d con // if the query box intersects our world extent then query for features mapnik::box2d box(pt, pt); box.pad(tol); + return geowave_datasource::query(box); +} + +mapnik::featureset_ptr geowave_datasource::query(mapnik::box2d const& box) const +{ if (extent_.intersects(box)) { GeometryFactory factory = java_new(); @@ -518,15 +497,22 @@ mapnik::featureset_ptr geowave_datasource::features_at_point(mapnik::coord2d con coordArray[3] = java_new(lonMin, latMax); coordArray[4] = java_new(lonMin, latMin); - VectorDataStore vector_datastore = java_new( + AccumuloDataStore accumulo_datastore = java_new( accumulo_operations_); - - return std::make_shared(vector_datastore.query(feature_data_adapter_, - IndexType_JaceIndexType::createSpatialVectorIndex(), - java_new(factory.createPolygon(coordArray)), - filter_, - Integer(0), - JArray(auths_)), + + QueryOptions query_options = java_new(feature_data_adapter_, + java_new().createPrimaryIndex()); + + query_options.setAuthorizations(JArray(auths_)); + + SpatialQuery spatial_query = java_new(factory.createPolygon(coordArray)); + + CQLQuery cql_query = java_new(spatial_query, + filter_, + feature_data_adapter_); + + return std::make_shared(accumulo_datastore.query(query_options, + cql_query), desc_.get_encoding(), ctx_); } diff --git a/test/geowave b/test/geowave index 6c8b8e7..589018f 100755 --- a/test/geowave +++ b/test/geowave @@ -3,7 +3,7 @@ cd "$( dirname "${BASH_SOURCE[0]}" )" cd ../ -CLASS_TO_RUN=mil.nga.giat.geowave.datastore.accumulo.app.GeoWaveDemoApp +GEOWAVE_DEMO_APP=mil.nga.giat.geowave.examples.accumulo.app.GeoWaveDemoApp GEOWAVE_CLI_MAIN=mil.nga.giat.geowave.core.cli.GeoWaveMain PID_PATH_NAME=$(pwd)/test/.geowave-pid @@ -20,7 +20,7 @@ SHAPEFILE=$(pwd)/test/data/shp/world_merc.shp geowave_start() { if [ ! -f $PID_PATH_NAME ]; then echo "Starting GeoWave ..." - nohup java -Dinteractive=false -cp $GEOWAVE_RUNTIME_JAR $CLASS_TO_RUN /tmp 2>> /dev/null >> /dev/null & + nohup java -Dinteractive=false -cp $GEOWAVE_EXAMPLE_JAR:$GEOWAVE_FORMAT_VECTOR_JAR:$GEOWAVE_RUNTIME_JAR:$GEOWAVE_TOOLS_JAR $GEOWAVE_DEMO_APP /tmp 2>> /dev/null >> /dev/null & echo $! > $PID_PATH_NAME sleep 5 echo "GeoWave started ..." @@ -31,7 +31,7 @@ geowave_start() { geowave_ingest() { if [ -f $PID_PATH_NAME ]; then - java -cp $GEOWAVE_INGEST_JAR $GEOWAVE_CLI_MAIN -localingest -z $ZOOKEEPER_URL -u $USERNAME -p $PASSWORD -i $INSTANCE_NAME -n $TABLE_NAMESPACE -f $ADAPTER_TYPE -b $SHAPEFILE + java -cp $GEOWAVE_EXAMPLE_JAR:$GEOWAVE_FORMAT_VECTOR_JAR:$GEOWAVE_RUNTIME_JAR:$GEOWAVE_TOOLS_JAR $GEOWAVE_CLI_MAIN -localingest -connectionParams "\"zookeeper=$ZOOKEEPER_URL;user=$USERNAME;password=$PASSWORD;instance=$INSTANCE_NAME\"" -gwNamespace $TABLE_NAMESPACE -f $ADAPTER_TYPE -b $SHAPEFILE else echo "GeoWave is not running ..." fi diff --git a/test/test.cpp b/test/test.cpp index 0457412..359ad7a 100644 --- a/test/test.cpp +++ b/test/test.cpp @@ -101,17 +101,30 @@ TEST(geowave_plugin, features_at_point) feature_ptr feature = fs->next(); - EXPECT_EQ(feature->get("FIPS").to_string(), "US"); - EXPECT_EQ(feature->get("ISO2").to_string(), "US"); - EXPECT_EQ(feature->get("ISO3").to_string(), "USA"); - EXPECT_EQ(feature->get("UN").to_int(), 840); - EXPECT_EQ(feature->get("NAME").to_string(), "United States"); - EXPECT_EQ(feature->get("AREA").to_int(), 915896); - EXPECT_EQ(feature->get("POP2005").to_int(), 299846449); - EXPECT_EQ(feature->get("REGION").to_int(), 19); - EXPECT_EQ(feature->get("SUBREGION").to_int(), 21); - EXPECT_EQ(feature->get("LON").to_double(), -98.606000); - EXPECT_EQ(feature->get("LAT").to_double(), 39.622000); + int count = 0; + bool found = false; + while (feature){ + count++; + + if (feature->get("UN").to_int() == 840){ + found = true; + EXPECT_EQ(feature->get("FIPS").to_string(), "US"); + EXPECT_EQ(feature->get("ISO2").to_string(), "US"); + EXPECT_EQ(feature->get("ISO3").to_string(), "USA"); + EXPECT_EQ(feature->get("UN").to_int(), 840); + EXPECT_EQ(feature->get("NAME").to_string(), "United States"); + EXPECT_EQ(feature->get("AREA").to_int(), 915896); + EXPECT_EQ(feature->get("POP2005").to_int(), 299846449); + EXPECT_EQ(feature->get("REGION").to_int(), 19); + EXPECT_EQ(feature->get("SUBREGION").to_int(), 21); + EXPECT_EQ(feature->get("LON").to_double(), -98.606000); + EXPECT_EQ(feature->get("LAT").to_double(), 39.622000); + } + feature = fs->next(); + } + + EXPECT_EQ(count, 6); + EXPECT_EQ(found, true); } TEST(geowave_plugin, cql_query_filter) @@ -139,17 +152,30 @@ TEST(geowave_plugin, cql_query_filter) feature_ptr feature = fs->next(); - EXPECT_EQ(feature->get("FIPS").to_string(), "EI"); - EXPECT_EQ(feature->get("ISO2").to_string(), "IE"); - EXPECT_EQ(feature->get("ISO3").to_string(), "IRL"); - EXPECT_EQ(feature->get("UN").to_int(), 372); - EXPECT_EQ(feature->get("NAME").to_string(), "Ireland"); - EXPECT_EQ(feature->get("AREA").to_int(), 6889); - EXPECT_EQ(feature->get("POP2005").to_int(), 4143294); - EXPECT_EQ(feature->get("REGION").to_int(), 150); - EXPECT_EQ(feature->get("SUBREGION").to_int(), 154); - EXPECT_EQ(feature->get("LON").to_double(), -8.152000); - EXPECT_EQ(feature->get("LAT").to_double(), 53.177000); + int count = 0; + bool found = false; + while (feature){ + count++; + + if (feature->get("UN").to_int() == 372){ + found = true; + EXPECT_EQ(feature->get("FIPS").to_string(), "EI"); + EXPECT_EQ(feature->get("ISO2").to_string(), "IE"); + EXPECT_EQ(feature->get("ISO3").to_string(), "IRL"); + EXPECT_EQ(feature->get("UN").to_int(), 372); + EXPECT_EQ(feature->get("NAME").to_string(), "Ireland"); + EXPECT_EQ(feature->get("AREA").to_int(), 6889); + EXPECT_EQ(feature->get("POP2005").to_int(), 4143294); + EXPECT_EQ(feature->get("REGION").to_int(), 150); + EXPECT_EQ(feature->get("SUBREGION").to_int(), 154); + EXPECT_EQ(feature->get("LON").to_double(), -8.152000); + EXPECT_EQ(feature->get("LAT").to_double(), 53.177000); + } + feature = fs->next(); + } + + EXPECT_EQ(count, 1); + EXPECT_EQ(found, true); } TEST(geowave_plugin, fields_and_types) From 59fe681da1952db9cc32736e43450f0170317b75 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 6 Apr 2016 17:05:36 -0700 Subject: [PATCH 2/2] temporary snapshot, private mason --- bootstrap.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index e754e1a..abab166 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -4,7 +4,7 @@ function setup_mason() { if [[ ! -d ./.mason ]]; then - git clone --depth 1 https://github.com/mapbox/mason.git ./.mason + git clone --depth 1 https://github.com/jwomeara/mason.git ./.mason else echo "Updating to latest mason" (cd ./.mason && git pull) @@ -16,9 +16,9 @@ function setup_mason() { } if [[ $(uname -s) == 'Darwin' ]]; then - FIND_PATTERN="\/Users\/travis\/build\/mapbox\/mason" + FIND_PATTERN="\/Users\/travis\/build\/jwomeara\/mason" else - FIND_PATTERN="\/home\/travis\/build\/mapbox\/mason" + FIND_PATTERN="\/home\/travis\/build\/jwomeara\/mason" fi REPLACE="$(pwd)" @@ -88,9 +88,9 @@ export GEOWAVE_TOOLS_JAR=${MASON_LINKED_ABS}/bin/geowave-tools.jar function make_config() { if [[ $(uname -s) == 'Darwin' ]]; then - local PATH_REPLACE="/Users/travis/build/mapbox/mason/mason_packages:./mason_packages" + local PATH_REPLACE="/Users/travis/build/jwomeara/mason/mason_packages:./mason_packages" else - local PATH_REPLACE="/home/travis/build/mapbox/mason/mason_packages:./mason_packages" + local PATH_REPLACE="/home/travis/build/jwomeara/mason/mason_packages:./mason_packages" fi echo "{