Skip to content

299362 CITUS Optimize retrieval and rendering of large geospatial data #269

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ChristosKalaitzis wants to merge 1 commit into
base: SandboxEnv
Choose a base branch
from
+29 −2
Open

299362 CITUS Optimize retrieval and rendering of large geospatial data #269

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions common-utitlities/src/main/java/org/eea/datalake/service/SpatialDataHandling.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ public interface SpatialDataHandling {

String processSpatialPayload(byte[] byteArray) throws IOException, ParseException;

String buildSpatialDescriptorJSON(Integer srid, String type, Double sizeMB, String dimension);

List<DataType> getGeoJsonEnums();

DataType getGeometryType(byte[] byteArray) throws ParseException;
Expand Down
2 changes: 1 addition & 1 deletion common-utitlities/src/main/java/org/eea/datalake/service/impl/SpatialDataHandlingImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,7 +233,7 @@ public String processSpatialPayload(byte[] byteArray) {
/**
* Builds lightweight spatial descriptor JSON for UI consumption.
*/
private String buildSpatialDescriptorJSON(Integer srid, String type, Double sizeMB, String dimension) {
public String buildSpatialDescriptorJSON(Integer srid, String type, Double sizeMB, String dimension) {

SpatialDataDescriptor descriptor = new SpatialDataDescriptor();
descriptor.setSrid(srid);
Expand Down
26 changes: 26 additions & 0 deletions ...rvice/src/main/java/org/eea/dataset/persistence/data/repository/RecordRepositoryImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@
import org.eea.interfaces.vo.recordstore.enums.ProcessTypeEnum;
import org.eea.multitenancy.TenantResolver;
import org.eea.utils.LiteralConstants;
import org.geolatte.geom.G2D;
import org.geolatte.geom.Geometry;
import org.hibernate.HibernateException;
import org.hibernate.Session;
import org.hibernate.annotations.QueryHints;
Expand Down Expand Up @@ -377,6 +379,12 @@ public class RecordRepositoryImpl implements RecordExtendedQueriesRepository {
private static final String RECORD_JSON_QUERY = "SELECT record_json from dataset_%s.temp_etlexport "
+ "WHERE filter_value= ? and id>= ? and id< ?";

private static final List<String> AFFECTED_TYPES = Arrays.asList(
DataType.POLYGON.getValue(),
DataType.MULTIPOLYGON.getValue(),
DataType.GEOMETRYCOLLECTION.getValue()
);

/**
* Find by table value with order.
*
Expand Down Expand Up @@ -1312,6 +1320,24 @@ private List<RecordValue> sanitizeRecords(List<RecordValue> records, String data
String idPk = (referenced != null) ? referenced.get("idPk").toString() : null;
fieldValue.setReferenceFieldSchemaId(idPk);
}
if (AFFECTED_TYPES.contains(fieldValue.getType().getValue())) {
// starting with this type of geometry to extract valuable info
//{"type":"Feature","geometry":{"type":"MultiPolygon","coordinates":[...]}
Geometry<G2D> geometry = fieldValue.getGeometry();
if (geometry != null) {
final double BYTES_IN_MB = 1024.0 * 1024.0; // conversion factor bytes -> megabytes

String type = geometry.getGeometryType().toString();
int srid = geometry.getSRID();
int dimensions = geometry.getDimension();

final int sizeBytes = fieldValue.getValue().getBytes(StandardCharsets.UTF_8).length;
final double sizeMB = Math.round((sizeBytes / BYTES_IN_MB) * 100.0) / 100.0;

String jsonValue = spatialDataHandling.buildSpatialDescriptorJSON(srid, type, sizeMB, dimensions +"D");
fieldValue.setValue(jsonValue);
}
}
}
sanitizedRecords.add(recordValue);
}
Expand Down
1 change: 0 additions & 1 deletion dataset-service/src/main/java/org/eea/dataset/service/impl/DatasetServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2066,7 +2066,6 @@ private TableVO calculatedErrorsAndRecordsToSee(final Long datasetId, final Stri
&& (idRules == null || idRules.length == 0) && fieldSchema == null && fieldValue == null) {
records = recordRepository.findByTableValueNoOrder(idTableSchema, pageable);
List<RecordVO> recordVOs = recordNoValidationMapper.entityListToClass(records);
spatialDataHandling.transformSpatialFields(recordVOs);
result.setTotalFilteredRecords(0L);
result.setRecords(recordVOs);
} else {
Expand Down