diff --git a/algorithms/java/src/org/openda/algorithms/kalmanFilter/EnKF.java b/algorithms/java/src/org/openda/algorithms/kalmanFilter/EnKF.java index 9fe1a93f1..0660ca613 100644 --- a/algorithms/java/src/org/openda/algorithms/kalmanFilter/EnKF.java +++ b/algorithms/java/src/org/openda/algorithms/kalmanFilter/EnKF.java @@ -718,7 +718,7 @@ protected void saveGain(FileBasedModelState algorithmState) { // write gain into files in a temp dir File kgStorage = new File(algorithmState.getDirContainingModelStateFiles(), "kgStorage_restart_tempdir"); double lastAnalysisTime = this.smoothedGainMatrix.previousAnalysisTime; - KalmanGainStorage gainStorage = new KalmanGainStorage(kgStorage, lastAnalysisTime, false); // here we should provide lastAnalysisTimeMJD + KalmanGainStorage gainStorage = new KalmanGainStorage(kgStorage, lastAnalysisTime); // here we should provide lastAnalysisTimeMJD gainStorage.writeKalmanGain(this.smoothedGainMatrix.lastGainMatrixHashMap, this.smoothedGainMatrix.obsIds, this.smoothedGainMatrix.lastObsTimeOffsets); FileBasedModelState gainState = new FileBasedModelState(); diff --git a/core/java/resources/netcdf/netcdf-4.3.23.jar b/core/java/resources/netcdf/netcdf-4.3.23.jar deleted file mode 100644 index c5d2420b5..000000000 Binary files a/core/java/resources/netcdf/netcdf-4.3.23.jar and /dev/null differ diff --git a/core/java/resources/netcdf/netcdfAll-5.9.1.jar b/core/java/resources/netcdf/netcdfAll-5.9.1.jar new file mode 100644 index 000000000..7b024764b Binary files /dev/null and b/core/java/resources/netcdf/netcdfAll-5.9.1.jar differ diff --git a/core/java/src/org/openda/exchange/dataobjects/NetcdfDataObject.java b/core/java/src/org/openda/exchange/dataobjects/NetcdfDataObject.java index a85f9d063..ddfa4f9bc 100644 --- a/core/java/src/org/openda/exchange/dataobjects/NetcdfDataObject.java +++ b/core/java/src/org/openda/exchange/dataobjects/NetcdfDataObject.java @@ -31,7 +31,8 @@ import ucar.ma2.InvalidRangeException; import ucar.nc2.Dimension; import ucar.nc2.NetcdfFile; -import ucar.nc2.NetcdfFileWriter; +import ucar.nc2.dataset.NetcdfDatasets; +import ucar.nc2.write.NetcdfFormatWriter; import ucar.nc2.Variable; import java.io.File; @@ -41,7 +42,7 @@ /** * DataObject for data that is stored in a NetCDF file in a format that adheres to the NetCDF CF conventions (see http://cfconventions.org/ ). * - * If this NecdfDataObject is insufficient for reading/writing a particular new NetCDF file, then there are two possibilities: + * If this NetcdfDataObject is insufficient for reading/writing a particular new NetCDF file, then there are two possibilities: * 1. If the file is in a format that is compliant with the NetCDF CF conventions (see http://cfconventions.org/ ), * then this class can be changed to include code for reading/writing the new CF compliant format. * 2. If the file is in a format that is not compliant with the NetCDF CF conventions, @@ -62,7 +63,7 @@ public class NetcdfDataObject implements IComposableDataObject, IComposableEnsem public enum GridStartCorner {NORTH_WEST, SOUTH_WEST, UNKNOWN} - //TODO Stef: remove this code. Variable names should not be hardcoded. AK + //TODO: remove this code. Variable names should not be hardcoded. AK protected String stationIdVarName = NetcdfUtils.STATION_ID_VARIABLE_NAME; // can be overruled by subclasses protected String CrossSectionIdVarName = NetcdfUtils.CROSS_SECTION_ID_VARIABLE_NAME; // can be overruled by subclasses @@ -73,7 +74,14 @@ public enum GridStartCorner {NORTH_WEST, SOUTH_WEST, UNKNOWN} private File file = null; - private NetcdfFileWriter netcdfFileWriter = null; + // netcdfWriterBuilder and netcdfWriter are linked: + // - if both null, no netCDF file is opened for writing. + // - after opening a new or existing NetCDF file: netcdfWriterBuilder is defined (the header data can be changed), netcdfWriter=null. + // - after setting netcdfWriter=netcdfWriterBuilder.build() the header cannot be changed anymore, data can be written to file now. + // - after closing the file, both should be reset to null. + private NetcdfFormatWriter.Builder netcdfBuilder = null; + private NetcdfFormatWriter netcdfWriter = null; + private NetcdfFile netcdfReader = null; protected List exchangeItems = new ArrayList(); /** * For each exchangeItemId contains a map with one exchangeItem per ensembleMemberIndex. @@ -115,10 +123,10 @@ public void setInternalGridStartCorner(GridStartCorner internalGridStartCorner) /** * @param workingDir path to work directory - * @param arguments required argument 1: the pathname of the data file relative to the given workingDir. - * @param arguments optional argument 2: lazyReading true/false. - * @param arguments optional argument 3: lazyWriting true/false. - * @param arguments optional argument >=2: allowTimeIndependentItems=true/false. + * @param arguments required argument 1: the pathname of the data file relative to the given workingDir. + * optional argument 2: lazyReading true/false. + * optional argument 3: lazyWriting true/false. + * optional argument >=2: allowTimeIndependentItems=true/false. */ public void initialize(File workingDir, String[] arguments) { String fileName = arguments[0]; @@ -152,12 +160,11 @@ public void initialize(File workingDir, String[] arguments) { } if (this.file.exists()) { - //open existing netcdf file. Always use NetcdfFileWriteable in case data needs to be written later. + this.netcdfBuilder = NetcdfFormatWriter.openExisting(this.file.getAbsolutePath()); try { - this.netcdfFileWriter = NetcdfFileWriter.openExisting(this.file.getAbsolutePath()); + this.netcdfReader = NetcdfDatasets.openFile(this.file.getAbsolutePath(), new NetcdfUtils.myCancelTask()); } catch (IOException e) { - throw new RuntimeException("Error while opening existing netcdf file '" + this.file.getAbsolutePath() - + "'. Message was: " + e.getMessage(), e); + throw new RuntimeException(e); } if (this.lazyReading) {//if lazyReading is true, then reading from netcdf file will happen later in exchangeItem.getValues methods. @@ -182,18 +189,10 @@ public void initialize(File workingDir, String[] arguments) { } else {//if file does not exist. //no data to read, but this dataObject can still be used for writing data to file. //create file here, exchange items need to be added using method addExchangeItem. - //create new netcdf file. - try { - //set fill to true, otherwise missing values will not be written for scalar time series variables that do not have data for all stations. - this.netcdfFileWriter = NetcdfFileWriter.createNew(NetcdfFileWriter.Version.netcdf3, this.file.getAbsolutePath()); - this.netcdfFileWriter.setFill(true); - } catch (IOException e) { - throw new RuntimeException("Error while creating handle for new netcdf file '" + this.file.getAbsolutePath() - + "'. Message was: " + e.getMessage(), e); - } - //always create large file, in case much data needs to be written. - this.netcdfFileWriter.setLargeFile(true); - NetcdfUtils.addGlobalAttributes(this.netcdfFileWriter); + this.netcdfBuilder = NetcdfFormatWriter.createNewNetcdf3(this.file.getAbsolutePath()); + //set fill to true, otherwise missing values will not be written for scalar time series variables that do not have data for all stations. + this.netcdfBuilder.setFill(true); + NetcdfUtils.addGlobalAttributes(this.netcdfBuilder); } } @@ -217,7 +216,7 @@ private boolean checkLazyReadingOrLazyWritingArguments(int i, String argument) { * Create exchange items that represent the data in the netcdf file * and know how to read/write this data from the netcdf file. * - * Currently this method can only create exchangeItems for scalar time series variables + * Currently, this method can only create exchangeItems for scalar time series variables * that are in the format of the Delft-FEWS scalar time series netcdf export. * For this copied and adapted code from class nl.wldelft.fews.system.plugin.dataImport.NetcdfTimeSeriesTSParser * @@ -231,7 +230,7 @@ private void createExchangeItems(Set requiredExchangeItemIds) throws IOE //get locationIds. //TODO MVL - NetcdfFile netcdfFile = this.netcdfFileWriter.getNetcdfFile(); + NetcdfFile netcdfFile = NetcdfDatasets.openFile(this.file.getAbsolutePath(), new NetcdfUtils.myCancelTask()); Map stationIndexIdMap = NetcdfUtils.readAndStoreStationIdsMap(netcdfFile, stationIdVarName); if (!stationIndexIdMap.isEmpty()) {//if stations found. Results.putMessage(this.getClass().getSimpleName() + ": station_id variable found in netcdf file " + this.file.getAbsolutePath()); @@ -243,7 +242,7 @@ private void createExchangeItems(Set requiredExchangeItemIds) throws IOE //in most netcdfFiles the time and spatial coordinate variables are shared between multiple data variables. - //Therefore cache timeInfo objects so that time coordinate variables + //Therefore, cache timeInfo objects so that time coordinate variables //are never read more than once. Map timeInfoCache = new HashMap(); for (Variable variable : netcdfFile.getVariables()) { @@ -267,7 +266,7 @@ private void createExchangeItems(Set requiredExchangeItemIds) throws IOE ITimeInfo timeInfo = NetcdfUtils.createTimeInfo(variable, netcdfFile, timeInfoCache); - //skip variables that are not two or three dimensional. + //skip variables that are not two or three-dimensional. int dimensionCount = variable.getDimensions().size(); // Test and correct for ensembleIndex named "realization". @@ -382,12 +381,12 @@ private void createExchangeItems(Set requiredExchangeItemIds) throws IOE //TODO remove, always read data lazily. AK private void readNetcdfFile() throws IOException { this.exchangeItems.clear(); - NetcdfFile netcdfFile = this.netcdfFileWriter.getNetcdfFile(); + NetcdfFile netcdfFile = this.netcdfReader; Results.putMessage(this.getClass().getSimpleName() + ": reading data from file " + this.file.getAbsolutePath()); //in most netcdfFiles the time and spatial coordinate variables are shared between multiple data variables. - //Therefore cache timeInfo objects so that time coordinate variables + //Therefore, cache timeInfo objects so that time coordinate variables //are never read more than once. Map timeInfoCache = new HashMap(); for (Variable variable : netcdfFile.getVariables()) { @@ -535,12 +534,12 @@ public IExchangeItem getDataObjectExchangeItem(String exchangeItemId, int ensemb * exchangeItem after adding it, but before calling finish, may alter the outcome. * Throws an UnsupportedOperationException when the dataObject does not support the addition of items. * A RuntimeException is thrown if the type of data can not be handled. Note that in general, it is also - * possible that the type of echchangeItem can be handled, but with degraded meta data. + * possible that the type of exchangeItem can be handled, but with degraded metadata. * * @param item exchangeItem to be duplicated */ public void addExchangeItem(IExchangeItem item) { - if (!this.netcdfFileWriter.isDefineMode()) { + if (this.netcdfWriter != null) { throw new RuntimeException(getClass().getSimpleName() + ": cannot add new exchangeItems to an existing netcdf file."); } if (!GeometryUtils.isScalar(item.getGeometryInfo())) { @@ -561,13 +560,13 @@ public void addExchangeItem(IExchangeItem item) { * exchangeItem after adding it, but before calling finish, may alter the outcome. * Throws an UnsupportedOperationException when the dataObject does not support the addition of items. * A RuntimeException is thrown if the type of data can not be handled. Note that in general, it is also - * possible that the type of echchangeItem can be handled, but with degraded meta data. + * possible that the type of exchangeItem can be handled, but with degraded metadata. * * @param item to be duplicated. * @param ensembleMemberIndex ensemble member index of the exchangeItem. */ public void addExchangeItem(IExchangeItem item, int ensembleMemberIndex) { - if (!this.netcdfFileWriter.isDefineMode()) { + if (this.netcdfWriter != null) { throw new RuntimeException(getClass().getSimpleName() + ": cannot add new exchangeItems to an existing netcdf file."); } if (!GeometryUtils.isScalar(item.getGeometryInfo())) { @@ -604,24 +603,24 @@ public void finish() { if (this.lazyWriting) { //write data to netcdf file. Results.putMessage(this.getClass().getSimpleName() + ": writing data to file " + this.file.getAbsolutePath()); - writeData(this.netcdfFileWriter, this.exchangeItems); + writeData(this.netcdfWriter, this.exchangeItems); } else {//if lazyWriting is false, then the variable data has already been written to netcdf file in exchangeItem.setValues methods, //and the metadata variable values have already been written to netcdf file in method createFile. //do nothing. } try { - this.netcdfFileWriter.close(); + this.netcdfWriter.close(); } catch (IOException e) { throw new RuntimeException("Error while closing netcdf file '" + this.file.getAbsolutePath() + "'. Message was: " + e.getMessage(), e); } } /** - * If the netcdf file has not yet been created, then creates it. Otherwise does nothing. + * If the netcdf file has not yet been created, then creates it. Otherwise, do nothing. */ public void makeSureFileHasBeenCreated() { - if (this.netcdfFileWriter.isDefineMode()) { + if (this.netcdfWriter == null) { createFile(); } } @@ -645,25 +644,27 @@ private void createFile() { uniqueEnsembleMemberIndices = Arrays.asList(BBUtils.box(getEnsembleMemberIndices())); //set station indices and realization indices in exchangeItems as soon as they are known. setStationAndRealizationIndicesForWriting(exchangeItems, ensembleExchangeItems); - NetcdfUtils.createMetadataAndDataVariablesForScalars(netcdfFileWriter, exchangeItems, ensembleExchangeItems, timeInfoTimeDimensionMap, stationIdVarName, + NetcdfUtils.createMetadataAndDataVariablesForScalars(this.netcdfBuilder, exchangeItems, ensembleExchangeItems, timeInfoTimeDimensionMap, stationIdVarName, stationDimensionVarName, uniqueStationIds.size(), uniqueEnsembleMemberIndices.size()); } else {//if grids. - NetcdfUtils.createMetadataAndDataVariablesForGrids(this.netcdfFileWriter, this.exchangeItems, this.timeInfoTimeDimensionMap, this.geometryInfoGridVariablePropertiesMap); + NetcdfUtils.createMetadataAndDataVariablesForGrids(this.netcdfBuilder, this.exchangeItems, this.timeInfoTimeDimensionMap, this.geometryInfoGridVariablePropertiesMap); } } - + // header information is finished, create writer to start writing data try { - this.netcdfFileWriter.create(); + this.netcdfWriter = this.netcdfBuilder.build(); } catch (IOException e) { - throw new RuntimeException("Error while creating new netcdf file '" + this.netcdfFileWriter.getNetcdfFile().getLocation() + "'. Message was: " + e.getMessage(), e); + throw new RuntimeException("Error finishing header information for file '" + this.file.getAbsolutePath() + "'. Message was: " + e.getMessage(), e); } + // Once build() is called, do not use the Builder again. + this.netcdfBuilder = null; - //write metadata variable values for all exchange items, i.e. times and spatial coordinates. - //This is only needed if a new file has been created. If an existing file has been opened, then the file should already contain these values. + // Write values for all exchange items, i.e. times and spatial coordinates. + // This is only needed if a new file has been created. If an existing file has been opened, then the file should already contain these values. try { - NetcdfUtils.writeMetadata(netcdfFileWriter, timeInfoTimeDimensionMap, geometryInfoGridVariablePropertiesMap, stationIdVarName, uniqueStationIds, uniqueEnsembleMemberIndices); + NetcdfUtils.writeMetadata(this.netcdfWriter, timeInfoTimeDimensionMap, geometryInfoGridVariablePropertiesMap, stationIdVarName, uniqueStationIds, uniqueEnsembleMemberIndices); } catch (Exception e) { - throw new RuntimeException("Error while writing metadata values to netcdf file '" + this.file.getAbsolutePath() + "'. Message was: " + e.getMessage(), e); + throw new RuntimeException("Error while writing data values to netcdf file '" + this.file.getAbsolutePath() + "'. Message was: " + e.getMessage(), e); } } @@ -687,7 +688,7 @@ private void validateExchangeItems(boolean scalar, List exchangeI for (Map ensemble : ensembleExchangeItems.values()) { for (IExchangeItem item : ensemble.values()) { if (GeometryUtils.isScalar(item.getGeometryInfo()) != scalar) { - throw new RuntimeException(getClass().getSimpleName() + ": Not all exchange items are of the same geometryInfo type. " + throw new RuntimeException(getClass().getSimpleName() + ": Not all ensemble exchange items are of the same geometryInfo type. " + getClass().getSimpleName() + " can only write a NetCDF file for exchange items that are all of the same geometryInfo type."); } } @@ -713,17 +714,17 @@ private void setStationAndRealizationIndicesForWriting(List excha * Writes all data for the given exchangeItems to the given netcdfFile. * Only executed when this.lazyWriting = true * - * @param netcdfFileWriter + * @param NetcdfFormatWriter Object to writes Netcdf 3 or 4 formatted files to disk. * @param exchangeItems to write. */ //TODO remove. This is only used for SWAN state files (see SwanStateNetcdfFileTest.testSwanNetcdfStateFile_1). AK - private void writeData(NetcdfFileWriter netcdfFileWriter, List exchangeItems) { + private void writeData(NetcdfFormatWriter NetcdfFormatWriter, List exchangeItems) { for (IExchangeItem exchangeItem : exchangeItems){ if (exchangeItem.getGeometryInfo() == null){ //TODO Julius: please remove this hack for SWAN state files. AK //TODO: replace this SWAN specific implementation with the above generic ones. String exchangeItemId = exchangeItem.getId(); - //TODO: add netcdf writers for various data type / exchangeitems. + //TODO: add netcdf writers for various data type / exchange items. //For SWAN state file, only wave_spectrum is modified and rewritten. if (exchangeItemId.equalsIgnoreCase("wave_spectrum")){ double[] dblValues = exchangeItem.getValuesAsDoubles(); @@ -734,8 +735,7 @@ private void writeData(NetcdfFileWriter netcdfFileWriter, List ex // get dimensions: // TODO: in the future, dimensions should be available in the exchangeItem as part of meta data // This will avoid having to read the netcdf file for obtaining the dimensions. - List dimensions = netcdfFileWriter.getNetcdfFile().getDimensions(); - int nDim = dimensions.size(); + List dimensions = NetcdfFormatWriter.getOutputFile().getRootGroup().getDimensions(); for (Dimension dimension : dimensions) { if ("my".equalsIgnoreCase(dimension.getShortName())) { my = dimension.getLength(); @@ -745,8 +745,6 @@ private void writeData(NetcdfFileWriter netcdfFileWriter, List ex wave_frequency = dimension.getLength(); } else if ("wave_direction".equalsIgnoreCase(dimension.getShortName())) { wave_direction = dimension.getLength(); - } else { - continue; } } ArrayDouble.D4 values = new ArrayDouble.D4(my,mx,wave_frequency,wave_direction); @@ -762,8 +760,8 @@ private void writeData(NetcdfFileWriter netcdfFileWriter, List ex } } try { - Variable myVar = netcdfFileWriter.findVariable("wave_spectrum"); - netcdfFileWriter.write(myVar,values); + Variable myVar = NetcdfFormatWriter.findVariable("wave_spectrum"); + NetcdfFormatWriter.write(myVar,values); } catch (IOException | InvalidRangeException e) { e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. } @@ -773,47 +771,47 @@ private void writeData(NetcdfFileWriter netcdfFileWriter, List ex } public double[] readDataForExchangeItemForSingleLocation(IExchangeItem item, int stationDimensionIndex, int stationIndex) { - Variable variable = NetcdfUtils.getVariableForExchangeItem(netcdfFileWriter.getNetcdfFile(), item); + Variable variable = NetcdfUtils.getVariableForExchangeItem(netcdfReader, item); return NetcdfUtils.readDataForVariableForSingleLocation(variable, stationDimensionIndex, stationIndex); } public double[] readDataForExchangeItemForSingleLocationSingleLayer(IExchangeItem item, int stationDimensionIndex, int stationIndex, int layerDimensionIndex, int layerIndex) { - Variable variable = NetcdfUtils.getVariableForExchangeItem(netcdfFileWriter.getNetcdfFile(), item); + Variable variable = NetcdfUtils.getVariableForExchangeItem(netcdfReader, item); return NetcdfUtils.readDataForVariableForSingleLocationAndLayer(variable, stationDimensionIndex, stationIndex, layerDimensionIndex, layerIndex); } public double[] readDataForExchangeItemForSingleLocationSingleRealization(IExchangeItem item, int stationDimensionIndex, int stationIndex, int realizationDimensionIndex, int realizationIndex) { - Variable variable = NetcdfUtils.getVariableForExchangeItem(netcdfFileWriter.getNetcdfFile(), item); + Variable variable = NetcdfUtils.getVariableForExchangeItem(netcdfReader, item); return NetcdfUtils.readDataForVariableForSingleLocationAndRealization(variable, stationDimensionIndex, stationIndex, realizationDimensionIndex, realizationIndex); } public double[] readDataForExchangeItemFor2DGridForSingleTime(IExchangeItem item, int timeDimensionIndex, int timeIndex, int dimensionIndexToFlip) { - Variable variable = NetcdfUtils.getVariableForExchangeItem(netcdfFileWriter.getNetcdfFile(), item); + Variable variable = NetcdfUtils.getVariableForExchangeItem(netcdfReader, item); return NetcdfUtils.readDataForVariableFor2DGridForSingleTime(variable, timeDimensionIndex, timeIndex, dimensionIndexToFlip); } public double[] readDataForExchangeItemFor2DGridForSingleTimeAndRealization( IExchangeItem item, int realizationDimensionIndex, int realizationIndex, int timeDimensionIndex, int timeIndex, int dimensionIndexToFlip) { - Variable variable = NetcdfUtils.getVariableForExchangeItem(netcdfFileWriter.getNetcdfFile(), item); + Variable variable = NetcdfUtils.getVariableForExchangeItem(this.netcdfReader, item); return NetcdfUtils.readDataForVariableFor2DGridForSingleTimeAndRealization(variable, realizationDimensionIndex, realizationIndex, timeDimensionIndex, timeIndex, dimensionIndexToFlip); } public void writeDataForExchangeItemForSingleTime(IExchangeItem item, int timeDimensionIndex, int timeIndex, double[] values) { - Variable variable = NetcdfUtils.getVariableForExchangeItem(netcdfFileWriter.getNetcdfFile(), item); - NetcdfUtils.writeDataForVariableForSingleTime(this.netcdfFileWriter, variable, timeDimensionIndex, timeIndex, values); + Variable variable = NetcdfUtils.getVariableForExchangeItem(this.netcdfWriter.getOutputFile(), item); + NetcdfUtils.writeDataForVariableForSingleTime(this.netcdfWriter, variable, timeDimensionIndex, timeIndex, values); } public void writeDataForExchangeItemForSingleTimeSingleLocation(IExchangeItem item, int timeIndex, int stationDimensionIndex, int stationIndex, double[] values) { - Variable variable = NetcdfUtils.getVariableForExchangeItem(netcdfFileWriter.getNetcdfFile(), item); - NetcdfUtils.writeDataForVariableForSingleTimeSingleLocation(this.netcdfFileWriter, variable, timeIndex, stationDimensionIndex, stationIndex, values); + Variable variable = NetcdfUtils.getVariableForExchangeItem(netcdfWriter.getOutputFile(), item); + NetcdfUtils.writeDataForVariableForSingleTimeSingleLocation(this.netcdfWriter, variable, timeIndex, stationDimensionIndex, stationIndex, values); } public void writeDataForExchangeItemForSingleTimeSingleLocationSingleRealization(IExchangeItem item, int timeIndex, int realizationDimensionIndex, int realizationIndex, int stationDimensionIndex, int stationIndex, double[] values) { - Variable variable = NetcdfUtils.getVariableForExchangeItem(netcdfFileWriter.getNetcdfFile(), item); - NetcdfUtils.writeDataForVariableForSingleTimeSingleLocationSingleRealization(this.netcdfFileWriter, variable, timeIndex, realizationDimensionIndex, realizationIndex, stationDimensionIndex, stationIndex, values); + Variable variable = NetcdfUtils.getVariableForExchangeItem(netcdfWriter.getOutputFile(), item); + NetcdfUtils.writeDataForVariableForSingleTimeSingleLocationSingleRealization(this.netcdfWriter, variable, timeIndex, realizationDimensionIndex, realizationIndex, stationDimensionIndex, stationIndex, values); } } diff --git a/core/java/src/org/openda/exchange/dataobjects/NetcdfFileConcatenater.java b/core/java/src/org/openda/exchange/dataobjects/NetcdfFileConcatenator.java similarity index 87% rename from core/java/src/org/openda/exchange/dataobjects/NetcdfFileConcatenater.java rename to core/java/src/org/openda/exchange/dataobjects/NetcdfFileConcatenator.java index 18d6a5dbc..da970aba2 100644 --- a/core/java/src/org/openda/exchange/dataobjects/NetcdfFileConcatenater.java +++ b/core/java/src/org/openda/exchange/dataobjects/NetcdfFileConcatenator.java @@ -24,26 +24,32 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import ucar.ma2.*; -import ucar.nc2.*; +import ucar.nc2.Attribute; +import ucar.nc2.Dimension; +import ucar.nc2.Variable; +import ucar.nc2.Variable.Builder; import ucar.nc2.units.DateUnit; - +import ucar.nc2.NetcdfFile; +import ucar.nc2.NetcdfFiles; import java.io.File; import java.io.IOException; import java.util.*; +import ucar.nc2.write.NetcdfFormatWriter; -public class NetcdfFileConcatenater { +public class NetcdfFileConcatenator { - private static final Logger LOGGER = LoggerFactory.getLogger(NetcdfFileConcatenater.class); + private static final Logger LOGGER = LoggerFactory.getLogger(NetcdfFileConcatenator.class); public static void main(String[] arguments) { if (arguments.length < 2) { - throw new IllegalArgumentException("NetcdfFileConcatenater expects at least two arguments:\n" + + throw new IllegalArgumentException("NetcdfFileConcatenator expects at least two arguments:\n" + " and "); } boolean useOldValueOnOverlap = false; for (int i = 2; i < arguments.length; i++) { String argument = arguments[i]; String[] keyValue = StringUtilities.getKeyValuePair(argument); + assert keyValue != null; String key = keyValue[0]; String value = keyValue[1]; if (key.equals("useOldValueOnOverlap")) { @@ -52,10 +58,10 @@ public static void main(String[] arguments) { throw new RuntimeException("Unknown key " + key + ". Please specify only useOldValueOnOverlap as key=value pair"); } } - // file must be there. If it is indeed, the path has been made absolute by BBaction + // file must be there. If it is indeed, the path has been made absolute by BB-action File netcdfFileToBeAdded = new File(arguments[1]); - // file may be there. If so, the path has been made absolute by BBaction + // file may be there. If so, the path has been made absolute by BB-action File targetNetcdfFile = new File(arguments[0]); if (!targetNetcdfFile.isAbsolute()) { // target file is not there yet; make path absolute @@ -96,31 +102,27 @@ public static void main(String[] arguments) { } private static void addNetcdfFile(File targetNetcdfFile, File netcdfFileToBeAdded) throws IOException { - NetcdfFile netcdfToAdd = null; - try { - netcdfToAdd = NetcdfFile.open(netcdfFileToBeAdded.getAbsolutePath()); + try (NetcdfFile netcdfToAdd = NetcdfFiles.open(netcdfFileToBeAdded.getAbsolutePath())) { Dimension time = netcdfToAdd.findDimension("time"); if (time != null && time.isUnlimited()) { BBUtils.copyFile(netcdfFileToBeAdded, targetNetcdfFile); } else { rewriteNetcdfFile(targetNetcdfFile, netcdfToAdd); } - } finally { - if (netcdfToAdd != null) netcdfToAdd.close(); } } private static void concatenateNetcdfFiles(boolean useOldValueOnOverlap, List netcdfFilesToBeAdded, File targetNetcdfFile) throws IOException { NetcdfFile sourceNetcdfFile = null; - NetcdfFileWriter netcdfFileWriter = null; + NetcdfFormatWriter netcdfFileWriter = null; try { - netcdfFileWriter = NetcdfFileWriter.openExisting(targetNetcdfFile.getAbsolutePath()); + netcdfFileWriter = NetcdfFormatWriter.openExisting(targetNetcdfFile.getAbsolutePath()).build(); Map variableArraysMap = new HashMap<>(); Map timeVariableArraysMap = new HashMap<>(); for (File netcdfFileToBeAdded : netcdfFilesToBeAdded) { - sourceNetcdfFile = NetcdfFile.open(netcdfFileToBeAdded.getAbsolutePath()); + sourceNetcdfFile = NetcdfFiles.open(netcdfFileToBeAdded.getAbsolutePath()); List variablesToBeAdded = sourceNetcdfFile.getVariables(); concatenateVariables(useOldValueOnOverlap, sourceNetcdfFile, variablesToBeAdded, netcdfFileWriter, variableArraysMap, timeVariableArraysMap); @@ -145,11 +147,11 @@ private static void concatenateNetcdfFiles(boolean useOldValueOnOverlap, List variablesToBeAdded = sourceNetcdfFile.getVariables(); - netcdfFileWriter = NetcdfFileWriter.openExisting(targetNetcdfFile.getAbsolutePath()); + netcdfFileWriter = NetcdfFormatWriter.openExisting(targetNetcdfFile.getAbsolutePath()).build(); Map variableArraysMap = new HashMap<>(); Map timeVariableArraysMap = new HashMap<>(); @@ -172,14 +174,14 @@ private static void concatenateNetcdfFiles(boolean useOldValueOnOverlap, File ne } } - private static void concatenateVariables(boolean useOldValueOnOverlap, NetcdfFile sourceNetCdfFile, List variablesToBeAdded, NetcdfFileWriter netcdfFileWriter, Map variableArraysMap, Map timeVariableArraysMap) throws IOException { + private static void concatenateVariables(boolean useOldValueOnOverlap, NetcdfFile sourceNetCdfFile, List variablesToBeAdded, NetcdfFormatWriter netcdfFileWriter, Map variableArraysMap, Map timeVariableArraysMap) throws IOException { for (Variable variable : variablesToBeAdded) { if (NetcdfUtils.isTimeVariable(variable)) continue; Variable timeVariableToBeAdded = NetcdfUtils.findTimeVariableForVariable(variable, sourceNetCdfFile); if (timeVariableToBeAdded == null) continue; - Variable targetVariable = netcdfFileWriter.findVariable(variable.getFullNameEscaped()); + Variable targetVariable = netcdfFileWriter.findVariable(variable.getFullName()); if (targetVariable == null) continue; - Variable timeVariableTarget = NetcdfUtils.findTimeVariableForVariable(targetVariable, netcdfFileWriter.getNetcdfFile()); + Variable timeVariableTarget = NetcdfUtils.findTimeVariableForVariable(targetVariable, netcdfFileWriter.getOutputFile()); if (timeVariableTarget == null) continue; List addedDimensions = variable.getDimensions(); @@ -197,13 +199,18 @@ private static void concatenateVariables(boolean useOldValueOnOverlap, NetcdfFil int targetLocationDimensionLength = targetLocationDimension.getLength(); if (addedLocationDimension.getLength() != targetLocationDimensionLength) throw new RuntimeException("Variables from source and target must have same location dimension size"); } - double[] targetValues = (double[]) targetVariable.read().get1DJavaArray(Double.TYPE); - double[] addedValues = (double[]) variable.read().get1DJavaArray(Double.TYPE); - - Array read = timeVariableTarget.read(); - double[] timesTarget = (double[]) read.get1DJavaArray(Double.TYPE); + NetcdfFile reader = NetcdfFiles.open(netcdfFileWriter.getOutputFile().getLocation()); + Variable var_to_read = reader.findVariable(targetVariable.getFullName()); + assert var_to_read != null; + double[] targetValues = (double[]) var_to_read.read().get1DJavaArray(DataType.DOUBLE); + double[] addedValues = (double[]) variable.read().get1DJavaArray(DataType.DOUBLE); + + var_to_read = reader.findVariable(timeVariableTarget.getFullName()); + assert var_to_read != null; + double[] timesTarget = (double[]) var_to_read.read().get1DJavaArray(DataType.DOUBLE); + reader.close(); String timeVariableTargetUnitsString = timeVariableTarget.getUnitsString(); - double[] timesToBeAdded = (double[]) timeVariableToBeAdded.read().get1DJavaArray(Double.TYPE); + double[] timesToBeAdded = (double[]) timeVariableToBeAdded.read().get1DJavaArray(DataType.DOUBLE); String timeVariableToBeAddedUnitsString = timeVariableToBeAdded.getUnitsString(); DateUnit targetDateUnit; DateUnit toBeAddedDateUnit; @@ -238,14 +245,14 @@ private static void concatenateVariables(boolean useOldValueOnOverlap, NetcdfFil private static void rewriteNetcdfFile(File targetNetcdfFile, NetcdfFile netcdfToAdd) throws IOException { - NetcdfFileWriter netcdfWriter = null; + NetcdfFormatWriter netcdfWriter = null; try { - netcdfWriter = NetcdfFileWriter.createNew(NetcdfFileWriter.Version.netcdf3, targetNetcdfFile.getCanonicalPath()); + NetcdfFormatWriter.Builder netcdfBuilder = NetcdfFormatWriter.createNewNetcdf3(targetNetcdfFile.getCanonicalPath()); - redefineVariablesAndDimensions(netcdfToAdd, netcdfWriter); + redefineVariablesAndDimensions(netcdfToAdd, netcdfBuilder); - netcdfWriter.create(); + netcdfWriter = netcdfBuilder.build(); writeValues(netcdfToAdd, netcdfWriter); } catch (InvalidRangeException e) { @@ -255,11 +262,11 @@ private static void rewriteNetcdfFile(File targetNetcdfFile, NetcdfFile netcdfTo } } - private static void writeValues(NetcdfFile netcdfToAdd, NetcdfFileWriter netcdfWriter) throws IOException, InvalidRangeException { + private static void writeValues(NetcdfFile netcdfToAdd, NetcdfFormatWriter netcdfWriter) throws IOException, InvalidRangeException { List variables = netcdfToAdd.getVariables(); for (Variable variable : variables) { - String fullNameEscaped = variable.getFullNameEscaped(); - Variable netcdfFileVariable = netcdfWriter.findVariable(fullNameEscaped); + String fullName = variable.getFullName(); + Variable netcdfFileVariable = netcdfWriter.findVariable(fullName); Array read = variable.read(); netcdfWriter.write(netcdfFileVariable, read); } @@ -348,13 +355,13 @@ private static void fillFirstPartOf2DArray(double[] targetValues, double[] times } } - private static void redefineVariablesAndDimensions(NetcdfFile source, NetcdfFileWriter target) { + private static void redefineVariablesAndDimensions(NetcdfFile source, NetcdfFormatWriter.Builder target) { - List dimensions = source.getDimensions(); + List dimensions = source.getRootGroup().getDimensions(); for (Dimension dimension : dimensions) { - String fullNameEscaped = dimension.getFullNameEscaped(); - if ("time".equals(fullNameEscaped)) continue; - target.addDimension(null, fullNameEscaped, dimension.getLength()); + String fullName = dimension.getName(); + if ("time".equals(fullName)) continue; + target.addDimension(fullName, dimension.getLength()); } Dimension timeDimension = target.addUnlimitedDimension("time"); @@ -362,23 +369,22 @@ private static void redefineVariablesAndDimensions(NetcdfFile source, NetcdfFile rewriteVariables(target, variables, timeDimension); } - private static void rewriteVariables(NetcdfFileWriter netcdf, List variables, Dimension timeDimension) { + private static void rewriteVariables(NetcdfFormatWriter.Builder netcdf, List variables, Dimension timeDimension) { for (Variable variable : variables) { List newDimensions = getDimensions(variable, timeDimension); - String fullNameEscaped = variable.getFullNameEscaped(); - Variable newVariable = netcdf.addVariable(null, fullNameEscaped, variable.getDataType(), newDimensions); - List attributes = variable.getAttributes(); - for (Attribute attribute : attributes) { - netcdf.addVariableAttribute(newVariable, attribute); + String fullName = variable.getFullName(); + Builder newVariable = netcdf.addVariable(fullName, variable.getDataType(), newDimensions); + for (Attribute attribute : variable.attributes()) { + newVariable.addAttribute(attribute); } } } private static List getDimensions(Variable variable, Dimension timeDimension) { - List dimensionsAll = variable.getDimensionsAll(); + List dimensionsAll = variable.getDimensions(); List newDimensions = new ArrayList<>(dimensionsAll.size()); for (Dimension dimension : dimensionsAll) { - if (dimension.getFullNameEscaped().equals("time")) { + if (dimension.getName().equals("time")) { newDimensions.add(timeDimension); } else { newDimensions.add(dimension); diff --git a/core/java/src/org/openda/exchange/dataobjects/NetcdfUtils.java b/core/java/src/org/openda/exchange/dataobjects/NetcdfUtils.java index e818108d1..375641320 100644 --- a/core/java/src/org/openda/exchange/dataobjects/NetcdfUtils.java +++ b/core/java/src/org/openda/exchange/dataobjects/NetcdfUtils.java @@ -29,15 +29,15 @@ import org.openda.utils.geometry.GeometryUtils; import ucar.ma2.*; import ucar.nc2.NetcdfFile; -import ucar.nc2.NetcdfFileWriter; +import ucar.nc2.write.NetcdfFormatWriter; import ucar.nc2.Variable; import ucar.nc2.Dimension; import ucar.nc2.Attribute; -import ucar.nc2.NCdumpW; - +import ucar.nc2.util.CancelTask; +import ucar.nc2.write.Ncdump; import ucar.nc2.units.DateUnit; - +import ucar.nc2.dataset.NetcdfDatasets; import java.io.File; import java.io.IOException; import java.io.StringWriter; @@ -112,6 +112,7 @@ public class NetcdfUtils { //dimension lengths. private static final int CHARLENGTH_ID = 64; + private static final boolean isUnsigned = false; /** * Converts the data in the given netcdf file to text format and returns this as a String. @@ -124,15 +125,19 @@ public class NetcdfUtils { * @throws IOException */ public static String netcdfFileToString(File netcdfFile, String variableNames) throws IOException { - boolean printAllVariables = false; + String options; if (variableNames == null || variableNames.isEmpty()) { - printAllVariables = true; - variableNames = null; + options = "-vall"; + } else { + options = "-v " + variableNames; } + Writer writer = new StringWriter(); - NCdumpW.print(netcdfFile.getAbsolutePath(), writer, printAllVariables, false, false, true, variableNames, null); - return writer.toString(); + try (NetcdfFile nc = NetcdfDatasets.openFile(netcdfFile.getAbsolutePath(), new myCancelTask())) { + Ncdump.ncdump(nc, options, writer, null); + return writer.toString(); + } } /** @@ -571,7 +576,7 @@ private static double[] readTimes(Variable timeVariable) throws IOException { if ((timeVariable != null) && timeVariable.isCoordinateVariable()) { //read times. ucar.ma2.Array timesArray = timeVariable.read(); - double[] times = (double[]) timesArray.get1DJavaArray(double.class); + double[] times = (double[]) timesArray.get1DJavaArray(DataType.DOUBLE); //convert times. convertedTimes = new double[times.length]; @@ -602,7 +607,7 @@ private static double[] readTimes(Variable timeVariable) throws IOException { public static Object readData(Variable variable) { double[] values; try { - values = (double[]) variable.read().get1DJavaArray(double.class); + values = (double[]) variable.read().get1DJavaArray(DataType.DOUBLE); } catch (IOException e) { throw new RuntimeException("Error while reading data from netcdf variable '" + variable.getShortName() + "'. Message was: " + e.getMessage(), e); @@ -643,7 +648,7 @@ public static double[] readSelectedData(Variable variable, int[] origin, int[] s array = array.flip(dimensionIndexToFlip); } - double[] values = (double[]) array.get1DJavaArray(Double.class); + double[] values = (double[]) array.get1DJavaArray(DataType.DOUBLE); //apply scale factor and offset and replace missing values with Double.NaN. double missingValue = getMissingValueDouble(variable); @@ -715,7 +720,7 @@ public static double[] readDataForVariableFor2DGridForSingleTimeAndRealization(V return readSelectedData(variable, origin, sizeArray, dimensionIndexToFlip); } - public static void writeDataForVariableForSingleTime(NetcdfFileWriter netcdfFileWriter, Variable variable, int timeDimensionIndex, int timeIndex, double[] values) { + public static void writeDataForVariableForSingleTime(NetcdfFormatWriter NetcdfFormatWriter, Variable variable, int timeDimensionIndex, int timeIndex, double[] values) { int[] origin = createOrigin(variable); int[] sizeArray = variable.getShape(); @@ -723,10 +728,10 @@ public static void writeDataForVariableForSingleTime(NetcdfFileWriter netcdfFile origin[timeDimensionIndex] = timeIndex; sizeArray[timeDimensionIndex] = 1; - writeSelectedData(netcdfFileWriter, variable, origin, sizeArray, values); + writeSelectedData(NetcdfFormatWriter, variable, origin, sizeArray, values); } - public static void writeDataForVariableForSingleTimeSingleLocation(NetcdfFileWriter netcdfFileWriter, Variable variable, int timeIndex, int stationDimensionIndex, int stationIndex, double[] values) { + public static void writeDataForVariableForSingleTimeSingleLocation(NetcdfFormatWriter NetcdfFormatWriter, Variable variable, int timeIndex, int stationDimensionIndex, int stationIndex, double[] values) { int[] origin = createOrigin(variable); int[] sizeArray = variable.getShape(); @@ -738,10 +743,10 @@ public static void writeDataForVariableForSingleTimeSingleLocation(NetcdfFileWri origin[stationDimensionIndex] = stationIndex; sizeArray[stationDimensionIndex] = 1; - writeSelectedData(netcdfFileWriter, variable, origin, sizeArray, values); + writeSelectedData(NetcdfFormatWriter, variable, origin, sizeArray, values); } - public static void writeDataForVariableForSingleTimeSingleLocationSingleRealization(NetcdfFileWriter netcdfFileWriter, Variable variable, + public static void writeDataForVariableForSingleTimeSingleLocationSingleRealization(NetcdfFormatWriter NetcdfFormatWriter, Variable variable, int timeIndex, int realizationDimensionIndex, int realizationIndex, int stationDimensionIndex, int stationIndex, double[] values) { int[] origin = createOrigin(variable); int[] sizeArray = variable.getShape(); @@ -756,10 +761,10 @@ public static void writeDataForVariableForSingleTimeSingleLocationSingleRealizat origin[stationDimensionIndex] = stationIndex; sizeArray[stationDimensionIndex] = 1; - writeSelectedData(netcdfFileWriter, variable, origin, sizeArray, values); + writeSelectedData(NetcdfFormatWriter, variable, origin, sizeArray, values); } - public static void writeSelectedData(NetcdfFileWriter netcdfFileWriter, Variable variable, int[] origin, int[] sizeArray, double[] values) { + public static void writeSelectedData(NetcdfFormatWriter NetcdfFormatWriter, Variable variable, int[] origin, int[] sizeArray, double[] values) { //replace NaN values with missing value for variable. double missingValue = getMissingValueDouble(variable); if (!Double.isNaN(missingValue)) { @@ -780,10 +785,10 @@ public static void writeSelectedData(NetcdfFileWriter netcdfFileWriter, Variable //write data. try { - netcdfFileWriter.write(variable, origin, array); + NetcdfFormatWriter.write(variable, origin, array); } catch (IOException | InvalidRangeException e) { throw new RuntimeException("Error while writing data to netcdf variable '" + variable.getShortName() - + "' in netcdf file " + netcdfFileWriter.getNetcdfFile().getLocation() + ". Message was: " + e.getMessage(), e); + + "' in netcdf file " + NetcdfFormatWriter.getOutputFile().getLocation() + ". Message was: " + e.getMessage(), e); } } @@ -981,28 +986,28 @@ private static DataType getDataType(IExchangeItem.ValueType valueType) { * Each exchangeItem stores a scalar timeseries for a single location (and a single ensemble member). * All exchangeItems must use the same ensemble member indices. */ - public static void createMetadataAndDataVariablesForScalars(NetcdfFileWriter netcdfFileWriter, + public static void createMetadataAndDataVariablesForScalars(NetcdfFormatWriter.Builder NetcdfWriterBuilder, List exchangeItems, Map> ensembleExchangeItems, Map timeInfoTimeDimensionMap, String stationNameVarName, String stationDimensionVarName, int stationCount, int ensembleMemberCount) { //create stations variable. - Dimension stationDimension = createStationsVariable(netcdfFileWriter, stationNameVarName, stationDimensionVarName, stationCount); + Dimension stationDimension = createStationsVariable(NetcdfWriterBuilder, stationNameVarName, stationDimensionVarName, stationCount); //create realization variable. Dimension realizationDimension = null; if (ensembleMemberCount > 0) { - realizationDimension = createRealizationVariable(netcdfFileWriter, ensembleMemberCount); + realizationDimension = createRealizationVariable(NetcdfWriterBuilder, ensembleMemberCount); } //create data variables. - NetcdfUtils.createDataVariables(netcdfFileWriter, exchangeItems, ensembleExchangeItems, realizationDimension, stationDimension, timeInfoTimeDimensionMap, null); + NetcdfUtils.createDataVariables(NetcdfWriterBuilder, exchangeItems, ensembleExchangeItems, realizationDimension, stationDimension, timeInfoTimeDimensionMap, null); } /** * Creates metadata for the given exchangeItem in the given netcdfFile, if not present yet. */ - public static void createMetadataAndDataVariablesForGrids(NetcdfFileWriter netcdfFileWriter, List exchangeItems, Map timeInfoTimeDimensionMap, + public static void createMetadataAndDataVariablesForGrids(NetcdfFormatWriter.Builder NetcdfWriterBuilder, List exchangeItems, Map timeInfoTimeDimensionMap, Map geometryInfoGridVariablePropertiesMap) { int[] uniqueTimeVariableCount = new int[]{0}; @@ -1012,22 +1017,22 @@ public static void createMetadataAndDataVariablesForGrids(NetcdfFileWriter netcd Dimension timeDimension = null; ITimeInfo timeInfo = exchangeItem.getTimeInfo(); if (timeInfo != null && timeInfo.getTimes() != null) {//if variable depends on time. - timeDimension = createTimeVariable(netcdfFileWriter, timeInfo, uniqueTimeVariableCount, timeInfoTimeDimensionMap); + timeDimension = createTimeVariable(NetcdfWriterBuilder, timeInfo, uniqueTimeVariableCount, timeInfoTimeDimensionMap); } //create spatial coordinate variables, if not present yet. //this only adds spatial dimensions, this does not add spatial variables with coordinates, //because the coordinates are usually not available in exchangeItems that come from models. if (!GeometryUtils.isScalar(exchangeItem.getGeometryInfo())) {//if grid. - createGridVariables(netcdfFileWriter, exchangeItem.getGeometryInfo(), uniqueGeometryCount, geometryInfoGridVariablePropertiesMap); + createGridVariables(NetcdfWriterBuilder, exchangeItem.getGeometryInfo(), uniqueGeometryCount, geometryInfoGridVariablePropertiesMap); } //create data variable. - createDataVariable(netcdfFileWriter, exchangeItem, timeDimension, null, null, geometryInfoGridVariablePropertiesMap); + createDataVariable(NetcdfWriterBuilder, exchangeItem, timeDimension, null, null, geometryInfoGridVariablePropertiesMap); } } - private static Dimension createTimeVariable(NetcdfFileWriter netcdfFileWriter, ITimeInfo timeInfo, int[] uniqueTimeVariableCount, Map timeInfoTimeDimensionMap) { + private static Dimension createTimeVariable(NetcdfFormatWriter.Builder NetcdfWriterBuilder, ITimeInfo timeInfo, int[] uniqueTimeVariableCount, Map timeInfoTimeDimensionMap) { Dimension timeDimension = timeInfoTimeDimensionMap.get(timeInfo); if (timeDimension == null) {//if timeVariable with correct times not yet present. @@ -1038,7 +1043,7 @@ private static Dimension createTimeVariable(NetcdfFileWriter netcdfFileWriter, I String postfix = uniqueTimeVariableCount[0] <= 1 ? "" : String.valueOf(uniqueTimeVariableCount[0]); String timeVariableName = TIME_VARIABLE_NAME + postfix; int timeCount = timeInfo.getTimes().length; - timeDimension = createTimeVariable(netcdfFileWriter, timeVariableName, timeCount, NetcdfUtils.createTimeUnitString()); + timeDimension = createTimeVariable(NetcdfWriterBuilder, timeVariableName, timeCount, NetcdfUtils.createTimeUnitString()); //put timeDimension in map so that it can be re-used later by other variables in the same file. timeInfoTimeDimensionMap.put(timeInfo, timeDimension); @@ -1055,23 +1060,23 @@ private static Dimension createTimeVariable(NetcdfFileWriter netcdfFileWriter, I * @param timeUnitString * @return timeDimension */ - public static Dimension createTimeVariable(NetcdfFileWriter dataFile, String timeVariableName, int timeCount, String timeUnitString) { + public static Dimension createTimeVariable(NetcdfFormatWriter.Builder dataFile, String timeVariableName, int timeCount, String timeUnitString) { //create time dimension. Dimension timeDimension; if (timeCount == -1 || timeCount == 0) { timeDimension = dataFile.addUnlimitedDimension(timeVariableName); } else { - timeDimension = dataFile.addDimension(null,timeVariableName, timeCount); + timeDimension = dataFile.addDimension(timeVariableName, timeCount); } //create time variable. - Variable myVar = dataFile.addVariable(null,timeVariableName, ucar.ma2.DataType.DOUBLE, Arrays.asList(timeDimension) ); - dataFile.addVariableAttribute(myVar, new Attribute (STANDARD_NAME_ATTRIBUTE_NAME, TIME_VARIABLE_NAME) ); - dataFile.addVariableAttribute(myVar, new Attribute(LONG_NAME_ATTRIBUTE_NAME, TIME_VARIABLE_NAME)); - dataFile.addVariableAttribute(myVar, new Attribute(UNITS_ATTRIBUTE_NAME, timeUnitString)); + Variable.Builder myVar = dataFile.addVariable(timeVariableName, DataType.DOUBLE, Arrays.asList(timeDimension) ); + myVar.addAttribute(new Attribute (STANDARD_NAME_ATTRIBUTE_NAME, TIME_VARIABLE_NAME) ); + myVar.addAttribute(new Attribute(LONG_NAME_ATTRIBUTE_NAME, TIME_VARIABLE_NAME)); + myVar.addAttribute(new Attribute(UNITS_ATTRIBUTE_NAME, timeUnitString)); //use default calendar. - dataFile.addVariableAttribute(myVar, new Attribute(CALENDAR_ATTRIBUTE_NAME, DEFAULT_CALENDAR_ATTRIBUTE_VALUE)); - dataFile.addVariableAttribute(myVar, new Attribute(AXIS_ATTRIBUTE_NAME, T_AXIS)); + dataFile.addAttribute(new Attribute(CALENDAR_ATTRIBUTE_NAME, DEFAULT_CALENDAR_ATTRIBUTE_VALUE)); + dataFile.addAttribute(new Attribute(AXIS_ATTRIBUTE_NAME, T_AXIS)); return timeDimension; } @@ -1080,11 +1085,11 @@ public static Dimension createTimeVariable(NetcdfFileWriter dataFile, String tim * and stores the properties of the created grid variables in the returned geometryGridVariablePropertiesMap. * These properties are used later in the method NetcdfUtils.createDataVariables. * - * @param netcdfFileWriter + * @param NetcdfWriterBuilder * @param geometryInfos * @return geometryGridVariablePropertiesMap */ - public static Map createGridVariables(NetcdfFileWriter netcdfFileWriter, IGeometryInfo[] geometryInfos) { + public static Map createGridVariables(NetcdfFormatWriter.Builder NetcdfWriterBuilder, IGeometryInfo[] geometryInfos) { Map geometryGridVariablePropertiesMap = new LinkedHashMap<>(); int[] uniqueGeometryCount = new int[]{0}; @@ -1093,13 +1098,13 @@ public static Map createGridVariables(Net continue; } - createGridVariables(netcdfFileWriter, geometryInfo, uniqueGeometryCount, geometryGridVariablePropertiesMap); + createGridVariables(NetcdfWriterBuilder, geometryInfo, uniqueGeometryCount, geometryGridVariablePropertiesMap); } return geometryGridVariablePropertiesMap; } - private static void createGridVariables(NetcdfFileWriter netcdfFileWriter, IGeometryInfo geometryInfo, int[] uniqueGeometryCount, + private static void createGridVariables(NetcdfFormatWriter.Builder NetcdfWriterBuilder, IGeometryInfo geometryInfo, int[] uniqueGeometryCount, Map geometryInfoGridVariablePropertiesMap) { GridVariableProperties gridVariableProperties = geometryInfoGridVariablePropertiesMap.get(geometryInfo); @@ -1115,7 +1120,7 @@ private static void createGridVariables(NetcdfFileWriter netcdfFileWriter, IGeom //create face dimension. int gridCellCount = ((IrregularGridGeometryInfo) geometryInfo).getCellCount(); - Dimension faceDimension = netcdfFileWriter.addDimension(null, faceDimensionName, gridCellCount); + Dimension faceDimension = NetcdfWriterBuilder.addDimension(faceDimensionName, gridCellCount); //put faceDimension in map so that it can be re-used later by other variables in the same file. gridVariableProperties = new GridVariableProperties(); @@ -1128,11 +1133,11 @@ private static void createGridVariables(NetcdfFileWriter netcdfFileWriter, IGeom //create face dimension. int gridCellCount = ((LayeredIrregularGridGeometryInfo) geometryInfo).getCellCount(); - Dimension faceDimension = netcdfFileWriter.addDimension(null,faceDimensionName, gridCellCount); + Dimension faceDimension = NetcdfWriterBuilder.addDimension(faceDimensionName, gridCellCount); //create z dimension. int layerCount = ((LayeredIrregularGridGeometryInfo) geometryInfo).getLayerCount(); - Dimension layerDimension = netcdfFileWriter.addDimension(null,layerDimensionName, layerCount); + Dimension layerDimension = NetcdfWriterBuilder.addDimension(layerDimensionName, layerCount); //put dimensions in map so that they can be re-used later by other variables in the same file. gridVariableProperties = new GridVariableProperties(); @@ -1141,7 +1146,7 @@ private static void createGridVariables(NetcdfFileWriter netcdfFileWriter, IGeom geometryInfoGridVariablePropertiesMap.put(geometryInfo, gridVariableProperties); //create z variable. - createZVariable(netcdfFileWriter, layerDimension); + createZVariable(NetcdfWriterBuilder, layerDimension); } else if (geometryInfo instanceof ArrayGeometryInfo && !(geometryInfo instanceof PointGeometryInfo)) { String yDimensionName = Y_VARIABLE_NAME + postfix; @@ -1150,8 +1155,8 @@ private static void createGridVariables(NetcdfFileWriter netcdfFileWriter, IGeom //create y,x dimensions. int rowCount = ((ArrayGeometryInfo) geometryInfo).getLatitudeArray().length(); int columnCount = ((ArrayGeometryInfo) geometryInfo).getLongitudeArray().length(); - Dimension yDimension = netcdfFileWriter.addDimension(null,yDimensionName, rowCount); - Dimension xDimension = netcdfFileWriter.addDimension(null,xDimensionName, columnCount); + Dimension yDimension = NetcdfWriterBuilder.addDimension(yDimensionName, rowCount); + Dimension xDimension = NetcdfWriterBuilder.addDimension(xDimensionName, columnCount); //put dimensions in map so that these can be re-used later by other variables in the same file. gridVariableProperties = new GridVariableProperties(); @@ -1162,7 +1167,7 @@ private static void createGridVariables(NetcdfFileWriter netcdfFileWriter, IGeom IQuantityInfo xQuantityInfo = ((ArrayGeometryInfo) geometryInfo).getLongitudeQuantityInfo(); if (yQuantityInfo != null && xQuantityInfo != null) { //create x and y variables. - createYX1DVariables(netcdfFileWriter, yDimension, xDimension, PROJECTION_Y_COORDINATE, PROJECTION_X_COORDINATE, + createYX1DVariables(NetcdfWriterBuilder, yDimension, xDimension, PROJECTION_Y_COORDINATE, PROJECTION_X_COORDINATE, yQuantityInfo.getQuantity(), xQuantityInfo.getQuantity(), yQuantityInfo.getUnit(), xQuantityInfo.getUnit(), gridVariableProperties); //TODO add grid mapping variable. This requires information about the coordinate system used for the grid. AK @@ -1189,31 +1194,31 @@ private static void createGridVariables(NetcdfFileWriter netcdfFileWriter, IGeom } } - private static void createDataVariables(NetcdfFileWriter netcdfFileWriter, List exchangeItems, Map> ensembleExchangeItems, + private static void createDataVariables(NetcdfFormatWriter.Builder NetcdfWriterBuilder, List exchangeItems, Map> ensembleExchangeItems, Dimension realizationDimension, Dimension stationDimension, Map timeInfoTimeDimensionMap, Map geometryInfoGridVariablePropertiesMap) { //non-ensemble exchange items. int[] uniqueTimeVariableCount = new int[]{0}; - createDataVariables(netcdfFileWriter, exchangeItems, null, stationDimension, timeInfoTimeDimensionMap, geometryInfoGridVariablePropertiesMap, uniqueTimeVariableCount); + createDataVariables(NetcdfWriterBuilder, exchangeItems, null, stationDimension, timeInfoTimeDimensionMap, geometryInfoGridVariablePropertiesMap, uniqueTimeVariableCount); //ensemble exchange items. if (ensembleExchangeItems != null) { for (Map ensemble : ensembleExchangeItems.values()) { Collection items = ensemble.values(); - createDataVariables(netcdfFileWriter, items, realizationDimension, stationDimension, timeInfoTimeDimensionMap, geometryInfoGridVariablePropertiesMap, uniqueTimeVariableCount); + createDataVariables(NetcdfWriterBuilder, items, realizationDimension, stationDimension, timeInfoTimeDimensionMap, geometryInfoGridVariablePropertiesMap, uniqueTimeVariableCount); } } } - private static void createDataVariables(NetcdfFileWriter netcdfFileWriter, Collection exchangeItems, Dimension realizationDimension, Dimension stationDimension, + private static void createDataVariables(NetcdfFormatWriter.Builder NetcdfWriterBuilder, Collection exchangeItems, Dimension realizationDimension, Dimension stationDimension, Map timeInfoTimeDimensionMap, Map geometryInfoGridVariablePropertiesMap, int[] uniqueTimeVariableCount) { for (IExchangeItem item : exchangeItems) { String variableName = getVariableName(item); - if (netcdfFileWriter.findVariable(variableName) != null) {//if variable already exists. + //if (NetcdfWriter.findVariable(variableName) != null) {//if variable already exists. //if the variable already exists, we do not need to add it again. This can happen for scalar time series. - continue; - } + // continue; + //} //if variable does not exist yet. //create time coordinate variable, if not present yet. @@ -1221,28 +1226,28 @@ private static void createDataVariables(NetcdfFileWriter netcdfFileWriter, Colle Dimension timeDimension = null; ITimeInfo timeInfo = item.getTimeInfo(); if (timeInfo != null && timeInfo.getTimes() != null) {//if variable depends on time. - timeDimension = createTimeVariable(netcdfFileWriter, timeInfo, uniqueTimeVariableCount, timeInfoTimeDimensionMap); + timeDimension = createTimeVariable(NetcdfWriterBuilder, timeInfo, uniqueTimeVariableCount, timeInfoTimeDimensionMap); } - NetcdfUtils.createDataVariable(netcdfFileWriter, item, timeDimension, realizationDimension, stationDimension, geometryInfoGridVariablePropertiesMap); + NetcdfUtils.createDataVariable(NetcdfWriterBuilder, item, timeDimension, realizationDimension, stationDimension, geometryInfoGridVariablePropertiesMap); } } - public static void createDataVariables(NetcdfFileWriter netcdfFileWriter, Collection exchangeItems, + public static void createDataVariables(NetcdfFormatWriter.Builder NetcdfWriterBuilder, Collection exchangeItems, Dimension timeDimension, Dimension realizationDimension, Dimension stationDimension, Map geometryInfoGridVariablePropertiesMap) { for (IExchangeItem item : exchangeItems) { String variableName = getVariableName(item); - if (netcdfFileWriter.findVariable(variableName) != null) {//if variable already exists. + //if (NetcdfWriter.findVariable(variableName) != null) {//if variable already exists. //if the variable already exists, we do not need to add it again. This can happen for scalar time series. - continue; - } + // continue; + //} //if variable does not exist yet. - NetcdfUtils.createDataVariable(netcdfFileWriter, item, timeDimension, realizationDimension, stationDimension, geometryInfoGridVariablePropertiesMap); + NetcdfUtils.createDataVariable(NetcdfWriterBuilder, item, timeDimension, realizationDimension, stationDimension, geometryInfoGridVariablePropertiesMap); } } - private static void createDataVariable(NetcdfFileWriter netcdfFileWriter, IExchangeItem exchangeItem, Dimension timeDimension, Dimension realizationDimension, Dimension stationDimension, + private static void createDataVariable(NetcdfFormatWriter.Builder NetcdfWriterBuilder, IExchangeItem exchangeItem, Dimension timeDimension, Dimension realizationDimension, Dimension stationDimension, Map geometryInfoGridVariablePropertiesMap) { List dimensions = new ArrayList(); if (timeDimension != null) { @@ -1263,11 +1268,11 @@ private static void createDataVariable(NetcdfFileWriter netcdfFileWriter, IExcha DataType dataType = getDataType(exchangeItem.getValuesType()); String variableName = getVariableName(exchangeItem); - Variable myVar = netcdfFileWriter.addVariable(null,variableName, dataType, dimensions); + Variable.Builder myVar = NetcdfWriterBuilder.addVariable(variableName, dataType, dimensions); //at this point standard_name of data is unknown, so cannot add the optional standard_name attribute here. - netcdfFileWriter.addVariableAttribute(myVar, new Attribute(LONG_NAME_ATTRIBUTE_NAME, variableName)); - netcdfFileWriter.addVariableAttribute(myVar, new Attribute(UNITS_ATTRIBUTE_NAME, exchangeItem.getQuantityInfo().getUnit() )); - netcdfFileWriter.addVariableAttribute(myVar, new Attribute(FILL_VALUE_ATTRIBUTE_NAME, DEFAULT_FILL_VALUE_DOUBLE)); + myVar.addAttribute(new Attribute(LONG_NAME_ATTRIBUTE_NAME, variableName)); + myVar.addAttribute(new Attribute(UNITS_ATTRIBUTE_NAME, exchangeItem.getQuantityInfo().getUnit() )); + myVar.addAttribute(new Attribute(FILL_VALUE_ATTRIBUTE_NAME, DEFAULT_FILL_VALUE_DOUBLE)); //TODO add grid mapping attribute. AK // if (gridVariableProperties != null) { @@ -1281,7 +1286,7 @@ private static void createDataVariable(NetcdfFileWriter netcdfFileWriter, IExcha /** * Creates y and x variables with the given properties. * - * @param netcdfFileWriter + * @param NetcdfWriterBuilder * @param yDimension * @param xDimension * @param yStandardName @@ -1292,29 +1297,29 @@ private static void createDataVariable(NetcdfFileWriter netcdfFileWriter, IExcha * @param xUnits * @param gridVariableProperties */ - private static void createYX1DVariables(NetcdfFileWriter netcdfFileWriter, + private static void createYX1DVariables(NetcdfFormatWriter.Builder NetcdfWriterBuilder, Dimension yDimension, Dimension xDimension, String yStandardName, String xStandardName, String yLongName, String xLongName, String yUnits, String xUnits, GridVariableProperties gridVariableProperties) { String yVariableName = yDimension.getShortName(); String xVariableName = xDimension.getShortName(); - createCoordinateVariable(netcdfFileWriter, yVariableName, yDimension, yStandardName, yLongName, yUnits, Y_AXIS, + createCoordinateVariable(NetcdfWriterBuilder, yVariableName, yDimension, yStandardName, yLongName, yUnits, Y_AXIS, DEFAULT_FILL_VALUE_DOUBLE); - createCoordinateVariable(netcdfFileWriter, xVariableName, xDimension, xStandardName, xLongName, xUnits, X_AXIS, + createCoordinateVariable(NetcdfWriterBuilder, xVariableName, xDimension, xStandardName, xLongName, xUnits, X_AXIS, DEFAULT_FILL_VALUE_DOUBLE); gridVariableProperties.setY1DVariableName(yVariableName); gridVariableProperties.setX1DVariableName(xVariableName); } - private static void createZVariable(NetcdfFileWriter netcdfFileWriter, Dimension zDimension) { + private static void createZVariable(NetcdfFormatWriter.Builder NetcdfWriterBuilder, Dimension zDimension) { String zVariableName = zDimension.getShortName(); ArrayList dimensions = new ArrayList<>(); dimensions.add(zDimension); - Variable myVar = netcdfFileWriter.addVariable(null,zVariableName, DataType.DOUBLE, dimensions); - netcdfFileWriter.addVariableAttribute(myVar,new Attribute(UNITS_ATTRIBUTE_NAME, Z_UNITS)); - netcdfFileWriter.addVariableAttribute(myVar,new Attribute(AXIS_ATTRIBUTE_NAME, Z_AXIS)); - netcdfFileWriter.addVariableAttribute(myVar,new Attribute(POSITIVE_ATTRIBUTE_NAME, Z_POSITIVE)); - netcdfFileWriter.addVariableAttribute(myVar,new Attribute(FILL_VALUE_ATTRIBUTE_NAME, DEFAULT_FILL_VALUE_DOUBLE)); + Variable.Builder myVar = NetcdfWriterBuilder.addVariable(zVariableName, DataType.DOUBLE, dimensions); + myVar.addAttribute(new Attribute(UNITS_ATTRIBUTE_NAME, Z_UNITS)); + myVar.addAttribute(new Attribute(AXIS_ATTRIBUTE_NAME, Z_AXIS)); + myVar.addAttribute(new Attribute(POSITIVE_ATTRIBUTE_NAME, Z_POSITIVE)); + myVar.addAttribute(new Attribute(FILL_VALUE_ATTRIBUTE_NAME, DEFAULT_FILL_VALUE_DOUBLE)); } /** @@ -1328,83 +1333,83 @@ private static void createZVariable(NetcdfFileWriter netcdfFileWriter, Dimension * @param units * @param axis */ - private static void createCoordinateVariable(NetcdfFileWriter dataFile, + private static void createCoordinateVariable(NetcdfFormatWriter.Builder dataFile, String variableName, Dimension dimension, String standardName, String longName, String units, String axis, double fillValue) { ArrayList dimensions = new ArrayList(); dimensions.add(dimension); - Variable myVar = dataFile.addVariable(null, variableName, DataType.DOUBLE, dimensions); - dataFile.addVariableAttribute(myVar,new Attribute(STANDARD_NAME_ATTRIBUTE_NAME, standardName)); - dataFile.addVariableAttribute(myVar,new Attribute(LONG_NAME_ATTRIBUTE_NAME, longName)); - dataFile.addVariableAttribute(myVar,new Attribute(UNITS_ATTRIBUTE_NAME, units)); - dataFile.addVariableAttribute(myVar,new Attribute(AXIS_ATTRIBUTE_NAME, axis)); - dataFile.addVariableAttribute(myVar,new Attribute( FILL_VALUE_ATTRIBUTE_NAME, fillValue)); + Variable.Builder myVar = dataFile.addVariable(variableName, DataType.DOUBLE, dimensions); + myVar.addAttribute(new Attribute(STANDARD_NAME_ATTRIBUTE_NAME, standardName)); + myVar.addAttribute(new Attribute(LONG_NAME_ATTRIBUTE_NAME, longName)); + myVar.addAttribute(new Attribute(UNITS_ATTRIBUTE_NAME, units)); + myVar.addAttribute(new Attribute(AXIS_ATTRIBUTE_NAME, axis)); + myVar.addAttribute(new Attribute( FILL_VALUE_ATTRIBUTE_NAME, fillValue)); } /** * Writes all metadata variables for the given maps to the given netcdfFile, i.e. times and spatial coordinates. */ - public static void writeMetadata(NetcdfFileWriter netcdfFileWriter, Map timeInfoTimeDimensionMap, + public static void writeMetadata(NetcdfFormatWriter NetcdfFormatWriter, Map timeInfoTimeDimensionMap, Map geometryInfoGridVariablePropertiesMap, String stationNameVarName, List stationIdList, List ensembleMemberIndexList) throws Exception { //write time variable values. - NetcdfUtils.writeTimeVariablesValues(netcdfFileWriter, timeInfoTimeDimensionMap); + NetcdfUtils.writeTimeVariablesValues(NetcdfFormatWriter, timeInfoTimeDimensionMap); //write geometry variable values. - NetcdfUtils.writeGridVariablesValues(netcdfFileWriter, geometryInfoGridVariablePropertiesMap); + NetcdfUtils.writeGridVariablesValues(NetcdfFormatWriter, geometryInfoGridVariablePropertiesMap); //write station_id if available. - NetcdfUtils.writeStationIdVariableValues(netcdfFileWriter, stationNameVarName, stationIdList); + NetcdfUtils.writeStationIdVariableValues(NetcdfFormatWriter, stationNameVarName, stationIdList); //write realization variable values. - NetcdfUtils.writeRealizationVariableValues(netcdfFileWriter, ensembleMemberIndexList); + NetcdfUtils.writeRealizationVariableValues(NetcdfFormatWriter, ensembleMemberIndexList); } - public static void writeStationIdVariableValues(NetcdfFileWriter netcdfFileWriter, + public static void writeStationIdVariableValues(NetcdfFormatWriter NetcdfWriter, String stationNameVarName, List stationIdList) throws Exception { if (stationIdList == null || stationIdList.isEmpty()) return; - ArrayObject.D1 statidsArray = new ArrayObject.D1(String.class, stationIdList.size()); + ArrayObject.D1 statidsArray = new ArrayObject.D1(DataType.STRING, String.class, false, stationIdList.size()); for (int i=0; i ensembleMemberIndexList) throws Exception { + private static void writeRealizationVariableValues(NetcdfFormatWriter NetcdfFormatWriter, List ensembleMemberIndexList) throws Exception { if (ensembleMemberIndexList == null || ensembleMemberIndexList.isEmpty()) return; - ArrayInt.D1 ensembleMemberIndices = new ArrayInt.D1(ensembleMemberIndexList.size()); + ArrayInt.D1 ensembleMemberIndices = new ArrayInt.D1(ensembleMemberIndexList.size(), isUnsigned); for (int n = 0; n < ensembleMemberIndexList.size(); n++) { ensembleMemberIndices.set(n, ensembleMemberIndexList.get(n)); } - Variable myVar = netcdfFileWriter.findVariable(REALIZATION_VARIABLE_NAME); - netcdfFileWriter.write(myVar, ensembleMemberIndices); + Variable myVar = NetcdfFormatWriter.findVariable(REALIZATION_VARIABLE_NAME); + NetcdfFormatWriter.write(myVar, ensembleMemberIndices); } /** * Write values for all time variables that are present. * - * @param netcdfFileWriter + * @param NetcdfFormatWriter * @param timeInfoTimeDimensionMap * @throws Exception */ - public static void writeTimeVariablesValues(NetcdfFileWriter netcdfFileWriter, + public static void writeTimeVariablesValues(NetcdfFormatWriter NetcdfFormatWriter, Map timeInfoTimeDimensionMap) throws Exception { for (Map.Entry entry : timeInfoTimeDimensionMap.entrySet()) { ITimeInfo timeInfo = entry.getKey(); Dimension timeDimension = entry.getValue(); - Variable timeVariable = netcdfFileWriter.findVariable(timeDimension.getShortName()); + Variable timeVariable = NetcdfFormatWriter.findVariable(timeDimension.getShortName()); String timeUnitString = timeVariable.findAttribute(UNITS_ATTRIBUTE_NAME).getStringValue(); DateUnit dateUnit = new DateUnit(timeUnitString); @@ -1415,11 +1420,11 @@ public static void writeTimeVariablesValues(NetcdfFileWriter netcdfFileWriter, timesArray.set(n, newTime); } - netcdfFileWriter.write(timeVariable, timesArray); + NetcdfFormatWriter.write(timeVariable, timesArray); } } - public static void writeTimeVariableSingleValue(NetcdfFileWriter netcdfFileWriter, Variable timeVariable, int timeIndex, double time) throws Exception { + public static void writeTimeVariableSingleValue(NetcdfFormatWriter NetcdfFormatWriter, Variable timeVariable, int timeIndex, double time) throws Exception { String timeUnitString = timeVariable.findAttribute(UNITS_ATTRIBUTE_NAME).getStringValue(); DateUnit dateUnit = new DateUnit(timeUnitString); @@ -1428,17 +1433,17 @@ public static void writeTimeVariableSingleValue(NetcdfFileWriter netcdfFileWrite timeArray.set(0, newTime); int[] origin = new int[]{timeIndex}; - netcdfFileWriter.write(timeVariable, origin, timeArray); + NetcdfFormatWriter.write(timeVariable, origin, timeArray); } /** * Write values for all grid variables that are present. * - * @param netcdfFileWriter + * @param NetcdfFormatWriter * @param geometryGridVariablePropertiesMap * @throws Exception */ - public static void writeGridVariablesValues(NetcdfFileWriter netcdfFileWriter, + public static void writeGridVariablesValues(NetcdfFormatWriter NetcdfFormatWriter, Map geometryGridVariablePropertiesMap) throws Exception { //this currently only writes YX1DVariables and only for ArrayGeometryInfo. @@ -1450,13 +1455,13 @@ public static void writeGridVariablesValues(NetcdfFileWriter netcdfFileWriter, String y1DVariableName = gridVariableProperties.getY1DVariableName(); String x1DVariableName = gridVariableProperties.getX1DVariableName(); if (y1DVariableName != null && x1DVariableName != null && geometryInfo instanceof ArrayGeometryInfo) { - writeYX1DVariableValues(netcdfFileWriter, (ArrayGeometryInfo) geometryInfo, y1DVariableName, x1DVariableName); + writeYX1DVariableValues(NetcdfFormatWriter, (ArrayGeometryInfo) geometryInfo, y1DVariableName, x1DVariableName); } //write z variable values (if present). String zVariableName = gridVariableProperties.getZVariableName(); if (zVariableName != null && geometryInfo instanceof LayeredIrregularGridGeometryInfo) { - writeZVariableValues(netcdfFileWriter, (LayeredIrregularGridGeometryInfo) geometryInfo, zVariableName); + writeZVariableValues(NetcdfFormatWriter, (LayeredIrregularGridGeometryInfo) geometryInfo, zVariableName); } } } @@ -1466,13 +1471,13 @@ public static void writeGridVariablesValues(NetcdfFileWriter netcdfFileWriter, * The values are taken from the given geometry and are written as coordinates * in the coordinate system that is used by the given geometry. * - * @param netcdfFileWriter + * @param NetcdfFormatWriter * @param geometryInfo * @param yVariableName * @param xVariableName * @throws Exception */ - private static void writeYX1DVariableValues(NetcdfFileWriter netcdfFileWriter, + private static void writeYX1DVariableValues(NetcdfFormatWriter NetcdfFormatWriter, ArrayGeometryInfo geometryInfo, String yVariableName, String xVariableName) throws Exception { IArray latitudeArray = geometryInfo.getLatitudeArray(); @@ -1485,8 +1490,8 @@ private static void writeYX1DVariableValues(NetcdfFileWriter netcdfFileWriter, } yArray.set(index, y); } - Variable myYVar = netcdfFileWriter.findVariable(yVariableName); - netcdfFileWriter.write(myYVar, yArray); + Variable myYVar = NetcdfFormatWriter.findVariable(yVariableName); + NetcdfFormatWriter.write(myYVar, yArray); IArray longitudeArray = geometryInfo.getLongitudeArray(); int columnCount = longitudeArray.length(); @@ -1498,18 +1503,18 @@ private static void writeYX1DVariableValues(NetcdfFileWriter netcdfFileWriter, } xArray.set(index, x); } - Variable myXVar = netcdfFileWriter.findVariable(xVariableName); - netcdfFileWriter.write(myXVar, xArray); + Variable myXVar = NetcdfFormatWriter.findVariable(xVariableName); + NetcdfFormatWriter.write(myXVar, xArray); } - private static void writeZVariableValues(NetcdfFileWriter netcdfFileWriter, LayeredIrregularGridGeometryInfo geometryInfo, String zVariableName) throws Exception { + private static void writeZVariableValues(NetcdfFormatWriter NetcdfFormatWriter, LayeredIrregularGridGeometryInfo geometryInfo, String zVariableName) throws Exception { IVector zCoordinates = geometryInfo.getZCoordinates(); ArrayDouble.D1 zArray = new ArrayDouble.D1(zCoordinates.getSize()); for (int n = 0; n < zArray.getSize(); n++) { zArray.set(n, zCoordinates.getValue(n)); } - Variable myZVar = netcdfFileWriter.findVariable(zVariableName); - netcdfFileWriter.write(myZVar, zArray); + Variable myZVar = NetcdfFormatWriter.findVariable(zVariableName); + NetcdfFormatWriter.write(myZVar, zArray); } /** @@ -1607,9 +1612,9 @@ private static boolean isAscending(double[] array) { return true; } - public static void addGlobalAttributes(NetcdfFileWriter netcdfFileWriter) { - netcdfFileWriter.addGroupAttribute(null,new Attribute("title", "Netcdf data")); - addGeneralGlobalAttributes(netcdfFileWriter); + public static void addGlobalAttributes(NetcdfFormatWriter.Builder NetcdfWriterBuilder) { + NetcdfWriterBuilder.addAttribute(new Attribute("title", "Netcdf data")); + addGeneralGlobalAttributes(NetcdfWriterBuilder); } public static double[] addMissingValuesForNonActiveGridCells(IGeometryInfo geometryInfo, double[] values) { @@ -1666,16 +1671,16 @@ public static List getStationIds(List exchangeItems, Map< /** * Add variable for station_id. */ - public static Dimension createStationsVariable(NetcdfFileWriter netcdfFileWriter, String stationNameVarName, String stationDimensionVarName, int stationCount) { - Dimension stationDimension = netcdfFileWriter.addDimension(null,stationDimensionVarName, stationCount); - Dimension charDimension = netcdfFileWriter.addDimension(null,CHAR_LEN_ID, CHARLENGTH_ID); + public static Dimension createStationsVariable(NetcdfFormatWriter.Builder NetcdfWriterBuilder, String stationNameVarName, String stationDimensionVarName, int stationCount) { + Dimension stationDimension = NetcdfWriterBuilder.addDimension(stationDimensionVarName, stationCount); + Dimension charDimension = NetcdfWriterBuilder.addDimension(CHAR_LEN_ID, CHARLENGTH_ID); ArrayList dimensions = new ArrayList(); dimensions.add(stationDimension); dimensions.add(charDimension); - Variable myVar = netcdfFileWriter.addVariable(null, stationNameVarName, DataType.CHAR, dimensions); - netcdfFileWriter.addVariableAttribute( myVar, new Attribute( LONG_NAME_ATTRIBUTE_NAME, "station identification code")); - netcdfFileWriter.addVariableAttribute( myVar, new Attribute(CF_ROLE_ATTRIBUTE_NAME, NetcdfUtils.TIME_SERIES_ID_CF_ROLE)); + Variable.Builder myVar = NetcdfWriterBuilder.addVariable(stationNameVarName, DataType.CHAR, dimensions); + myVar.addAttribute(new Attribute( LONG_NAME_ATTRIBUTE_NAME, "station identification code")); + myVar.addAttribute(new Attribute(CF_ROLE_ATTRIBUTE_NAME, NetcdfUtils.TIME_SERIES_ID_CF_ROLE)); return stationDimension; } @@ -1683,7 +1688,7 @@ public static Dimension createStationsVariable(NetcdfFileWriter netcdfFileWriter /** * Creates a realization dimension and variable. */ - public static Dimension createRealizationVariable(NetcdfFileWriter netcdfFileWriter, int ensembleMemberCount) { + public static Dimension createRealizationVariable(NetcdfFormatWriter.Builder NetcdfWriterBuilder, int ensembleMemberCount) { //See http://cf-pcmdi.llnl.gov/documents/cf-standard-names/standard-name-table/18/cf-standard-name-table.html //standard name "realization": //Realization is used to label a dimension that can be thought of as a statistical sample, @@ -1698,24 +1703,54 @@ public static Dimension createRealizationVariable(NetcdfFileWriter netcdfFileWri //float physical_field(time, realization, level, latitude, longitude) ; //create realization dimension. - Dimension realizationDimension = netcdfFileWriter.addDimension(null, REALIZATION_VARIABLE_NAME, ensembleMemberCount); + Dimension realizationDimension = NetcdfWriterBuilder.addDimension(REALIZATION_VARIABLE_NAME, ensembleMemberCount); //create realization variable. ArrayList realizationDimensionList = new ArrayList<>(); realizationDimensionList.add(realizationDimension); - Variable myVar = netcdfFileWriter.addVariable(null, REALIZATION_VARIABLE_NAME, DataType.INT, realizationDimensionList); - netcdfFileWriter.addVariableAttribute(myVar, new Attribute(STANDARD_NAME_ATTRIBUTE_NAME, REALIZATION_VARIABLE_NAME)); - netcdfFileWriter.addVariableAttribute(myVar, new Attribute(LONG_NAME_ATTRIBUTE_NAME, "Index of an ensemble member within an ensemble")); - netcdfFileWriter.addVariableAttribute(myVar, new Attribute(UNITS_ATTRIBUTE_NAME, "1")); + Variable.Builder myVar = NetcdfWriterBuilder.addVariable(REALIZATION_VARIABLE_NAME, DataType.INT, realizationDimensionList); + myVar.addAttribute(new Attribute(STANDARD_NAME_ATTRIBUTE_NAME, REALIZATION_VARIABLE_NAME)); + myVar.addAttribute(new Attribute(LONG_NAME_ATTRIBUTE_NAME, "Index of an ensemble member within an ensemble")); + myVar.addAttribute(new Attribute(UNITS_ATTRIBUTE_NAME, "1")); return realizationDimension; } - public static void addGeneralGlobalAttributes(NetcdfFileWriter netcdfFileWriter) { - netcdfFileWriter.addGroupAttribute(null, new Attribute("institution", "OpenDA Association")); - netcdfFileWriter.addGroupAttribute(null, new Attribute("source", "Written by OpenDA")); - netcdfFileWriter.addGroupAttribute(null, new Attribute("history", "Created at " + new Date(System.currentTimeMillis()))); - netcdfFileWriter.addGroupAttribute(null, new Attribute("references", "http://www.openda.org")); - netcdfFileWriter.addGroupAttribute(null,new Attribute("Conventions", "CF-1.6")); + public static void addGeneralGlobalAttributes(NetcdfFormatWriter.Builder NetcdfWriterBuilder) { + NetcdfWriterBuilder.addAttribute(new Attribute("institution", "OpenDA Association")); + NetcdfWriterBuilder.addAttribute(new Attribute("source", "Written by OpenDA")); + NetcdfWriterBuilder.addAttribute(new Attribute("history", "Created at " + new Date(System.currentTimeMillis()))); + NetcdfWriterBuilder.addAttribute(new Attribute("references", "http://www.openda.org")); + NetcdfWriterBuilder.addAttribute(new Attribute("Conventions", "CF-1.6")); + } + + public static class myCancelTask implements CancelTask { + + @Override + public boolean isCancel(){ + return false; + } + + @Override + public boolean isDone() { + return false; + } + + @Override + public void setDone(boolean b) { + } + + @Override + public void setSuccess() { + CancelTask.super.setSuccess(); + } + + @Override + public void setError(String s) { + } + + @Override + public void setProgress(String s, int i) { + } } } diff --git a/core/java/src/org/openda/exchange/iotools/DataCopier.java b/core/java/src/org/openda/exchange/iotools/DataCopier.java index 50d938ef8..f2dd058d1 100644 --- a/core/java/src/org/openda/exchange/iotools/DataCopier.java +++ b/core/java/src/org/openda/exchange/iotools/DataCopier.java @@ -228,33 +228,6 @@ private static void copyValuesForNamedEnsembleItem(IExchangeItem inputExchangeIt outputExchangeItem.copyValuesFromItem(inputExchangeItem); } -// /** -// * Copies the values from the exchangeItems with the given inputExchangeItemIds from the input IDataObject -// * to the exchangeItems with the given outputExchangeItemIds in the output IDataObject. -// */ -// private void copyValuesForNamedItems(String[] inputExchangeItemIds, String[] outputExchangeItemIds) { -// if (inputExchangeItemIds.length != outputExchangeItemIds.length) { -// throw new RuntimeException(getClass().getSimpleName() + ": inputExchangeItemIds length (" + -// inputExchangeItemIds.length + ") and outputExchangeItemIds length (" + -// outputExchangeItemIds.length + ") should be the same."); -// } -// -// //copy exchangeItems. -// for (int n = 0; n < inputExchangeItemIds.length; n++) { -// copyValuesForNamedItem(inputExchangeItemIds[n], outputExchangeItemIds[n]); -// } -// } -// -// /** -// * Copy the values for the exchangeItems with the given ids from input to output IDataObject. -// */ -// private void copyValuesForNamedItems(String[] exchangeItemIds) { -// //copy exchangeItems. -// for (String exchangeItemId : exchangeItemIds) { -// copyValuesForNamedItem(exchangeItemId, exchangeItemId); -// } -// } - /** * Copies all exchangeItems from the input dataObject to the output dataObject. */ @@ -322,11 +295,11 @@ public void finish() { /** * Help text for the command line - * @return + * @return text string */ private static String getUsageMessage() { - StringBuffer message = new StringBuffer(); - if (BBUtils.RUNNING_ON_LINUX) { + StringBuilder message = new StringBuilder(); + if (BBUtils.RUNNING_ON_LINUX) { message.append("NAME\n" + "\t oda_copy.sh - a tool for copying data between OpenDA data-objects\n"); } else { @@ -354,7 +327,7 @@ private static String getUsageMessage() { /** * Gathers the following variables from the given arguments. - * + *

* inputFilePath: full pathname of input file. * inputClassName: fully qualified class name of input IDataObject to use to read data from the input file. * inputArguments: optional one or more arguments that are passed to the input IDataObject initialize method. @@ -372,7 +345,7 @@ private void processArguments(String[] arguments) { inputClassName=null; inputArgs=new String[0]; while(nextArg.startsWith("-")){ - String argValue=null; + String argValue; if((argIndex+1) * Depending on the nature of the input/output exchangeItems, different value(s) are copied. * If e.g an outputExchangeItem can only store value(s) for a single time and the corresponding inputExchangeItem stores * values for an entire time series, then when this method is called the outputExchangeItem only copies the value(s) for the diff --git a/core/java/src/org/openda/resultwriters/NetcdfResultWriterPureJava.java b/core/java/src/org/openda/resultwriters/NetcdfResultWriterPureJava.java index 4844affa2..2df1b912f 100644 --- a/core/java/src/org/openda/resultwriters/NetcdfResultWriterPureJava.java +++ b/core/java/src/org/openda/resultwriters/NetcdfResultWriterPureJava.java @@ -12,18 +12,14 @@ import java.util.Map; /** - * Created by nils on 13/11/15. + * @author nils on 13/11/15. */ public class NetcdfResultWriterPureJava implements IResultWriter { - private int defaultMaxSize = Integer.MAX_VALUE; - private String netcdfnameprefix; - private Map netcdfFiles = new HashMap(); - private Map notSupportedObjects = new HashMap(); - private Map timeIndex = new HashMap(); - - - private File workingDir; + private final String netcdfnameprefix; + private final Map netcdfFiles = new HashMap<>(); + private final Map notSupportedObjects = new HashMap<>(); + private final Map timeIndex = new HashMap<>(); public NetcdfResultWriterPureJava(File workingDir, String configString) { @@ -36,12 +32,11 @@ public NetcdfResultWriterPureJava(File workingDir, String configString) { throw new RuntimeException(error); } } - else{ + else { workingDir.mkdir(); } - this.workingDir = workingDir; - //Check the config string + //Check the config string if (configString.startsWith("... * * @author Arno Kockx */ public class GridExchangeItemNetcdfWriter { private final IExchangeItem[] exchangeItems; - private final NetcdfFileWriter netcdfFileWriter; + private final NetcdfFormatWriter NetcdfWriter; private final Variable timeVariable; private int currentTimeIndex = -1; @@ -60,17 +60,11 @@ public GridExchangeItemNetcdfWriter(IExchangeItem[] exchangeItems, File outputFi this.exchangeItems = exchangeItems; //create netcdf file. - try { - netcdfFileWriter = NetcdfFileWriter.createNew(NetcdfFileWriter.Version.netcdf3, outputFile.getAbsolutePath()); - } catch (IOException e) { - throw new RuntimeException(getClass().getSimpleName() + ": Error while opening netcdf file " + outputFile.getAbsolutePath() + " Message was: " + e.getMessage(), e); - } - //always create large file, in case much data needs to be written. - netcdfFileWriter.setLargeFile(true); + NetcdfFormatWriter.Builder netcdfBuilder = NetcdfFormatWriter.createNewNetcdf3(outputFile.getAbsolutePath()); //create time dimension and variable. - Dimension timeDimension = NetcdfUtils.createTimeVariable(netcdfFileWriter, NetcdfUtils.TIME_VARIABLE_NAME, -1, NetcdfUtils.createTimeUnitString()); - timeVariable = netcdfFileWriter.findVariable(NetcdfUtils.TIME_VARIABLE_NAME); + Dimension timeDimension = NetcdfUtils.createTimeVariable(netcdfBuilder, NetcdfUtils.TIME_VARIABLE_NAME, -1, NetcdfUtils.createTimeUnitString()); + timeVariable = netcdfBuilder.getRootGroup().build().findVariableOrInParent(NetcdfUtils.TIME_VARIABLE_NAME); //create grid dimensions and variables. //gather geometryInfos. @@ -85,25 +79,25 @@ public GridExchangeItemNetcdfWriter(IExchangeItem[] exchangeItems, File outputFi //create spatial coordinate variables, if not present yet. //this only adds spatial dimensions, this does not add spatial variables with coordinates, //because the coordinates are usually not available in exchangeItems that come from models. - Map geometryInfoGridVariablePropertiesMap = NetcdfUtils.createGridVariables(netcdfFileWriter, geometryInfos.toArray(new IGeometryInfo[geometryInfos.size()])); + Map geometryInfoGridVariablePropertiesMap = NetcdfUtils.createGridVariables(netcdfBuilder, geometryInfos.toArray(new IGeometryInfo[geometryInfos.size()])); //create data variables. - NetcdfUtils.createDataVariables(netcdfFileWriter, Arrays.asList(exchangeItems), timeDimension, null, null, geometryInfoGridVariablePropertiesMap); + NetcdfUtils.createDataVariables(netcdfBuilder, Arrays.asList(exchangeItems), timeDimension, null, null, geometryInfoGridVariablePropertiesMap); //add global metadata. - NetcdfUtils.addGlobalAttributes(netcdfFileWriter); + NetcdfUtils.addGlobalAttributes(netcdfBuilder); try { - netcdfFileWriter.create(); + NetcdfWriter = netcdfBuilder.build(); } catch (Exception e) { - throw new RuntimeException(getClass().getSimpleName() + ": Error while creating netcdf file " + netcdfFileWriter.getNetcdfFile().getLocation() + " Message was: " + e.getMessage(), e); + throw new RuntimeException(getClass().getSimpleName() + ": Error while creating netcdf file " + outputFile.getAbsolutePath() + " Message was: " + e.getMessage(), e); } //write grid variables values. try { - NetcdfUtils.writeGridVariablesValues(netcdfFileWriter, geometryInfoGridVariablePropertiesMap); + NetcdfUtils.writeGridVariablesValues(NetcdfWriter, geometryInfoGridVariablePropertiesMap); } catch (Exception e) { - throw new RuntimeException(getClass().getSimpleName() + ": Error while writing grid variable values to netcdf file " + netcdfFileWriter.getNetcdfFile().getLocation() + " Message was: " + e.getMessage(), e); + throw new RuntimeException(getClass().getSimpleName() + ": Error while writing grid variable values to netcdf file " + NetcdfWriter.getOutputFile().getLocation() + " Message was: " + e.getMessage(), e); } } @@ -134,21 +128,21 @@ private void writeTime() { } else { if (time != currentTime) { throw new RuntimeException(getClass().getSimpleName() + ": Cannot write data. Exchange items have different times for time index " + currentTimeIndex - + " in netcdf file " + netcdfFileWriter.getNetcdfFile().getLocation()); + + " in netcdf file " + NetcdfWriter.getOutputFile().getLocation()); } } } //write current time. if (timesWrittenSoFar.contains(currentTime)) { - throw new RuntimeException(getClass().getSimpleName() + ": Cannot write data. Output netcdf file " + netcdfFileWriter.getNetcdfFile().getLocation() + throw new RuntimeException(getClass().getSimpleName() + ": Cannot write data. Output netcdf file " + NetcdfWriter.getOutputFile().getLocation() + " already contains data for current time " + new Date(Time.mjdToMillies(currentTime))); } try { - NetcdfUtils.writeTimeVariableSingleValue(netcdfFileWriter, timeVariable, currentTimeIndex, currentTime); + NetcdfUtils.writeTimeVariableSingleValue(NetcdfWriter, timeVariable, currentTimeIndex, currentTime); } catch (Exception e) { throw new RuntimeException(getClass().getSimpleName() + ": Error while writing time value for current time " + new Date(Time.mjdToMillies(currentTime)) - + " to netcdf file " + netcdfFileWriter.getNetcdfFile().getLocation() + " Message was: " + e.getMessage(), e); + + " to netcdf file " + NetcdfWriter.getOutputFile().getLocation() + " Message was: " + e.getMessage(), e); } timesWrittenSoFar.add(currentTime); } @@ -212,16 +206,16 @@ private void writeData() { //write values for current time. origin[0] = currentTimeIndex; - Variable dataVariable = NetcdfUtils.getVariableForExchangeItem(netcdfFileWriter.getNetcdfFile(), item); - NetcdfUtils.writeSelectedData(netcdfFileWriter, dataVariable, origin, dimensions, values); + Variable dataVariable = NetcdfUtils.getVariableForExchangeItem(NetcdfWriter.getOutputFile(), item); + NetcdfUtils.writeSelectedData(NetcdfWriter, dataVariable, origin, dimensions, values); } } public void close() { try { - netcdfFileWriter.close(); + NetcdfWriter.close(); } catch (IOException e) { - throw new RuntimeException("Error while closing netcdf file " + netcdfFileWriter.getNetcdfFile().getLocation() + " Message was: " + e.getMessage(), e); + throw new RuntimeException("Error while closing netcdf file " + NetcdfWriter.getOutputFile().getLocation() + " Message was: " + e.getMessage(), e); } } } diff --git a/core/java/src/org/openda/utils/io/KalmanGainStorage.java b/core/java/src/org/openda/utils/io/KalmanGainStorage.java index ae2f64bbe..acdd2dcb1 100644 --- a/core/java/src/org/openda/utils/io/KalmanGainStorage.java +++ b/core/java/src/org/openda/utils/io/KalmanGainStorage.java @@ -34,8 +34,13 @@ import org.openda.utils.*; import ucar.ma2.Array; import ucar.ma2.*; -import ucar.nc2.*; +import ucar.nc2.Attribute; +import ucar.nc2.Dimension; +import ucar.nc2.Variable; import ucar.nc2.dataset.NetcdfDataset; +import ucar.nc2.dataset.NetcdfDatasets; +import ucar.nc2.write.NetcdfFormatWriter; +import ucar.nc2.write.NetcdfFormatWriter.*; import java.io.*; import java.util.*; @@ -64,7 +69,7 @@ public class KalmanGainStorage { // set in constructor private File workingDir; - private double timeStampAsMJD = Double.MIN_VALUE;; + private double timeStampAsMJD = Double.MIN_VALUE; // properties that be can be overridden before writing private String storageDirPrefix = "kgStorage_"; @@ -103,21 +108,6 @@ public KalmanGainStorage(File workingDir, double timeStampAsMJD) { this.timeStampAsMJD = timeStampAsMJD; } - /** - * Constructor for creating and writing kalman gain storage object, used to read or to write the kalman gain. - * The kalman gain is stored in a separate directory, with a separate file for each kalman gain column - * in the kalman gain matrix. - * - * @param workingDir The directory under which the directory for the kalman gain for the - * specified time stamp must be created - * @param timeStampAsMJD The time stamp for which the kalman gain is valid - */ - public KalmanGainStorage(File workingDir, double timeStampAsMJD, boolean useTimeStampInDirectoryName) { - this.workingDir = workingDir; - this.timeStampAsMJD = timeStampAsMJD; - this.useTimeStampInDirectoryName = false; - } - /** * Constructor for reading kalman gain storage object, used to read the kalman gain. * The kalman gain is stored with a separate file for each kalman gain column @@ -138,14 +128,6 @@ public void setStorageDirPrefix(String storageDirPrefix) { this.storageDirPrefix = storageDirPrefix; } - /** - * Set the prefix to be used for the directory name for storing the kalman gain column files. - * @param columnFilePrefix The file prefix for. - */ - public void setColumnFilePrefix(String columnFilePrefix) { - this.columnFilePrefix = columnFilePrefix; - } - /** * Set the file type for storage of Kalman gain vectors * @param fileType @@ -188,9 +170,9 @@ public void setComment(String comment) { } /** - * Write the kalman gain matrix to xml file and, in case of large column vectors, to netdcf files. - * A HashMap is used with Id's for each column. This helps to find the right column matching an observation - * @param gainVectors each vector corresponds to a column of the the Kalman gain and matches an observation + * Write the kalman gain matrix to XML file and, in case of large column vectors, to NetCDF files. + * A HashMap is used with Ids for each column. This helps to find the right column matching an observation + * @param gainVectors each vector corresponds to a column of the Kalman gain and matches an observation * @param obsId combination of location and quantity to map observations to the right column * @param obsTimeOffset For observations at a time preceding the analysis time this gives the offset in days. */ @@ -214,7 +196,7 @@ public void writeKalmanGain(HashMap gainVectors, } /** - * Write the kalman gain matrix to xml file and, in case of large column vectors, to netdcf files. + * Write the kalman gain matrix to XML file and, in case of large column vectors, to NetCDF files. * * @param observationIds The observation identifiers * @param observationOffsetsInDays The time offset of the observations, expressed in days, @@ -302,32 +284,30 @@ private void fillObservationsXML(String[] observationIds, double[] observationOf } private void writeKalmanGainToNetcdfCF(String[] observationIds, double[] observationOffsetsInDays, IVector[] kalmanGainColumns, File directoryForStorage, double[][] hk) { - NetcdfFileWriter netcdfFileWriter = null; + NetcdfFormatWriter netcdfFileWriter = null; try { File file = new File(directoryForStorage, kalmanGainStorageFileName); - netcdfFileWriter = NetcdfFileWriter.createNew(NetcdfFileWriter.Version.netcdf3, file.getAbsolutePath()); - netcdfFileWriter.setLargeFile(true); - addGlobalAttributes(netcdfFileWriter); - Variable timeStampVariable = createTimeStampVariable(netcdfFileWriter); - Dimension stationDimension = netcdfFileWriter.addDimension(null, STATION_DIMENSION, observationIds.length); - netcdfFileWriter.addDimension(null, CHAR_LENGTH_ID, STATION_ID_CHAR_LENGTH); - Variable observationOffsetVariable = createObservationOffsetVariable(netcdfFileWriter); - Variable hkVariable = hk != null ? createHKVariable(netcdfFileWriter) : null; - Variable stationVariable = createStationVariable(netcdfFileWriter); - - KalmanGainVariableData[] kalmanGainVariableData = getKalmanGainVariableData(kalmanGainColumns, netcdfFileWriter, stationDimension); - netcdfFileWriter.create(); - - writeStationData(observationIds, netcdfFileWriter, stationVariable); - writeTimeStampData(netcdfFileWriter, timeStampVariable); - writeObservationOffsetData(observationOffsetsInDays, netcdfFileWriter, observationOffsetVariable); - if (hkVariable != null) writeHK(hk, netcdfFileWriter, hkVariable); - + NetcdfFormatWriter.Builder netcdfBuilder = NetcdfFormatWriter.createNewNetcdf3(file.getAbsolutePath()); + addGlobalAttributes(netcdfBuilder); + createTimeStampVariable(netcdfBuilder); + Dimension stationDimension = netcdfBuilder.addDimension(STATION_DIMENSION, observationIds.length); + netcdfBuilder.addDimension(CHAR_LENGTH_ID, STATION_ID_CHAR_LENGTH); + createObservationOffsetVariable(netcdfBuilder); + createStationVariable(netcdfBuilder); + boolean hkVariable = hk != null ? createHKVariable(netcdfBuilder) : false; + + KalmanGainVariableData[] kalmanGainVariableData = getKalmanGainVariableData(kalmanGainColumns, netcdfBuilder, stationDimension); + netcdfFileWriter = netcdfBuilder.build(); + + writeStationData(observationIds, netcdfFileWriter); + writeTimeStampData(netcdfFileWriter); + writeObservationOffsetData(observationOffsetsInDays, netcdfFileWriter); + if (hkVariable) writeHK(hk, netcdfFileWriter); for (int observationIndex = 0; observationIndex < kalmanGainColumns.length; observationIndex++) { ITreeVector kalmanGainColumnForObservation = (ITreeVector) kalmanGainColumns[observationIndex]; for (KalmanGainVariableData kalmanGainVariableDatum : kalmanGainVariableData) { - Variable variable = kalmanGainVariableDatum.getVariable(); + String variableId = kalmanGainVariableDatum.getVariableId(); ITreeVector dataVector = getDataVector(kalmanGainColumnForObservation, kalmanGainVariableDatum); ArrayList dimensions = kalmanGainVariableDatum.getDimensions(); Array values = null; @@ -339,7 +319,7 @@ private void writeKalmanGainToNetcdfCF(String[] observationIds, double[] observa int[] origin = new int[dimensions.size() + 1]; origin[0] = observationIndex; try { - netcdfFileWriter.write(variable, origin, values); + netcdfFileWriter.write(variableId, origin, values); } catch (IOException ioe) { throw new RuntimeException(ioe); } @@ -376,40 +356,40 @@ private Array getData1D(ITreeVector dataVector, ArrayList dimensions) return arrayDoubleD2; } - private void writeObservationOffsetData(double[] observationOffsetsInDays, NetcdfFileWriter netcdfFileWriter, Variable observationOffsetVariable) throws IOException, InvalidRangeException { + private void writeObservationOffsetData(double[] observationOffsetsInDays, NetcdfFormatWriter netcdfFileWriter) throws IOException, InvalidRangeException { ArrayDouble.D1 observationOffsetArray = new ArrayDouble.D1(observationOffsetsInDays.length); for (int i = 0; i < observationOffsetsInDays.length; i++) { observationOffsetArray.set(i, observationOffsetsInDays[i]); } - netcdfFileWriter.write(observationOffsetVariable, observationOffsetArray); + netcdfFileWriter.write(OBSERVATION_OFFSET, observationOffsetArray); } - private void writeHK(double[][] hk, NetcdfFileWriter netcdfFileWriter, Variable hkVariable) throws IOException, InvalidRangeException { + private void writeHK(double[][] hk, NetcdfFormatWriter netcdfFileWriter) throws IOException, InvalidRangeException { ArrayDouble.D2 hkArray = new ArrayDouble.D2(hk.length, hk.length); for (int i = 0; i < hk.length; i++) { for (int j = 0; j < hk.length; j++) { hkArray.set(i, j, hk[i][j]); } } - netcdfFileWriter.write(hkVariable, hkArray); + netcdfFileWriter.write("HK", hkArray); } - private void writeStationData(String[] observationIds, NetcdfFileWriter netcdfFileWriter, Variable stationVariable) throws IOException, InvalidRangeException { + private void writeStationData(String[] observationIds, NetcdfFormatWriter netcdfFileWriter) throws IOException, InvalidRangeException { ArrayChar.D2 stationArray = new ArrayChar.D2(observationIds.length, STATION_ID_CHAR_LENGTH); for (int i = 0; i < observationIds.length; i++) { String observationId = observationIds[i]; stationArray.setString(i, observationId); } - netcdfFileWriter.write(stationVariable, stationArray); + netcdfFileWriter.write(STATION_ID, stationArray); } - private void writeTimeStampData(NetcdfFileWriter netcdfFileWriter, Variable timeStampVariable) throws IOException, InvalidRangeException { + private void writeTimeStampData(NetcdfFormatWriter netcdfFileWriter) throws IOException, InvalidRangeException { ArrayDouble.D1 timeStampArray = new ArrayDouble.D1(1); timeStampArray.setDouble(0, timeStampAsMJD); - netcdfFileWriter.write(timeStampVariable, timeStampArray); + netcdfFileWriter.write(TIME_STAMP, timeStampArray); } - private KalmanGainVariableData[] getKalmanGainVariableData(IVector[] kalmanGainColumns, NetcdfFileWriter netcdfFileWriter, Dimension stationDimension) { + private KalmanGainVariableData[] getKalmanGainVariableData(IVector[] kalmanGainColumns, NetcdfFormatWriter.Builder netcdfFileWriter, Dimension stationDimension) { TreeVector kalmanGainColumn = (TreeVector) kalmanGainColumns[0]; ArrayList subTreeVectorIds = kalmanGainColumn.getSubTreeVectorIds(); KalmanGainVariableData[] kalmanGainVariableData = new KalmanGainVariableData[subTreeVectorIds.size()]; @@ -456,14 +436,14 @@ private KalmanGainVariableData[] getKalmanGainVariableData(IVector[] kalmanGainC return kalmanGainVariableData; } - private KalmanGainVariableData createKalmanGainVariableData(NetcdfFileWriter netcdfFileWriter, Dimension stationDimension, ArrayList vectorIds, String variableId, int[] shape, ITreeVector dataVector) { + private KalmanGainVariableData createKalmanGainVariableData(NetcdfFormatWriter.Builder netcdfFileWriter, Dimension stationDimension, ArrayList vectorIds, String variableId, int[] shape, ITreeVector dataVector) { ArrayList dimensions = new ArrayList<>(); dimensions.add(stationDimension); for (int i = 0; i < shape.length; i++) { - Dimension dim = netcdfFileWriter.addDimension(null, variableId + "_dimension_" + i, shape[i]); + Dimension dim = netcdfFileWriter.addDimension(variableId + "_dimension_" + i, shape[i]); dimensions.add(dim); } - Variable variable = netcdfFileWriter.addVariable(null, variableId, DataType.DOUBLE, dimensions); + Variable.Builder variable = netcdfFileWriter.addVariable(variableId, DataType.DOUBLE, dimensions); if (vectorIds.size() > 1) { String parentVectorId = vectorIds.get(0); variable.addAttribute(new Attribute(PARENT_VECTOR_ID, parentVectorId)); @@ -473,44 +453,41 @@ private KalmanGainVariableData createKalmanGainVariableData(NetcdfFileWriter net // Remove station dimension again because stations data are written one by one Dimension removed = dimensions.remove(0); assert removed.getShortName().equals(STATION_DIMENSION); - return new KalmanGainVariableData(variableId, dimensions, variable, dataVector, vectorIds); + return new KalmanGainVariableData(variableId, dimensions, dataVector, vectorIds); } - private Variable createStationVariable(NetcdfFileWriter netcdfFileWriter) { - Variable stationVariable = netcdfFileWriter.addVariable(null, STATION_ID, DataType.CHAR, STATION_DIMENSION + ' ' + CHAR_LENGTH_ID); + private void createStationVariable(NetcdfFormatWriter.Builder netcdfFileWriter) { + Variable.Builder stationVariable = netcdfFileWriter.addVariable(STATION_ID, DataType.CHAR, STATION_DIMENSION + ' ' + CHAR_LENGTH_ID); stationVariable.addAttribute(STATION_IDENTIFICATION_CODE_ATT); stationVariable.addAttribute(TIME_SERIES_ID_ATT); - return stationVariable; } - private Variable createObservationOffsetVariable(NetcdfFileWriter netcdfFileWriter) { - Variable observationOffsetVariable = netcdfFileWriter.addVariable(null, OBSERVATION_OFFSET, DataType.DOUBLE, STATION_DIMENSION); + private void createObservationOffsetVariable(NetcdfFormatWriter.Builder netcdfFileWriter) { + Variable.Builder observationOffsetVariable = netcdfFileWriter.addVariable(OBSERVATION_OFFSET, DataType.DOUBLE, STATION_DIMENSION); observationOffsetVariable.addAttribute(OBSERVATION_OFFSET_LONG_NAME_ATT); observationOffsetVariable.addAttribute(TIME_STAMP_UNIT_ATT); - return observationOffsetVariable; } - private Variable createHKVariable(NetcdfFileWriter netcdfFileWriter) { - Variable observationOffsetVariable = netcdfFileWriter.addVariable(null, "HK", DataType.DOUBLE, STATION_DIMENSION + ' ' + STATION_DIMENSION); + private boolean createHKVariable(NetcdfFormatWriter.Builder netcdfFileWriter) { + Variable.Builder observationOffsetVariable = netcdfFileWriter.addVariable("HK", DataType.DOUBLE, STATION_DIMENSION + ' ' + STATION_DIMENSION); observationOffsetVariable.addAttribute(HK_LONG_NAME_ATT); observationOffsetVariable.addAttribute(FRACTIONS_UNIT_ATT); - return observationOffsetVariable; + return true; } - private Variable createTimeStampVariable(NetcdfFileWriter netcdfFileWriter) { - Dimension timeStampDimension = netcdfFileWriter.addDimension(null, "time_stamp_dimension", 1); + private void createTimeStampVariable(Builder netcdfFileWriter) { + Dimension timeStampDimension = netcdfFileWriter.addDimension("time_stamp_dimension", 1); ArrayList dimensionList = new ArrayList<>(); dimensionList.add(timeStampDimension); - Variable timeStampVariable = netcdfFileWriter.addVariable(null, TIME_STAMP, DataType.DOUBLE, dimensionList); + Variable.Builder timeStampVariable = netcdfFileWriter.addVariable(TIME_STAMP, DataType.DOUBLE, dimensionList); timeStampVariable.addAttribute(TIME_STAMP_LONG_NAME_ATT); timeStampVariable.addAttribute(TIME_STAMP_UNIT_ATT); - return timeStampVariable; } - private void addGlobalAttributes(NetcdfFileWriter netcdfFileWriter) { - netcdfFileWriter.addGroupAttribute(null, new Attribute("title", "Kalman gain data")); - NetcdfUtils.addGeneralGlobalAttributes(netcdfFileWriter); - if (comment != null && !comment.isEmpty()) netcdfFileWriter.addGroupAttribute(null, new Attribute("Comment", comment)); + private void addGlobalAttributes(NetcdfFormatWriter.Builder netcdfBuilder) { + netcdfBuilder.addAttribute(new Attribute("title", "Kalman gain data")); + NetcdfUtils.addGeneralGlobalAttributes(netcdfBuilder); + if (comment != null && !comment.isEmpty()) netcdfBuilder.addAttribute(new Attribute("Comment", comment)); } private ITreeVector getDataVector(ITreeVector kalmanGainColumnForObservation, KalmanGainVariableData kalmanGainVariableDatum) { @@ -631,10 +608,10 @@ public void readKalmanGain(IVector templateTreeVector) { } private void readKalmanGainFromNetcdfCF(File directoryForStorage) { - NetcdfFile netcdfFile; + NetcdfDataset netcdfFile; try { File file = new File(directoryForStorage, kalmanGainStorageFileName); - netcdfFile = NetcdfDataset.openDataset(file.getAbsolutePath()); + netcdfFile = NetcdfDatasets.openDataset(file.getAbsolutePath()); List variables = netcdfFile.getVariables(); Variable stationVariable = netcdfFile.findVariable(STATION_ID); int stationLength = stationVariable.getShape(0); @@ -644,6 +621,7 @@ private void readKalmanGainFromNetcdfCF(File directoryForStorage) { } readVariables(variables, stationLength, kalmanGainColumns); this.kalmanGainColumns = kalmanGainColumns; + netcdfFile.close(); } catch (Exception e) { throw new RuntimeException(e); } @@ -658,11 +636,11 @@ private void readVariables(List variables, int stationLength, TreeVect continue; } if (OBSERVATION_OFFSET.equals(shortName)) { - this.observationOffsetInDays = (double[]) variable.read().get1DJavaArray(double.class); + this.observationOffsetInDays = (double[]) variable.read().get1DJavaArray(DataType.DOUBLE); continue; } if (TIME_STAMP.equals(shortName)) { - this.timeStampAsMJD = ((double[]) variable.read().get1DJavaArray(double.class))[0]; + this.timeStampAsMJD = ((double[]) variable.read().get1DJavaArray(DataType.DOUBLE))[0]; continue; } if ("HK".equals(shortName)) { @@ -688,7 +666,7 @@ private void readVariables(List variables, int stationLength, TreeVect } private String[] read1DimensionalStringArray(Variable variable) throws IOException { - Object[] array = (Object[]) variable.read().get1DJavaArray(String.class); + Object[] array = (Object[]) variable.read().get1DJavaArray(DataType.STRING); String[] strings = new String[array.length]; for (int i = 0; i < array.length; i++) { strings[i] = (String) array[i]; @@ -720,7 +698,7 @@ static String[] read2DCharVariable(Variable variable) throws IOException { private void read2DimensionalArray(Variable variable, String shortName, int[] shape, int[] shapeForRead, int[] originForRead, TreeVector kalmanGainColumns) throws IOException, InvalidRangeException { Array read = variable.read(originForRead, shapeForRead); - double[] doubleArray = (double[]) read.get1DJavaArray(Double.TYPE); + double[] doubleArray = (double[]) read.get1DJavaArray(DataType.DOUBLE); DimensionIndex[] dimensionIndices = {new DimensionIndex(shape[1]), new DimensionIndex(shape[2])}; Vector doubleVector = new Vector(doubleArray); Attribute parentVectorIdAttribute = variable.findAttribute(PARENT_VECTOR_ID); @@ -732,7 +710,7 @@ private void read2DimensionalArray(Variable variable, String shortName, int[] sh private void read1DimensionalArray(Variable variable, String shortName, int[] shape, int[] shapeForRead, int[] originForRead, TreeVector kalmanGainColumns) throws IOException, InvalidRangeException { Array read = variable.read(originForRead, shapeForRead); - double[] doubleArray = (double[]) read.get1DJavaArray(Double.TYPE); + double[] doubleArray = (double[]) read.get1DJavaArray(DataType.DOUBLE); DimensionIndex dimensionIndex = new DimensionIndex(shape[1]); Vector doubleVector = new Vector(doubleArray); TreeVector vector = new TreeVector(shortName, doubleVector, new DimensionIndex[]{dimensionIndex}); @@ -954,14 +932,12 @@ public double getTimeStampAsMjd(){ private class KalmanGainVariableData { private final String variableId; private final ArrayList dimensions; - private final Variable variable; private final ITreeVector dataVector; private ArrayList vectorIds; - public KalmanGainVariableData(String variableId, ArrayList dimensions, Variable variable, ITreeVector dataVector, ArrayList vectorIds) { + public KalmanGainVariableData(String variableId, ArrayList dimensions, ITreeVector dataVector, ArrayList vectorIds) { this.variableId = variableId; this.dimensions = dimensions; - this.variable = variable; this.dataVector = dataVector; this.vectorIds = vectorIds; } @@ -974,10 +950,6 @@ public ArrayList getDimensions() { return dimensions; } - public Variable getVariable() { - return variable; - } - public ITreeVector getDataVector() { return dataVector; } diff --git a/core/java/src/org/openda/utils/io/NetCDFFile.java b/core/java/src/org/openda/utils/io/NetCDFFile.java index ce7f54006..61e0be8c0 100644 --- a/core/java/src/org/openda/utils/io/NetCDFFile.java +++ b/core/java/src/org/openda/utils/io/NetCDFFile.java @@ -20,25 +20,23 @@ package org.openda.utils.io; import ucar.ma2.*; -import ucar.nc2.*; +import ucar.nc2.Dimension; +import ucar.nc2.Variable; +import ucar.nc2.write.NetcdfFormatWriter; import java.io.IOException; import java.io.File; /** - * Created by nils van Velzen on 25/09/15. - * * This is a simple utility class to write variables to NetCDF. The current implementation is extremely simple but can/will be extended in the future. * + * @author nils van Velzen on 25/09/15. */ public class NetCDFFile { - private File fileName; - private NetcdfFileWriter netcdfFileWriter; + private final File fileName; private boolean fileIsNotCreated=true; private Dimension nDim; - private Dimension timeDim; - private Variable variable; //Create and initialize a new NetCDFFile. Nothing happens yet (no files are created) @@ -47,32 +45,33 @@ public NetCDFFile(File fileName) { } - //Write an array to file. Currently we only support a single array per file. The timeindex species the additional axis along we can extend the array in an existing NetCDF file - public void writeArray(double[] vals, int[] nDims, int iTime, String shortName) throws IOException, InvalidRangeException { - //Check wether we have to define de header and create the file on first write + //Write an array to file. Currently, we only support a single array per file. The time-index specifies the additional axis along we can extend the array in an existing NetCDF file + public void writeArray(double[] vals, int iTime, String shortName) throws IOException, InvalidRangeException { + //Check whether we have to define de header and create the file on first write int n=vals.length; - NetcdfFileWriter netcdfFileWriter; - - + NetcdfFormatWriter.Builder NetcdfBuilder; + NetcdfFormatWriter NetcdfWriter; Variable myVar; + if (this.fileIsNotCreated){ // Create a new file - netcdfFileWriter = NetcdfFileWriter.createNew(NetcdfFileWriter.Version.netcdf3, fileName.getAbsolutePath()); + NetcdfBuilder = NetcdfFormatWriter.createNewNetcdf3(fileName.getAbsolutePath()); // Setup Header - this.nDim = netcdfFileWriter.addDimension(null,"n", n); - this.timeDim = netcdfFileWriter.addUnlimitedDimension("time"); - myVar = netcdfFileWriter.addVariable(null, shortName, DataType.DOUBLE, "time n"); + this.nDim = NetcdfBuilder.addDimension("n", n); + NetcdfBuilder.addUnlimitedDimension("time"); + NetcdfBuilder.addVariable(shortName, DataType.DOUBLE, "time n"); // create the file - netcdfFileWriter.create(); + NetcdfWriter = NetcdfBuilder.build(); this.fileIsNotCreated=false; } else { - netcdfFileWriter = NetcdfFileWriter.openExisting(this.fileName.getAbsolutePath()); - myVar = netcdfFileWriter.findVariable(shortName); + NetcdfBuilder = NetcdfFormatWriter.openExisting(this.fileName.getAbsolutePath()); + NetcdfWriter = NetcdfBuilder.build(); } + myVar = NetcdfWriter.findVariable(shortName); ArrayDouble.D2 values = new ArrayDouble.D2(1, nDim.getLength()); // Copy the values @@ -82,10 +81,10 @@ public void writeArray(double[] vals, int[] nDims, int iTime, String shortName) int[] origin = new int[]{iTime, 0}; - netcdfFileWriter.write(myVar, origin, values); + NetcdfWriter.write(myVar, origin, values); //Always close - netcdfFileWriter.close(); + NetcdfWriter.close(); } } diff --git a/core/java/src/org/openda/utils/io/ScalarExchangeItemNetcdfWriter.java b/core/java/src/org/openda/utils/io/ScalarExchangeItemNetcdfWriter.java index 270dd530f..7c9cbefa9 100644 --- a/core/java/src/org/openda/utils/io/ScalarExchangeItemNetcdfWriter.java +++ b/core/java/src/org/openda/utils/io/ScalarExchangeItemNetcdfWriter.java @@ -26,7 +26,7 @@ import org.openda.interfaces.IGeometryInfo; import org.openda.utils.Time; import ucar.nc2.Dimension; -import ucar.nc2.NetcdfFileWriter; +import ucar.nc2.write.NetcdfFormatWriter; import ucar.nc2.Variable; import ucar.nc2.Attribute; @@ -37,18 +37,18 @@ /** * Writes scalar data from one or more IExchangeItems to a NetCDF file. The data is written per time step. * This can only handle exchangeItems that contain scalar data for a single timeStep at a time, e.g. a model state scalar exchange item. - * The written NetCDF files are compliant with the NetCDF CF conventions as much as possible, see http://cfconventions.org/ + * The written NetCDF files are compliant with the NetCDF CF conventions as much as possible, see ... * * @author Arno Kockx */ public class ScalarExchangeItemNetcdfWriter { private final IExchangeItem[] exchangeItems; - private final NetcdfFileWriter netcdfFileWriter; + private final NetcdfFormatWriter netcdfWriter; private final List stationIds; private final Variable timeVariable; private int currentTimeIndex = -1; - private final List timesWrittenSoFar = new ArrayList(); + private final List timesWrittenSoFar = new ArrayList<>(); public ScalarExchangeItemNetcdfWriter(IExchangeItem[] exchangeItems, File outputFile, String stationIdVarName, String stationDimensionVarName) { if (exchangeItems == null) throw new IllegalArgumentException("exchangeItems == null"); @@ -60,44 +60,38 @@ public ScalarExchangeItemNetcdfWriter(IExchangeItem[] exchangeItems, File output //validate that all exchange items are scalars. validateExchangeItems(exchangeItems); - //create netcdf file. - try { - //set fill to true, otherwise missing values will not be written for variables that do not have data for all stations. - netcdfFileWriter = NetcdfFileWriter.createNew(NetcdfFileWriter.Version.netcdf3, outputFile.getAbsolutePath()); - netcdfFileWriter.setFill(true); - } catch (IOException e) { - throw new RuntimeException(getClass().getSimpleName() + ": Error while opening netcdf file " + outputFile.getAbsolutePath() + " Message was: " + e.getMessage(), e); - } + //set fill to true, otherwise missing values will not be written for variables that do not have data for all stations. + NetcdfFormatWriter.Builder netcdfWriterBuilder = NetcdfFormatWriter.createNewNetcdf3(outputFile.getAbsolutePath()); + netcdfWriterBuilder.setFill(true); //create time dimension and variable. - Dimension timeDimension = NetcdfUtils.createTimeVariable(netcdfFileWriter, NetcdfUtils.TIME_VARIABLE_NAME, -1, NetcdfUtils.createTimeUnitString()); - timeVariable = netcdfFileWriter.findVariable(NetcdfUtils.TIME_VARIABLE_NAME); + Dimension timeDimension = NetcdfUtils.createTimeVariable(netcdfWriterBuilder, NetcdfUtils.TIME_VARIABLE_NAME, -1, NetcdfUtils.createTimeUnitString()); + timeVariable = netcdfWriterBuilder.getRootGroup().build().findVariableOrInParent(NetcdfUtils.TIME_VARIABLE_NAME); //create station dimension and variable. //this only adds a station id variable, this does not add spatial variables with coordinates, //because the coordinates are usually not available in exchangeItems that come from models. //gather locationIds. stationIds = NetcdfUtils.getStationIds(Arrays.asList(exchangeItems), null); - Dimension stationDimension = NetcdfUtils.createStationsVariable(netcdfFileWriter, stationIdVarName, stationDimensionVarName, stationIds.size()); + Dimension stationDimension = NetcdfUtils.createStationsVariable(netcdfWriterBuilder, stationIdVarName, stationDimensionVarName, stationIds.size()); //create data variables. - NetcdfUtils.createDataVariables(netcdfFileWriter, Arrays.asList(exchangeItems), timeDimension, null, stationDimension, null); + NetcdfUtils.createDataVariables(netcdfWriterBuilder, Arrays.asList(exchangeItems), timeDimension, null, stationDimension, null); //add global metadata. - NetcdfUtils.addGlobalAttributes(netcdfFileWriter); - netcdfFileWriter.addGroupAttribute(null, new Attribute(NetcdfUtils.FEATURE_TYPE_ATTRIBUTE_NAME, NetcdfUtils.TIME_SERIES_FEATURE_TYPE)); - + NetcdfUtils.addGlobalAttributes(netcdfWriterBuilder); + netcdfWriterBuilder.addAttribute(new Attribute(NetcdfUtils.FEATURE_TYPE_ATTRIBUTE_NAME, NetcdfUtils.TIME_SERIES_FEATURE_TYPE)); try { - netcdfFileWriter.create(); + netcdfWriter = netcdfWriterBuilder.build(); } catch (Exception e) { - throw new RuntimeException(getClass().getSimpleName() + ": Error while creating netcdf file " + netcdfFileWriter.getNetcdfFile().getLocation() + " Message was: " + e.getMessage(), e); + throw new RuntimeException(getClass().getSimpleName() + ": Error while creating netcdf file " + outputFile.getAbsolutePath() + " Message was: " + e.getMessage(), e); } //write station variable values. try { - NetcdfUtils.writeStationIdVariableValues(netcdfFileWriter, stationIdVarName, stationIds); + NetcdfUtils.writeStationIdVariableValues(netcdfWriter, stationIdVarName, stationIds); } catch (Exception e) { - throw new RuntimeException(getClass().getSimpleName() + ": Error while writing station variable values to netcdf file " + netcdfFileWriter.getNetcdfFile().getLocation() + " Message was: " + e.getMessage(), e); + throw new RuntimeException(getClass().getSimpleName() + ": Error while writing station variable values to netcdf file " + netcdfWriter.getOutputFile().getLocation() + " Message was: " + e.getMessage(), e); } } @@ -125,6 +119,23 @@ public void writeDataForCurrentTimeStep() { */ private void writeTime() { //get current time. + double currentTime = getCurrentTime(); + + //write current time. + if (timesWrittenSoFar.contains(currentTime)) { + throw new RuntimeException(getClass().getSimpleName() + ": Cannot write data. Output netcdf file " + netcdfWriter.getOutputFile().getLocation() + + " already contains data for current time " + new Date(Time.mjdToMillies(currentTime))); + } + try { + NetcdfUtils.writeTimeVariableSingleValue(netcdfWriter, timeVariable, currentTimeIndex, currentTime); + } catch (Exception e) { + throw new RuntimeException(getClass().getSimpleName() + ": Error while writing time value for current time " + new Date(Time.mjdToMillies(currentTime)) + + " to netcdf file " + netcdfWriter.getOutputFile().getLocation() + " Message was: " + e.getMessage(), e); + } + timesWrittenSoFar.add(currentTime); + } + + private double getCurrentTime() { double currentTime = Double.NaN; for (IExchangeItem item : exchangeItems) { if (item.getTimeInfo() == null || item.getTimeInfo().getTimes() == null || item.getTimeInfo().getTimes().length != 1 || Double.isNaN(item.getTimeInfo().getTimes()[0])) { @@ -138,23 +149,11 @@ private void writeTime() { } else { if (time != currentTime) { throw new RuntimeException(getClass().getSimpleName() + ": Cannot write data. Exchange items have different times for time index " + currentTimeIndex - + " in netcdf file " + netcdfFileWriter.getNetcdfFile().getLocation()); + + " in netcdf file " + netcdfWriter.getOutputFile().getLocation()); } } } - - //write current time. - if (timesWrittenSoFar.contains(currentTime)) { - throw new RuntimeException(getClass().getSimpleName() + ": Cannot write data. Output netcdf file " + netcdfFileWriter.getNetcdfFile().getLocation() - + " already contains data for current time " + new Date(Time.mjdToMillies(currentTime))); - } - try { - NetcdfUtils.writeTimeVariableSingleValue(netcdfFileWriter, timeVariable, currentTimeIndex, currentTime); - } catch (Exception e) { - throw new RuntimeException(getClass().getSimpleName() + ": Error while writing time value for current time " + new Date(Time.mjdToMillies(currentTime)) - + " to netcdf file " + netcdfFileWriter.getNetcdfFile().getLocation() + " Message was: " + e.getMessage(), e); - } - timesWrittenSoFar.add(currentTime); + return currentTime; } /** @@ -184,16 +183,16 @@ private void writeData() { origin[1] = stationIndex; //write values for current time. - Variable dataVariable = NetcdfUtils.getVariableForExchangeItem(netcdfFileWriter.getNetcdfFile(), item); - NetcdfUtils.writeSelectedData(netcdfFileWriter, dataVariable, origin, dimensions, values); + Variable dataVariable = NetcdfUtils.getVariableForExchangeItem(netcdfWriter.getOutputFile(), item); + NetcdfUtils.writeSelectedData(netcdfWriter, dataVariable, origin, dimensions, values); } } public void close() { try { - netcdfFileWriter.close(); + netcdfWriter.close(); } catch (IOException e) { - throw new RuntimeException("Error while closing netcdf file " + netcdfFileWriter.getNetcdfFile().getLocation() + " Message was: " + e.getMessage(), e); + throw new RuntimeException("Error while closing netcdf file " + netcdfWriter.getOutputFile().getLocation() + " Message was: " + e.getMessage(), e); } } } diff --git a/core/java/test/org/openda/exchange/dataobjects/NetcdfFileConcatenaterTest.java b/core/java/test/org/openda/exchange/dataobjects/NetcdfFileConcatenaterTest.java index 15f784386..b9dc60bf6 100644 --- a/core/java/test/org/openda/exchange/dataobjects/NetcdfFileConcatenaterTest.java +++ b/core/java/test/org/openda/exchange/dataobjects/NetcdfFileConcatenaterTest.java @@ -22,7 +22,9 @@ import junit.framework.TestCase; import org.openda.blackbox.config.BBUtils; import org.openda.utils.OpenDaTestSupport; +import ucar.ma2.DataType; import ucar.nc2.NetcdfFile; +import ucar.nc2.NetcdfFiles; import java.io.File; import java.io.IOException; @@ -43,15 +45,17 @@ public void testDelft3dHisFileConcatenation() throws IOException { File targetFile = new File(this.testRunDataDir, "westerscheldt_his.nc"); if (targetFile.exists()) BBUtils.deleteFileOrDir(targetFile); assertFalse(targetFile.exists()); - NetcdfFileConcatenater.main(new String[]{targetFile.getAbsolutePath(), firstFile.getAbsolutePath()}); + NetcdfFileConcatenator.main(new String[]{targetFile.getAbsolutePath(), firstFile.getAbsolutePath()}); assertTrue(targetFile.exists()); File secondFile = new File(this.testRunDataDir, "westerscheldt_part2_his.nc"); - NetcdfFileConcatenater.main(new String[]{targetFile.getAbsolutePath(), secondFile.getAbsolutePath()}); + NetcdfFileConcatenator.main(new String[]{targetFile.getAbsolutePath(), secondFile.getAbsolutePath()}); // find size of time array for original files and concatenated file - long size1 = NetcdfFile.open(firstFile.toString()).findVariable("time").read().getSize(); - long size2 = NetcdfFile.open(secondFile.toString()).findVariable("time").read().getSize(); - long size3 = NetcdfFile.open(targetFile.toString()).findVariable("time").read().getSize(); - assertEquals(size3, size1 + size2); + int[] size1 = Objects.requireNonNull(NetcdfFiles.open(firstFile.toString()).findVariable("time")).getShape(); + int[] size2 = Objects.requireNonNull(NetcdfFiles.open(secondFile.toString()).findVariable("time")).getShape(); + int[] size3 = Objects.requireNonNull(NetcdfFiles.open(targetFile.toString()).findVariable("time")).getShape(); + for (int i = 0; i < size3.length; ++i) { + assertEquals(size3[i], size1[i] + size2[i]); + } } public void testDflowfmHisfileConcatenation() throws IOException { @@ -59,15 +63,17 @@ public void testDflowfmHisfileConcatenation() throws IOException { File targetFile = new File(this.testRunDataDir, "simple_waal_his.nc"); if (targetFile.exists()) BBUtils.deleteFileOrDir(targetFile); assertFalse(targetFile.exists()); - NetcdfFileConcatenater.main(new String[]{targetFile.getAbsolutePath(), firstFile.getAbsolutePath()}); + NetcdfFileConcatenator.main(new String[]{targetFile.getAbsolutePath(), firstFile.getAbsolutePath()}); assertTrue(targetFile.exists()); File secondFile = new File(this.testRunDataDir, "simple_waal_part2_his.nc"); - NetcdfFileConcatenater.main(new String[]{targetFile.getAbsolutePath(), secondFile.getAbsolutePath()}); + NetcdfFileConcatenator.main(new String[]{targetFile.getAbsolutePath(), secondFile.getAbsolutePath()}); // find size of time array for original files and concatenated file - long size1 = NetcdfFile.open(firstFile.toString()).findVariable("time").read().getSize(); - long size2 = NetcdfFile.open(secondFile.toString()).findVariable("time").read().getSize(); - long size3 = NetcdfFile.open(targetFile.toString()).findVariable("time").read().getSize(); - assertEquals(size3, size1 + size2); + int[] size1 = Objects.requireNonNull(NetcdfFiles.open(firstFile.toString()).findVariable("time")).getShape(); + int[] size2 = Objects.requireNonNull(NetcdfFiles.open(secondFile.toString()).findVariable("time")).getShape(); + int[] size3 = Objects.requireNonNull(NetcdfFiles.open(targetFile.toString()).findVariable("time")).getShape(); + for (int i = 0; i < size3.length; ++i) { + assertEquals(size3[i], size1[i] + size2[i]); + } } public void testNetcdfFixedTimeDimensionConcatenation() { @@ -76,7 +82,7 @@ public void testNetcdfFixedTimeDimensionConcatenation() { File targetFile = new File(testRunDataSubDir, "concatenated.nc"); if (targetFile.exists()) BBUtils.deleteFileOrDir(targetFile); assertFalse(targetFile.exists()); - NetcdfFileConcatenater.main(new String[]{targetFile.getAbsolutePath(), firstFile.getAbsolutePath()}); + NetcdfFileConcatenator.main(new String[]{targetFile.getAbsolutePath(), firstFile.getAbsolutePath()}); assertTrue(targetFile.exists()); } @@ -86,10 +92,10 @@ public void testNetcdfFixedTimeDimensionInt() throws IOException { File targetFile = new File(testRunDataSubDir, "concatenated_timeInt.nc"); if (targetFile.exists()) BBUtils.deleteFileOrDir(targetFile); assertFalse(targetFile.exists()); - NetcdfFileConcatenater.main(new String[]{targetFile.getAbsolutePath(), firstFile.getAbsolutePath()}); + NetcdfFileConcatenator.main(new String[]{targetFile.getAbsolutePath(), firstFile.getAbsolutePath()}); assertTrue(targetFile.exists()); File secondFile = new File(testRunDataSubDir, "rrunoff_201257_timeInt.nc"); - NetcdfFileConcatenater.main(new String[]{targetFile.getAbsolutePath(), secondFile.getAbsolutePath()}); + NetcdfFileConcatenator.main(new String[]{targetFile.getAbsolutePath(), secondFile.getAbsolutePath()}); assertTrue(targetFile.exists()); checkConcatenatedValues(firstFile, targetFile, secondFile, 6); @@ -101,10 +107,10 @@ public void testNetcdfFixedTimeDimensionUseNewValueOnOverlapConcatenation() thro File targetFile = new File(testRunDataSubDir, "concatenated_rrrunoff.nc"); if (targetFile.exists()) BBUtils.deleteFileOrDir(targetFile); assertFalse(targetFile.exists()); - NetcdfFileConcatenater.main(new String[]{targetFile.getAbsolutePath(), firstFile.getAbsolutePath()}); + NetcdfFileConcatenator.main(new String[]{targetFile.getAbsolutePath(), firstFile.getAbsolutePath()}); assertTrue(targetFile.exists()); File secondFile = new File(testRunDataSubDir, "rrunoff_201257.nc"); - NetcdfFileConcatenater.main(new String[]{targetFile.getAbsolutePath(), secondFile.getAbsolutePath()}); + NetcdfFileConcatenator.main(new String[]{targetFile.getAbsolutePath(), secondFile.getAbsolutePath()}); assertTrue(targetFile.exists()); checkConcatenatedValues(firstFile, targetFile, secondFile, 6); @@ -116,10 +122,10 @@ public void testNetcdfFloats() throws IOException { File targetFile = new File(testRunDataSubDir, "concatenated_rrrunoff_floats.nc"); if (targetFile.exists()) BBUtils.deleteFileOrDir(targetFile); assertFalse(targetFile.exists()); - NetcdfFileConcatenater.main(new String[]{targetFile.getAbsolutePath(), firstFile.getAbsolutePath()}); + NetcdfFileConcatenator.main(new String[]{targetFile.getAbsolutePath(), firstFile.getAbsolutePath()}); assertTrue(targetFile.exists()); File secondFile = new File(testRunDataSubDir, "rrunoff_201257_floats.nc"); - NetcdfFileConcatenater.main(new String[]{targetFile.getAbsolutePath(), secondFile.getAbsolutePath()}); + NetcdfFileConcatenator.main(new String[]{targetFile.getAbsolutePath(), secondFile.getAbsolutePath()}); assertTrue(targetFile.exists()); checkConcatenatedValues(firstFile, targetFile, secondFile, 6); @@ -132,9 +138,9 @@ public void testNetcdf3Dhis() throws IOException { File secondFile = new File(testRunDataSubDir, "FlowFM_his.nc"); if (targetFile.exists()) BBUtils.deleteFileOrDir(targetFile); assertFalse(targetFile.exists()); - NetcdfFileConcatenater.main(new String[]{targetFile.getAbsolutePath(), firstFile.getAbsolutePath()}); + NetcdfFileConcatenator.main(new String[]{targetFile.getAbsolutePath(), firstFile.getAbsolutePath()}); assertTrue(targetFile.exists()); - NetcdfFileConcatenater.main(new String[]{targetFile.getAbsolutePath(), secondFile.getAbsolutePath()}); + NetcdfFileConcatenator.main(new String[]{targetFile.getAbsolutePath(), secondFile.getAbsolutePath()}); checkConcatenated3dVariable(firstFile, targetFile, secondFile, "temperature", 18000); } @@ -146,24 +152,20 @@ public void testNetcdf3Dmap() throws IOException { File secondFile = new File(testRunDataSubDir, "FlowFM_map.nc"); if (targetFile.exists()) BBUtils.deleteFileOrDir(targetFile); assertFalse(targetFile.exists()); - NetcdfFileConcatenater.main(new String[]{targetFile.getAbsolutePath(), firstFile.getAbsolutePath()}); + NetcdfFileConcatenator.main(new String[]{targetFile.getAbsolutePath(), firstFile.getAbsolutePath()}); assertTrue(targetFile.exists()); - NetcdfFileConcatenater.main(new String[]{targetFile.getAbsolutePath(), secondFile.getAbsolutePath()}); + NetcdfFileConcatenator.main(new String[]{targetFile.getAbsolutePath(), secondFile.getAbsolutePath()}); checkConcatenated3dVariable(firstFile, targetFile, secondFile, "tem1", 3000); } private static void checkConcatenated3dVariable(File firstFile, File targetFile, File secondFile, String variableName, int split) throws IOException { - NetcdfFile firstNetcdf = null; - NetcdfFile secondNetcdf = null; - NetcdfFile concatenatedNetcdf = null; - try { - firstNetcdf = NetcdfFile.open(firstFile.toString()); - secondNetcdf = NetcdfFile.open(secondFile.toString()); - concatenatedNetcdf = NetcdfFile.open(targetFile.toString()); - double[] firstValues = (double[]) firstNetcdf.findVariable(variableName).read().get1DJavaArray(Double.TYPE); - double[] secondValues = (double[]) secondNetcdf.findVariable(variableName).read().get1DJavaArray(Double.TYPE); - double[] concatenatedValues = (double[]) concatenatedNetcdf.findVariable(variableName).read().get1DJavaArray(Double.TYPE); + try (NetcdfFile firstNetcdf = NetcdfFiles.open(firstFile.toString()); + NetcdfFile secondNetcdf = NetcdfFiles.open(secondFile.toString()); + NetcdfFile concatenatedNetcdf = NetcdfFiles.open(targetFile.toString())) { + double[] firstValues = (double[]) Objects.requireNonNull(firstNetcdf.findVariable(variableName)).read().get1DJavaArray(DataType.DOUBLE); + double[] secondValues = (double[]) Objects.requireNonNull(secondNetcdf.findVariable(variableName)).read().get1DJavaArray(DataType.DOUBLE); + double[] concatenatedValues = (double[]) Objects.requireNonNull(concatenatedNetcdf.findVariable(variableName)).read().get1DJavaArray(DataType.DOUBLE); assertEquals(firstValues.length + secondValues.length - 125, concatenatedValues.length); for (int i = 0; i < split; i++) { assertEquals(firstValues[i], concatenatedValues[i]); @@ -171,10 +173,6 @@ private static void checkConcatenated3dVariable(File firstFile, File targetFile, for (int i = split; i < concatenatedValues.length; i++) { assertEquals(secondValues[i - split], concatenatedValues[i]); } - } finally { - if (firstNetcdf != null) firstNetcdf.close(); - if (secondNetcdf != null) secondNetcdf.close(); - if (concatenatedNetcdf != null) concatenatedNetcdf.close(); } } @@ -184,10 +182,10 @@ public void testNetcdfFixedTimeDimensionUseOldValueOnOverlapConcatenation() thro File targetFile = new File(testRunDataSubDir, "concatenated_rrrunoff_oldValueOverlap.nc"); if (targetFile.exists()) BBUtils.deleteFileOrDir(targetFile); assertFalse(targetFile.exists()); - NetcdfFileConcatenater.main(new String[]{targetFile.getAbsolutePath(), firstFile.getAbsolutePath(), "useOldValueOnOverlap=true"}); + NetcdfFileConcatenator.main(new String[]{targetFile.getAbsolutePath(), firstFile.getAbsolutePath(), "useOldValueOnOverlap=true"}); assertTrue(targetFile.exists()); File secondFile = new File(testRunDataSubDir, "rrunoff_201257.nc"); - NetcdfFileConcatenater.main(new String[]{targetFile.getAbsolutePath(), secondFile.getAbsolutePath(), "useOldValueOnOverlap=true"}); + NetcdfFileConcatenator.main(new String[]{targetFile.getAbsolutePath(), secondFile.getAbsolutePath(), "useOldValueOnOverlap=true"}); assertTrue(targetFile.exists()); checkConcatenatedValues(firstFile, targetFile, secondFile, 7); @@ -201,16 +199,16 @@ public void testConcatenationOfManyFiles() throws IOException { File[] files = Objects.requireNonNull(inputDirectory.listFiles()); Arrays.sort(files); for(File file : files) { - NetcdfFileConcatenater.main(new String[]{targetFile.getAbsolutePath(), file.getAbsolutePath(), "useOldValueOnOverlap=true"}); + NetcdfFileConcatenator.main(new String[]{targetFile.getAbsolutePath(), file.getAbsolutePath(), "useOldValueOnOverlap=true"}); } File expectedFile = new File(testRunDataSubDir, "expected_averaged.nc"); NetcdfFile expectedNetcdf = null; NetcdfFile targetNetcdf = null; try { - expectedNetcdf = NetcdfFile.open(expectedFile.toString()); - long expected = expectedNetcdf.findVariable("time").read().getSize(); - targetNetcdf = NetcdfFile.open(targetFile.toString()); - long target = targetNetcdf.findVariable("time").read().getSize(); + expectedNetcdf = NetcdfFiles.open(expectedFile.toString()); + long expected = Objects.requireNonNull(expectedNetcdf.findVariable("time")).read().getSize(); + targetNetcdf = NetcdfFiles.open(targetFile.toString()); + long target = Objects.requireNonNull(targetNetcdf.findVariable("time")).read().getSize(); assertEquals("Time data matches", expected, target); } finally { if (expectedNetcdf != null) expectedNetcdf.close(); @@ -224,16 +222,16 @@ public void testConcatenationOfManyFilesInOneGo() throws IOException { File targetFile = new File(testRunDataSubDir, "concatenated_averaged.nc"); if (targetFile.exists()) BBUtils.deleteFileOrDir(targetFile); - NetcdfFileConcatenater.main(new String[]{targetFile.getAbsolutePath(), inputDirectory.getAbsolutePath(), "useOldValueOnOverlap=true"}); + NetcdfFileConcatenator.main(new String[]{targetFile.getAbsolutePath(), inputDirectory.getAbsolutePath(), "useOldValueOnOverlap=true"}); assertTrue(targetFile.exists()); File expectedFile = new File(testRunDataSubDir, "expected_averaged.nc"); NetcdfFile expectedNetcdf = null; NetcdfFile targetNetcdf = null; try { - expectedNetcdf = NetcdfFile.open(expectedFile.toString()); - long expected = expectedNetcdf.findVariable("time").read().getSize(); - targetNetcdf = NetcdfFile.open(targetFile.toString()); - long target = targetNetcdf.findVariable("time").read().getSize(); + expectedNetcdf = NetcdfFiles.open(expectedFile.toString()); + long expected = Objects.requireNonNull(expectedNetcdf.findVariable("time")).read().getSize(); + targetNetcdf = NetcdfFiles.open(targetFile.toString()); + long target = Objects.requireNonNull(targetNetcdf.findVariable("time")).read().getSize(); assertEquals("Time data matches", expected, target); } finally { if (expectedNetcdf != null) expectedNetcdf.close(); @@ -242,16 +240,12 @@ public void testConcatenationOfManyFilesInOneGo() throws IOException { } private void checkConcatenatedValues(File firstFile, File targetFile, File secondFile, int split) throws IOException { - NetcdfFile firstNetcdf = null; - NetcdfFile secondNetcdf = null; - NetcdfFile concatenatedNetcdf = null; - try { - firstNetcdf = NetcdfFile.open(firstFile.toString()); - secondNetcdf = NetcdfFile.open(secondFile.toString()); - concatenatedNetcdf = NetcdfFile.open(targetFile.toString()); - double[] firstValues = (double[]) firstNetcdf.findVariable("Runoff").read().get1DJavaArray(Double.TYPE); - double[] secondValues = (double[]) secondNetcdf.findVariable("Runoff").read().get1DJavaArray(Double.TYPE); - double[] concatenatedValues = (double[]) concatenatedNetcdf.findVariable("Runoff").read().get1DJavaArray(Double.TYPE); + try (NetcdfFile firstNetcdf = NetcdfFiles.open(firstFile.toString()); + NetcdfFile secondNetcdf = NetcdfFiles.open(secondFile.toString()); + NetcdfFile concatenatedNetcdf = NetcdfFiles.open(targetFile.toString())) { + double[] firstValues = (double[]) Objects.requireNonNull(firstNetcdf.findVariable("Runoff")).read().get1DJavaArray(DataType.DOUBLE); + double[] secondValues = (double[]) Objects.requireNonNull(secondNetcdf.findVariable("Runoff")).read().get1DJavaArray(DataType.DOUBLE); + double[] concatenatedValues = (double[]) Objects.requireNonNull(concatenatedNetcdf.findVariable("Runoff")).read().get1DJavaArray(DataType.DOUBLE); assertEquals(firstValues.length + secondValues.length - 1, concatenatedValues.length); for (int i = 0; i < split; i++) { assertEquals(firstValues[i], concatenatedValues[i]); @@ -259,10 +253,6 @@ private void checkConcatenatedValues(File firstFile, File targetFile, File secon for (int i = split; i < concatenatedValues.length; i++) { assertEquals(secondValues[i - 6], concatenatedValues[i]); } - } finally { - if (firstNetcdf != null) firstNetcdf.close(); - if (secondNetcdf != null) secondNetcdf.close(); - if (concatenatedNetcdf != null) concatenatedNetcdf.close(); } } } diff --git a/core/java/test/org/openda/utils/io/KalmanGainStorageTest.java b/core/java/test/org/openda/utils/io/KalmanGainStorageTest.java index 09d09c2c8..51d8c5bf3 100644 --- a/core/java/test/org/openda/utils/io/KalmanGainStorageTest.java +++ b/core/java/test/org/openda/utils/io/KalmanGainStorageTest.java @@ -20,14 +20,11 @@ package org.openda.utils.io; import junit.framework.TestCase; import org.openda.blackbox.config.BBUtils; -import org.openda.costa.CtaTreeVector; -import org.openda.costa.CtaVector; import org.openda.exchange.timeseries.TimeUtils; import org.openda.interfaces.ITreeVector; import org.openda.interfaces.IVector; import org.openda.utils.OpenDaTestSupport; import org.openda.utils.TreeVector; -import org.openda.utils.Vector; import java.io.File; import java.io.IOException; @@ -47,7 +44,7 @@ protected void setUp() throws IOException { testRunDataDir = testData.getTestRunDataDir(); } - public void testReadWriteKalmanGainXML() throws Exception { + public void testReadWriteKalmanGainXML() { File kgStorageDir = new File(testRunDataDir, "kgStorage_201102181800"); File kgStorageDirCopy = new File(testRunDataDir, "kgStorage_201102181800_copy"); @@ -244,7 +241,7 @@ private void checkKalmanGainContentsDoubleState(double timeAsMJD, KalmanGainStor assertEquals("NoiseY", secondStateSubTreeVectorIds.get(0)); double[] secondStateValues = secondStateTreeVector.getValues(); - assertEquals(120 - 0, secondStateValues[0], delta); + assertEquals(120 , secondStateValues[0], delta); assertEquals(120 - 1, secondStateValues[1], delta); assertEquals(120 - 2, secondStateValues[2], delta); assertEquals(120 - 13, secondStateValues[13], delta); @@ -263,107 +260,4 @@ private void checkKalmanGainContentsDoubleState(double timeAsMJD, KalmanGainStor assertEquals(-0.038791, s1Values[4], delta); } - private void writeAndReadKalmanGain(double timeAsMJD, int columnSize) { - // write the kalman gain - IVector[] kalmanGainColumnsOut = createKalmanGainColumns(columnSize); - String[] observationIdsOut = {"loc-A", "loc-B", "loc-C", "loc-D"}; - double[] observationOffsetsInDaysOut = {-0.1d, -3d, 0d, 0d}; - String commentOut = "this is a four column kalman gain"; - KalmanGainStorage kgStorageOut = new KalmanGainStorage(testRunDataDir, timeAsMJD); - kgStorageOut.setComment(commentOut); - kgStorageOut.setKalmanGainStorageFileName("fourColumnStorage.xml"); - kgStorageOut.writeKalmanGain(observationIdsOut, observationOffsetsInDaysOut, kalmanGainColumnsOut, null); - - // read the kalman gain - KalmanGainStorage kgStorageIn = new KalmanGainStorage(testRunDataDir, timeAsMJD); - kgStorageIn.setKalmanGainStorageFileName("fourColumnStorage.xml"); - kgStorageIn.readKalmanGain(); - String commentIn = kgStorageIn.getComment(); - assertEquals("Comment out/in", commentOut, commentIn); - - String[] observationIdsIn = kgStorageIn.getObservationIds(); - for (int i = 0; i < observationIdsIn.length; i++) { - assertEquals("observationId out/in", observationIdsOut[i], observationIdsIn[i]); - } - - double[] observationOffsetsInDaysIn = kgStorageIn.getObservationOffsetInDays(); - for (int i = 0; i < observationOffsetsInDaysIn.length; i++) { - assertEquals("observationOffsetInDays out/in", - observationOffsetsInDaysOut[i], observationOffsetsInDaysIn[i]); - } - - IVector[] kalmanGainColumnsIn = kgStorageIn.getKalmanGainColumns(); - for (int i = 0; i < kalmanGainColumnsIn.length; i++) { - for (int j = 0; j < kalmanGainColumnsIn[i].getSize(); j++) { - assertEquals("Value out/in", kalmanGainColumnsOut[i].getValue(j), kalmanGainColumnsIn[i].getValue(j)); - } - } - } - - private IVector[] createKalmanGainColumns(int columnSize) { - // kalman gain column that will be written - IVector[] kalmanGainColumns = new IVector[4]; - // first kalman gain column - kalmanGainColumns[0] = new Vector(columnSize); - kalmanGainColumns[0].setConstant(321.0d); - // second kalman gain column - kalmanGainColumns[1] = new Vector(columnSize); - for (int i = 0; i < kalmanGainColumns[1].getSize(); i++) { - kalmanGainColumns[1].setValue(i, 101d + 1); - } - // third kalman gain column (cta-tree vector) - kalmanGainColumns[2] = createKgColumnTreeVector("kgColumn3XML", columnSize); - // third kalman gain column (cta-tree vector) - kalmanGainColumns[3] = createKgColumnCTATreeVector("kgColumn4Cta", columnSize); - return kalmanGainColumns; - } - - private TreeVector createKgColumnTreeVector(String treeVectorId, int totalSize) { - TreeVector treeVector = new TreeVector(treeVectorId); - - // tree vector part 1 - double[] part1Values = {999d, 888d, 777d, 666d, 555d, 444d, 333d, 222d, 111d}; - treeVector.addChild("part1", part1Values); - - // tree vector part 3 - IVector part3Vector = new Vector(16); - part3Vector.setConstant(23d); - treeVector.addChild("part2", part3Vector.getValues()); - - // tree vector part 2 - double[] part2Values = new double[totalSize- part1Values.length-part3Vector.getSize()]; - for (int i = 0; i < part2Values.length; i++) { - part2Values[i] = 1010d + i * 10; - } - treeVector.addChild("part3", part2Values); - return treeVector; - } - - private CtaTreeVector createKgColumnCTATreeVector(String treeVectorId, int totalSize) { - - // tree vector part 1 - double[] part1Values = {999d, 888d, 777d, 666d, 555d, 444d, 333d, 222d, 111d}; - IVector vec1 = new CtaVector(9); - vec1.setValues(part1Values); - CtaTreeVector cta_part1 = new CtaTreeVector("first part", "first", vec1); - - // tree vector part 3 - IVector part3Vector = new CtaVector(16); - part3Vector.setConstant(23d); - CtaTreeVector cta_part3 = new CtaTreeVector("third part", "third", part3Vector); - - // tree vector part 2 - double[] part2Values = new double[totalSize- part1Values.length-part3Vector.getSize()]; - for (int i = 0; i < part2Values.length; i++) { - part2Values[i] = 1010d + i * 10; - } - IVector vec2 = new CtaVector(part2Values.length); - vec2.setValues(part2Values); - CtaTreeVector cta_part2 = new CtaTreeVector("second part", "second", vec1); - - CtaTreeVector[] subVectors = {cta_part1, cta_part2, cta_part3}; - return new CtaTreeVector(treeVectorId,treeVectorId,subVectors); - - } - } diff --git a/model_delft3d/java/src/org/openda/model_delft3d/NetcdfD3dHisDataObject.java b/model_delft3d/java/src/org/openda/model_delft3d/NetcdfD3dHisDataObject.java index 74e914a52..8c6023f3e 100644 --- a/model_delft3d/java/src/org/openda/model_delft3d/NetcdfD3dHisDataObject.java +++ b/model_delft3d/java/src/org/openda/model_delft3d/NetcdfD3dHisDataObject.java @@ -22,9 +22,10 @@ import org.openda.exchange.dataobjects.NetcdfUtils; import org.openda.interfaces.*; import ucar.ma2.Array; +import ucar.ma2.DataType; import ucar.nc2.Dimension; import ucar.nc2.NetcdfFile; -import ucar.nc2.NetcdfFileWriter; +import ucar.nc2.write.NetcdfFormatWriter; import ucar.nc2.Variable; import java.io.File; @@ -100,7 +101,7 @@ public IExchangeItem getDataObjectExchangeItem(String exchangeItemID) { private void readNetCdfVariables() { //in most netcdfFiles the time and spatial coordinate variables are shared between multiple data variables. - //Therefore cache timeInfo objects so that time coordinate variables are never read more than once. + //Therefore, cache timeInfo objects so that time coordinate variables are never read more than once. Map timeInfoCache = new HashMap(); // Loading of variable containing stations names @@ -116,7 +117,7 @@ private void readNetCdfVariables() { e.printStackTrace(); } // Extraction of station names - char[] nameCharArray = (char[]) stationNamesChar.get1DJavaArray(char.class); + char[] nameCharArray = (char[]) stationNamesChar.get1DJavaArray(DataType.CHAR); int stringLength=stationNamesChar.getShape()[1]; for (Variable variable : this.netcdfFile.getVariables()) { @@ -140,7 +141,7 @@ private void readNetCdfVariables() { throw new RuntimeException("NetcdfD3dHisDataObject could not read time variable " + timeVariable.getShortName() + "from netcdf file " + netcdfFile.getLocation()); } - timesInNetcdfFile = (double[]) timesArray.get1DJavaArray(double.class); + timesInNetcdfFile = (double[]) timesArray.get1DJavaArray(DataType.DOUBLE); } timeInfoForAllTimeDepVars = new TimeInfo(timesInNetcdfFile); @@ -217,7 +218,7 @@ private void readNetCdfVariables() { throw new RuntimeException("NetcdfD3dHisDataObject could not read time variable " + timeVariable.getShortName() + "from netcdf file " + netcdfFile.getLocation()); } - timesInNetcdfFile = (double[]) timesArray.get1DJavaArray(double.class); + timesInNetcdfFile = (double[]) timesArray.get1DJavaArray(DataType.DOUBLE); } timeInfoForAllTimeDepVars = new TimeInfo(timesInNetcdfFile); @@ -502,17 +503,17 @@ public static void hisFileWriter(String varName, double[] values,File workingDir } - NetcdfFileWriter netcdfFileWriter = null; + NetcdfFormatWriter NetcdfWriter = null; try { - netcdfFileWriter = NetcdfFileWriter.openExisting(netcdfFilePath.getAbsolutePath()); + NetcdfWriter = (NetcdfFormatWriter.openExisting(netcdfFilePath.getAbsolutePath())).build(); } catch (IOException e) { - e.printStackTrace(); + throw new RuntimeException(e); } - NetcdfUtils.writeSelectedData(netcdfFileWriter,variable, origin, sizeArray, stationValues); + NetcdfUtils.writeSelectedData(NetcdfWriter,variable, origin, sizeArray, stationValues); try { - netcdfFileWriter.close(); + NetcdfWriter.close(); netcdfHisFile.close(); } catch (IOException e) { e.printStackTrace(); diff --git a/model_delft3d/java/src/org/openda/model_delft3d/NetcdfD3dMapDataObject.java b/model_delft3d/java/src/org/openda/model_delft3d/NetcdfD3dMapDataObject.java index ba3866494..a0b69cb98 100644 --- a/model_delft3d/java/src/org/openda/model_delft3d/NetcdfD3dMapDataObject.java +++ b/model_delft3d/java/src/org/openda/model_delft3d/NetcdfD3dMapDataObject.java @@ -22,9 +22,11 @@ import org.openda.exchange.TimeInfo; import org.openda.exchange.dataobjects.NetcdfUtils; import org.openda.interfaces.*; +import ucar.ma2.DataType; import ucar.nc2.Dimension; import ucar.nc2.NetcdfFile; -import ucar.nc2.NetcdfFileWriter; +import ucar.nc2.NetcdfFiles; +import ucar.nc2.write.NetcdfFormatWriter; import ucar.nc2.Variable; import java.io.File; @@ -78,7 +80,7 @@ public void initialize(File workingDir, String[] arguments) { this.runID = arguments[0].substring(5,arguments[0].length()-3); try { - netcdfFile = NetcdfFile.open(netcdfFilePath.getAbsolutePath()); + netcdfFile = NetcdfFiles.open(netcdfFilePath.getAbsolutePath()); } catch (IOException e) { throw new RuntimeException("NetcdfD3dMapDataObject could not open netcdf file " + netcdfFilePath.getAbsolutePath()); } @@ -184,7 +186,7 @@ private void readNetCdfVariables() { throw new RuntimeException("NetcdfD3dMapDataObject could not read time variable " + timeVariable.getShortName() + "from netcdf file " + netcdfFile.getLocation()); } - timesInNetcdfFile = (double[]) timesArray.get1DJavaArray(double.class); + timesInNetcdfFile = (double[]) timesArray.get1DJavaArray(DataType.DOUBLE); } // get the number of spatial dimensions @@ -401,18 +403,18 @@ public void writeExchangeItemValues(String varName, double[] values) { // origin[kmaxOutRestrDimensionIndex] = 0; //} - NetcdfFileWriter netcdfFileWriter= null; + NetcdfFormatWriter NetcdfFormatWriter= null; try { - netcdfFileWriter = NetcdfFileWriter.openExisting(this.netcdfFilePath.getAbsolutePath()); + NetcdfFormatWriter = NetcdfFormatWriter.openExisting(this.netcdfFilePath.getAbsolutePath()).build(); } catch (IOException e) { e.printStackTrace(); } - NetcdfUtils.writeSelectedData(netcdfFileWriter,variable, origin, sizeArray, values); + NetcdfUtils.writeSelectedData(NetcdfFormatWriter,variable, origin, sizeArray, values); try { - netcdfFileWriter.close(); + NetcdfFormatWriter.close(); //if (binRestartFile != null) { //binRestartFile.close(); //} @@ -483,7 +485,7 @@ public double[] back2RealDomain(double[] expandedDomain, int LastTimeIndex) { private double[][][] from1dTo3dArray(double[] oneDArray){ - double Domain3D[][][] = new double[mMax][nMax][nLay]; + double[][][] Domain3D = new double[mMax][nMax][nLay]; int k=0; for (int lay = 0; lay < nLay; lay++) { diff --git a/model_dflowfm_blackbox/java/src/org/openda/model_dflowfm/DFlowFMRestartFileWrapper.java b/model_dflowfm_blackbox/java/src/org/openda/model_dflowfm/DFlowFMRestartFileWrapper.java index 87806d06d..34ffa224d 100644 --- a/model_dflowfm_blackbox/java/src/org/openda/model_dflowfm/DFlowFMRestartFileWrapper.java +++ b/model_dflowfm_blackbox/java/src/org/openda/model_dflowfm/DFlowFMRestartFileWrapper.java @@ -30,7 +30,10 @@ import ucar.ma2.Array; import ucar.ma2.DataType; import ucar.ma2.InvalidRangeException; -import ucar.nc2.*; +import ucar.nc2.NetcdfFile; +import ucar.nc2.Attribute; +import ucar.nc2.Variable; +import ucar.nc2.write.NetcdfFormatWriter; import java.io.File; import java.io.IOException; @@ -325,8 +328,9 @@ public IExchangeItem getDataObjectExchangeItem(String ExchangeItemID) { // Write the NetCDF restart file public void finish() { try { - NetcdfFileWriter netcdfFileWriter= NetcdfFileWriter.openExisting(netcdffileName); - netcdfFileWriter.setFill(true); + NetcdfFormatWriter.Builder NetcdfBuilder= NetcdfFormatWriter.openExisting(netcdffileName); + NetcdfBuilder.setFill(true); + NetcdfFormatWriter NetcdfWriter = NetcdfBuilder.build(); //Loop over all exchangeItems Set keys = this.ExchangeItems.keySet(); @@ -351,9 +355,9 @@ public void finish() { for (int i=0; i - +