Skip to content

454 upgrade the netcdf library #484

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
aljavrieling wants to merge 9 commits into
base: master
Choose a base branch
from
Draft

454 upgrade the netcdf library #484

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
bc69709
Remove *.dia files from dflowfm input directories
vrielin Mar 12, 2026
9557637
Upgrade to netcdfAll-5.9.1.jar; no build erroros, lots of deprecated …
vrielin Mar 12, 2026
103b3d0
Replace deprecated method with ucar.nc2.write.Ncdump
vrielin Mar 26, 2026
8a23ff8
Change datatype to eliminate warnings for deprecated method get1DJava…
vrielin Mar 26, 2026
26a7b83
Replace deprecated method ucar.nc2.NetcdfFileWriter with ucar.nc2.wri…
vrielin Apr 23, 2026
352180f
Explicit import statements for ucar.nc2.*; some warnings eliminated
vrielin Apr 23, 2026
91ddbfe
Eliminate warnings about deprecated methods (other than NetcdfFileWri…
vrielin Apr 23, 2026
37ba40a
Replace deprecated methods in NetcdfFileConcatenator.java and Tests
vrielin Apr 23, 2026
ff32336
Remove one more deprecated method
vrielin Apr 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion algorithms/java/src/org/openda/algorithms/kalmanFilter/EnKF.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -718,7 +718,7 @@ protected void saveGain(FileBasedModelState algorithmState) {
// write gain into files in a temp dir
File kgStorage = new File(algorithmState.getDirContainingModelStateFiles(), "kgStorage_restart_tempdir");
double lastAnalysisTime = this.smoothedGainMatrix.previousAnalysisTime;
KalmanGainStorage gainStorage = new KalmanGainStorage(kgStorage, lastAnalysisTime, false); // here we should provide lastAnalysisTimeMJD
KalmanGainStorage gainStorage = new KalmanGainStorage(kgStorage, lastAnalysisTime); // here we should provide lastAnalysisTimeMJD
gainStorage.writeKalmanGain(this.smoothedGainMatrix.lastGainMatrixHashMap, this.smoothedGainMatrix.obsIds, this.smoothedGainMatrix.lastObsTimeOffsets);

FileBasedModelState gainState = new FileBasedModelState();
Expand Down
Binary file removed core/java/resources/netcdf/netcdf-4.3.23.jar
View file
Open in desktop
Binary file not shown.
Binary file added core/java/resources/netcdf/netcdfAll-5.9.1.jar
View file
Open in desktop
Binary file not shown.
134 changes: 66 additions & 68 deletions core/java/src/org/openda/exchange/dataobjects/NetcdfDataObject.java
View file
Open in desktop

Large diffs are not rendered by default.

100 changes: 53 additions & 47 deletions ...e/dataobjects/NetcdfFileConcatenater.java → ...e/dataobjects/NetcdfFileConcatenator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,26 +24,32 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ucar.ma2.*;
import ucar.nc2.*;
import ucar.nc2.Attribute;
import ucar.nc2.Dimension;
import ucar.nc2.Variable;
import ucar.nc2.Variable.Builder;
import ucar.nc2.units.DateUnit;

import ucar.nc2.NetcdfFile;
import ucar.nc2.NetcdfFiles;
import java.io.File;
import java.io.IOException;
import java.util.*;
import ucar.nc2.write.NetcdfFormatWriter;

public class NetcdfFileConcatenater {
public class NetcdfFileConcatenator {

private static final Logger LOGGER = LoggerFactory.getLogger(NetcdfFileConcatenater.class);
private static final Logger LOGGER = LoggerFactory.getLogger(NetcdfFileConcatenator.class);

public static void main(String[] arguments) {
if (arguments.length < 2) {
throw new IllegalArgumentException("NetcdfFileConcatenater expects at least two arguments:\n" +
throw new IllegalArgumentException("NetcdfFileConcatenator expects at least two arguments:\n" +
"<targetNetcdfFile> and <netcdfFileToBeAdded>");
}
boolean useOldValueOnOverlap = false;
for (int i = 2; i < arguments.length; i++) {
String argument = arguments[i];
String[] keyValue = StringUtilities.getKeyValuePair(argument);
assert keyValue != null;
String key = keyValue[0];
String value = keyValue[1];
if (key.equals("useOldValueOnOverlap")) {
Expand All @@ -52,10 +58,10 @@
throw new RuntimeException("Unknown key " + key + ". Please specify only useOldValueOnOverlap as key=value pair");
}
}
// file must be there. If it is indeed, the path has been made absolute by BBaction
// file must be there. If it is indeed, the path has been made absolute by BB-action
File netcdfFileToBeAdded = new File(arguments[1]);

// file may be there. If so, the path has been made absolute by BBaction
// file may be there. If so, the path has been made absolute by BB-action
File targetNetcdfFile = new File(arguments[0]);
if (!targetNetcdfFile.isAbsolute()) {
// target file is not there yet; make path absolute
Expand Down Expand Up @@ -96,31 +102,27 @@
}

private static void addNetcdfFile(File targetNetcdfFile, File netcdfFileToBeAdded) throws IOException {
NetcdfFile netcdfToAdd = null;
try {
netcdfToAdd = NetcdfFile.open(netcdfFileToBeAdded.getAbsolutePath());
try (NetcdfFile netcdfToAdd = NetcdfFiles.open(netcdfFileToBeAdded.getAbsolutePath())) {
Dimension time = netcdfToAdd.findDimension("time");
if (time != null && time.isUnlimited()) {
BBUtils.copyFile(netcdfFileToBeAdded, targetNetcdfFile);
} else {
rewriteNetcdfFile(targetNetcdfFile, netcdfToAdd);
}
} finally {
if (netcdfToAdd != null) netcdfToAdd.close();
}
}

private static void concatenateNetcdfFiles(boolean useOldValueOnOverlap, List<File> netcdfFilesToBeAdded, File targetNetcdfFile) throws IOException {
NetcdfFile sourceNetcdfFile = null;
NetcdfFileWriter netcdfFileWriter = null;
NetcdfFormatWriter netcdfFileWriter = null;
try {
netcdfFileWriter = NetcdfFileWriter.openExisting(targetNetcdfFile.getAbsolutePath());
netcdfFileWriter = NetcdfFormatWriter.openExisting(targetNetcdfFile.getAbsolutePath()).build();

Map<Variable, Array> variableArraysMap = new HashMap<>();
Map<Variable, Array> timeVariableArraysMap = new HashMap<>();

for (File netcdfFileToBeAdded : netcdfFilesToBeAdded) {
sourceNetcdfFile = NetcdfFile.open(netcdfFileToBeAdded.getAbsolutePath());
sourceNetcdfFile = NetcdfFiles.open(netcdfFileToBeAdded.getAbsolutePath());
List<Variable> variablesToBeAdded = sourceNetcdfFile.getVariables();

concatenateVariables(useOldValueOnOverlap, sourceNetcdfFile, variablesToBeAdded, netcdfFileWriter, variableArraysMap, timeVariableArraysMap);
Expand All @@ -145,11 +147,11 @@

private static void concatenateNetcdfFiles(boolean useOldValueOnOverlap, File netcdfFileToBeAdded, File targetNetcdfFile) throws IOException {
NetcdfFile sourceNetcdfFile = null;
NetcdfFileWriter netcdfFileWriter = null;
NetcdfFormatWriter netcdfFileWriter = null;
try {
sourceNetcdfFile = NetcdfFile.open(netcdfFileToBeAdded.getAbsolutePath());
sourceNetcdfFile = NetcdfFiles.open(netcdfFileToBeAdded.getAbsolutePath());
List<Variable> variablesToBeAdded = sourceNetcdfFile.getVariables();
netcdfFileWriter = NetcdfFileWriter.openExisting(targetNetcdfFile.getAbsolutePath());
netcdfFileWriter = NetcdfFormatWriter.openExisting(targetNetcdfFile.getAbsolutePath()).build();

Map<Variable, Array> variableArraysMap = new HashMap<>();
Map<Variable, Array> timeVariableArraysMap = new HashMap<>();
Expand All @@ -172,14 +174,14 @@
}
}

private static void concatenateVariables(boolean useOldValueOnOverlap, NetcdfFile sourceNetCdfFile, List<Variable> variablesToBeAdded, NetcdfFileWriter netcdfFileWriter, Map<Variable, Array> variableArraysMap, Map<Variable, Array> timeVariableArraysMap) throws IOException {
private static void concatenateVariables(boolean useOldValueOnOverlap, NetcdfFile sourceNetCdfFile, List<Variable> variablesToBeAdded, NetcdfFormatWriter netcdfFileWriter, Map<Variable, Array> variableArraysMap, Map<Variable, Array> timeVariableArraysMap) throws IOException {

Check warning on line 177 in core/java/src/org/openda/exchange/dataobjects/NetcdfFileConcatenator.java

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

core/java/src/org/openda/exchange/dataobjects/NetcdfFileConcatenator.java#L177 

The method 'concatenateVariables(boolean, NetcdfFile, List, NetcdfFormatWriter, Map, Map)' has an NPath complexity of 9217, current threshold is 200
for (Variable variable : variablesToBeAdded) {
if (NetcdfUtils.isTimeVariable(variable)) continue;
Variable timeVariableToBeAdded = NetcdfUtils.findTimeVariableForVariable(variable, sourceNetCdfFile);
if (timeVariableToBeAdded == null) continue;
Variable targetVariable = netcdfFileWriter.findVariable(variable.getFullNameEscaped());
Variable targetVariable = netcdfFileWriter.findVariable(variable.getFullName());
if (targetVariable == null) continue;
Variable timeVariableTarget = NetcdfUtils.findTimeVariableForVariable(targetVariable, netcdfFileWriter.getNetcdfFile());
Variable timeVariableTarget = NetcdfUtils.findTimeVariableForVariable(targetVariable, netcdfFileWriter.getOutputFile());
if (timeVariableTarget == null) continue;

List<Dimension> addedDimensions = variable.getDimensions();
Expand All @@ -197,13 +199,18 @@
int targetLocationDimensionLength = targetLocationDimension.getLength();
if (addedLocationDimension.getLength() != targetLocationDimensionLength) throw new RuntimeException("Variables from source and target must have same location dimension size");
}
double[] targetValues = (double[]) targetVariable.read().get1DJavaArray(Double.TYPE);
double[] addedValues = (double[]) variable.read().get1DJavaArray(Double.TYPE);

Array read = timeVariableTarget.read();
double[] timesTarget = (double[]) read.get1DJavaArray(Double.TYPE);
NetcdfFile reader = NetcdfFiles.open(netcdfFileWriter.getOutputFile().getLocation());
Variable var_to_read = reader.findVariable(targetVariable.getFullName());
assert var_to_read != null;
double[] targetValues = (double[]) var_to_read.read().get1DJavaArray(DataType.DOUBLE);
double[] addedValues = (double[]) variable.read().get1DJavaArray(DataType.DOUBLE);

var_to_read = reader.findVariable(timeVariableTarget.getFullName());
assert var_to_read != null;
double[] timesTarget = (double[]) var_to_read.read().get1DJavaArray(DataType.DOUBLE);
reader.close();
String timeVariableTargetUnitsString = timeVariableTarget.getUnitsString();
double[] timesToBeAdded = (double[]) timeVariableToBeAdded.read().get1DJavaArray(Double.TYPE);
double[] timesToBeAdded = (double[]) timeVariableToBeAdded.read().get1DJavaArray(DataType.DOUBLE);
String timeVariableToBeAddedUnitsString = timeVariableToBeAdded.getUnitsString();
DateUnit targetDateUnit;
DateUnit toBeAddedDateUnit;
Expand Down Expand Up @@ -238,14 +245,14 @@


private static void rewriteNetcdfFile(File targetNetcdfFile, NetcdfFile netcdfToAdd) throws IOException {
NetcdfFileWriter netcdfWriter = null;
NetcdfFormatWriter netcdfWriter = null;

try {
netcdfWriter = NetcdfFileWriter.createNew(NetcdfFileWriter.Version.netcdf3, targetNetcdfFile.getCanonicalPath());
NetcdfFormatWriter.Builder netcdfBuilder = NetcdfFormatWriter.createNewNetcdf3(targetNetcdfFile.getCanonicalPath());

redefineVariablesAndDimensions(netcdfToAdd, netcdfWriter);
redefineVariablesAndDimensions(netcdfToAdd, netcdfBuilder);

netcdfWriter.create();
netcdfWriter = netcdfBuilder.build();

writeValues(netcdfToAdd, netcdfWriter);
} catch (InvalidRangeException e) {
Expand All @@ -255,11 +262,11 @@
}
}

private static void writeValues(NetcdfFile netcdfToAdd, NetcdfFileWriter netcdfWriter) throws IOException, InvalidRangeException {
private static void writeValues(NetcdfFile netcdfToAdd, NetcdfFormatWriter netcdfWriter) throws IOException, InvalidRangeException {
List<Variable> variables = netcdfToAdd.getVariables();
for (Variable variable : variables) {
String fullNameEscaped = variable.getFullNameEscaped();
Variable netcdfFileVariable = netcdfWriter.findVariable(fullNameEscaped);
String fullName = variable.getFullName();
Variable netcdfFileVariable = netcdfWriter.findVariable(fullName);
Array read = variable.read();
netcdfWriter.write(netcdfFileVariable, read);
}
Expand Down Expand Up @@ -348,37 +355,36 @@
}
}

private static void redefineVariablesAndDimensions(NetcdfFile source, NetcdfFileWriter target) {
private static void redefineVariablesAndDimensions(NetcdfFile source, NetcdfFormatWriter.Builder target) {

List<Dimension> dimensions = source.getDimensions();
List<Dimension> dimensions = source.getRootGroup().getDimensions();
for (Dimension dimension : dimensions) {
String fullNameEscaped = dimension.getFullNameEscaped();
if ("time".equals(fullNameEscaped)) continue;
target.addDimension(null, fullNameEscaped, dimension.getLength());
String fullName = dimension.getName();
if ("time".equals(fullName)) continue;
target.addDimension(fullName, dimension.getLength());
}

Dimension timeDimension = target.addUnlimitedDimension("time");
List<Variable> variables = source.getVariables();
rewriteVariables(target, variables, timeDimension);
}

private static void rewriteVariables(NetcdfFileWriter netcdf, List<Variable> variables, Dimension timeDimension) {
private static void rewriteVariables(NetcdfFormatWriter.Builder netcdf, List<Variable> variables, Dimension timeDimension) {
for (Variable variable : variables) {
List<Dimension> newDimensions = getDimensions(variable, timeDimension);
String fullNameEscaped = variable.getFullNameEscaped();
Variable newVariable = netcdf.addVariable(null, fullNameEscaped, variable.getDataType(), newDimensions);
List<Attribute> attributes = variable.getAttributes();
for (Attribute attribute : attributes) {
netcdf.addVariableAttribute(newVariable, attribute);
String fullName = variable.getFullName();
Builder newVariable = netcdf.addVariable(fullName, variable.getDataType(), newDimensions);
for (Attribute attribute : variable.attributes()) {
newVariable.addAttribute(attribute);
}
}
}

private static List<Dimension> getDimensions(Variable variable, Dimension timeDimension) {
List<Dimension> dimensionsAll = variable.getDimensionsAll();
List<Dimension> dimensionsAll = variable.getDimensions();
List<Dimension> newDimensions = new ArrayList<>(dimensionsAll.size());
for (Dimension dimension : dimensionsAll) {
if (dimension.getFullNameEscaped().equals("time")) {
if (dimension.getName().equals("time")) {
newDimensions.add(timeDimension);
} else {
newDimensions.add(dimension);
Expand Down
Loading
Loading