From fa4236ae6ef9494bcd38d6e53e38373fba99d37e Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Mon, 1 Feb 2021 15:56:42 +0100 Subject: [PATCH 01/67] data preprocessing --- src/mlmc/metamodel/__init__.py | 0 src/mlmc/metamodel/main.py | 110 +++++++++++++++++++++++++++++++++ src/mlmc/tool/hdf5.py | 10 ++- 3 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 src/mlmc/metamodel/__init__.py create mode 100644 src/mlmc/metamodel/main.py diff --git a/src/mlmc/metamodel/__init__.py b/src/mlmc/metamodel/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/mlmc/metamodel/main.py b/src/mlmc/metamodel/main.py new file mode 100644 index 00000000..74db545c --- /dev/null +++ b/src/mlmc/metamodel/main.py @@ -0,0 +1,110 @@ +import os +import numpy as np +import pandas as pd +from mlmc.tool.hdf5 import HDF5 +import matplotlib.pyplot as plt + +from sklearn.model_selection import train_test_split + +DATA_PATH = "/home/martin/Documents/metamodels/data" + +def get_inputs(dir): + fields_file = os.path.join(dir, "fine_fields_sample.msh") + input = [] + with open(fields_file, "r") as r: + fields = r.readlines() + for f in fields[12:]: + line = f.split(" ") + if len(line) > 1: + input.append(float(line[1])) + else: + break + return input + + +def preprocess_data(): + dir_path = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/output" + hdf = HDF5(file_path="/home/martin/Documents/metamodels/data/L1/test/01_cond_field/mlmc_1.hdf5", + load_from_file=True) + level_group = hdf.add_level_group(level_id=str(0)) + collected = zip(level_group.get_collected_ids(), level_group.collected()) + + df_values = [] + for sample_id, col_values in collected: + output_value = col_values[0, 0] + sample_dir = os.path.join(dir_path, sample_id) + if os.path.isdir(sample_dir): + input = get_inputs(sample_dir) + d = {'x': np.array(input), 'y': output_value} + df_values.append(d) + + df = pd.DataFrame(df_values) + df.to_pickle(os.path.join(DATA_PATH, "data.pkl")) + + +def load_data(): + df = pd.read_pickle(os.path.join(DATA_PATH, "data.pkl")) + return df + + +def data_analysis(df): + print(df.info()) + print(df.y.describe()) + + # df.y.plot.hist(bins=50, logx=True) + # plt.show() + df.y.plot.kde(bw_method=0.3) + plt.xlim([-5, df.y.max()]) + plt.show() + + +def support_vector_regression(df): + from sklearn.svm import SVR + from sklearn.preprocessing import StandardScaler + print("df. info ", df.info) + train, test = train_test_split(df, test_size=0.2) + + print("train describe", train.describe()) + print("test describe ", test.describe()) + + + + # sc_X = StandardScaler() + # sc_y = StandardScaler() + # X = sc_X.fit_transform(train.x) + # y = sc_y.fit_transform(train.y) + + x = np.stack(train.x.to_numpy(), axis=0) + y = train.y.to_numpy() + + svr_rbf = SVR(kernel='rbf', gamma='auto') # 'linear' kernel fitting is never-ending and 'poly' kernel gives very bad score (e.g. -2450), sigmoid gives also bad score (e.g. -125) + svr_rbf.fit(x, y) + + train_error = svr_rbf.score(np.stack(train.x.to_numpy(), axis=0), train.y.to_numpy()) + test_error = svr_rbf.score(np.stack(test.x.to_numpy(), axis=0), test.y.to_numpy()) + + print("train error ", train_error) + print("test error ", test_error) + + +def svr_run(): + # preprocess_data() + df = load_data() + # data_analysis(df) + support_vector_regression(df) + + +if __name__ == "__main__": + # import cProfile + # import pstats + # pr = cProfile.Profile() + # pr.enable() + svr_run() + # my_result = svr_run() + # + # pr.disable() + # ps = pstats.Stats(pr).sort_stats('cumtime') + # ps.print_stats() + + + diff --git a/src/mlmc/tool/hdf5.py b/src/mlmc/tool/hdf5.py index d012000d..620c9972 100644 --- a/src/mlmc/tool/hdf5.py +++ b/src/mlmc/tool/hdf5.py @@ -401,6 +401,12 @@ def collected_n_items(self): collected_n_items = len(dataset[()]) return collected_n_items + def get_collected_ids(self): + collected_ids = [] + with h5py.File(self.file_name, 'r') as hdf_file: + collected_ids = [sample[0].decode() for sample in hdf_file[self.level_group_path][self.collected_ids_dset][()]] + return collected_ids + def get_finished_ids(self): """ Get collected and failed samples ids @@ -408,8 +414,8 @@ def get_finished_ids(self): """ with h5py.File(self.file_name, 'r') as hdf_file: failed_ids = [sample[0].decode() for sample in hdf_file[self.level_group_path][self.failed_dset][()]] - successful_ids = [sample[0].decode() for sample in hdf_file[self.level_group_path][self.collected_ids_dset][()]] - return np.concatenate((np.array(successful_ids), np.array(failed_ids)), axis=0) + collected_ids = [sample[0].decode() for sample in hdf_file[self.level_group_path][self.collected_ids_dset][()]] + return np.concatenate((np.array(collected_ids), np.array(failed_ids)), axis=0) def get_unfinished_ids(self): """ From 0f5220c51b3a175c6e93a0acc1f6379f5f96ed5b Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Wed, 10 Feb 2021 16:28:49 +0100 Subject: [PATCH 02/67] create graph --- src/mlmc/metamodel/GCN.py | 0 src/mlmc/metamodel/create_graph.py | 107 +++++++++++++++++++++++++++++ src/mlmc/metamodel/mnist.py | 0 3 files changed, 107 insertions(+) create mode 100644 src/mlmc/metamodel/GCN.py create mode 100644 src/mlmc/metamodel/create_graph.py create mode 100644 src/mlmc/metamodel/mnist.py diff --git a/src/mlmc/metamodel/GCN.py b/src/mlmc/metamodel/GCN.py new file mode 100644 index 00000000..e69de29b diff --git a/src/mlmc/metamodel/create_graph.py b/src/mlmc/metamodel/create_graph.py new file mode 100644 index 00000000..88ca05da --- /dev/null +++ b/src/mlmc/metamodel/create_graph.py @@ -0,0 +1,107 @@ +import os +import os.path +import numpy as np +from mlmc.tool import gmsh_io + +MESH = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/l_step_0.055_common_files/mesh.msh" +#FIELDS_SAMPLE_MESH = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/output/L00_S0000000/fine_fields_sample.msh" +FIELDS_SAMPLE = "fine_fields_sample.msh" +OUTPUT_DIR = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/output/" + + +def extract_mesh_gmsh_io(mesh_file): + """ + Extract mesh from file + :param mesh_file: Mesh file path + :return: Dict + """ + mesh = gmsh_io.GmshIO(mesh_file) + is_bc_region = {} + region_map = {} + for name, (id, _) in mesh.physical.items(): + unquoted_name = name.strip("\"'") + is_bc_region[id] = (unquoted_name[0] == '.') + region_map[unquoted_name] = id + + bulk_elements = [] + + for id, el in mesh.elements.items(): + _, tags, i_nodes = el + region_id = tags[0] + if not is_bc_region[region_id]: + bulk_elements.append(id) + + n_bulk = len(bulk_elements) + centers = np.empty((n_bulk, 3)) + ele_ids = np.zeros(n_bulk, dtype=int) + ele_nodes = {} + point_region_ids = np.zeros(n_bulk, dtype=int) + + for i, id_bulk in enumerate(bulk_elements): + _, tags, i_nodes = mesh.elements[id_bulk] + region_id = tags[0] + centers[i] = np.average(np.array([mesh.nodes[i_node] for i_node in i_nodes]), axis=0) + point_region_ids[i] = region_id + ele_ids[i] = id_bulk + ele_nodes[id_bulk] = i_nodes + + return ele_nodes + + +def get_node_features(fields_mesh): + mesh = gmsh_io.GmshIO(fields_mesh) + element_data = mesh.current_elem_data + features = list(element_data.values()) + return features + + +def create_adjacency_matrix(ele_nodes): + adjacency_matrix = np.zeros((len(ele_nodes), len(ele_nodes))) + #adjacency_matrix = sparse.csr_matrix((len(ele_nodes), len(ele_nodes))) # + + nodes = list(ele_nodes.values()) + for i in range(adjacency_matrix.shape[0]): + ele_nodes = nodes[i] + for j in range(i+1, len(nodes)): + #print("i: {}, j: {}".format(i, j)) + if i == j: + continue + ele_n = nodes[j] + if len(list(set(ele_nodes).intersection(ele_n))) == 2: + adjacency_matrix[j][i] = adjacency_matrix[i][j] = 1 + + print(np.count_nonzero(adjacency_matrix)) + assert np.allclose(adjacency_matrix, adjacency_matrix.T) # symmetry + return adjacency_matrix + + +def extract_mesh(): + adjacency_matrix = create_adjacency_matrix(extract_mesh_gmsh_io(MESH)) + np.save(os.path.join(OUTPUT_DIR, "adjacency_matrix"), adjacency_matrix, allow_pickle=True) + loaded_adjacency_matrix = np.load(os.path.join(OUTPUT_DIR, "adjacency_matrix.npy"), allow_pickle=True) + + print("loaded adjacency matrix ", loaded_adjacency_matrix) + ele_nodes = extract_mesh_gmsh_io(MESH) + + for s_dir in os.listdir(OUTPUT_DIR): + if os.path.isdir(os.path.join(OUTPUT_DIR, s_dir)): + sample_dir = os.path.join(OUTPUT_DIR, s_dir) + field_mesh = os.path.join(sample_dir, FIELDS_SAMPLE) + features = get_node_features(field_mesh) + + np.save(os.path.join(sample_dir, "nodes_features"), features) + #loaded_features = np.load(os.path.join(sample_dir, "nodes_features.npy")) + #print("loaded features ", loaded_features) + + +if __name__ == "__main__": + import cProfile + import pstats + pr = cProfile.Profile() + pr.enable() + + my_result = extract_mesh() + + pr.disable() + ps = pstats.Stats(pr).sort_stats('cumtime') + ps.print_stats() diff --git a/src/mlmc/metamodel/mnist.py b/src/mlmc/metamodel/mnist.py new file mode 100644 index 00000000..e69de29b From 975713762da4f385734ab8915a3514c3f6e36030 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Thu, 18 Feb 2021 15:21:22 +0100 Subject: [PATCH 03/67] dnn --- src/mlmc/metamodel/flow_dataset.py | 76 +++++++++++ src/mlmc/metamodel/flow_task_GNN.py | 199 ++++++++++++++++++++++++++++ src/mlmc/metamodel/flow_task_NN.py | 0 3 files changed, 275 insertions(+) create mode 100644 src/mlmc/metamodel/flow_dataset.py create mode 100644 src/mlmc/metamodel/flow_task_GNN.py create mode 100644 src/mlmc/metamodel/flow_task_NN.py diff --git a/src/mlmc/metamodel/flow_dataset.py b/src/mlmc/metamodel/flow_dataset.py new file mode 100644 index 00000000..fc27137a --- /dev/null +++ b/src/mlmc/metamodel/flow_dataset.py @@ -0,0 +1,76 @@ +import os +import numpy as np +import pandas as pd +from mlmc.tool import gmsh_io +from spektral.data import Dataset, Graph + +MESH = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/l_step_0.055_common_files/mesh.msh" +FIELDS_SAMPLE = "fine_fields_sample.msh" +OUTPUT_DIR = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/output/" + + +class FlowDataset(Dataset): + """ + """ + def __init__(self, **kwargs): + self.adjacency_matrix = np.load(os.path.join(OUTPUT_DIR, "adjacency_matrix.npy"), allow_pickle=True) # adjacency matrix + self.data = [] + super().__init__(**kwargs) + self.a = self.adjacency_matrix + + self.dataset = pd.DataFrame(self.data) + + def read(self): + graphs = [] + for s_dir in os.listdir(OUTPUT_DIR): + if os.path.isdir(os.path.join(OUTPUT_DIR, s_dir)): + sample_dir = os.path.join(OUTPUT_DIR, s_dir) + if os.path.exists(os.path.join(sample_dir, "nodes_features.npy")): + features = np.load(os.path.join(sample_dir, "nodes_features.npy")) + output = np.load(os.path.join(sample_dir, "output.npy")) + graphs.append(Graph(x=features, y=output))#, a=self.adjacency_matrix)) + + # Save data for pandas dataframe creation, not used with Graph neural network + self.data.append({'x': features, 'y': output}) + return graphs + + + + +def extract_mesh_gmsh_io(mesh_file): + """ + Extract mesh from file + :param mesh_file: Mesh file path + :return: Dict + """ + mesh = gmsh_io.GmshIO(mesh_file) + is_bc_region = {} + region_map = {} + for name, (id, _) in mesh.physical.items(): + unquoted_name = name.strip("\"'") + is_bc_region[id] = (unquoted_name[0] == '.') + region_map[unquoted_name] = id + + bulk_elements = [] + + for id, el in mesh.elements.items(): + _, tags, i_nodes = el + region_id = tags[0] + if not is_bc_region[region_id]: + bulk_elements.append(id) + + n_bulk = len(bulk_elements) + centers = np.empty((n_bulk, 3)) + ele_ids = np.zeros(n_bulk, dtype=int) + ele_nodes = {} + point_region_ids = np.zeros(n_bulk, dtype=int) + + for i, id_bulk in enumerate(bulk_elements): + _, tags, i_nodes = mesh.elements[id_bulk] + region_id = tags[0] + centers[i] = np.average(np.array([mesh.nodes[i_node] for i_node in i_nodes]), axis=0) + point_region_ids[i] = region_id + ele_ids[i] = id_bulk + ele_nodes[id_bulk] = i_nodes + + return ele_nodes diff --git a/src/mlmc/metamodel/flow_task_GNN.py b/src/mlmc/metamodel/flow_task_GNN.py new file mode 100644 index 00000000..83bb63c3 --- /dev/null +++ b/src/mlmc/metamodel/flow_task_GNN.py @@ -0,0 +1,199 @@ +import os +import numpy as np +import matplotlib.pyplot as plt +#os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only +import tensorflow as tf +from tensorflow.keras import Model +from tensorflow.keras.layers import Dense +from tensorflow.keras.losses import MeanSquaredError, SparseCategoricalCrossentropy, KLDivergence +from tensorflow.keras.metrics import mean_squared_error, kl_divergence +from tensorflow.keras.optimizers import Adam +from tensorflow.keras.regularizers import l2 + +from spektral.data import MixedLoader +from mlmc.metamodel.flow_dataset import FlowDataset +from spektral.layers import GCNConv, GlobalSumPool, ChebConv, GraphSageConv, ARMAConv, GATConv, APPNPConv, GINConv +from spektral.layers.ops import sp_matrix_to_sp_tensor + + + +#conv_layer = GCNConv +conv_layer = ChebConv # Seems better than GCNConv, good distribution of predictions +#conv_layer = GraphSageConv # Seems better than ChebConv, good loss but very narrow distribution of predictions +#conv_layer = ARMAConv # Seems worse than GraphSageConv +#conv_layer = GATConv # Slow and not better than GraphSageConv +# conv_layer = APPNPConv # Not bad but worse than GraphSageConv +# conv_layer = GINConv # it is comparable to APPNPConv +act_func = "relu"#"tanh"#"elu" # ReLU keep predictions above zero + +optimizer = Adam() +loss_fn = MeanSquaredError() +#loss_fn = KLDivergence() + +acc_fn = mean_squared_error +acc_fn = kl_divergence + +print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU'))) + +# Parameters +batch_size = 1000 # Batch size +epochs = 1000 # Number of training epochs +patience = 30 # Patience for early stopping +l2_reg = 0#5e-4 # Regularization rate for l2 + + +# Load data +data = FlowDataset() + +#print("data.a ", data.a) + +data.a = conv_layer.preprocess(data.a) +data.a = sp_matrix_to_sp_tensor(data.a) + +# Train/valid/test split +data_tr, data_te = data[:10000], data[10000:], +np.random.shuffle(data_tr) +data_tr, data_va = data_tr[:8000], data_tr[8000:] + +# We use a MixedLoader since the dataset is in mixed mode +loader_tr = MixedLoader(data_tr, batch_size=batch_size, epochs=epochs) +loader_va = MixedLoader(data_va, batch_size=batch_size) +loader_te = MixedLoader(data_te, batch_size=batch_size) + +# Build model +class Net(Model): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.conv1 = conv_layer(32, activation=act_func, kernel_regularizer=l2(l2_reg)) + self.conv2 = conv_layer(32, activation=act_func, kernel_regularizer=l2(l2_reg)) + self.flatten = GlobalSumPool() + self.fc1 = Dense(512, activation="relu") + self.fc2 = Dense(1, activation="linear") # linear activation for output neuron + + def call(self, inputs): + x, a = inputs + x = self.conv1([x, a]) + x = self.conv2([x, a]) + output = self.flatten(x) + output = self.fc1(output) + output = self.fc2(output) + + return output + + +# Create model +model = Net() + + + +# Training function +@tf.function +def train_on_batch(inputs, target): + with tf.GradientTape() as tape: + print("inputs data shape ", inputs[0].shape) # (number of train samples, number of vertices, number of properties for each vertex) + predictions = model(inputs, training=True) + #@TODO: zkusit pridat k loss function KLDivergence + # print(KLDivergence(target, predictions)) + # exit() + loss = loss_fn(target, predictions) + sum(model.losses)# + KLDivergence(target, predictions)#+ sum(model.losses) + acc = tf.reduce_mean(acc_fn(target, predictions)) + + gradients = tape.gradient(loss, model.trainable_variables) + optimizer.apply_gradients(zip(gradients, model.trainable_variables)) + return loss, acc + + +# Evaluation function +def evaluate(loader): + step = 0 + results = [] + for batch in loader: + step += 1 + inputs, target = batch + print("loader ", loader) + print("inputs data shape ", inputs[0].shape) # (number of validation or test samples, number of vertices, number of properties for each vertex) + predictions = model(inputs, training=False) + + loss = loss_fn(target, predictions) + acc = tf.reduce_mean(acc_fn(target, predictions)) + results.append((loss, acc, len(target))) # Keep track of batch size + if step == loader.steps_per_epoch: + results = np.array(results) + return np.average(results[:, :-1], axis=0, weights=results[:, -1]), target, predictions + + + +def analyze_results(target, predictions): + from scipy.stats import ks_2samp + statistics, pvalue = ks_2samp(target, predictions) + print("KS statistics: {}, pvalue: {}".format(statistics, pvalue)) + # The closer KS statistic is to 0 the more likely it is that the two samples were drawn from the same distribution + + print("len(target) ", len(target)) + print("len(predictions) ", len(predictions)) + + plt.hist(target, bins=10, alpha=0.5, label='target', density=True) + plt.hist(predictions, bins=10, alpha=0.5, label='predictions', density=True) + plt.legend(loc='upper right') + plt.show() + + + +# Setup training +best_val_loss = np.inf +current_patience = patience +step = 0 + +# Training loop +results_tr = [] +for batch in loader_tr: + step += 1 + + # Training step + inputs, target = batch + loss, acc = train_on_batch(inputs, target) + results_tr.append((loss, acc, len(target))) + + all_targets = [] + all_predictions = [] + + if step == loader_tr.steps_per_epoch: + results_va, target, predictions = evaluate(loader_va) + if results_va[0] < best_val_loss: + best_val_loss = results_va[0] + current_patience = patience + results_te, target, predictions = evaluate(loader_te) + + print("target ", target) + print("predictions ", np.squeeze(predictions.numpy())) + print("len(target) ", len(target)) + print("len(predictions) ", len(np.squeeze(predictions.numpy()))) + + analyze_results(target, np.squeeze(predictions.numpy())) + + + else: + current_patience -= 1 + if current_patience == 0: + print("Early stopping") + break + + # Print results + results_tr = np.array(results_tr) + results_tr = np.average(results_tr[:, :-1], 0, weights=results_tr[:, -1]) + print( + "Train loss: {:.4f}, acc: {:.4f} | " + "Valid loss: {:.4f}, acc: {:.4f} | " + "Test loss: {:.4f}, acc: {:.4f}".format( + *results_tr, *results_va, *results_te + ) + ) + + # Reset epoch + results_tr = [] + step = 0 + +analyze_results(target, np.squeeze(predictions.numpy())) + + + diff --git a/src/mlmc/metamodel/flow_task_NN.py b/src/mlmc/metamodel/flow_task_NN.py new file mode 100644 index 00000000..e69de29b From 64fddf21b2f89c4e43cb8fb32e0adc2885df449e Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Fri, 19 Feb 2021 15:53:41 +0100 Subject: [PATCH 04/67] dnn abs activation --- src/mlmc/metamodel/analyze_nn.py | 198 +++++++++++++++++++++++++++ src/mlmc/metamodel/custom_methods.py | 5 + src/mlmc/metamodel/flow_task_NN.py | 167 ++++++++++++++++++++++ src/mlmc/metamodel/postprocessing.py | 25 ++++ 4 files changed, 395 insertions(+) create mode 100644 src/mlmc/metamodel/analyze_nn.py create mode 100644 src/mlmc/metamodel/custom_methods.py create mode 100644 src/mlmc/metamodel/postprocessing.py diff --git a/src/mlmc/metamodel/analyze_nn.py b/src/mlmc/metamodel/analyze_nn.py new file mode 100644 index 00000000..95e4ed9e --- /dev/null +++ b/src/mlmc/metamodel/analyze_nn.py @@ -0,0 +1,198 @@ +import os +import numpy as np +import random +#os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only +from mlmc.metamodel.flow_dataset import FlowDataset +# Make numpy printouts easier to read. +np.set_printoptions(precision=3, suppress=True) +import tensorflow as tf +from scipy.stats import ks_2samp + +from mlmc.tool import plot +from mlmc.moments import Legendre +import mlmc.tool.simple_distribution +import mlmc.estimator +from mlmc.sample_storage import Memory +from mlmc.quantity_spec import QuantitySpec, ChunkSpec +from mlmc.quantity import make_root_quantity +from mlmc.quantity_estimate import estimate_mean +import sklearn.model_selection +from mlmc.metamodel.custom_methods import abs_activation +from mlmc.metamodel.postprocessing import analyze_results, plot_loss +from mlmc.metamodel.flow_task_NN import DNN +print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU'))) + +epochs = 100 + + +def prepare_dataset(dataset): + # Load data + dataset = dataset.dropna() + train_x, test_x, train_y, test_y = sklearn.model_selection.train_test_split(dataset.x, dataset.y, + test_size=0.2, random_state=123) + train_x = np.squeeze(np.stack(train_x.to_numpy(), axis=0)) + train_x = np.asarray(train_x).astype('float64') + train_y = train_y.to_numpy() + train_y = np.asarray(train_y).astype('float64') + + test_x = np.squeeze(np.stack(test_x.to_numpy(), axis=0)) + test_x = np.asarray(test_x).astype('float64') + test_y = test_y.to_numpy() + test_y = np.asarray(test_y).astype('float64') + + return train_x, train_y, test_x, test_y + + +def estimate_density(values, title="Density"): + sample_storage = Memory() + n_levels = 1 + n_moments = 15 + distr_accuracy = 1e-7 + + distr_plot = plot.Distribution(title=title, + log_density=True) + + result_format = [QuantitySpec(name="flow", unit="m", shape=(1,), times=[0], locations=['0'])] + + sample_storage.save_global_data(result_format=result_format, level_parameters=np.ones(n_levels)) + + successful_samples = {} + failed_samples = {} + n_ops = {} + n_successful = len(values) + for l_id in range(n_levels): + sizes = [] + for quantity_spec in result_format: + sizes.append(np.prod(quantity_spec.shape) * len(quantity_spec.times) * len(quantity_spec.locations)) + + # Dict[level_id, List[Tuple[sample_id:str, Tuple[fine_result: ndarray, coarse_result: ndarray]]]] + successful_samples[l_id] = [] + for sample_id in range(len(values)): + successful_samples[l_id].append((str(sample_id), (values[sample_id], 0))) + + n_ops[l_id] = [random.random(), n_successful] + + sample_storage.save_scheduled_samples(l_id, samples=["S{:07d}".format(i) for i in range(n_successful)]) + + print("len successful samples ", len(successful_samples[0])) + sample_storage.save_samples(successful_samples, failed_samples) + sample_storage.save_n_ops(list(n_ops.items())) + + quantity = make_root_quantity(storage=sample_storage, q_specs=result_format) + length = quantity['flow'] + time = length[0] + location = time['0'] + value_quantity = location[0] + + quantile = 0.001 + true_domain = mlmc.estimator.Estimate.estimate_domain(value_quantity, sample_storage, quantile=quantile) + moments_fn = Legendre(n_moments, true_domain) + + estimator = mlmc.estimator.Estimate(quantity=value_quantity, sample_storage=sample_storage, moments_fn=moments_fn) + + reg_param = 0 + target_var = 1e-4 + distr_obj, info, result, moments_fn = estimator.construct_density( + tol=distr_accuracy, + reg_param=reg_param, + orth_moments_tol=target_var) + + samples = value_quantity.samples(ChunkSpec(level_id=0, n_samples=sample_storage.get_n_collected()[0]))[..., 0] + + print("samples ", np.array(samples).shape) + print("np.squeeze(samples) ", np.squeeze(samples).shape) + + distr_plot.add_raw_samples(np.squeeze(samples)) + + distr_plot.add_distribution(distr_obj, label="") + + # kl = mlmc.tool.simple_distribution.KL_divergence(self.cut_distr.pdf, distr_obj.density, + # self.cut_distr.domain[0], self.cut_distr.domain[1]) + #kl_divergences.append(kl) + + distr_plot.show(file=None) + + +def run(): + # Parameters + loss = "mean_squared_error" + optimizer = tf.optimizers.Adam(learning_rate=0.001) + + data = FlowDataset() + dataset = data.dataset[:50000] + train_input, train_output, test_input, test_output = prepare_dataset(dataset) + + print("len test(output) ", len(test_output)) + + dnn = DNN(loss=loss, optimizer=optimizer, output_activation=abs_activation) + dnn.fit(train_input, train_output) + + predictions = dnn.predict(test_input) + predictions = np.squeeze(predictions) + + print("len(predictions) ", len(predictions)) + + plot_loss(dnn.history) + analyze_results(test_output, predictions) + + estimate_density(test_output) + estimate_density(predictions) + +def bootstrap(): + loss = "mean_squared_error" + optimizer = tf.optimizers.Adam(learning_rate=0.001) + n_subsamples = 5 + size = 5000 + + train_losses = [] + val_losses = [] + all_test_outputs = [] + all_predictions = [] + ks_statistics = [] + ks_p_values = [] + + data = FlowDataset() + dataset = data.dataset.dropna() + + for i in range(n_subsamples): + dset = dataset.sample(size, replace=True) + print("len dset ", len(dset)) + train_input, train_output, test_input, test_output = prepare_dataset(dset) + + print("Size TRAIN in: {}, out: {}, TEST in: {}, out: {}".format(len(train_input), len(train_output), + len(test_input), len(test_output))) + + dnn = DNN(loss=loss, optimizer=optimizer, output_activation=abs_activation) + dnn.fit(train_input, train_output) + + predictions = dnn.predict(test_input) + predictions = np.squeeze(predictions) + + train_losses.append(dnn.history.history['loss']) + val_losses.append(dnn.history.history['val_loss']) + + all_test_outputs.append(test_output) + all_predictions.append(predictions) + + statistics, pvalue = ks_2samp(test_output, predictions) + ks_statistics.append(statistics) + ks_p_values.append(pvalue) + + + analyze_results(np.mean(all_test_outputs, axis=0), np.mean(all_predictions, axis=0)) + # + estimate_density(np.mean(all_test_outputs, axis=0), title="Test outputs") + estimate_density(np.mean(all_predictions, axis=0), title="Predictions") + + print("len(dataset) ", len(dataset)) + print("dataset[:10] ", dataset[:10]) + + + + + + +if __name__ == "__main__": + #run() + + bootstrap() \ No newline at end of file diff --git a/src/mlmc/metamodel/custom_methods.py b/src/mlmc/metamodel/custom_methods.py new file mode 100644 index 00000000..f00a84b7 --- /dev/null +++ b/src/mlmc/metamodel/custom_methods.py @@ -0,0 +1,5 @@ +from tensorflow.keras import backend as K + + +def abs_activation(x): + return K.abs(x) diff --git a/src/mlmc/metamodel/flow_task_NN.py b/src/mlmc/metamodel/flow_task_NN.py index e69de29b..d2f51159 100644 --- a/src/mlmc/metamodel/flow_task_NN.py +++ b/src/mlmc/metamodel/flow_task_NN.py @@ -0,0 +1,167 @@ +import numpy as np +from mlmc.metamodel.flow_dataset import FlowDataset +import tensorflow as tf +from tensorflow import keras +from tensorflow.keras import layers +from tensorflow.keras.layers.experimental import preprocessing + + +# epochs = 200 +# +# +# # Parameters +# loss_fn = "mean_squared_error" +# optimizer_nn = tf.optimizers.Adam(learning_rate=0.1) +# optimizer_dnn = tf.optimizers.Adam(learning_rate=0.001) +# +# +# # Load data +# data = FlowDataset() +# +# dataset = data.dataset[:5000] +# print("len(dataset) ", len(dataset)) +# dataset = dataset.dropna() +# +# train_dataset = dataset.sample(frac=0.8, random_state=0) +# test_dataset = dataset.drop(train_dataset.index) +# +# train_x = np.squeeze(np.stack(train_dataset.x.to_numpy(), axis=0)) +# train_x = np.asarray(train_x).astype('float64') +# train_y = train_dataset.y.to_numpy() +# train_y = np.asarray(train_y).astype('float64') +# +# test_x = np.squeeze(np.stack(test_dataset.x.to_numpy(), axis=0)) +# test_x = np.asarray(test_x).astype('float32') +# test_y = test_dataset.y.to_numpy() +# test_y = np.asarray(test_y).astype('float32') + + +# #################### +# ## Neural network - very bad for our purposes ## +# #################### +# +# normalizer = preprocessing.Normalization() +# linear_model = tf.keras.Sequential([ +# normalizer, +# layers.Dense(units=1) +# ]) +# +# linear_model.compile( +# optimizer=optimizer_nn, +# loss=loss_fn) +# # +# # history = linear_model.fit( +# # train_x, train_y, +# # epochs=100, +# # # suppress logging +# # verbose=0, +# # # Calculate validation results on 20% of the training data +# # validation_split=0.2) +# # +# # linear_model.summary() +# # +# # +# # plot_loss(history) +# # predictions = np.squeeze(linear_model.predict(test_x)) +# # analyze_results(target=test_y, predictions=predictions) + + + +######################### +# Deep neural network # +######################### + +class DNN: + def __init__(self, **kwargs): + self._epochs = kwargs.get('epochs', 100) + self._val_split = kwargs.get('var_split', 0.2) + self._verbose = kwargs.get('verbose', False) + + self._hidden_activation = kwargs.get('hidden_activation', 'relu') + self._output_activation = kwargs.get('output_activation', 'linear') + self._n_hidden_layers = kwargs.get('n_hidden_layers', 1) + self._n_hidden_neurons = kwargs.get('n_hidden_neurons', [64]) # Number of hidden neurons for each hidden layer + + self._loss = kwargs.get('loss', 'mean_squared_error') + self._optimizer = kwargs.get('optimizer', tf.optimizers.Adam(learning_rate=0.001)) + self._normalizer = kwargs.get('normalizer', preprocessing.Normalization()) + + self.history = None # Set in fit method + self._create_model() + + def _create_model(self): + hidden_layers = [] + for i in range(self._n_hidden_layers): + hidden_layers.append(layers.Dense(self._n_hidden_neurons[i], activation=self._hidden_activation)) + + self._model = keras.Sequential([ + self._normalizer, + *hidden_layers, + layers.Dense(1, activation=self._output_activation) + ]) + + self._model.compile(loss=self._loss, optimizer=self._optimizer) + + def fit(self, train_input, train_output): + self.history = self._model.fit(train_input, train_output, validation_split=self._val_split, + verbose=self._verbose, epochs=self._epochs) + + def predict(self, test_input): + return self._model.predict(test_input) + + def summary(self): + """ + Should be called after fit method + """ + return self._model.summary() + + +# def build_and_compile_model(normalizer): +# model = keras.Sequential([ +# normalizer, +# layers.Dense(450, activation='relu'), #64 +# #layers.Dense(64, activation='relu'), +# layers.Dense(1, activation=abs_activation) +# ]) +# +# model.compile(loss=loss_fn, +# optimizer=optimizer_dnn) +# return model +# +# +# normalizer = preprocessing.Normalization() +# dnn_model = build_and_compile_model(normalizer) +# dnn_history = dnn_model.fit( +# train_x, train_y, +# validation_split=0.2, +# verbose=0, epochs=epochs) +# dnn_model.summary() +# +# plot_loss(dnn_history) +# +# predictions = np.squeeze(dnn_model.predict(test_x)) +# +# print("target ", test_y) +# print("predictions ", predictions) +# +# +# for index, (t, p) in enumerate(zip(test_y, predictions)): +# if index > 100: +# break +# print("t: {}, p: {}".format(t, p)) +# +# print("target mean ", np.mean(test_y)) +# print("predictions mean ", np.mean(predictions)) +# +# print("target var ", np.var(test_y)) +# print("predictions var ", np.var(predictions)) +# +# analyze_results(target=test_y, predictions=predictions) + + + + + + + + diff --git a/src/mlmc/metamodel/postprocessing.py b/src/mlmc/metamodel/postprocessing.py new file mode 100644 index 00000000..6b4459ce --- /dev/null +++ b/src/mlmc/metamodel/postprocessing.py @@ -0,0 +1,25 @@ +import matplotlib.pyplot as plt +from scipy.stats import ks_2samp + + +def plot_loss(history): + plt.plot(history.history['loss'], label='loss') + plt.plot(history.history['val_loss'], label='val_loss') + #plt.ylim([0, 10]) + plt.yscale("log") + plt.xlabel('Epoch') + plt.ylabel('Error') + plt.legend() + plt.grid(True) + plt.show() + + +def analyze_results(target, predictions): + statistics, pvalue = ks_2samp(target, predictions) + print("KS statistics: {}, pvalue: {}".format(statistics, pvalue)) + # The closer KS statistic is to 0 the more likely it is that the two samples were drawn from the same distribution + + plt.hist(target, alpha=0.5, label='target', density=True) + plt.hist(predictions, alpha=0.5, label='predictions', density=True) + plt.legend(loc='upper right') + plt.show() \ No newline at end of file From f6358bd3758642845ba33134378514d05644de21 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Sun, 21 Feb 2021 14:06:16 +0100 Subject: [PATCH 05/67] GNN model class --- src/mlmc/metamodel/analyze_nn.py | 27 ++-- src/mlmc/metamodel/create_graph.py | 39 +++-- src/mlmc/metamodel/flow_dataset.py | 2 +- src/mlmc/metamodel/flow_task_GNN_2.py | 196 ++++++++++++++++++++++++++ src/mlmc/metamodel/flow_task_NN.py | 10 +- src/mlmc/metamodel/graph_models.py | 23 +++ src/mlmc/metamodel/postprocessing.py | 19 ++- 7 files changed, 285 insertions(+), 31 deletions(-) create mode 100644 src/mlmc/metamodel/flow_task_GNN_2.py create mode 100644 src/mlmc/metamodel/graph_models.py diff --git a/src/mlmc/metamodel/analyze_nn.py b/src/mlmc/metamodel/analyze_nn.py index 95e4ed9e..bcecd139 100644 --- a/src/mlmc/metamodel/analyze_nn.py +++ b/src/mlmc/metamodel/analyze_nn.py @@ -124,7 +124,7 @@ def run(): print("len test(output) ", len(test_output)) - dnn = DNN(loss=loss, optimizer=optimizer, output_activation=abs_activation) + dnn = DNN(loss=loss, optimizer=optimizer, output_activation=abs_activation, hidden_activation='relu') dnn.fit(train_input, train_output) predictions = dnn.predict(test_input) @@ -132,17 +132,18 @@ def run(): print("len(predictions) ", len(predictions)) - plot_loss(dnn.history) + plot_loss(dnn.history.history['loss'], dnn.history.history['val_loss']) analyze_results(test_output, predictions) estimate_density(test_output) estimate_density(predictions) + def bootstrap(): - loss = "mean_squared_error" + loss = "mean_absolute_error" optimizer = tf.optimizers.Adam(learning_rate=0.001) - n_subsamples = 5 - size = 5000 + n_subsamples = 10 + size = 10000 train_losses = [] val_losses = [] @@ -156,13 +157,12 @@ def bootstrap(): for i in range(n_subsamples): dset = dataset.sample(size, replace=True) - print("len dset ", len(dset)) train_input, train_output, test_input, test_output = prepare_dataset(dset) print("Size TRAIN in: {}, out: {}, TEST in: {}, out: {}".format(len(train_input), len(train_output), len(test_input), len(test_output))) - dnn = DNN(loss=loss, optimizer=optimizer, output_activation=abs_activation) + dnn = DNN(loss=loss, optimizer=optimizer, output_activation=abs_activation, hidden_activation='relu') dnn.fit(train_input, train_output) predictions = dnn.predict(test_input) @@ -178,18 +178,11 @@ def bootstrap(): ks_statistics.append(statistics) ks_p_values.append(pvalue) - analyze_results(np.mean(all_test_outputs, axis=0), np.mean(all_predictions, axis=0)) + analyze_results(np.var(all_test_outputs, axis=0), np.var(all_predictions, axis=0)) # - estimate_density(np.mean(all_test_outputs, axis=0), title="Test outputs") - estimate_density(np.mean(all_predictions, axis=0), title="Predictions") - - print("len(dataset) ", len(dataset)) - print("dataset[:10] ", dataset[:10]) - - - - + # estimate_density(np.mean(all_test_outputs, axis=0), title="Test outputs") + # estimate_density(np.mean(all_predictions, axis=0), title="Predictions") if __name__ == "__main__": diff --git a/src/mlmc/metamodel/create_graph.py b/src/mlmc/metamodel/create_graph.py index 88ca05da..c6e9bb8b 100644 --- a/src/mlmc/metamodel/create_graph.py +++ b/src/mlmc/metamodel/create_graph.py @@ -1,12 +1,17 @@ import os import os.path import numpy as np +import networkx as nx +import matplotlib.pyplot as plt from mlmc.tool import gmsh_io +from mlmc.tool.hdf5 import HDF5 + MESH = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/l_step_0.055_common_files/mesh.msh" #FIELDS_SAMPLE_MESH = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/output/L00_S0000000/fine_fields_sample.msh" FIELDS_SAMPLE = "fine_fields_sample.msh" -OUTPUT_DIR = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/output/" +OUTPUT_DIR = "/home/martin/Documents/metamodels/data/1000_ele/test/01_cond_field/output/" +HDF_PATH = "/home/martin/Documents/metamodels/data/1000_ele/test/01_cond_field/mlmc_1.hdf5" def extract_mesh_gmsh_io(mesh_file): @@ -56,17 +61,19 @@ def get_node_features(fields_mesh): def create_adjacency_matrix(ele_nodes): + adjacency_matrix = np.zeros((len(ele_nodes), len(ele_nodes))) #adjacency_matrix = sparse.csr_matrix((len(ele_nodes), len(ele_nodes))) # nodes = list(ele_nodes.values()) for i in range(adjacency_matrix.shape[0]): ele_nodes = nodes[i] + for j in range(i+1, len(nodes)): - #print("i: {}, j: {}".format(i, j)) if i == j: continue ele_n = nodes[j] + if len(list(set(ele_nodes).intersection(ele_n))) == 2: adjacency_matrix[j][i] = adjacency_matrix[i][j] = 1 @@ -75,21 +82,35 @@ def create_adjacency_matrix(ele_nodes): return adjacency_matrix +def plot_graph(adjacency_matrix): + #G = nx.from_scipy_sparse_matrix(adjacency_matrix) + G = nx.from_numpy_matrix(adjacency_matrix) + nx.draw_kamada_kawai(G, with_labels=True, node_size=1, font_size=6) + plt.axis('equal') + plt.show() + + def extract_mesh(): adjacency_matrix = create_adjacency_matrix(extract_mesh_gmsh_io(MESH)) np.save(os.path.join(OUTPUT_DIR, "adjacency_matrix"), adjacency_matrix, allow_pickle=True) loaded_adjacency_matrix = np.load(os.path.join(OUTPUT_DIR, "adjacency_matrix.npy"), allow_pickle=True) - print("loaded adjacency matrix ", loaded_adjacency_matrix) - ele_nodes = extract_mesh_gmsh_io(MESH) + plot_graph(loaded_adjacency_matrix) - for s_dir in os.listdir(OUTPUT_DIR): - if os.path.isdir(os.path.join(OUTPUT_DIR, s_dir)): - sample_dir = os.path.join(OUTPUT_DIR, s_dir) - field_mesh = os.path.join(sample_dir, FIELDS_SAMPLE) - features = get_node_features(field_mesh) + hdf = HDF5(file_path=HDF_PATH, + load_from_file=True) + level_group = hdf.add_level_group(level_id=str(0)) + collected = zip(level_group.get_collected_ids(), level_group.collected()) + for sample_id, col_values in collected: + output_value = col_values[0, 0] + sample_dir = os.path.join(OUTPUT_DIR, sample_id) + field_mesh = os.path.join(sample_dir, FIELDS_SAMPLE) + if os.path.exists(field_mesh): + features = get_node_features(field_mesh) np.save(os.path.join(sample_dir, "nodes_features"), features) + np.save(os.path.join(sample_dir, "output"), output_value) + #loaded_features = np.load(os.path.join(sample_dir, "nodes_features.npy")) #print("loaded features ", loaded_features) diff --git a/src/mlmc/metamodel/flow_dataset.py b/src/mlmc/metamodel/flow_dataset.py index fc27137a..829f8693 100644 --- a/src/mlmc/metamodel/flow_dataset.py +++ b/src/mlmc/metamodel/flow_dataset.py @@ -6,7 +6,7 @@ MESH = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/l_step_0.055_common_files/mesh.msh" FIELDS_SAMPLE = "fine_fields_sample.msh" -OUTPUT_DIR = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/output/" +OUTPUT_DIR = "/home/martin/Documents/metamodels/data/1000_ele/test/01_cond_field/output/" class FlowDataset(Dataset): diff --git a/src/mlmc/metamodel/flow_task_GNN_2.py b/src/mlmc/metamodel/flow_task_GNN_2.py new file mode 100644 index 00000000..0b9ff557 --- /dev/null +++ b/src/mlmc/metamodel/flow_task_GNN_2.py @@ -0,0 +1,196 @@ +import os +import numpy as np +#os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only +import tensorflow as tf +from tensorflow.keras.losses import MeanSquaredError, SparseCategoricalCrossentropy, KLDivergence +from tensorflow.keras.metrics import mean_squared_error, kl_divergence +from tensorflow.keras.callbacks import History +from tensorflow.keras.optimizers import Adam +from tensorflow.keras.regularizers import l2 +from mlmc.metamodel.postprocessing import analyze_results, plot_loss +from spektral.data import MixedLoader +from mlmc.metamodel.flow_dataset import FlowDataset +from spektral.layers import GCNConv, GlobalSumPool, ChebConv, GraphSageConv, ARMAConv, GATConv, APPNPConv, GINConv +from spektral.layers.ops import sp_matrix_to_sp_tensor +from tensorflow.keras.layers.experimental import preprocessing +from mlmc.metamodel.custom_methods import abs_activation + +from mlmc.metamodel.graph_models import Net1 + + +class GNN: + def __init__(self, **kwargs): + + self._epochs = kwargs.get('epochs', 100) + #self._val_split = kwargs.get('var_split', 0.2) + #self._verbose = kwargs.get('verbose', False) + + self._hidden_activation = kwargs.get('hidden_activation', 'relu') + self._hidden_regularizer = kwargs.get('hidden_reqularizer', None) + self._output_activation = kwargs.get('output_activation', 'linear') + self._conv_layer = kwargs.get('conv_layer', None) + #self._n_hidden_layers = kwargs.get('n_hidden_layers', 1) + #self._n_hidden_neurons = kwargs.get('n_hidden_neurons', [64]) # Number of hidden neurons for each hidden layer + + self._loss = kwargs.get('loss', mean_squared_error) + self._accuracy_func = kwargs.get('accuracy_func', mean_squared_error) + self._optimizer = kwargs.get('optimizer', tf.optimizers.Adam(learning_rate=0.001)) + self._normalizer = kwargs.get('normalizer', preprocessing.Normalization()) + self._patience = kwargs.get('patience', 20) + + self._train_loss = [] + self._val_loss = [] + + self._model = Net1(conv_layer=self._conv_layer, hidden_activation=self._hidden_activation, + output_activation=self._output_activation, + kernel_regularization=self._hidden_regularizer) + + def fit(self, loader_tr, loader_va, loader_te): + """ + Training procedure + """ + # Setup training + best_val_loss = np.inf + current_patience = self._patience + step = 0 + + # Training loop + results_tr = [] + for batch in loader_tr: + step += 1 + + # Training step + inputs, target = batch + loss, acc = self.train_on_batch(inputs, target) + self._train_loss.append(loss) + results_tr.append((loss, acc, len(target))) + + results_va = self.evaluate(loader_va) + self._val_loss.append(results_va[0]) + + if step == loader_tr.steps_per_epoch: # step_per_epoch = int(np.ceil(len(self.dataset) / self.batch_size)) + # results_va = self.evaluate(loader_va) + # self._val_loss.append(results_va[0]) + if results_va[0] < best_val_loss: + best_val_loss = results_va[0] + current_patience = self._patience + results_te = self.evaluate(loader_te) + else: + current_patience -= 1 + if current_patience == 0: + print("Early stopping") + break + + # Print results + results_tr = np.array(results_tr) + results_tr = np.average(results_tr[:, :-1], 0, weights=results_tr[:, -1]) + print( + "Train loss: {:.4f}, acc: {:.4f} | " + "Valid loss: {:.4f}, acc: {:.4f} | " + "Test loss: {:.4f}, acc: {:.4f}".format( + *results_tr, *results_va, *results_te + ) + ) + # Reset epoch + results_tr = [] + step = 0 + + # Training function + @tf.function + def train_on_batch(self, inputs, target): + with tf.GradientTape() as tape: + predictions = self._model(inputs, training=True) + # @TODO: try to add KLDivergence to loss + # print(KLDivergence(target, predictions)) + loss = self._loss(target, predictions) + sum(self._model.losses) # + KLDivergence(target, predictions)#+ sum(model.losses) + acc = tf.reduce_mean(self._accuracy_func(target, predictions)) + + gradients = tape.gradient(loss, self._model.trainable_variables) + optimizer.apply_gradients(zip(gradients, self._model.trainable_variables)) + return loss, acc + + def evaluate(self, loader): + step = 0 + results = [] + for batch in loader: + step += 1 + inputs, target = batch + predictions = self._model(inputs, training=False) + + loss = self._loss(target, predictions) + acc = tf.reduce_mean(self._accuracy_func(target, predictions)) + results.append((loss, acc, len(target))) # Keep track of batch size + if step == loader.steps_per_epoch: + results = np.array(results) + return np.average(results[:, :-1], axis=0, weights=results[:, -1]) + + def predict(self, loader): + targets = [] + predictions = [] + step = 0 + for batch in loader: + step += 1 + + + print("predict") + inputs, target = batch + print("len(inputs) ", len(inputs[0])) + + print("step ", step) + + targets.extend(target) + predictions.extend(self._model(inputs, training=False)) + + if step == loader.steps_per_epoch: + return targets, predictions + + +if __name__ == "__main__": + # Parameters + #conv_layer = GCNConv + conv_layer = ChebConv # Seems better than GCNConv, good distribution of predictions + # # conv_layer = GraphSageConv # Seems better than ChebConv, good loss but very narrow distribution of predictions + # # conv_layer = ARMAConv # Seems worse than GraphSageConv + # # conv_layer = GATConv # Slow and not better than GraphSageConv + # # conv_layer = APPNPConv # Not bad but worse than GraphSageConv + # # conv_layer = GINConv # it is comparable to APPNPConv + # act_func = "relu" # "tanh"#"elu" # ReLU keep predictions above zero + loss = MeanSquaredError() + optimizer = tf.optimizers.Adam(learning_rate=0.001) + batch_size = 1000 + epochs = 100 + + # Load data + data = FlowDataset() + data = data[:10000] + data.a = conv_layer.preprocess(data.a) + data.a = sp_matrix_to_sp_tensor(data.a) + + train_data_len = int(len(data) * 0.8) + + # Train/valid/test split + data_tr, data_te = data[:train_data_len], data[train_data_len:], + np.random.shuffle(data_tr) + + val_data_len = int(len(data_tr) * 0.2) + data_tr, data_va = data_tr[:-val_data_len], data_tr[-val_data_len:] + + print("data_tr len ", len(data_tr)) + print("data_va len ", len(data_va)) + print("data_te len ", len(data_te)) + + # We use a MixedLoader since the dataset is in mixed mode + loader_tr = MixedLoader(data_tr, batch_size=batch_size, epochs=epochs) + loader_va = MixedLoader(data_va, batch_size=batch_size) + loader_te = MixedLoader(data_te, batch_size=batch_size) + + + gnn = GNN(loss=loss, optimizer=optimizer, conv_layer=conv_layer, output_activation=abs_activation, + hidden_activation='relu', patience=10) + gnn.fit(loader_tr, loader_va, loader_te) + + targets, predictions = gnn.predict(loader_te) + predictions = np.squeeze(predictions) + + plot_loss(gnn._train_loss, gnn._val_loss) + analyze_results(targets, predictions) diff --git a/src/mlmc/metamodel/flow_task_NN.py b/src/mlmc/metamodel/flow_task_NN.py index d2f51159..e565f862 100644 --- a/src/mlmc/metamodel/flow_task_NN.py +++ b/src/mlmc/metamodel/flow_task_NN.py @@ -78,6 +78,7 @@ def __init__(self, **kwargs): self._verbose = kwargs.get('verbose', False) self._hidden_activation = kwargs.get('hidden_activation', 'relu') + self._hidden_regularizer = kwargs.get('hidden_reqularizer', None) self._output_activation = kwargs.get('output_activation', 'linear') self._n_hidden_layers = kwargs.get('n_hidden_layers', 1) self._n_hidden_neurons = kwargs.get('n_hidden_neurons', [64]) # Number of hidden neurons for each hidden layer @@ -92,7 +93,14 @@ def __init__(self, **kwargs): def _create_model(self): hidden_layers = [] for i in range(self._n_hidden_layers): - hidden_layers.append(layers.Dense(self._n_hidden_neurons[i], activation=self._hidden_activation)) + if self._hidden_regularizer is not None: + hidden_layers.append( + layers.Dense(self._n_hidden_neurons[i], + kernel_regularizer=self._hidden_regularizer, + activation=self._hidden_activation)) + else: + hidden_layers.append( + layers.Dense(self._n_hidden_neurons[i],activation=self._hidden_activation)) self._model = keras.Sequential([ self._normalizer, diff --git a/src/mlmc/metamodel/graph_models.py b/src/mlmc/metamodel/graph_models.py new file mode 100644 index 00000000..7dc03ac5 --- /dev/null +++ b/src/mlmc/metamodel/graph_models.py @@ -0,0 +1,23 @@ +from tensorflow.keras import Model +from tensorflow.keras.layers import Dense +from spektral.layers import GlobalSumPool + + +# Build model +class Net1(Model): + def __init__(self, conv_layer, hidden_activation, output_activation, kernel_regularization, **kwargs): + super().__init__(**kwargs) + self.conv1 = conv_layer(32, activation=hidden_activation, kernel_regularizer=kernel_regularization) + self.conv2 = conv_layer(32, activation=hidden_activation, kernel_regularizer=kernel_regularization) + self.flatten = GlobalSumPool() + self.fc1 = Dense(512, activation=hidden_activation) + self.fc2 = Dense(1, activation=output_activation) # linear activation for output neuron + + def call(self, inputs): + x, a = inputs + x = self.conv1([x, a]) + x = self.conv2([x, a]) + output = self.flatten(x) + output = self.fc1(output) + output = self.fc2(output) + return output \ No newline at end of file diff --git a/src/mlmc/metamodel/postprocessing.py b/src/mlmc/metamodel/postprocessing.py index 6b4459ce..3f401df3 100644 --- a/src/mlmc/metamodel/postprocessing.py +++ b/src/mlmc/metamodel/postprocessing.py @@ -1,10 +1,11 @@ +import numpy as np import matplotlib.pyplot as plt from scipy.stats import ks_2samp -def plot_loss(history): - plt.plot(history.history['loss'], label='loss') - plt.plot(history.history['val_loss'], label='val_loss') +def plot_loss(train_loss, val_loss): + plt.plot(train_loss, label='loss') + plt.plot(val_loss, label='val_loss') #plt.ylim([0, 10]) plt.yscale("log") plt.xlabel('Epoch') @@ -16,6 +17,18 @@ def plot_loss(history): def analyze_results(target, predictions): statistics, pvalue = ks_2samp(target, predictions) + + print("Target mean: {}, var: {}, Q25: {}, Q50: {}, Q75: {}".format(np.mean(target), + np.var(target), + np.quantile(target, 0.25), + np.quantile(target, 0.5), + np.quantile(target, 0.75))) + print("Predic mean: {}, var: {}, Q25: {}, Q50: {}, Q75: {}".format(np.mean(predictions), + np.var(predictions), + np.quantile(predictions, 0.25), + np.quantile(predictions, 0.5), + np.quantile(predictions, 0.75))) + print("KS statistics: {}, pvalue: {}".format(statistics, pvalue)) # The closer KS statistic is to 0 the more likely it is that the two samples were drawn from the same distribution From 29f175bd1383fbb10e6f8fa6bd569ff361bae264 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Mon, 22 Feb 2021 16:03:11 +0100 Subject: [PATCH 06/67] CNN --- src/mlmc/metamodel/flow_task_CNN.py | 100 ++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 src/mlmc/metamodel/flow_task_CNN.py diff --git a/src/mlmc/metamodel/flow_task_CNN.py b/src/mlmc/metamodel/flow_task_CNN.py new file mode 100644 index 00000000..e4d5805b --- /dev/null +++ b/src/mlmc/metamodel/flow_task_CNN.py @@ -0,0 +1,100 @@ +import numpy as np +from mlmc.metamodel.flow_dataset import FlowDataset +import tensorflow as tf +from tensorflow import keras +from tensorflow.keras import layers +from tensorflow.keras.layers.experimental import preprocessing + +# Following 3 lines prevent "Failed to get convolution algorithm. This is probably because cuDNN failed to initialize" +config = tf.compat.v1.ConfigProto() +config.gpu_options.allow_growth = True +session = tf.compat.v1.InteractiveSession(config=config) + +import os +import numpy as np +#os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only +import tensorflow as tf +from tensorflow.keras.losses import MeanSquaredError, SparseCategoricalCrossentropy, KLDivergence +from tensorflow.keras.metrics import mean_squared_error, kl_divergence +from tensorflow.keras.callbacks import History +from tensorflow.keras.optimizers import Adam +from tensorflow.keras.regularizers import l2 +from mlmc.metamodel.postprocessing import analyze_results, plot_loss +from spektral.data import MixedLoader +from mlmc.metamodel.flow_dataset import FlowDataset +from spektral.layers import GCNConv, GlobalSumPool, ChebConv, GraphSageConv, ARMAConv, GATConv, APPNPConv, GINConv +from spektral.layers.ops import sp_matrix_to_sp_tensor +from tensorflow.keras.layers.experimental import preprocessing +from mlmc.metamodel.custom_methods import abs_activation + +from mlmc.metamodel.graph_models import Net1 + + +################################## +# Convolutional neural network # +################################## + +class CNN: + def __init__(self, **kwargs): + self._epochs = kwargs.get('epochs', 100) + self._val_split = kwargs.get('var_split', 0.2) + self._verbose = kwargs.get('verbose', False) + + self._hidden_activation = kwargs.get('hidden_activation', 'relu') + self._hidden_regularizer = kwargs.get('hidden_reqularizer', None) + self._output_activation = kwargs.get('output_activation', 'linear') + self._n_hidden_layers = kwargs.get('n_hidden_layers', 1) + self._n_hidden_neurons = kwargs.get('n_hidden_neurons', [64]) # Number of hidden neurons for each hidden layer + + self._loss = kwargs.get('loss', 'mean_squared_error') + self._optimizer = kwargs.get('optimizer', tf.optimizers.Adam(learning_rate=0.001)) + self._normalizer = kwargs.get('normalizer', preprocessing.Normalization()) + + self.history = None # Set in fit method + self._create_model() + + def _create_model(self): + hidden_layers = [] + for i in range(self._n_hidden_layers): + if self._hidden_regularizer is not None: + hidden_layers.append( + layers.Dense(self._n_hidden_neurons[i], + kernel_regularizer=self._hidden_regularizer, + activation=self._hidden_activation)) + else: + hidden_layers.append( + layers.Dense(self._n_hidden_neurons[i],activation=self._hidden_activation)) + + self._model = keras.Sequential([ + #@TODO: Try normalization + #self._normalizer, # Seems worse results with normalization + layers.Conv1D(filters=64, kernel_size=3, activation='relu', input_shape=(958, 1)), + #layers.BatchNormalization(), + layers.MaxPooling1D(pool_size=2), + layers.Conv1D(filters=64, kernel_size=3, activation='relu'), + #layers.BatchNormalization(), + layers.MaxPooling1D(pool_size=2), + layers.Flatten(), + layers.Dense(64, activation='relu'), + layers.Dense(1, activation=self._output_activation) + ]) + + self._model.compile(loss=self._loss, optimizer=self._optimizer) + + def fit(self, train_input, train_output): + self.history = self._model.fit(train_input, train_output, validation_split=self._val_split, + verbose=self._verbose, epochs=self._epochs) + + def predict(self, test_input): + return self._model.predict(test_input) + + def summary(self): + """ + Should be called after fit method + """ + return self._model.summary() + + + + + From bcec4d6df03f5cc4a0f83f5ccb2ffee0dd934835 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Tue, 2 Mar 2021 16:55:21 +0100 Subject: [PATCH 07/67] postprocessing - compare results --- src/mlmc/metamodel/analyze_nn.py | 372 ++++++++++++++++++++------ src/mlmc/metamodel/flow_task_GNN_2.py | 133 +++++---- src/mlmc/metamodel/postprocessing.py | 320 +++++++++++++++++++++- 3 files changed, 681 insertions(+), 144 deletions(-) diff --git a/src/mlmc/metamodel/analyze_nn.py b/src/mlmc/metamodel/analyze_nn.py index bcecd139..42293291 100644 --- a/src/mlmc/metamodel/analyze_nn.py +++ b/src/mlmc/metamodel/analyze_nn.py @@ -4,135 +4,114 @@ #os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only from mlmc.metamodel.flow_dataset import FlowDataset # Make numpy printouts easier to read. -np.set_printoptions(precision=3, suppress=True) + +#np.set_printoptions(precision=9, suppress=True) import tensorflow as tf from scipy.stats import ks_2samp - -from mlmc.tool import plot -from mlmc.moments import Legendre -import mlmc.tool.simple_distribution -import mlmc.estimator -from mlmc.sample_storage import Memory -from mlmc.quantity_spec import QuantitySpec, ChunkSpec -from mlmc.quantity import make_root_quantity -from mlmc.quantity_estimate import estimate_mean import sklearn.model_selection from mlmc.metamodel.custom_methods import abs_activation -from mlmc.metamodel.postprocessing import analyze_results, plot_loss +from mlmc.metamodel.postprocessing import analyze_results, plot_loss, estimate_density, diff_moments, process_mlmc from mlmc.metamodel.flow_task_NN import DNN +from mlmc.metamodel.flow_task_CNN import CNN + + +from mlmc.metamodel.flow_task_GNN_2 import GNN +from spektral.layers import GCNConv, GlobalSumPool, ChebConv, GraphSageConv, ARMAConv, GATConv, APPNPConv, GINConv +from tensorflow.keras.losses import MeanSquaredError +from spektral.data import MixedLoader +from spektral.layers.ops import sp_matrix_to_sp_tensor print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU'))) epochs = 100 -def prepare_dataset(dataset): +def prepare_data(data): + data = np.squeeze(np.stack(data.to_numpy(), axis=0)) + return np.asarray(data).astype('float64') + + +def split_dataset(dataset): # Load data dataset = dataset.dropna() train_x, test_x, train_y, test_y = sklearn.model_selection.train_test_split(dataset.x, dataset.y, test_size=0.2, random_state=123) - train_x = np.squeeze(np.stack(train_x.to_numpy(), axis=0)) - train_x = np.asarray(train_x).astype('float64') - train_y = train_y.to_numpy() - train_y = np.asarray(train_y).astype('float64') - test_x = np.squeeze(np.stack(test_x.to_numpy(), axis=0)) - test_x = np.asarray(test_x).astype('float64') - test_y = test_y.to_numpy() - test_y = np.asarray(test_y).astype('float64') - - return train_x, train_y, test_x, test_y + train_x = prepare_data(train_x) + train_y = prepare_data(train_y) + test_x = prepare_data(test_x) + test_y = prepare_data(test_y) -def estimate_density(values, title="Density"): - sample_storage = Memory() - n_levels = 1 - n_moments = 15 - distr_accuracy = 1e-7 - - distr_plot = plot.Distribution(title=title, - log_density=True) - - result_format = [QuantitySpec(name="flow", unit="m", shape=(1,), times=[0], locations=['0'])] - - sample_storage.save_global_data(result_format=result_format, level_parameters=np.ones(n_levels)) - - successful_samples = {} - failed_samples = {} - n_ops = {} - n_successful = len(values) - for l_id in range(n_levels): - sizes = [] - for quantity_spec in result_format: - sizes.append(np.prod(quantity_spec.shape) * len(quantity_spec.times) * len(quantity_spec.locations)) - - # Dict[level_id, List[Tuple[sample_id:str, Tuple[fine_result: ndarray, coarse_result: ndarray]]]] - successful_samples[l_id] = [] - for sample_id in range(len(values)): - successful_samples[l_id].append((str(sample_id), (values[sample_id], 0))) - - n_ops[l_id] = [random.random(), n_successful] + return train_x, train_y, test_x, test_y - sample_storage.save_scheduled_samples(l_id, samples=["S{:07d}".format(i) for i in range(n_successful)]) - print("len successful samples ", len(successful_samples[0])) - sample_storage.save_samples(successful_samples, failed_samples) - sample_storage.save_n_ops(list(n_ops.items())) +def run(): + # Parameters + loss = "mean_squared_error" + optimizer = tf.optimizers.Adam(learning_rate=0.001) - quantity = make_root_quantity(storage=sample_storage, q_specs=result_format) - length = quantity['flow'] - time = length[0] - location = time['0'] - value_quantity = location[0] + data = FlowDataset() + # dataset = data.dataset[:10000] + # test_dataset = data.dataset[10000:50000] - quantile = 0.001 - true_domain = mlmc.estimator.Estimate.estimate_domain(value_quantity, sample_storage, quantile=quantile) - moments_fn = Legendre(n_moments, true_domain) + dataset = data.dataset[:50000] + test_dataset = data.dataset[50000:] - estimator = mlmc.estimator.Estimate(quantity=value_quantity, sample_storage=sample_storage, moments_fn=moments_fn) + train_input = prepare_data(dataset.x) + train_output = prepare_data(dataset.y) - reg_param = 0 - target_var = 1e-4 - distr_obj, info, result, moments_fn = estimator.construct_density( - tol=distr_accuracy, - reg_param=reg_param, - orth_moments_tol=target_var) + #train_input, train_output, test__input, test_output = split_dataset(dataset) + #print("len test(output) ", len(test_output)) - samples = value_quantity.samples(ChunkSpec(level_id=0, n_samples=sample_storage.get_n_collected()[0]))[..., 0] + dnn = DNN(loss=loss, optimizer=optimizer, output_activation=abs_activation, hidden_activation='relu', epochs=150) + dnn.fit(train_input, train_output) - print("samples ", np.array(samples).shape) - print("np.squeeze(samples) ", np.squeeze(samples).shape) + test_input = prepare_data(test_dataset.x) + test_output = prepare_data(test_dataset.y) - distr_plot.add_raw_samples(np.squeeze(samples)) + predictions = dnn.predict(test_input) + predictions = np.squeeze(predictions) - distr_plot.add_distribution(distr_obj, label="") + print("len(predictions) ", len(predictions)) - # kl = mlmc.tool.simple_distribution.KL_divergence(self.cut_distr.pdf, distr_obj.density, - # self.cut_distr.domain[0], self.cut_distr.domain[1]) - #kl_divergences.append(kl) + plot_loss(dnn.history.history['loss'], dnn.history.history['val_loss']) + analyze_results(test_output, predictions) - distr_plot.show(file=None) + estimate_density(test_output) + estimate_density(predictions) -def run(): +def run_CNN(): # Parameters loss = "mean_squared_error" optimizer = tf.optimizers.Adam(learning_rate=0.001) data = FlowDataset() - dataset = data.dataset[:50000] - train_input, train_output, test_input, test_output = prepare_dataset(dataset) + dataset = data.dataset[:10000] + + + train_input, train_output, test_input, test_output = split_dataset(dataset) print("len test(output) ", len(test_output)) - dnn = DNN(loss=loss, optimizer=optimizer, output_activation=abs_activation, hidden_activation='relu') + train_input = np.expand_dims(train_input, axis=-1) + test_input = np.expand_dims(test_input, axis=-1) + + print("train input shape ", train_input.shape) + + dnn = CNN(loss=loss, optimizer=optimizer, output_activation=abs_activation, hidden_activation='relu') + dnn.fit(train_input, train_output) + test_dataset = data.dataset[10000:] + test_input = prepare_data(test_dataset.x) + test_output = prepare_data(test_dataset.y) + predictions = dnn.predict(test_input) predictions = np.squeeze(predictions) - print("len(predictions) ", len(predictions)) - plot_loss(dnn.history.history['loss'], dnn.history.history['val_loss']) + analyze_results(test_output, predictions) estimate_density(test_output) @@ -157,7 +136,7 @@ def bootstrap(): for i in range(n_subsamples): dset = dataset.sample(size, replace=True) - train_input, train_output, test_input, test_output = prepare_dataset(dset) + train_input, train_output, test_input, test_output = split_dataset(dset) print("Size TRAIN in: {}, out: {}, TEST in: {}, out: {}".format(len(train_input), len(train_output), len(test_input), len(test_output))) @@ -185,7 +164,228 @@ def bootstrap(): # estimate_density(np.mean(all_predictions, axis=0), title="Predictions") +def bootstrap_GNN(): + loss = "mean_absolute_error" + optimizer = tf.optimizers.Adam(learning_rate=0.001) + n_subsamples = 10 + size = 10000 + + train_losses = [] + val_losses = [] + all_test_outputs = [] + all_predictions = [] + ks_statistics = [] + ks_p_values = [] + + data = FlowDataset() + dataset = data.dataset.dropna() + + for i in range(n_subsamples): + # Parameters + # conv_layer = GCNConv + conv_layer = ChebConv # Seems better than GCNConv, good distribution of predictions + # # conv_layer = GraphSageConv # Seems better than ChebConv, good loss but very narrow distribution of predictions + # # conv_layer = ARMAConv # Seems worse than GraphSageConv + # # conv_layer = GATConv # Slow and not better than GraphSageConv + # # conv_layer = APPNPConv # Not bad but worse than GraphSageConv + # # conv_layer = GINConv # it is comparable to APPNPConv + # act_func = "relu" # "tanh"#"elu" # ReLU keep predictions above zero + loss = MeanSquaredError() + optimizer = tf.optimizers.Adam(learning_rate=0.001) + batch_size = 1000 + epochs = 100 + + # Load data + data = FlowDataset() + data = data # [:10000] + # data.a = conv_layer.preprocess(data.a) + data.a = sp_matrix_to_sp_tensor(data.a) + + train_data_len = int(len(data) * 0.8) + train_data_len = 10000 + + # Train/valid/test split + data_tr, data_te = data[:train_data_len], data[train_data_len:], + np.random.shuffle(data_tr) + + val_data_len = int(len(data_tr) * 0.2) + data_tr, data_va = data_tr[:-val_data_len], data_tr[-val_data_len:] + + print("data_tr len ", len(data_tr)) + print("data_va len ", len(data_va)) + print("data_te len ", len(data_te)) + + # We use a MixedLoader since the dataset is in mixed mode + loader_tr = MixedLoader(data_tr, batch_size=batch_size, epochs=epochs) + loader_va = MixedLoader(data_va, batch_size=batch_size) + loader_te = MixedLoader(data_te, batch_size=batch_size) + + gnn = GNN(loss=loss, optimizer=optimizer, conv_layer=conv_layer, output_activation=abs_activation, + hidden_activation='relu', patience=10) + gnn.fit(loader_tr, loader_va, loader_te) + + targets, predictions = gnn.predict(loader_te) + predictions = np.squeeze(predictions) + + all_test_outputs.append(targets) + all_predictions(predictions) + + train_losses.append(gnn._train_loss[-1]) + val_losses.append(gnn._val_loss[-1]) + + + plot_loss(train_losses, val_losses) + analyze_results(np.mean(all_test_outputs, axis=0), np.mean(all_predictions, axis=0)) + analyze_results(np.var(all_test_outputs, axis=0), np.var(all_predictions, axis=0)) + + estimate_density(np.mean(all_test_outputs, axis=0), title="Test outputs") + estimate_density(np.mean(all_predictions, axis=0), title="Predictions") + + +def run_GNN(): + loss = "mean_absolute_error" + optimizer = tf.optimizers.Adam(learning_rate=0.001) + n_subsamples = 10 + size = 10000 + + train_losses = [] + val_losses = [] + all_test_outputs = [] + all_predictions = [] + ks_statistics = [] + ks_p_values = [] + + data = FlowDataset() + dataset = data.dataset.dropna() + + # Parameters + #conv_layer = GCNConv + conv_layer = ChebConv # Seems better than GCNConv, good distribution of predictions + # # conv_layer = GraphSageConv # Seems better than ChebConv, good loss but very narrow distribution of predictions + # # conv_layer = ARMAConv # Seems worse than GraphSageConv + # conv_layer = GATConv # Slow and not better than GraphSageConv + # # conv_layer = APPNPConv # Not bad but worse than GraphSageConv + # # conv_layer = GINConv # it is comparable to APPNPConv + # act_func = "relu" # "tanh"#"elu" # ReLU keep predictions above zero + loss = MeanSquaredError() + optimizer = tf.optimizers.Adam(learning_rate=0.001) + batch_size = 1000 + epochs = 100 + + # Load data + data = FlowDataset() + data = data # [:10000] + # data.a = conv_layer.preprocess(data.a) + data.a = sp_matrix_to_sp_tensor(data.a) + + train_data_len = int(len(data) * 0.8) + train_data_len = 10000 + + # Train/valid/test split + data_tr, data_te = data[:train_data_len], data[train_data_len:], + np.random.shuffle(data_tr) + + val_data_len = int(len(data_tr) * 0.2) + data_tr, data_va = data_tr[:-val_data_len], data_tr[-val_data_len:] + + print("data_tr len ", len(data_tr)) + print("data_va len ", len(data_va)) + print("data_te len ", len(data_te)) + + # We use a MixedLoader since the dataset is in mixed mode + loader_tr = MixedLoader(data_tr, batch_size=batch_size, epochs=epochs) + loader_va = MixedLoader(data_va, batch_size=batch_size) + loader_te = MixedLoader(data_te, batch_size=batch_size) + + gnn = GNN(loss=loss, optimizer=optimizer, conv_layer=conv_layer, output_activation=abs_activation, + hidden_activation='relu', patience=20) + train_targets = gnn.fit(loader_tr, loader_va, loader_te) + + val_targets = gnn.val_targets + + targets, predictions = gnn.predict(loader_te) + predictions = np.squeeze(predictions) + + print("np.var(target-predictions) ", np.var(targets - predictions)) + + plot_loss(gnn._train_loss, gnn._val_loss) + analyze_results(targets, predictions) + + # target_means, target_vars = estimate_density(targets, title="Test outputs") + # pred_means, pred_vars = estimate_density(predictions, title="Predictions") + # + # print("target means ", target_means) + # print("predic means ", pred_means) + # + # print("target vars ", target_vars) + # print("predic vars ", pred_vars) + # + # diff_means, diff_vars = estimate_density(targets - predictions, title="diff") + # print("diff_means ", diff_means) + # print("diff vars ", diff_vars) + + diff_moments(targets, predictions) + + save_load_data(False, targets, predictions, train_targets, val_targets) + + +def save_load_data(load=False, targets=None, predictions=None, train_targets=None, val_targets=None): + path = "/home/martin/Documents/metamodels/data/" + + if load: + targets = None + predictions = None + if os.path.exists(os.path.join(path, "targets.npy")): + targets = np.load(os.path.join(path, "targets.npy")) + if os.path.exists(os.path.join(path, "predictions.npy")): + predictions = np.load(os.path.join(path, "predictions.npy")) + if os.path.exists(os.path.join(path, "train_targets.npy")): + train_targets = np.load(os.path.join(path, "train_targets.npy")) + if os.path.exists(os.path.join(path, "val_targets.npy")): + val_targets = np.load(os.path.join(path, "val_targets.npy")) + return targets, predictions, train_targets, val_targets + else: + if targets is not None: + np.save(os.path.join(path, "targets"), targets) + if predictions is not None: + np.save(os.path.join(path, "predictions"), predictions) + if train_targets is not None: + np.save(os.path.join(path, "train_targets"), train_targets) + if val_targets is not None: + np.save(os.path.join(path, "val_targets"), val_targets) + + +def process_results(): + targets, predictions, train_targets, val_targets = save_load_data(load=True) + + print("len targets ", len(targets)) + print("len predictions ", len(predictions)) + + print("len train targets ", len(train_targets)) + print("len val targets ", len(val_targets)) + + process_mlmc(targets, predictions, train_targets, val_targets) + + if __name__ == "__main__": + + # import cProfile + # import pstats + # pr = cProfile.Profile() + # pr.enable() + # + # my_result = run_GNN() + # + # pr.disable() + # ps = pstats.Stats(pr).sort_stats('cumtime') + # ps.print_stats() + + #run_GNN() + + process_results() + #bootstrap_GNN() #run() - bootstrap() \ No newline at end of file + #run_CNN() + + #bootstrap() \ No newline at end of file diff --git a/src/mlmc/metamodel/flow_task_GNN_2.py b/src/mlmc/metamodel/flow_task_GNN_2.py index 0b9ff557..6c9d6855 100644 --- a/src/mlmc/metamodel/flow_task_GNN_2.py +++ b/src/mlmc/metamodel/flow_task_GNN_2.py @@ -7,7 +7,7 @@ from tensorflow.keras.callbacks import History from tensorflow.keras.optimizers import Adam from tensorflow.keras.regularizers import l2 -from mlmc.metamodel.postprocessing import analyze_results, plot_loss +from mlmc.metamodel.postprocessing import analyze_results, plot_loss, estimate_density from spektral.data import MixedLoader from mlmc.metamodel.flow_dataset import FlowDataset from spektral.layers import GCNConv, GlobalSumPool, ChebConv, GraphSageConv, ARMAConv, GATConv, APPNPConv, GINConv @@ -41,6 +41,8 @@ def __init__(self, **kwargs): self._train_loss = [] self._val_loss = [] + self.val_targets = [] + self._model = Net1(conv_layer=self._conv_layer, hidden_activation=self._hidden_activation, output_activation=self._output_activation, kernel_regularization=self._hidden_regularizer) @@ -54,6 +56,9 @@ def fit(self, loader_tr, loader_va, loader_te): current_patience = self._patience step = 0 + train_targets = True + train_targets_list = [] + # Training loop results_tr = [] for batch in loader_tr: @@ -61,6 +66,11 @@ def fit(self, loader_tr, loader_va, loader_te): # Training step inputs, target = batch + + if train_targets: + train_targets_list.extend(target) + print("len(train targets ", len(train_targets_list)) + loss, acc = self.train_on_batch(inputs, target) self._train_loss.append(loss) results_tr.append((loss, acc, len(target))) @@ -69,6 +79,7 @@ def fit(self, loader_tr, loader_va, loader_te): self._val_loss.append(results_va[0]) if step == loader_tr.steps_per_epoch: # step_per_epoch = int(np.ceil(len(self.dataset) / self.batch_size)) + train_targets = False # results_va = self.evaluate(loader_va) # self._val_loss.append(results_va[0]) if results_va[0] < best_val_loss: @@ -95,6 +106,8 @@ def fit(self, loader_tr, loader_va, loader_te): results_tr = [] step = 0 + return train_targets_list + # Training function @tf.function def train_on_batch(self, inputs, target): @@ -106,15 +119,25 @@ def train_on_batch(self, inputs, target): acc = tf.reduce_mean(self._accuracy_func(target, predictions)) gradients = tape.gradient(loss, self._model.trainable_variables) - optimizer.apply_gradients(zip(gradients, self._model.trainable_variables)) + self._optimizer.apply_gradients(zip(gradients, self._model.trainable_variables)) return loss, acc def evaluate(self, loader): step = 0 results = [] + + if len(self.val_targets) > 0: + val_targets = False + else: + val_targets = True + for batch in loader: step += 1 inputs, target = batch + + if val_targets: + self.val_targets.extend(target) + predictions = self._model(inputs, training=False) loss = self._loss(target, predictions) @@ -131,12 +154,7 @@ def predict(self, loader): for batch in loader: step += 1 - - print("predict") inputs, target = batch - print("len(inputs) ", len(inputs[0])) - - print("step ", step) targets.extend(target) predictions.extend(self._model(inputs, training=False)) @@ -145,52 +163,55 @@ def predict(self, loader): return targets, predictions -if __name__ == "__main__": - # Parameters - #conv_layer = GCNConv - conv_layer = ChebConv # Seems better than GCNConv, good distribution of predictions - # # conv_layer = GraphSageConv # Seems better than ChebConv, good loss but very narrow distribution of predictions - # # conv_layer = ARMAConv # Seems worse than GraphSageConv - # # conv_layer = GATConv # Slow and not better than GraphSageConv - # # conv_layer = APPNPConv # Not bad but worse than GraphSageConv - # # conv_layer = GINConv # it is comparable to APPNPConv - # act_func = "relu" # "tanh"#"elu" # ReLU keep predictions above zero - loss = MeanSquaredError() - optimizer = tf.optimizers.Adam(learning_rate=0.001) - batch_size = 1000 - epochs = 100 - - # Load data - data = FlowDataset() - data = data[:10000] - data.a = conv_layer.preprocess(data.a) - data.a = sp_matrix_to_sp_tensor(data.a) - - train_data_len = int(len(data) * 0.8) - - # Train/valid/test split - data_tr, data_te = data[:train_data_len], data[train_data_len:], - np.random.shuffle(data_tr) - - val_data_len = int(len(data_tr) * 0.2) - data_tr, data_va = data_tr[:-val_data_len], data_tr[-val_data_len:] - - print("data_tr len ", len(data_tr)) - print("data_va len ", len(data_va)) - print("data_te len ", len(data_te)) - - # We use a MixedLoader since the dataset is in mixed mode - loader_tr = MixedLoader(data_tr, batch_size=batch_size, epochs=epochs) - loader_va = MixedLoader(data_va, batch_size=batch_size) - loader_te = MixedLoader(data_te, batch_size=batch_size) - - - gnn = GNN(loss=loss, optimizer=optimizer, conv_layer=conv_layer, output_activation=abs_activation, - hidden_activation='relu', patience=10) - gnn.fit(loader_tr, loader_va, loader_te) - - targets, predictions = gnn.predict(loader_te) - predictions = np.squeeze(predictions) - - plot_loss(gnn._train_loss, gnn._val_loss) - analyze_results(targets, predictions) +# if __name__ == "__main__": +# # Parameters +# #conv_layer = GCNConv +# conv_layer = ChebConv # Seems better than GCNConv, good distribution of predictions +# # # conv_layer = GraphSageConv # Seems better than ChebConv, good loss but very narrow distribution of predictions +# # # conv_layer = ARMAConv # Seems worse than GraphSageConv +# # # conv_layer = GATConv # Slow and not better than GraphSageConv +# # # conv_layer = APPNPConv # Not bad but worse than GraphSageConv +# # # conv_layer = GINConv # it is comparable to APPNPConv +# # act_func = "relu" # "tanh"#"elu" # ReLU keep predictions above zero +# loss = MeanSquaredError() +# optimizer = tf.optimizers.Adam(learning_rate=0.001) +# batch_size = 500 +# epochs = 100 +# +# # Load data +# data = FlowDataset() +# data = data#[:10000] +# #data.a = conv_layer.preprocess(data.a) +# data.a = sp_matrix_to_sp_tensor(data.a) +# +# train_data_len = int(len(data) * 0.8) +# train_data_len = 10000 +# +# # Train/valid/test split +# data_tr, data_te = data[:train_data_len], data[train_data_len:], +# np.random.shuffle(data_tr) +# +# val_data_len = int(len(data_tr) * 0.2) +# data_tr, data_va = data_tr[:-val_data_len], data_tr[-val_data_len:] +# +# print("data_tr len ", len(data_tr)) +# print("data_va len ", len(data_va)) +# print("data_te len ", len(data_te)) +# +# # We use a MixedLoader since the dataset is in mixed mode +# loader_tr = MixedLoader(data_tr, batch_size=batch_size, epochs=epochs) +# loader_va = MixedLoader(data_va, batch_size=batch_size) +# loader_te = MixedLoader(data_te, batch_size=batch_size) +# +# gnn = GNN(loss=loss, optimizer=optimizer, conv_layer=conv_layer, output_activation=abs_activation, +# hidden_activation='relu', patience=20) +# gnn.fit(loader_tr, loader_va, loader_te) +# +# targets, predictions = gnn.predict(loader_te) +# predictions = np.squeeze(predictions) +# +# plot_loss(gnn._train_loss, gnn._val_loss) +# analyze_results(targets, predictions) +# +# estimate_density(targets) +# estimate_density(predictions) diff --git a/src/mlmc/metamodel/postprocessing.py b/src/mlmc/metamodel/postprocessing.py index 3f401df3..ed3dd405 100644 --- a/src/mlmc/metamodel/postprocessing.py +++ b/src/mlmc/metamodel/postprocessing.py @@ -1,6 +1,17 @@ -import numpy as np +import random import matplotlib.pyplot as plt from scipy.stats import ks_2samp +from mlmc.tool import plot +import mlmc.tool.simple_distribution +import mlmc.estimator +import mlmc.quantity_estimate as qe +from mlmc.sample_storage import Memory +from mlmc.quantity_spec import QuantitySpec, ChunkSpec +import numpy as np +from mlmc.sample_storage_hdf import SampleStorageHDF +from mlmc.moments import Legendre, Monomial +from mlmc.quantity import make_root_quantity +import mlmc.tool.simple_distribution def plot_loss(train_loss, val_loss): @@ -35,4 +46,309 @@ def analyze_results(target, predictions): plt.hist(target, alpha=0.5, label='target', density=True) plt.hist(predictions, alpha=0.5, label='predictions', density=True) plt.legend(loc='upper right') - plt.show() \ No newline at end of file + plt.show() + + +def estimate_density(values, title="Density"): + sample_storage = Memory() + n_levels = 1 + n_moments = 25 + distr_accuracy = 1e-7 + + distr_plot = plot.Distribution(title=title, + log_density=True) + + result_format = [QuantitySpec(name="flow", unit="m", shape=(1,), times=[0], locations=['0'])] + + sample_storage.save_global_data(result_format=result_format, level_parameters=np.ones(n_levels)) + + successful_samples = {} + failed_samples = {} + n_ops = {} + n_successful = len(values) + for l_id in range(n_levels): + sizes = [] + for quantity_spec in result_format: + sizes.append(np.prod(quantity_spec.shape) * len(quantity_spec.times) * len(quantity_spec.locations)) + + # Dict[level_id, List[Tuple[sample_id:str, Tuple[fine_result: ndarray, coarse_result: ndarray]]]] + successful_samples[l_id] = [] + for sample_id in range(len(values)): + successful_samples[l_id].append((str(sample_id), (values[sample_id], 0))) + + n_ops[l_id] = [random.random(), n_successful] + + sample_storage.save_scheduled_samples(l_id, samples=["S{:07d}".format(i) for i in range(n_successful)]) + + sample_storage.save_samples(successful_samples, failed_samples) + sample_storage.save_n_ops(list(n_ops.items())) + + quantity = make_root_quantity(storage=sample_storage, q_specs=result_format) + length = quantity['flow'] + time = length[0] + location = time['0'] + value_quantity = location[0] + + quantile = 0.001 + true_domain = mlmc.estimator.Estimate.estimate_domain(value_quantity, sample_storage, quantile=quantile) + print("true domain ", true_domain) + moments_fn = Legendre(n_moments, true_domain) + + estimator = mlmc.estimator.Estimate(quantity=value_quantity, sample_storage=sample_storage, moments_fn=moments_fn) + + reg_param = 0 + target_var = 1e-4 + distr_obj, info, result, moments_fn = estimator.construct_density( + tol=distr_accuracy, + reg_param=reg_param, + orth_moments_tol=target_var) + + samples = value_quantity.samples(ChunkSpec(level_id=0, n_samples=sample_storage.get_n_collected()[0]))[..., 0] + + distr_plot.add_raw_samples(np.squeeze(samples)) + + distr_plot.add_distribution(distr_obj, label="") + + # kl = mlmc.tool.simple_distribution.KL_divergence(self.cut_distr.pdf, distr_obj.density, + # self.cut_distr.domain[0], self.cut_distr.domain[1]) + #kl_divergences.append(kl) + + distr_plot.show(file=None) + + + return estimator.estimate_moments() + + +def create_quantity(target, predictions): + sample_storage = Memory() + n_levels = 2 + + result_format = [QuantitySpec(name="flow", unit="m", shape=(1,), times=[0], locations=['0'])] + + sample_storage.save_global_data(result_format=result_format, level_parameters=np.ones(n_levels)) + + successful_samples = {} + failed_samples = {} + n_ops = {} + n_successful = len(target) + for l_id in range(n_levels): + sizes = [] + for quantity_spec in result_format: + sizes.append(np.prod(quantity_spec.shape) * len(quantity_spec.times) * len(quantity_spec.locations)) + + successful_samples[l_id] = [] + for sample_id in range(n_successful): + if l_id == 0: + fine_result = predictions[sample_id] + coarse_result = (np.zeros((np.sum(sizes),))) + else: + fine_result = target[sample_id] + coarse_result = predictions[sample_id] + + successful_samples[l_id].append((str(sample_id), (fine_result, coarse_result))) + + n_ops[l_id] = [random.random(), n_successful] + sample_storage.save_scheduled_samples(l_id, samples=["S{:07d}".format(i) for i in range(n_successful)]) + + sample_storage.save_samples(successful_samples, failed_samples) + sample_storage.save_n_ops(list(n_ops.items())) + + quantity = make_root_quantity(storage=sample_storage, q_specs=result_format) + length = quantity['flow'] + time = length[0] + location = time['0'] + value_quantity = location[0] + + return value_quantity, sample_storage + + +def diff_moments(target, predictions): + n_moments = 25 + quantity, target_sample_storage = create_quantity(target, predictions) + + quantile = 0.001 + true_domain = mlmc.estimator.Estimate.estimate_domain(quantity, target_sample_storage, quantile=quantile) + + moments_fn = Legendre(n_moments, true_domain) + + quantity_moments = qe.moments(quantity, moments_fn) + + + moments_mean = qe.estimate_mean(quantity_moments) + + print("moments l means ", moments_mean.l_means) + print("moments l vars ", moments_mean.l_vars) + + print("np.max values mean l vars ", np.max(moments_mean.l_vars, axis=1)) + + print("moments mean ", moments_mean.mean) + print("moments var ", moments_mean.var) + + +def create_quantity_mlmc(data): + sample_storage = Memory() + n_levels = len(data) + + result_format = [QuantitySpec(name="conductivity", unit="m", shape=(1, 1), times=[1], locations=['0'])] + + sample_storage.save_global_data(result_format=result_format, level_parameters=np.ones(n_levels)) + + successful_samples = {} + failed_samples = {} + n_ops = {} + n_successful = 15 + sizes = [] + for l_id in range(n_levels): + n_successful = data[l_id].shape[1] + sizes = [] + for quantity_spec in result_format: + sizes.append(np.prod(quantity_spec.shape) * len(quantity_spec.times) * len(quantity_spec.locations)) + + # Dict[level_id, List[Tuple[sample_id:str, Tuple[fine_result: ndarray, coarse_result: ndarray]]]] + successful_samples[l_id] = [] + for sample_id in range(n_successful): + + fine_result = data[l_id][:, sample_id, 0] + if l_id == 0: + coarse_result = (np.zeros((np.sum(sizes),))) + else: + coarse_result = data[l_id][:, sample_id, 1] + successful_samples[l_id].append((str(sample_id), (fine_result, coarse_result))) + + n_ops[l_id] = [random.random(), n_successful] + + sample_storage.save_scheduled_samples(l_id, samples=["S{:07d}".format(i) for i in range(n_successful)]) + + sample_storage.save_samples(successful_samples, failed_samples) + sample_storage.save_n_ops(list(n_ops.items())) + + return sample_storage + + +def estimate_moments(sample_storage): + n_moments = 25 + result_format = sample_storage.load_result_format() + root_quantity = make_root_quantity(sample_storage, result_format) + + conductivity = root_quantity['conductivity'] + time = conductivity[1] # times: [1] + location = time['0'] # locations: ['0'] + q_value = location[0, 0] + + # @TODO: How to estimate true_domain? + quantile = 0.001 + true_domain = mlmc.estimator.Estimate.estimate_domain(q_value, sample_storage, quantile=quantile) + print("true domain ", true_domain) + moments_fn = Legendre(n_moments, true_domain) + + estimator = mlmc.estimator.Estimate(quantity=q_value, sample_storage=sample_storage, moments_fn=moments_fn) + means, vars = estimator.estimate_moments(moments_fn) + + # moments_quantity = moments(root_quantity, moments_fn=moments_fn, mom_at_bottom=True) + # moments_mean = estimate_mean(moments_quantity) + # conductivity_mean = moments_mean['conductivity'] + # time_mean = conductivity_mean[1] # times: [1] + # location_mean = time_mean['0'] # locations: ['0'] + # values_mean = location_mean[0] # result shape: (1,) + # + # print("values_mean.n_samples ", values_mean.n_samples) + # print("values_mean.l_means ", values_mean.l_means) + # + # print("l_means / n samples ", values_mean.l_means / values_mean.n_samples[:, None]) + # print("values mean. l_vars ", values_mean.l_vars) + # + # print("np.max values mean l vars ", np.max(values_mean.l_vars, axis=1)) + # + # print("values_mean mean ", values_mean.mean) + # print("values_mean var ", values_mean.var) + + return means, vars, estimator + + +# def construct_density(estimator): +# +# distr_obj, info, result, moments_fn = estimator.construct_density( +# tol=distr_accuracy, +# reg_param=reg_param, +# orth_moments_tol=target_var) +# +# samples = estimator.quantity.samples(ChunkSpec(level_id=0, n_samples=estimator._sample_storage.get_n_collected()[0]))[..., 0] +# +# return distr_obj, samples + + +def compare_densities(estimator_1, estimator_2, label_1="", label_2=""): + + distr_plot = plot.ArticleDistribution(title="densities", log_density=True) + tol = 1e-10 + reg_param = 0 + + distr_obj, result, _, _ = estimator_2.construct_density(tol=tol, reg_param=reg_param) + #distr_plot.add_raw_samples(np.squeeze(samples)) + distr_plot.add_distribution(distr_obj, label=label_1, color="blue") + + distr_obj, result, _, _ = estimator_2.construct_density(tol=tol, reg_param=reg_param) + #distr_plot.add_raw_samples(np.squeeze(samples)) + distr_plot.add_distribution(distr_obj, label=label_2, color="red") + # + + distr_plot.show(file=None) + + +def get_quantity_estimator(sample_storage): + n_moments = 25 + result_format = sample_storage.load_result_format() + root_quantity = make_root_quantity(sample_storage, result_format) + conductivity = root_quantity['conductivity'] + time = conductivity[1] # times: [1] + location = time['0'] # locations: ['0'] + quantity = location[0, 0] + + quantile = 0.001 + true_domain = mlmc.estimator.Estimate.estimate_domain(quantity, sample_storage, quantile=quantile) + moments_fn = Legendre(n_moments, true_domain) + + return mlmc.estimator.Estimate(quantity=quantity, sample_storage=sample_storage, moments_fn=moments_fn) + + +def process_mlmc(targets, predictions, train_targets, val_targets): + n_levels = 5 + mlmc_file = "/home/martin/Documents/metamodels/data/cl_0_3_s_4/L5/mlmc_5.hdf5" + + sample_storage = SampleStorageHDF(file_path=mlmc_file) + original_means, original_vars, estimator = estimate_moments(sample_storage) + + # Test storage creation + data = [] + for l_id in range(n_levels): + level_samples = estimator.get_level_samples(level_id=l_id) + data.append(level_samples) + sample_storage_2 = create_quantity_mlmc(data) + means_2, vars_2, estimator_2 = estimate_moments(sample_storage_2) + assert np.allclose(original_means, means_2) + assert np.allclose(original_vars, vars_2) + + data = [] + for l_id in range(n_levels): + level_samples = estimator.get_level_samples(level_id=l_id) + if l_id == 0: + level_samples = np.concatenate((train_targets.reshape(1, len(train_targets), 1), + targets.reshape(1, len(targets), 1)), axis=1) + data.append(level_samples) + sample_storage_nn = create_quantity_mlmc(data) + means_nn, vars_nn, estimator_nn = estimate_moments(sample_storage_nn) + + + print("original means ", original_means) + print("means nn ", means_nn) + + print("original vars ", original_vars) + print("vars nn ", vars_nn) + assert np.allclose(original_means, means_nn, atol=1e-3) + assert np.allclose(original_vars, vars_nn, atol=1e-3) + + original_q_estimator = get_quantity_estimator(sample_storage) + no_val_samples_q_estimator = get_quantity_estimator(sample_storage_nn) + + compare_densities(estimator, no_val_samples_q_estimator, label_1="original", label_2="without validation values") + From eb0b9b221962decb204d14943bac08fdf689665f Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Fri, 5 Mar 2021 17:03:04 +0100 Subject: [PATCH 08/67] nn mlmc --- src/mlmc/metamodel/analyze_nn.py | 97 ++++++---- src/mlmc/metamodel/postprocessing.py | 276 ++++++++++++++++++++------- 2 files changed, 267 insertions(+), 106 deletions(-) diff --git a/src/mlmc/metamodel/analyze_nn.py b/src/mlmc/metamodel/analyze_nn.py index 42293291..217ddddd 100644 --- a/src/mlmc/metamodel/analyze_nn.py +++ b/src/mlmc/metamodel/analyze_nn.py @@ -1,7 +1,9 @@ import os import numpy as np +import time import random #os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only +from mlmc.metamodel.create_graph import graph_creator from mlmc.metamodel.flow_dataset import FlowDataset # Make numpy printouts easier to read. @@ -18,6 +20,7 @@ from mlmc.metamodel.flow_task_GNN_2 import GNN from spektral.layers import GCNConv, GlobalSumPool, ChebConv, GraphSageConv, ARMAConv, GATConv, APPNPConv, GINConv from tensorflow.keras.losses import MeanSquaredError +from tensorflow.keras.regularizers import l2 from spektral.data import MixedLoader from spektral.layers.ops import sp_matrix_to_sp_tensor print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU'))) @@ -243,23 +246,11 @@ def bootstrap_GNN(): def run_GNN(): - loss = "mean_absolute_error" - optimizer = tf.optimizers.Adam(learning_rate=0.001) - n_subsamples = 10 - size = 10000 - - train_losses = [] - val_losses = [] - all_test_outputs = [] - all_predictions = [] - ks_statistics = [] - ks_p_values = [] - - data = FlowDataset() - dataset = data.dataset.dropna() + output_dir = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L1/test/01_cond_field/output/" + hdf_path = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L1/mlmc_1.hdf5" # Parameters - #conv_layer = GCNConv + conv_layer = GCNConv conv_layer = ChebConv # Seems better than GCNConv, good distribution of predictions # # conv_layer = GraphSageConv # Seems better than ChebConv, good loss but very narrow distribution of predictions # # conv_layer = ARMAConv # Seems worse than GraphSageConv @@ -268,18 +259,22 @@ def run_GNN(): # # conv_layer = GINConv # it is comparable to APPNPConv # act_func = "relu" # "tanh"#"elu" # ReLU keep predictions above zero loss = MeanSquaredError() - optimizer = tf.optimizers.Adam(learning_rate=0.001) + optimizer = tf.optimizers.Adam(learning_rate=0.01) batch_size = 1000 epochs = 100 + hidden_regularization = None#l2(2e-10) + + + graph_creator(output_dir, hdf_path) # Load data - data = FlowDataset() + data = FlowDataset(output_dir=output_dir) data = data # [:10000] # data.a = conv_layer.preprocess(data.a) data.a = sp_matrix_to_sp_tensor(data.a) train_data_len = int(len(data) * 0.8) - train_data_len = 10000 + train_data_len = 2000 # Train/valid/test split data_tr, data_te = data[:train_data_len], data[train_data_len:], @@ -298,7 +293,7 @@ def run_GNN(): loader_te = MixedLoader(data_te, batch_size=batch_size) gnn = GNN(loss=loss, optimizer=optimizer, conv_layer=conv_layer, output_activation=abs_activation, - hidden_activation='relu', patience=20) + hidden_activation='relu', patience=20, hidden_reqularizer=hidden_regularization) train_targets = gnn.fit(loader_tr, loader_va, loader_te) val_targets = gnn.val_targets @@ -324,17 +319,33 @@ def run_GNN(): # print("diff_means ", diff_means) # print("diff vars ", diff_vars) - diff_moments(targets, predictions) + #diff_moments(targets, predictions) + + + l_0_targets, l_0_predictions = predict_level_zero(gnn, batch_size) + + save_load_data(False, targets, predictions, train_targets, val_targets, l_0_targets, l_0_predictions) + + +def predict_level_zero(nn, batch_size=1000): + output_dir = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L1/test/01_cond_field/output/" + # Load data + data = FlowDataset(output_dir=output_dir) + data = data # [:10000] + # data.a = conv_layer.preprocess(data.a) + data.a = sp_matrix_to_sp_tensor(data.a) + + loader_te = MixedLoader(data, batch_size=batch_size) - save_load_data(False, targets, predictions, train_targets, val_targets) + targets, predictions = nn.predict(loader_te) + return targets, predictions -def save_load_data(load=False, targets=None, predictions=None, train_targets=None, val_targets=None): +def save_load_data(load=False, targets=None, predictions=None, train_targets=None, val_targets=None, l_0_targets=None, + l_0_predictions=None): path = "/home/martin/Documents/metamodels/data/" if load: - targets = None - predictions = None if os.path.exists(os.path.join(path, "targets.npy")): targets = np.load(os.path.join(path, "targets.npy")) if os.path.exists(os.path.join(path, "predictions.npy")): @@ -343,7 +354,11 @@ def save_load_data(load=False, targets=None, predictions=None, train_targets=Non train_targets = np.load(os.path.join(path, "train_targets.npy")) if os.path.exists(os.path.join(path, "val_targets.npy")): val_targets = np.load(os.path.join(path, "val_targets.npy")) - return targets, predictions, train_targets, val_targets + if os.path.exists(os.path.join(path, "l_0_targets.npy")): + l_0_targets = np.load(os.path.join(path, "l_0_targets.npy")) + if os.path.exists(os.path.join(path, "l_0_predictions.npy")): + l_0_predictions = np.load(os.path.join(path, "l_0_predictions.npy")) + return targets, predictions, train_targets, val_targets, l_0_targets, l_0_predictions else: if targets is not None: np.save(os.path.join(path, "targets"), targets) @@ -353,10 +368,14 @@ def save_load_data(load=False, targets=None, predictions=None, train_targets=Non np.save(os.path.join(path, "train_targets"), train_targets) if val_targets is not None: np.save(os.path.join(path, "val_targets"), val_targets) + if l_0_targets is not None: + np.save(os.path.join(path, "l_0_targets"), l_0_targets) + if l_0_predictions is not None: + np.save(os.path.join(path, "l_0_predictions"), l_0_predictions) def process_results(): - targets, predictions, train_targets, val_targets = save_load_data(load=True) + targets, predictions, train_targets, val_targets, l_0_targets, l_0_predictions = save_load_data(load=True) print("len targets ", len(targets)) print("len predictions ", len(predictions)) @@ -364,25 +383,25 @@ def process_results(): print("len train targets ", len(train_targets)) print("len val targets ", len(val_targets)) - process_mlmc(targets, predictions, train_targets, val_targets) + process_mlmc(targets, predictions, train_targets, val_targets, l_0_targets, l_0_predictions) if __name__ == "__main__": - # import cProfile - # import pstats - # pr = cProfile.Profile() - # pr.enable() - # - # my_result = run_GNN() - # - # pr.disable() - # ps = pstats.Stats(pr).sort_stats('cumtime') - # ps.print_stats() + import cProfile + import pstats + pr = cProfile.Profile() + pr.enable() - #run_GNN() + my_result = run_GNN() - process_results() + pr.disable() + ps = pstats.Stats(pr).sort_stats('cumtime') + ps.print_stats() + + # run_GNN() + # + # process_results() #bootstrap_GNN() #run() diff --git a/src/mlmc/metamodel/postprocessing.py b/src/mlmc/metamodel/postprocessing.py index ed3dd405..17eb154a 100644 --- a/src/mlmc/metamodel/postprocessing.py +++ b/src/mlmc/metamodel/postprocessing.py @@ -91,7 +91,6 @@ def estimate_density(values, title="Density"): quantile = 0.001 true_domain = mlmc.estimator.Estimate.estimate_domain(value_quantity, sample_storage, quantile=quantile) - print("true domain ", true_domain) moments_fn = Legendre(n_moments, true_domain) estimator = mlmc.estimator.Estimate(quantity=value_quantity, sample_storage=sample_storage, moments_fn=moments_fn) @@ -185,13 +184,16 @@ def diff_moments(target, predictions): print("moments var ", moments_mean.var) -def create_quantity_mlmc(data): +def create_quantity_mlmc(data, level_parameters=None): sample_storage = Memory() n_levels = len(data) result_format = [QuantitySpec(name="conductivity", unit="m", shape=(1, 1), times=[1], locations=['0'])] - sample_storage.save_global_data(result_format=result_format, level_parameters=np.ones(n_levels)) + if level_parameters is None: + level_parameters = np.ones(n_levels) + + sample_storage.save_global_data(result_format=result_format, level_parameters=level_parameters) successful_samples = {} failed_samples = {} @@ -225,7 +227,7 @@ def create_quantity_mlmc(data): return sample_storage -def estimate_moments(sample_storage): +def estimate_moments(sample_storage, true_domain=None): n_moments = 25 result_format = sample_storage.load_result_format() root_quantity = make_root_quantity(sample_storage, result_format) @@ -235,67 +237,72 @@ def estimate_moments(sample_storage): location = time['0'] # locations: ['0'] q_value = location[0, 0] - # @TODO: How to estimate true_domain? - quantile = 0.001 - true_domain = mlmc.estimator.Estimate.estimate_domain(q_value, sample_storage, quantile=quantile) + if true_domain is None: + quantile = 0.001 + true_domain = mlmc.estimator.Estimate.estimate_domain(q_value, sample_storage, quantile=quantile) print("true domain ", true_domain) moments_fn = Legendre(n_moments, true_domain) estimator = mlmc.estimator.Estimate(quantity=q_value, sample_storage=sample_storage, moments_fn=moments_fn) - means, vars = estimator.estimate_moments(moments_fn) - - # moments_quantity = moments(root_quantity, moments_fn=moments_fn, mom_at_bottom=True) - # moments_mean = estimate_mean(moments_quantity) - # conductivity_mean = moments_mean['conductivity'] - # time_mean = conductivity_mean[1] # times: [1] - # location_mean = time_mean['0'] # locations: ['0'] - # values_mean = location_mean[0] # result shape: (1,) - # - # print("values_mean.n_samples ", values_mean.n_samples) - # print("values_mean.l_means ", values_mean.l_means) - # - # print("l_means / n samples ", values_mean.l_means / values_mean.n_samples[:, None]) - # print("values mean. l_vars ", values_mean.l_vars) - # - # print("np.max values mean l vars ", np.max(values_mean.l_vars, axis=1)) - # - # print("values_mean mean ", values_mean.mean) - # print("values_mean var ", values_mean.var) - - return means, vars, estimator - - -# def construct_density(estimator): -# -# distr_obj, info, result, moments_fn = estimator.construct_density( -# tol=distr_accuracy, -# reg_param=reg_param, -# orth_moments_tol=target_var) -# -# samples = estimator.quantity.samples(ChunkSpec(level_id=0, n_samples=estimator._sample_storage.get_n_collected()[0]))[..., 0] -# -# return distr_obj, samples - - -def compare_densities(estimator_1, estimator_2, label_1="", label_2=""): + #means, vars = estimator.estimate_moments(moments_fn) + + moments_mean = qe.estimate_mean(qe.moments(q_value, moments_fn)) + return moments_mean, estimator, true_domain, q_value + + +def ref_distr(): + tol = 1e-10 + reg_param = 0 + mlmc_file = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L1_benchmark/mlmc_1.hdf5" + + sample_storage = SampleStorageHDF(file_path=mlmc_file) + original_moments, estimator, original_true_domain, _ = estimate_moments(sample_storage) + + print("original moments ", original_moments.mean) + print("original moments ", original_moments.var) + exit() + + distr_obj, result, _, _ = estimator.construct_density(tol=tol, reg_param=reg_param) + + + + return distr_obj + + +def compare_densities(estimator_1, estimator_2, label_1="", label_2="", ref_density=False): distr_plot = plot.ArticleDistribution(title="densities", log_density=True) tol = 1e-10 reg_param = 0 - distr_obj, result, _, _ = estimator_2.construct_density(tol=tol, reg_param=reg_param) + distr_obj_1, result, _, _ = estimator_1.construct_density(tol=tol, reg_param=reg_param) #distr_plot.add_raw_samples(np.squeeze(samples)) - distr_plot.add_distribution(distr_obj, label=label_1, color="blue") + distr_plot.add_distribution(distr_obj_1, label=label_1, color="blue") - distr_obj, result, _, _ = estimator_2.construct_density(tol=tol, reg_param=reg_param) + distr_obj_2, result, _, _ = estimator_2.construct_density(tol=tol, reg_param=reg_param) #distr_plot.add_raw_samples(np.squeeze(samples)) - distr_plot.add_distribution(distr_obj, label=label_2, color="red") - # + distr_plot.add_distribution(distr_obj_2, label=label_2, color="red", line_style="--") + + #if ref_density: + ref_distr_obj = ref_distr() + distr_plot.add_distribution(ref_distr_obj, label="L1 benchmark", color="black", line_style=":") + + domain = [np.min([ref_distr_obj.domain[0], distr_obj_1.domain[0]]), np.max([ref_distr_obj.domain[1], distr_obj_1.domain[1]])] + kl_div = mlmc.tool.simple_distribution.KL_divergence(ref_distr_obj.density, distr_obj_1.density, domain[0], domain[1]) + + print("KL div ref|mlmc: {}".format(kl_div)) + + domain = [np.min([ref_distr_obj.domain[0], distr_obj_2.domain[0]]), + np.max([ref_distr_obj.domain[1], distr_obj_2.domain[1]])] + kl_div = mlmc.tool.simple_distribution.KL_divergence(ref_distr_obj.density, distr_obj_2.density, domain[0], + domain[1]) + + print("KL div ref|mlmc prediction: {}".format(kl_div)) distr_plot.show(file=None) -def get_quantity_estimator(sample_storage): +def get_quantity_estimator(sample_storage, true_domain=None): n_moments = 25 result_format = sample_storage.load_result_format() root_quantity = make_root_quantity(sample_storage, result_format) @@ -304,19 +311,22 @@ def get_quantity_estimator(sample_storage): location = time['0'] # locations: ['0'] quantity = location[0, 0] - quantile = 0.001 - true_domain = mlmc.estimator.Estimate.estimate_domain(quantity, sample_storage, quantile=quantile) + if true_domain is None: + quantile = 0.001 + true_domain = mlmc.estimator.Estimate.estimate_domain(quantity, sample_storage, quantile=quantile) + moments_fn = Legendre(n_moments, true_domain) return mlmc.estimator.Estimate(quantity=quantity, sample_storage=sample_storage, moments_fn=moments_fn) -def process_mlmc(targets, predictions, train_targets, val_targets): +def process_mlmc(targets, predictions, train_targets, val_targets, l_0_targets=None, l_0_predictions=None): n_levels = 5 - mlmc_file = "/home/martin/Documents/metamodels/data/cl_0_3_s_4/L5/mlmc_5.hdf5" + #mlmc_file = "/home/martin/Documents/metamodels/data/cl_0_3_s_4/L5/mlmc_5.hdf5" + mlmc_file = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L5/mlmc_5.hdf5" sample_storage = SampleStorageHDF(file_path=mlmc_file) - original_means, original_vars, estimator = estimate_moments(sample_storage) + original_moments, estimator, original_true_domain, _ = estimate_moments(sample_storage) # Test storage creation data = [] @@ -324,10 +334,11 @@ def process_mlmc(targets, predictions, train_targets, val_targets): level_samples = estimator.get_level_samples(level_id=l_id) data.append(level_samples) sample_storage_2 = create_quantity_mlmc(data) - means_2, vars_2, estimator_2 = estimate_moments(sample_storage_2) - assert np.allclose(original_means, means_2) - assert np.allclose(original_vars, vars_2) + moments_2, estimator_2, _, _ = estimate_moments(sample_storage_2) + assert np.allclose(original_moments.mean, moments_2.mean) + assert np.allclose(original_moments.var, moments_2.var) + # Use train and test data without validation data data = [] for l_id in range(n_levels): level_samples = estimator.get_level_samples(level_id=l_id) @@ -336,19 +347,150 @@ def process_mlmc(targets, predictions, train_targets, val_targets): targets.reshape(1, len(targets), 1)), axis=1) data.append(level_samples) sample_storage_nn = create_quantity_mlmc(data) - means_nn, vars_nn, estimator_nn = estimate_moments(sample_storage_nn) + moments_nn, estimator_nn, _, _ = estimate_moments(sample_storage_nn, true_domain=original_true_domain) + + # Use predicted data as zero level results and level one coarse results + if l_0_targets is not None and l_0_predictions is not None: + data = [] + for l_id in range(n_levels + 1): + if l_id == 0: + level_samples = l_0_predictions.reshape(1, len(l_0_predictions), 1) + else: + level_samples = estimator.get_level_samples(level_id=l_id - 1) + if l_id == 1: + coarse_level_samples = predictions.reshape(1, len(predictions), 1) + fine_level_samples = targets.reshape(1, len(targets), 1) + level_samples = np.concatenate((fine_level_samples, coarse_level_samples), axis=2) + data.append(level_samples) + + # n0 = 1000 + # nL = 10 + # num_levels = n_levels + 1 + # initial_n_samples = np.round(np.exp2(np.linspace(np.log2(n0), np.log2(nL), num_levels))).astype(int) + # if len(initial_n_samples) == len(data): + # for i in range(len(data)): + # print(data[i].shape) + # data[i] = data[i][:, :initial_n_samples[i], :] + # print("data[i].shape ", data[i].shape) + + level_params = [sample_storage.get_level_parameters()[0], *sample_storage.get_level_parameters()] + sample_storage_predict = create_quantity_mlmc(data, level_parameters=level_params) + + moments_predict, estimator_predict, _, quantity = estimate_moments(sample_storage_predict, true_domain=original_true_domain) + + target_var = 1e-5 + + #n_level_samples = initial_n_samples #sample_storage_predict.get_n_collected() + n_level_samples = sample_storage_predict.get_n_collected() + custom_n_ops = [sample_storage.get_n_ops()[0], *sample_storage.get_n_ops()] + print("custom n ops ", custom_n_ops) + + # @TODO: test + # New estimation according to already finished samples + variances, n_ops = estimator_predict.estimate_diff_vars_regression(n_level_samples) + print("variances ", variances) + print("n ops ", n_ops) + n_ops = custom_n_ops + n_estimated = mlmc.estimator.estimate_n_samples_for_target_variance(target_var, variances, n_ops, + n_levels=len(n_level_samples)) + + print("n estimated ", n_estimated) + + + original_q_estimator = get_quantity_estimator(sample_storage) + predict_q_estimator = get_quantity_estimator(sample_storage_predict) + + print("means nn ", moments_nn.mean) + print("means_predict ", moments_predict.mean) + + print("means nn - means predict ", moments_nn.mean - moments_predict.mean) + print("abs means nn - means predict ", np.abs(moments_nn.mean - moments_predict.mean)) + + print("vars nn ", moments_nn.var) + print("vars predict ", moments_predict.var) + + print("moments_nn.l_means ", moments_nn.l_means[0]) + print("moments_predict.l_means ", moments_predict.l_means[0]) + + print("moments nn n samples ", moments_nn.n_samples) + print("moments nn n removed samples ", moments_predict.n_rm_samples) + print("moments predict n samples ", moments_predict.n_samples) + print("moments predict n removed samples ", moments_predict.n_rm_samples) + + for l_id, (l_mom, l_mom_pred) in enumerate(zip(moments_nn.l_means, moments_predict.l_means)): + print("L id: {}, mom diff: {}".format(l_id, l_mom - l_mom_pred)) + + compare_densities(original_q_estimator, predict_q_estimator, label_1="orig N: {}".format(moments_nn.n_samples), + label_2="gnn N: {}".format(moments_predict.n_samples)) + + # Use predicted data as zero level results instead of original ones + else: + data = [] + for l_id in range(n_levels): + level_samples = estimator.get_level_samples(level_id=l_id) + if l_id == 0: + level_samples = np.concatenate((train_targets.reshape(1, len(train_targets), 1), + predictions.reshape(1, len(predictions), 1)), axis=1) + data.append(level_samples) + sample_storage_predict = create_quantity_mlmc(data) + moments_predict, estimator_predict, _, _ = estimate_moments(sample_storage_predict, true_domain=original_true_domain) + + original_q_estimator = get_quantity_estimator(sample_storage) + predict_q_estimator = get_quantity_estimator(sample_storage_predict) + + + print("means nn ", moments_nn.mean) + print("means_predict ", moments_predict.mean) + + + print("means nn - means predict ", moments_nn.mean - moments_predict.mean) + print("abs means nn - means predict ", np.abs(moments_nn.mean - moments_predict.mean)) + print("vars nn ", moments_nn.var) + print("vars predict ", moments_predict.var) - print("original means ", original_means) - print("means nn ", means_nn) - print("original vars ", original_vars) - print("vars nn ", vars_nn) - assert np.allclose(original_means, means_nn, atol=1e-3) - assert np.allclose(original_vars, vars_nn, atol=1e-3) + print("moments_nn.l_means ", moments_nn.l_means[0]) + print("moments_predict.l_means ", moments_predict.l_means[0]) - original_q_estimator = get_quantity_estimator(sample_storage) - no_val_samples_q_estimator = get_quantity_estimator(sample_storage_nn) + print("moments nn n samples ", moments_nn.n_samples) + print("moments nn n removed samples ", moments_predict.n_rm_samples) + print("moments predict n samples ", moments_predict.n_samples) + print("moments predict n removed samples ", moments_predict.n_rm_samples) + + for l_id, (l_mom, l_mom_pred) in enumerate(zip(moments_nn.l_means, moments_predict.l_means)): + print("L id: {}, mom diff: {}".format(l_id, l_mom - l_mom_pred)) + + compare_densities(original_q_estimator, predict_q_estimator, label_1="original", label_2="level 0 predictions") + + +def analyze_mlmc_data(): + n_levels = 5 + # mlmc_file = "/home/martin/Documents/metamodels/data/cl_0_3_s_4/L5/mlmc_5.hdf5" + mlmc_file = "/home/martin/Documents/metamodels/data/cl_0_1_s_1/L5/mlmc_5.hdf5" + + sample_storage = SampleStorageHDF(file_path=mlmc_file) + original_moments, estimator, original_true_domain = estimate_moments(sample_storage) + + # Test storage creation + data = [] + for l_id in range(n_levels): + level_samples = estimator.get_level_samples(level_id=l_id) + + + l_fine = np.squeeze(level_samples[..., 0]) + + print("mean l_fine ", np.mean(l_fine)) + plt.hist(l_fine, alpha=0.5, label='{}'.format(l_id), density=True) + data.append(level_samples) + + plt.legend(loc='upper right') + plt.show() + sample_storage_2 = create_quantity_mlmc(data) + moments_2, estimator_2, _ = estimate_moments(sample_storage_2) + assert np.allclose(original_moments.mean, moments_2.mean) + assert np.allclose(original_moments.var, moments_2.var) - compare_densities(estimator, no_val_samples_q_estimator, label_1="original", label_2="without validation values") +if __name__ == "__main__": + analyze_mlmc_data() From caba88b0b4dacd84b3bf461eab71f4ca757781ed Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Sat, 6 Mar 2021 18:11:56 +0100 Subject: [PATCH 09/67] compare results --- src/mlmc/metamodel/analyze_nn.py | 115 ++++++++---- src/mlmc/metamodel/create_graph.py | 37 +++- src/mlmc/metamodel/flow_dataset.py | 38 +++- src/mlmc/metamodel/postprocessing.py | 256 ++++++++++++++++++++------- 4 files changed, 336 insertions(+), 110 deletions(-) diff --git a/src/mlmc/metamodel/analyze_nn.py b/src/mlmc/metamodel/analyze_nn.py index 217ddddd..33032f92 100644 --- a/src/mlmc/metamodel/analyze_nn.py +++ b/src/mlmc/metamodel/analyze_nn.py @@ -245,10 +245,7 @@ def bootstrap_GNN(): estimate_density(np.mean(all_predictions, axis=0), title="Predictions") -def run_GNN(): - output_dir = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L1/test/01_cond_field/output/" - hdf_path = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L1/mlmc_1.hdf5" - +def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path): # Parameters conv_layer = GCNConv conv_layer = ChebConv # Seems better than GCNConv, good distribution of predictions @@ -264,12 +261,15 @@ def run_GNN(): epochs = 100 hidden_regularization = None#l2(2e-10) - + preprocess_start_time = time.process_time() graph_creator(output_dir, hdf_path) # Load data data = FlowDataset(output_dir=output_dir) data = data # [:10000] + preprocess_time = time.process_time() - preprocess_start_time + + learning_time_start = time.process_time() # data.a = conv_layer.preprocess(data.a) data.a = sp_matrix_to_sp_tensor(data.a) @@ -283,9 +283,9 @@ def run_GNN(): val_data_len = int(len(data_tr) * 0.2) data_tr, data_va = data_tr[:-val_data_len], data_tr[-val_data_len:] - print("data_tr len ", len(data_tr)) - print("data_va len ", len(data_va)) - print("data_te len ", len(data_te)) + # print("data_tr len ", len(data_tr)) + # print("data_va len ", len(data_va)) + # print("data_te len ", len(data_te)) # We use a MixedLoader since the dataset is in mixed mode loader_tr = MixedLoader(data_tr, batch_size=batch_size, epochs=epochs) @@ -300,11 +300,12 @@ def run_GNN(): targets, predictions = gnn.predict(loader_te) predictions = np.squeeze(predictions) + learning_time = time.process_time() - learning_time_start - print("np.var(target-predictions) ", np.var(targets - predictions)) + #print("np.var(target-predictions) ", np.var(targets - predictions)) - plot_loss(gnn._train_loss, gnn._val_loss) - analyze_results(targets, predictions) + #plot_loss(gnn._train_loss, gnn._val_loss) + #analyze_results(targets, predictions) # target_means, target_vars = estimate_density(targets, title="Test outputs") # pred_means, pred_vars = estimate_density(predictions, title="Predictions") @@ -321,14 +322,17 @@ def run_GNN(): #diff_moments(targets, predictions) + predict_l_0_start_time = time.process_time() + l_0_targets, l_0_predictions = predict_level_zero(gnn, l_0_output_dir, l_0_hdf_path, batch_size) + predict_l_0_time = time.process_time() - predict_l_0_start_time - l_0_targets, l_0_predictions = predict_level_zero(gnn, batch_size) + save_times(save_path, False, (preprocess_time, len(data)), learning_time, (predict_l_0_time, len(l_0_targets))) + save_load_data(save_path, False, targets, predictions, train_targets, val_targets, l_0_targets, l_0_predictions) - save_load_data(False, targets, predictions, train_targets, val_targets, l_0_targets, l_0_predictions) +def predict_level_zero(nn, output_dir, hdf_path, batch_size=1000): + graph_creator(output_dir, hdf_path) -def predict_level_zero(nn, batch_size=1000): - output_dir = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L1/test/01_cond_field/output/" # Load data data = FlowDataset(output_dir=output_dir) data = data # [:10000] @@ -338,12 +342,41 @@ def predict_level_zero(nn, batch_size=1000): loader_te = MixedLoader(data, batch_size=batch_size) targets, predictions = nn.predict(loader_te) + predictions = np.squeeze(predictions) + #analyze_results(targets, predictions) return targets, predictions -def save_load_data(load=False, targets=None, predictions=None, train_targets=None, val_targets=None, l_0_targets=None, +def save_times(path, load=False, preprocess=None, learning_time=None, predict_l_0=None): + if load: + preprocess_time = None + preprocess_n = None + predict_time = None + predict_n = None + if os.path.exists(os.path.join(path, "preprocess_time.npy")): + preprocess_time = np.load(os.path.join(path, "preprocess_time.npy")) + if os.path.exists(os.path.join(path, "preprocess_n.npy")): + preprocess_n = np.load(os.path.join(path, "preprocess_n.npy")) + if os.path.exists(os.path.join(path, "learning_time.npy")): + learning_time = np.load(os.path.join(path, "learning_time.npy")) + if os.path.exists(os.path.join(path, "predict_l_0_time.npy")): + predict_time = np.load(os.path.join(path, "predict_l_0_time.npy")) + if os.path.exists(os.path.join(path, "predict_l_0_n.npy")): + predict_n = np.load(os.path.join(path, "predict_l_0_n.npy")) + return preprocess_time, preprocess_n, learning_time, predict_time, predict_n + else: + if preprocess is not None: + np.save(os.path.join(path, "preprocess_time"), preprocess[0]) + np.save(os.path.join(path, "preprocess_n"), preprocess[1]) + if learning_time is not None: + np.save(os.path.join(path, "learning_time"), learning_time) + if preprocess is not None: + np.save(os.path.join(path, "predict_l_0_time"), predict_l_0[0]) + np.save(os.path.join(path, "predict_l_0_n"), predict_l_0[1]) + + +def save_load_data(path, load=False, targets=None, predictions=None, train_targets=None, val_targets=None, l_0_targets=None, l_0_predictions=None): - path = "/home/martin/Documents/metamodels/data/" if load: if os.path.exists(os.path.join(path, "targets.npy")): @@ -374,8 +407,21 @@ def save_load_data(load=False, targets=None, predictions=None, train_targets=Non np.save(os.path.join(path, "l_0_predictions"), l_0_predictions) -def process_results(): - targets, predictions, train_targets, val_targets, l_0_targets, l_0_predictions = save_load_data(load=True) +def process_results(hdf_path, sampling_info_path, save_path): + targets, predictions, train_targets, val_targets, l_0_targets, l_0_predictions = save_load_data(save_path, load=True) + preprocess_time, preprocess_n, learning_time, predict_l_0_time, predict_l_0_n = save_times(save_path, load=True) + + l1_sample_time = preprocess_time / preprocess_n + learning_time / preprocess_n + l0_sample_time = predict_l_0_time / predict_l_0_n + + print("preprocess_time ", preprocess_time) + print("preprocess_n ", preprocess_n) + print("learning_time ", learning_time) + print("predict_l_0_time ", predict_l_0_time) + print("predict_l_0_n ", predict_l_0_n) + + print("l1 sample time ", l1_sample_time) + print("l0 sample time ", l0_sample_time) print("len targets ", len(targets)) print("len predictions ", len(predictions)) @@ -383,25 +429,34 @@ def process_results(): print("len train targets ", len(train_targets)) print("len val targets ", len(val_targets)) - process_mlmc(targets, predictions, train_targets, val_targets, l_0_targets, l_0_predictions) + process_mlmc(hdf_path, sampling_info_path, targets, predictions, train_targets, val_targets, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time) if __name__ == "__main__": + output_dir = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L5/test/01_cond_field/output/" + hdf_path = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L5/mlmc_5.hdf5" - import cProfile - import pstats - pr = cProfile.Profile() - pr.enable() + save_path = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1" - my_result = run_GNN() + l_0_output_dir = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L1/test/01_cond_field/output/" + l_0_hdf_path = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L1/mlmc_1.hdf5" - pr.disable() - ps = pstats.Stats(pr).sort_stats('cumtime') - ps.print_stats() + sampling_info_path = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/sampling_info" + + # import cProfile + # import pstats + # pr = cProfile.Profile() + # pr.enable() + # + # my_result = run_GNN() + # + # pr.disable() + # ps = pstats.Stats(pr).sort_stats('cumtime') + # ps.print_stats() - # run_GNN() + #run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path) # - # process_results() + process_results(hdf_path, sampling_info_path, save_path) #bootstrap_GNN() #run() diff --git a/src/mlmc/metamodel/create_graph.py b/src/mlmc/metamodel/create_graph.py index c6e9bb8b..c0cd29a9 100644 --- a/src/mlmc/metamodel/create_graph.py +++ b/src/mlmc/metamodel/create_graph.py @@ -5,13 +5,24 @@ import matplotlib.pyplot as plt from mlmc.tool import gmsh_io from mlmc.tool.hdf5 import HDF5 +from spektral.data import Graph +from mlmc.metamodel.flow_dataset import FlowDataset MESH = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/l_step_0.055_common_files/mesh.msh" #FIELDS_SAMPLE_MESH = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/output/L00_S0000000/fine_fields_sample.msh" FIELDS_SAMPLE = "fine_fields_sample.msh" -OUTPUT_DIR = "/home/martin/Documents/metamodels/data/1000_ele/test/01_cond_field/output/" -HDF_PATH = "/home/martin/Documents/metamodels/data/1000_ele/test/01_cond_field/mlmc_1.hdf5" +# OUTPUT_DIR = "/home/martin/Documents/metamodels/data/1000_ele/test/01_cond_field/output/" +# HDF_PATH = "/home/martin/Documents/metamodels/data/1000_ele/test/01_cond_field/mlmc_1.hdf5" + +# OUTPUT_DIR = "/home/martin/Documents/metamodels/data/cl_0_3_s_4/L5/test/01_cond_field/output/" +# HDF_PATH = "/home/martin/Documents/metamodels/data/cl_0_3_s_4/L5/mlmc_5.hdf5" + +# OUTPUT_DIR = "/home/martin/Documents/metamodels/data/cl_0_1_s_1/L5/test/01_cond_field/output/" +# HDF_PATH = "/home/martin/Documents/metamodels/data/cl_0_1_s_1/L5/mlmc_5.hdf5" + +# OUTPUT_DIR = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L5/test/01_cond_field/output/" +# HDF_PATH = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L5/mlmc_5.hdf5" def extract_mesh_gmsh_io(mesh_file): @@ -90,30 +101,40 @@ def plot_graph(adjacency_matrix): plt.show() -def extract_mesh(): +def graph_creator(output_dir, hdf_path): adjacency_matrix = create_adjacency_matrix(extract_mesh_gmsh_io(MESH)) - np.save(os.path.join(OUTPUT_DIR, "adjacency_matrix"), adjacency_matrix, allow_pickle=True) - loaded_adjacency_matrix = np.load(os.path.join(OUTPUT_DIR, "adjacency_matrix.npy"), allow_pickle=True) + np.save(os.path.join(output_dir, "adjacency_matrix"), adjacency_matrix, allow_pickle=True) + loaded_adjacency_matrix = np.load(os.path.join(output_dir, "adjacency_matrix.npy"), allow_pickle=True) plot_graph(loaded_adjacency_matrix) - hdf = HDF5(file_path=HDF_PATH, + hdf = HDF5(file_path=hdf_path, load_from_file=True) level_group = hdf.add_level_group(level_id=str(0)) collected = zip(level_group.get_collected_ids(), level_group.collected()) + graphs = [] + data = [] + for sample_id, col_values in collected: output_value = col_values[0, 0] - sample_dir = os.path.join(OUTPUT_DIR, sample_id) + sample_dir = os.path.join(output_dir, sample_id) field_mesh = os.path.join(sample_dir, FIELDS_SAMPLE) if os.path.exists(field_mesh): features = get_node_features(field_mesh) np.save(os.path.join(sample_dir, "nodes_features"), features) np.save(os.path.join(sample_dir, "output"), output_value) + #graphs.append(Graph(x=features, y=output_value)) # , a=self.adjacency_matrix)) + # Save data for pandas dataframe creation, not used with Graph neural network + #data.append({'x': features, 'y': output_value}) + #loaded_features = np.load(os.path.join(sample_dir, "nodes_features.npy")) #print("loaded features ", loaded_features) + #FlowDataset.pickle_data(graphs, FlowDataset.GRAPHS_FILE) + #FlowDataset.pickle_data(data, FlowDataset.DATA_FILE) + if __name__ == "__main__": import cProfile @@ -121,7 +142,7 @@ def extract_mesh(): pr = cProfile.Profile() pr.enable() - my_result = extract_mesh() + my_result = graph_creator() pr.disable() ps = pstats.Stats(pr).sort_stats('cumtime') diff --git a/src/mlmc/metamodel/flow_dataset.py b/src/mlmc/metamodel/flow_dataset.py index 829f8693..e1c06c19 100644 --- a/src/mlmc/metamodel/flow_dataset.py +++ b/src/mlmc/metamodel/flow_dataset.py @@ -3,17 +3,25 @@ import pandas as pd from mlmc.tool import gmsh_io from spektral.data import Dataset, Graph +import pickle MESH = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/l_step_0.055_common_files/mesh.msh" FIELDS_SAMPLE = "fine_fields_sample.msh" -OUTPUT_DIR = "/home/martin/Documents/metamodels/data/1000_ele/test/01_cond_field/output/" +#OUTPUT_DIR = "/home/martin/Documents/metamodels/data/1000_ele/test/01_cond_field/output/" +#OUTPUT_DIR = "/home/martin/Documents/metamodels/data/cl_0_3_s_4/L5/test/01_cond_field/output/" +#OUTPUT_DIR = "/home/martin/Documents/metamodels/data/cl_0_1_s_1/L5/test/01_cond_field/output/" +OUTPUT_DIR = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L5/test/01_cond_field/output/" class FlowDataset(Dataset): - """ - """ - def __init__(self, **kwargs): - self.adjacency_matrix = np.load(os.path.join(OUTPUT_DIR, "adjacency_matrix.npy"), allow_pickle=True) # adjacency matrix + GRAPHS_FILE = "graphs" + DATA_FILE = "data" + + def __init__(self, output_dir=None, **kwargs): + self._output_dir = output_dir + if self._output_dir is None: + self._output_dir = OUTPUT_DIR + self.adjacency_matrix = np.load(os.path.join(self._output_dir, "adjacency_matrix.npy"), allow_pickle=True) # adjacency matrix self.data = [] super().__init__(**kwargs) self.a = self.adjacency_matrix @@ -21,10 +29,19 @@ def __init__(self, **kwargs): self.dataset = pd.DataFrame(self.data) def read(self): + # with open(os.path.join(OUTPUT_DIR, FlowDataset.GRAPHS_FILE), 'rb') as reader: + # graphs = pickle.loads(reader) + # + # if os.path.exists(os.path.join(OUTPUT_DIR, FlowDataset.DATA_FILE)): + # with open(os.path.join(OUTPUT_DIR, FlowDataset.DATA_FILE), 'rb') as reader: + # self.data = pickle.loads(reader) + # + # return graphs + graphs = [] - for s_dir in os.listdir(OUTPUT_DIR): - if os.path.isdir(os.path.join(OUTPUT_DIR, s_dir)): - sample_dir = os.path.join(OUTPUT_DIR, s_dir) + for s_dir in os.listdir(self._output_dir): + if os.path.isdir(os.path.join(self._output_dir, s_dir)): + sample_dir = os.path.join(self._output_dir, s_dir) if os.path.exists(os.path.join(sample_dir, "nodes_features.npy")): features = np.load(os.path.join(sample_dir, "nodes_features.npy")) output = np.load(os.path.join(sample_dir, "output.npy")) @@ -34,7 +51,10 @@ def read(self): self.data.append({'x': features, 'y': output}) return graphs - + @staticmethod + def pickle_data(data, file_path): + with open(os.path.join(OUTPUT_DIR, file_path), 'wb') as writer: + pickle.dump(data, writer) def extract_mesh_gmsh_io(mesh_file): diff --git a/src/mlmc/metamodel/postprocessing.py b/src/mlmc/metamodel/postprocessing.py index 17eb154a..23fc4893 100644 --- a/src/mlmc/metamodel/postprocessing.py +++ b/src/mlmc/metamodel/postprocessing.py @@ -1,3 +1,4 @@ +import os import random import matplotlib.pyplot as plt from scipy.stats import ks_2samp @@ -122,7 +123,7 @@ def create_quantity(target, predictions): sample_storage = Memory() n_levels = 2 - result_format = [QuantitySpec(name="flow", unit="m", shape=(1,), times=[0], locations=['0'])] + result_format = [QuantitySpec(name="conductivity", unit="m", shape=(1, 1), times=[1], locations=['0'])] sample_storage.save_global_data(result_format=result_format, level_parameters=np.ones(n_levels)) @@ -250,52 +251,94 @@ def estimate_moments(sample_storage, true_domain=None): return moments_mean, estimator, true_domain, q_value -def ref_distr(): +def ref_storage(): tol = 1e-10 reg_param = 0 mlmc_file = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L1_benchmark/mlmc_1.hdf5" sample_storage = SampleStorageHDF(file_path=mlmc_file) - original_moments, estimator, original_true_domain, _ = estimate_moments(sample_storage) + return sample_storage + + +def get_largest_domain(storages): + true_domains = [] + for storage in storages: + result_format = storage.load_result_format() + root_quantity = make_root_quantity(storage, result_format) + + conductivity = root_quantity['conductivity'] + time = conductivity[1] # times: [1] + location = time['0'] # locations: ['0'] + q_value = location[0, 0] + + + # @TODO: How to estimate true_domain? + quantile = 0.001 + domain = mlmc.estimator.Estimate.estimate_domain(q_value, storage, quantile=quantile) + print("domain ", domain) + + true_domains.append([domain[0], domain[1]]) + + true_domains = np.array(true_domains) + + print("true domains ", true_domains) + true_domain = [np.min(true_domains[:, 0]), np.max(true_domains[:, 1])] + #true_domain = [np.max(true_domains[:, 0]), np.min(true_domains[:, 1])] + #true_domain = [np.mean(true_domains[:, 0]), np.mean(true_domains[:, 1])] + print("true domain ", true_domain) + return true_domain + + +def compare_moments(original_q_estimator, predict_q_estimator, ref_estimator): + original_q_estimator.estimate_moments() + orig_moments_mean = original_q_estimator.moments_mean + + predict_q_estimator.estimate_moments() + predict_moments_mean = predict_q_estimator.moments_mean - print("original moments ", original_moments.mean) - print("original moments ", original_moments.var) - exit() + ref_estimator.estimate_moments() + ref_moments_mean = ref_estimator.moments_mean - distr_obj, result, _, _ = estimator.construct_density(tol=tol, reg_param=reg_param) + print("ref moments mean ", ref_moments_mean.mean) + print("orig moments mean ", orig_moments_mean.mean) + print("predict moments mean ", predict_moments_mean.mean) + print("ref orig mean SSE ", np.sum((ref_moments_mean.mean - orig_moments_mean.mean)**2)) + print("ref predict mean SSE ", np.sum((ref_moments_mean.mean - predict_moments_mean.mean) ** 2)) + print("ref moments var ", ref_moments_mean.var) + print("orig moments var ", orig_moments_mean.var) + print("predict moments var ", predict_moments_mean.var) - return distr_obj + print("ref orig var SSE ", np.sum((ref_moments_mean.var - orig_moments_mean.var) ** 2)) + print("ref predict var SSE ", np.sum((ref_moments_mean.var - predict_moments_mean.var) ** 2)) -def compare_densities(estimator_1, estimator_2, label_1="", label_2="", ref_density=False): +def compare_densities(estimator_1, estimator_2, ref_estimator, label_1="", label_2=""): distr_plot = plot.ArticleDistribution(title="densities", log_density=True) tol = 1e-10 reg_param = 0 + distr_obj_1, result, _, _ = estimator_1.construct_density(tol=tol, reg_param=reg_param) - #distr_plot.add_raw_samples(np.squeeze(samples)) distr_plot.add_distribution(distr_obj_1, label=label_1, color="blue") distr_obj_2, result, _, _ = estimator_2.construct_density(tol=tol, reg_param=reg_param) - #distr_plot.add_raw_samples(np.squeeze(samples)) distr_plot.add_distribution(distr_obj_2, label=label_2, color="red", line_style="--") - #if ref_density: - ref_distr_obj = ref_distr() + ref_distr_obj, result, _, _ = ref_estimator.construct_density(tol=tol, reg_param=reg_param) distr_plot.add_distribution(ref_distr_obj, label="L1 benchmark", color="black", line_style=":") - domain = [np.min([ref_distr_obj.domain[0], distr_obj_1.domain[0]]), np.max([ref_distr_obj.domain[1], distr_obj_1.domain[1]])] - kl_div = mlmc.tool.simple_distribution.KL_divergence(ref_distr_obj.density, distr_obj_1.density, domain[0], domain[1]) + + kl_div = mlmc.tool.simple_distribution.KL_divergence(ref_distr_obj.density, distr_obj_1.density, ref_distr_obj.domain[0], ref_distr_obj.domain[1]) print("KL div ref|mlmc: {}".format(kl_div)) - domain = [np.min([ref_distr_obj.domain[0], distr_obj_2.domain[0]]), - np.max([ref_distr_obj.domain[1], distr_obj_2.domain[1]])] - kl_div = mlmc.tool.simple_distribution.KL_divergence(ref_distr_obj.density, distr_obj_2.density, domain[0], - domain[1]) + # domain = [np.min([ref_distr_obj.domain[0], distr_obj_2.domain[0]]), + # np.max([ref_distr_obj.domain[1], distr_obj_2.domain[1]])] + kl_div = mlmc.tool.simple_distribution.KL_divergence(ref_distr_obj.density, distr_obj_2.density, ref_distr_obj.domain[0], + ref_distr_obj.domain[1]) print("KL div ref|mlmc prediction: {}".format(kl_div)) @@ -320,10 +363,12 @@ def get_quantity_estimator(sample_storage, true_domain=None): return mlmc.estimator.Estimate(quantity=quantity, sample_storage=sample_storage, moments_fn=moments_fn) -def process_mlmc(targets, predictions, train_targets, val_targets, l_0_targets=None, l_0_predictions=None): +def process_mlmc(mlmc_file, sampling_info_path, targets, predictions, train_targets, val_targets, l_0_targets=None, l_0_predictions=None, + l1_sample_time=None, l0_sample_time=None): n_levels = 5 #mlmc_file = "/home/martin/Documents/metamodels/data/cl_0_3_s_4/L5/mlmc_5.hdf5" - mlmc_file = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L5/mlmc_5.hdf5" + # mlmc_file = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L5/mlmc_5.hdf5" + # sampling_info_path = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/sampling_info" sample_storage = SampleStorageHDF(file_path=mlmc_file) original_moments, estimator, original_true_domain, _ = estimate_moments(sample_storage) @@ -338,16 +383,33 @@ def process_mlmc(targets, predictions, train_targets, val_targets, l_0_targets=N assert np.allclose(original_moments.mean, moments_2.mean) assert np.allclose(original_moments.var, moments_2.var) - # Use train and test data without validation data - data = [] - for l_id in range(n_levels): - level_samples = estimator.get_level_samples(level_id=l_id) - if l_id == 0: - level_samples = np.concatenate((train_targets.reshape(1, len(train_targets), 1), - targets.reshape(1, len(targets), 1)), axis=1) - data.append(level_samples) - sample_storage_nn = create_quantity_mlmc(data) - moments_nn, estimator_nn, _, _ = estimate_moments(sample_storage_nn, true_domain=original_true_domain) + print("moments_2.l_vars max ", np.max(moments_2.l_vars, axis=1)) + + n_ops, field_times, flow_times = get_sample_times(sampling_info_path) + + n_collected_times = n_ops * np.array(sample_storage_2.get_n_collected()) + # print("n collected times ", n_collected_times) + # print("total time ", np.sum(n_collected_times)) + # print("n ops ", n_ops) + # print("field_times ", field_times) + # print("flow_times ", flow_times) + # exit() + + n_ops_predict = [l0_sample_time + field_times[0], n_ops[0] + l1_sample_time, *n_ops[1:]] + + print("n ops ", n_ops) + print("n ops predict ", n_ops_predict) + + # # Use train and test data without validation data + # data = [] + # for l_id in range(n_levels): + # level_samples = estimator.get_level_samples(level_id=l_id) + # if l_id == 0: + # level_samples = np.concatenate((train_targets.reshape(1, len(train_targets), 1), + # targets.reshape(1, len(targets), 1)), axis=1) + # data.append(level_samples) + # sample_storage_nn = create_quantity_mlmc(data) + # moments_nn, estimator_nn, _, _ = estimate_moments(sample_storage_nn, true_domain=original_true_domain) # Use predicted data as zero level results and level one coarse results if l_0_targets is not None and l_0_predictions is not None: @@ -355,6 +417,7 @@ def process_mlmc(targets, predictions, train_targets, val_targets, l_0_targets=N for l_id in range(n_levels + 1): if l_id == 0: level_samples = l_0_predictions.reshape(1, len(l_0_predictions), 1) + #level_samples = level_samples[:, :100000, :] else: level_samples = estimator.get_level_samples(level_id=l_id - 1) if l_id == 1: @@ -363,7 +426,7 @@ def process_mlmc(targets, predictions, train_targets, val_targets, l_0_targets=N level_samples = np.concatenate((fine_level_samples, coarse_level_samples), axis=2) data.append(level_samples) - # n0 = 1000 + # n0 = 100 # nL = 10 # num_levels = n_levels + 1 # initial_n_samples = np.round(np.exp2(np.linspace(np.log2(n0), np.log2(nL), num_levels))).astype(int) @@ -377,52 +440,69 @@ def process_mlmc(targets, predictions, train_targets, val_targets, l_0_targets=N sample_storage_predict = create_quantity_mlmc(data, level_parameters=level_params) moments_predict, estimator_predict, _, quantity = estimate_moments(sample_storage_predict, true_domain=original_true_domain) - target_var = 1e-5 #n_level_samples = initial_n_samples #sample_storage_predict.get_n_collected() n_level_samples = sample_storage_predict.get_n_collected() - custom_n_ops = [sample_storage.get_n_ops()[0], *sample_storage.get_n_ops()] - print("custom n ops ", custom_n_ops) + # custom_n_ops = [sample_storage.get_n_ops()[0], *sample_storage.get_n_ops()] + # print("custom n ops ", custom_n_ops) + + print("n level samples ", n_level_samples) # @TODO: test # New estimation according to already finished samples variances, n_ops = estimator_predict.estimate_diff_vars_regression(n_level_samples) print("variances ", variances) + print("level max var ", np.max(variances, axis=1)) print("n ops ", n_ops) - n_ops = custom_n_ops + n_ops = n_ops_predict n_estimated = mlmc.estimator.estimate_n_samples_for_target_variance(target_var, variances, n_ops, n_levels=len(n_level_samples)) - print("n estimated ", n_estimated) - - - original_q_estimator = get_quantity_estimator(sample_storage) - predict_q_estimator = get_quantity_estimator(sample_storage_predict) - - print("means nn ", moments_nn.mean) - print("means_predict ", moments_predict.mean) - - print("means nn - means predict ", moments_nn.mean - moments_predict.mean) - print("abs means nn - means predict ", np.abs(moments_nn.mean - moments_predict.mean)) - - print("vars nn ", moments_nn.var) - print("vars predict ", moments_predict.var) - - print("moments_nn.l_means ", moments_nn.l_means[0]) - print("moments_predict.l_means ", moments_predict.l_means[0]) - - print("moments nn n samples ", moments_nn.n_samples) - print("moments nn n removed samples ", moments_predict.n_rm_samples) - print("moments predict n samples ", moments_predict.n_samples) - print("moments predict n removed samples ", moments_predict.n_rm_samples) - - for l_id, (l_mom, l_mom_pred) in enumerate(zip(moments_nn.l_means, moments_predict.l_means)): - print("L id: {}, mom diff: {}".format(l_id, l_mom - l_mom_pred)) - - compare_densities(original_q_estimator, predict_q_estimator, label_1="orig N: {}".format(moments_nn.n_samples), + print("original collected ", sample_storage_2.get_n_collected()) + + est_times = n_ops_predict * np.array(n_estimated) + + print("est times ", est_times) + print("original n collected times ", n_collected_times) + print("total est times ", np.sum(est_times)) + print("original total collected times ", np.sum(n_collected_times)) + + + ref_sample_storage = ref_storage() + domain = get_largest_domain([sample_storage, sample_storage_predict, ref_sample_storage]) + #domain = [0.001, 5] + + original_q_estimator = get_quantity_estimator(sample_storage, true_domain=domain) + predict_q_estimator = get_quantity_estimator(sample_storage_predict, true_domain=domain) + ref_estimator = get_quantity_estimator(ref_sample_storage, true_domain=domain) + + # print("means nn ", moments_nn.mean) + # print("means_predict ", moments_predict.mean) + # + # print("means nn - means predict ", moments_nn.mean - moments_predict.mean) + # print("abs means nn - means predict ", np.abs(moments_nn.mean - moments_predict.mean)) + # + # print("vars nn ", moments_nn.var) + # print("vars predict ", moments_predict.var) + # + # print("moments_nn.l_means ", moments_nn.l_means[0]) + # print("moments_predict.l_means ", moments_predict.l_means[0]) + # + # print("moments nn n samples ", moments_nn.n_samples) + # print("moments nn n removed samples ", moments_predict.n_rm_samples) + # print("moments predict n samples ", moments_predict.n_samples) + # print("moments predict n removed samples ", moments_predict.n_rm_samples) + # + # for l_id, (l_mom, l_mom_pred) in enumerate(zip(moments_nn.l_means, moments_predict.l_means)): + # print("L id: {}, mom diff: {}".format(l_id, l_mom - l_mom_pred)) + + compare_moments(original_q_estimator, predict_q_estimator, ref_estimator) + + compare_densities(original_q_estimator, predict_q_estimator, ref_estimator, label_1="orig N: {}".format(moments_2.n_samples), label_2="gnn N: {}".format(moments_predict.n_samples)) + # Use predicted data as zero level results instead of original ones else: data = [] @@ -492,5 +572,55 @@ def analyze_mlmc_data(): assert np.allclose(original_moments.var, moments_2.var) +def get_sample_times(sampling_info_path): + n_levels = [5] + for nl in n_levels: + # sampling_info_path = "/home/martin/Documents/MLMC_article/data/sampling_info" + #sampling_info_path = "/home/martin/Documents/metamodels/data/1000_ele/sampling_info" + + variances = [] + n_ops = [] + times = [] + + times_scheduled_samples = [] + running_times = [] + flow_running_times = [] + + for i in range(0, 100): + sampling_info_path_iter = os.path.join(sampling_info_path, str(i)) + if os.path.isdir(sampling_info_path_iter): + variances.append(np.load(os.path.join(sampling_info_path_iter, "variances.npy"))) + n_ops.append(np.load(os.path.join(sampling_info_path_iter, "n_ops.npy"))) + times.append(np.load(os.path.join(sampling_info_path_iter, "time.npy"))) + + running_times.append(np.load(os.path.join(sampling_info_path_iter, "running_times.npy"))) + flow_running_times.append(np.load(os.path.join(sampling_info_path_iter, "flow_running_times.npy"))) + if os.path.exists(os.path.join(sampling_info_path_iter, "scheduled_samples_time.npy")): + times_scheduled_samples.append( + np.load(os.path.join(sampling_info_path_iter, "scheduled_samples_time.npy"))) + else: + break + + def time_for_sample_func(data): + new_n_ops = [] + for nop in data: + nop = np.squeeze(nop) + if len(nop) > 0: + new_n_ops.append(nop[:, 0]/nop[:, 1]) + return new_n_ops + + n_ops = time_for_sample_func(n_ops) + running_times = time_for_sample_func(running_times) + flow_running_times = time_for_sample_func(flow_running_times) + + field_times = np.mean(np.array(running_times) - np.array(flow_running_times) - np.array(flow_running_times), + axis=0) + + flow_times = np.mean(np.array(flow_running_times), axis=0) + n_ops = np.mean(n_ops, axis=0) + + return n_ops, field_times, flow_times + + if __name__ == "__main__": analyze_mlmc_data() From 1290eac9e8f97de621d9ba4c88948bf8eca95e8f Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Mon, 8 Mar 2021 16:28:44 +0100 Subject: [PATCH 10/67] level 0 gnn --- src/mlmc/metamodel/analyze_nn.py | 37 +++++---- src/mlmc/metamodel/postprocessing.py | 120 ++++++++++++--------------- 2 files changed, 72 insertions(+), 85 deletions(-) diff --git a/src/mlmc/metamodel/analyze_nn.py b/src/mlmc/metamodel/analyze_nn.py index 33032f92..ef940a55 100644 --- a/src/mlmc/metamodel/analyze_nn.py +++ b/src/mlmc/metamodel/analyze_nn.py @@ -258,11 +258,11 @@ def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path): loss = MeanSquaredError() optimizer = tf.optimizers.Adam(learning_rate=0.01) batch_size = 1000 - epochs = 100 + epochs = 500 hidden_regularization = None#l2(2e-10) preprocess_start_time = time.process_time() - graph_creator(output_dir, hdf_path) + #graph_creator(output_dir, hdf_path) # Load data data = FlowDataset(output_dir=output_dir) @@ -274,7 +274,7 @@ def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path): data.a = sp_matrix_to_sp_tensor(data.a) train_data_len = int(len(data) * 0.8) - train_data_len = 2000 + train_data_len = 10000#2000 # Train/valid/test split data_tr, data_te = data[:train_data_len], data[train_data_len:], @@ -293,7 +293,7 @@ def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path): loader_te = MixedLoader(data_te, batch_size=batch_size) gnn = GNN(loss=loss, optimizer=optimizer, conv_layer=conv_layer, output_activation=abs_activation, - hidden_activation='relu', patience=20, hidden_reqularizer=hidden_regularization) + hidden_activation='relu', patience=35, hidden_reqularizer=hidden_regularization) train_targets = gnn.fit(loader_tr, loader_va, loader_te) val_targets = gnn.val_targets @@ -304,8 +304,8 @@ def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path): #print("np.var(target-predictions) ", np.var(targets - predictions)) - #plot_loss(gnn._train_loss, gnn._val_loss) - #analyze_results(targets, predictions) + plot_loss(gnn._train_loss, gnn._val_loss) + analyze_results(targets, predictions) # target_means, target_vars = estimate_density(targets, title="Test outputs") # pred_means, pred_vars = estimate_density(predictions, title="Predictions") @@ -331,7 +331,7 @@ def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path): def predict_level_zero(nn, output_dir, hdf_path, batch_size=1000): - graph_creator(output_dir, hdf_path) + #graph_creator(output_dir, hdf_path) # Load data data = FlowDataset(output_dir=output_dir) @@ -407,7 +407,7 @@ def save_load_data(path, load=False, targets=None, predictions=None, train_targe np.save(os.path.join(path, "l_0_predictions"), l_0_predictions) -def process_results(hdf_path, sampling_info_path, save_path): +def process_results(hdf_path, sampling_info_path, ref_mlmc_file, save_path): targets, predictions, train_targets, val_targets, l_0_targets, l_0_predictions = save_load_data(save_path, load=True) preprocess_time, preprocess_n, learning_time, predict_l_0_time, predict_l_0_n = save_times(save_path, load=True) @@ -429,19 +429,22 @@ def process_results(hdf_path, sampling_info_path, save_path): print("len train targets ", len(train_targets)) print("len val targets ", len(val_targets)) - process_mlmc(hdf_path, sampling_info_path, targets, predictions, train_targets, val_targets, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time) + process_mlmc(hdf_path, sampling_info_path, ref_mlmc_file, targets, predictions, train_targets, val_targets, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time) if __name__ == "__main__": - output_dir = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L5/test/01_cond_field/output/" - hdf_path = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L5/mlmc_5.hdf5" + cl = "cl_0_3_s_4" + output_dir = "/home/martin/Documents/metamodels/data/1000_ele/{}/L5/test/01_cond_field/output/".format(cl) + hdf_path = "/home/martin/Documents/metamodels/data/1000_ele/{}/L5/mlmc_5.hdf5".format(cl) + + save_path = "/home/martin/Documents/metamodels/data/1000_ele/{}".format(cl) - save_path = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1" + l_0_output_dir = "/home/martin/Documents/metamodels/data/1000_ele/{}/L1/test/01_cond_field/output/".format(cl) + l_0_hdf_path = "/home/martin/Documents/metamodels/data/1000_ele/{}/L1/mlmc_1.hdf5".format(cl) - l_0_output_dir = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L1/test/01_cond_field/output/" - l_0_hdf_path = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L1/mlmc_1.hdf5" + sampling_info_path = "/home/martin/Documents/metamodels/data/1000_ele/{}/sampling_info".format(cl) - sampling_info_path = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/sampling_info" + ref_mlmc_file = "/home/martin/Documents/metamodels/data/1000_ele/{}/L1_benchmark/mlmc_1.hdf5".format(cl) # import cProfile # import pstats @@ -454,9 +457,9 @@ def process_results(hdf_path, sampling_info_path, save_path): # ps = pstats.Stats(pr).sort_stats('cumtime') # ps.print_stats() - #run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path) + run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path) # - process_results(hdf_path, sampling_info_path, save_path) + process_results(hdf_path, sampling_info_path, ref_mlmc_file, save_path) #bootstrap_GNN() #run() diff --git a/src/mlmc/metamodel/postprocessing.py b/src/mlmc/metamodel/postprocessing.py index 23fc4893..0943345d 100644 --- a/src/mlmc/metamodel/postprocessing.py +++ b/src/mlmc/metamodel/postprocessing.py @@ -28,7 +28,7 @@ def plot_loss(train_loss, val_loss): def analyze_results(target, predictions): - statistics, pvalue = ks_2samp(target, predictions) + #statistics, pvalue = ks_2samp(target, predictions) print("Target mean: {}, var: {}, Q25: {}, Q50: {}, Q75: {}".format(np.mean(target), np.var(target), @@ -41,7 +41,7 @@ def analyze_results(target, predictions): np.quantile(predictions, 0.5), np.quantile(predictions, 0.75))) - print("KS statistics: {}, pvalue: {}".format(statistics, pvalue)) + #print("KS statistics: {}, pvalue: {}".format(statistics, pvalue)) # The closer KS statistic is to 0 the more likely it is that the two samples were drawn from the same distribution plt.hist(target, alpha=0.5, label='target', density=True) @@ -251,11 +251,7 @@ def estimate_moments(sample_storage, true_domain=None): return moments_mean, estimator, true_domain, q_value -def ref_storage(): - tol = 1e-10 - reg_param = 0 - mlmc_file = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L1_benchmark/mlmc_1.hdf5" - +def ref_storage(mlmc_file): sample_storage = SampleStorageHDF(file_path=mlmc_file) return sample_storage @@ -271,7 +267,6 @@ def get_largest_domain(storages): location = time['0'] # locations: ['0'] q_value = location[0, 0] - # @TODO: How to estimate true_domain? quantile = 0.001 domain = mlmc.estimator.Estimate.estimate_domain(q_value, storage, quantile=quantile) @@ -286,6 +281,9 @@ def get_largest_domain(storages): #true_domain = [np.max(true_domains[:, 0]), np.min(true_domains[:, 1])] #true_domain = [np.mean(true_domains[:, 0]), np.mean(true_domains[:, 1])] print("true domain ", true_domain) + + #true_domain = true_domain[-1] + return true_domain @@ -306,6 +304,9 @@ def compare_moments(original_q_estimator, predict_q_estimator, ref_estimator): print("ref orig mean SSE ", np.sum((ref_moments_mean.mean - orig_moments_mean.mean)**2)) print("ref predict mean SSE ", np.sum((ref_moments_mean.mean - predict_moments_mean.mean) ** 2)) + print("ref orig mean SE ", np.sum(np.abs((ref_moments_mean.mean - orig_moments_mean.mean)))) + print("ref predict mean SE ", np.sum(np.abs((ref_moments_mean.mean - predict_moments_mean.mean)))) + print("ref moments var ", ref_moments_mean.var) print("orig moments var ", orig_moments_mean.var) print("predict moments var ", predict_moments_mean.var) @@ -313,6 +314,9 @@ def compare_moments(original_q_estimator, predict_q_estimator, ref_estimator): print("ref orig var SSE ", np.sum((ref_moments_mean.var - orig_moments_mean.var) ** 2)) print("ref predict var SSE ", np.sum((ref_moments_mean.var - predict_moments_mean.var) ** 2)) + print("ref orig var SE ", np.sum(np.abs((ref_moments_mean.var - orig_moments_mean.var)))) + print("ref predict var SE ", np.sum(np.abs((ref_moments_mean.var - predict_moments_mean.var)))) + def compare_densities(estimator_1, estimator_2, ref_estimator, label_1="", label_2=""): @@ -320,27 +324,29 @@ def compare_densities(estimator_1, estimator_2, ref_estimator, label_1="", label tol = 1e-10 reg_param = 0 - distr_obj_1, result, _, _ = estimator_1.construct_density(tol=tol, reg_param=reg_param) - distr_plot.add_distribution(distr_obj_1, label=label_1, color="blue") + #distr_plot.add_distribution(distr_obj_1, label=label_1, color="blue") distr_obj_2, result, _, _ = estimator_2.construct_density(tol=tol, reg_param=reg_param) - distr_plot.add_distribution(distr_obj_2, label=label_2, color="red", line_style="--") + #distr_plot.add_distribution(distr_obj_2, label=label_2, color="red", line_style="--") ref_distr_obj, result, _, _ = ref_estimator.construct_density(tol=tol, reg_param=reg_param) - distr_plot.add_distribution(ref_distr_obj, label="L1 benchmark", color="black", line_style=":") - + #distr_plot.add_distribution(ref_distr_obj, label="MC reference", color="black", line_style=":") - kl_div = mlmc.tool.simple_distribution.KL_divergence(ref_distr_obj.density, distr_obj_1.density, ref_distr_obj.domain[0], ref_distr_obj.domain[1]) + kl_div_ref_mlmc = mlmc.tool.simple_distribution.KL_divergence(ref_distr_obj.density, distr_obj_1.density, ref_distr_obj.domain[0], ref_distr_obj.domain[1]) - print("KL div ref|mlmc: {}".format(kl_div)) + print("KL div ref|mlmc: {}".format(kl_div_ref_mlmc)) # domain = [np.min([ref_distr_obj.domain[0], distr_obj_2.domain[0]]), # np.max([ref_distr_obj.domain[1], distr_obj_2.domain[1]])] - kl_div = mlmc.tool.simple_distribution.KL_divergence(ref_distr_obj.density, distr_obj_2.density, ref_distr_obj.domain[0], + kl_div_ref_gnn = mlmc.tool.simple_distribution.KL_divergence(ref_distr_obj.density, distr_obj_2.density, ref_distr_obj.domain[0], ref_distr_obj.domain[1]) - print("KL div ref|mlmc prediction: {}".format(kl_div)) + print("KL div ref|mlmc prediction: {}".format(kl_div_ref_gnn)) + + distr_plot.add_distribution(distr_obj_1, label=label_1 + ", KL(ref|orig):{:0.4g}".format(kl_div_ref_mlmc), color="blue") + distr_plot.add_distribution(distr_obj_2, label=label_2 + ", KL(ref|gnn):{:0.4g}".format(kl_div_ref_gnn), color="red", line_style="--") + distr_plot.add_distribution(ref_distr_obj, label="MC reference", color="black", line_style=":") distr_plot.show(file=None) @@ -363,13 +369,15 @@ def get_quantity_estimator(sample_storage, true_domain=None): return mlmc.estimator.Estimate(quantity=quantity, sample_storage=sample_storage, moments_fn=moments_fn) -def process_mlmc(mlmc_file, sampling_info_path, targets, predictions, train_targets, val_targets, l_0_targets=None, l_0_predictions=None, +def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predictions, train_targets, val_targets, l_0_targets=None, l_0_predictions=None, l1_sample_time=None, l0_sample_time=None): n_levels = 5 #mlmc_file = "/home/martin/Documents/metamodels/data/cl_0_3_s_4/L5/mlmc_5.hdf5" # mlmc_file = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L5/mlmc_5.hdf5" # sampling_info_path = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/sampling_info" + level_zero = False + sample_storage = SampleStorageHDF(file_path=mlmc_file) original_moments, estimator, original_true_domain, _ = estimate_moments(sample_storage) @@ -411,19 +419,36 @@ def process_mlmc(mlmc_file, sampling_info_path, targets, predictions, train_targ # sample_storage_nn = create_quantity_mlmc(data) # moments_nn, estimator_nn, _, _ = estimate_moments(sample_storage_nn, true_domain=original_true_domain) + if level_zero is True: + n_lev = n_levels + n_ops_predict = [l0_sample_time + field_times[0], *n_ops[1:]] + level_params = sample_storage.get_level_parameters() + else: + n_lev = n_levels + 1 + level_params = [sample_storage.get_level_parameters()[0], *sample_storage.get_level_parameters()] + # Use predicted data as zero level results and level one coarse results + if l_0_targets is not None and l_0_predictions is not None: data = [] - for l_id in range(n_levels + 1): + for l_id in range(n_lev): if l_id == 0: level_samples = l_0_predictions.reshape(1, len(l_0_predictions), 1) #level_samples = level_samples[:, :100000, :] else: - level_samples = estimator.get_level_samples(level_id=l_id - 1) - if l_id == 1: - coarse_level_samples = predictions.reshape(1, len(predictions), 1) - fine_level_samples = targets.reshape(1, len(targets), 1) - level_samples = np.concatenate((fine_level_samples, coarse_level_samples), axis=2) + if level_zero is True: + level_id = l_id + level_samples = estimator.get_level_samples(level_id=level_id) + else: + level_id = l_id - 1 + level_samples = estimator.get_level_samples(level_id=level_id) + if l_id == 1: + coarse_level_samples = predictions.reshape(1, len(predictions), 1) + fine_level_samples = targets.reshape(1, len(targets), 1) + level_samples = np.concatenate((fine_level_samples, coarse_level_samples), axis=2) + + + data.append(level_samples) # n0 = 100 @@ -436,7 +461,7 @@ def process_mlmc(mlmc_file, sampling_info_path, targets, predictions, train_targ # data[i] = data[i][:, :initial_n_samples[i], :] # print("data[i].shape ", data[i].shape) - level_params = [sample_storage.get_level_parameters()[0], *sample_storage.get_level_parameters()] + #level_params = [sample_storage.get_level_parameters()[0], *sample_storage.get_level_parameters()] sample_storage_predict = create_quantity_mlmc(data, level_parameters=level_params) moments_predict, estimator_predict, _, quantity = estimate_moments(sample_storage_predict, true_domain=original_true_domain) @@ -452,7 +477,7 @@ def process_mlmc(mlmc_file, sampling_info_path, targets, predictions, train_targ # @TODO: test # New estimation according to already finished samples variances, n_ops = estimator_predict.estimate_diff_vars_regression(n_level_samples) - print("variances ", variances) + #print("variances ", variances) print("level max var ", np.max(variances, axis=1)) print("n ops ", n_ops) n_ops = n_ops_predict @@ -469,7 +494,7 @@ def process_mlmc(mlmc_file, sampling_info_path, targets, predictions, train_targ print("original total collected times ", np.sum(n_collected_times)) - ref_sample_storage = ref_storage() + ref_sample_storage = ref_storage(ref_mlmc_file) domain = get_largest_domain([sample_storage, sample_storage_predict, ref_sample_storage]) #domain = [0.001, 5] @@ -503,47 +528,6 @@ def process_mlmc(mlmc_file, sampling_info_path, targets, predictions, train_targ label_2="gnn N: {}".format(moments_predict.n_samples)) - # Use predicted data as zero level results instead of original ones - else: - data = [] - for l_id in range(n_levels): - level_samples = estimator.get_level_samples(level_id=l_id) - if l_id == 0: - level_samples = np.concatenate((train_targets.reshape(1, len(train_targets), 1), - predictions.reshape(1, len(predictions), 1)), axis=1) - data.append(level_samples) - sample_storage_predict = create_quantity_mlmc(data) - moments_predict, estimator_predict, _, _ = estimate_moments(sample_storage_predict, true_domain=original_true_domain) - - original_q_estimator = get_quantity_estimator(sample_storage) - predict_q_estimator = get_quantity_estimator(sample_storage_predict) - - - print("means nn ", moments_nn.mean) - print("means_predict ", moments_predict.mean) - - - print("means nn - means predict ", moments_nn.mean - moments_predict.mean) - print("abs means nn - means predict ", np.abs(moments_nn.mean - moments_predict.mean)) - - print("vars nn ", moments_nn.var) - print("vars predict ", moments_predict.var) - - - print("moments_nn.l_means ", moments_nn.l_means[0]) - print("moments_predict.l_means ", moments_predict.l_means[0]) - - print("moments nn n samples ", moments_nn.n_samples) - print("moments nn n removed samples ", moments_predict.n_rm_samples) - print("moments predict n samples ", moments_predict.n_samples) - print("moments predict n removed samples ", moments_predict.n_rm_samples) - - for l_id, (l_mom, l_mom_pred) in enumerate(zip(moments_nn.l_means, moments_predict.l_means)): - print("L id: {}, mom diff: {}".format(l_id, l_mom - l_mom_pred)) - - compare_densities(original_q_estimator, predict_q_estimator, label_1="original", label_2="level 0 predictions") - - def analyze_mlmc_data(): n_levels = 5 # mlmc_file = "/home/martin/Documents/metamodels/data/cl_0_3_s_4/L5/mlmc_5.hdf5" From aea47fea3b4e522b65c5e6d6ae5e5fa6410bbd7a Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Sun, 14 Mar 2021 17:55:02 +0100 Subject: [PATCH 11/67] compare nn in progress --- src/mlmc/metamodel/analyze_nn.py | 282 ++++++++++++---- src/mlmc/metamodel/flow_dataset.py | 32 +- src/mlmc/metamodel/flow_task_GNN_2.py | 13 +- src/mlmc/metamodel/graph_models.py | 62 +++- src/mlmc/metamodel/postprocessing.py | 421 +++++++++++++++++------- src/mlmc/metamodel/random_field_time.py | 0 6 files changed, 614 insertions(+), 196 deletions(-) create mode 100644 src/mlmc/metamodel/random_field_time.py diff --git a/src/mlmc/metamodel/analyze_nn.py b/src/mlmc/metamodel/analyze_nn.py index ef940a55..453e0e59 100644 --- a/src/mlmc/metamodel/analyze_nn.py +++ b/src/mlmc/metamodel/analyze_nn.py @@ -5,13 +5,15 @@ #os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only from mlmc.metamodel.create_graph import graph_creator from mlmc.metamodel.flow_dataset import FlowDataset + +from mlmc.metamodel.own_cheb_conv import OwnChebConv # Make numpy printouts easier to read. #np.set_printoptions(precision=9, suppress=True) import tensorflow as tf from scipy.stats import ks_2samp import sklearn.model_selection -from mlmc.metamodel.custom_methods import abs_activation +from mlmc.metamodel.custom_methods import abs_activation, var_loss_function, total_loss_function from mlmc.metamodel.postprocessing import analyze_results, plot_loss, estimate_density, diff_moments, process_mlmc from mlmc.metamodel.flow_task_NN import DNN from mlmc.metamodel.flow_task_CNN import CNN @@ -167,45 +169,53 @@ def bootstrap(): # estimate_density(np.mean(all_predictions, axis=0), title="Predictions") -def bootstrap_GNN(): - loss = "mean_absolute_error" - optimizer = tf.optimizers.Adam(learning_rate=0.001) +def bootstrap_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path): n_subsamples = 10 - size = 10000 - train_losses = [] val_losses = [] + test_losses = [] all_test_outputs = [] all_predictions = [] - ks_statistics = [] - ks_p_values = [] + learning_times = [] - data = FlowDataset() - dataset = data.dataset.dropna() + # Parameters + #conv_layer = GCNConv + conv_layer = ChebConv # Seems better than GCNConv, good distribution of predictions + # conv_layer = GraphSageConv # Seems better than ChebConv, good loss but very narrow distribution of predictions + # # conv_layer = ARMAConv # Seems worse than GraphSageConv + # conv_layer = GATConv # Slow and not better than GraphSageConv + # # conv_layer = APPNPConv # Not bad but worse than GraphSageConv + # # conv_layer = GINConv # it is comparable to APPNPConv + # act_func = "relu" # "tanh"#"elu" # ReLU keep predictions above zero + + save_path = os.path.join(save_path, conv_layer.__name__) + if not os.path.isdir(save_path): + os.makedirs(save_path) + else: + print("dir exists") + exit() for i in range(n_subsamples): - # Parameters - # conv_layer = GCNConv - conv_layer = ChebConv # Seems better than GCNConv, good distribution of predictions - # # conv_layer = GraphSageConv # Seems better than ChebConv, good loss but very narrow distribution of predictions - # # conv_layer = ARMAConv # Seems worse than GraphSageConv - # # conv_layer = GATConv # Slow and not better than GraphSageConv - # # conv_layer = APPNPConv # Not bad but worse than GraphSageConv - # # conv_layer = GINConv # it is comparable to APPNPConv - # act_func = "relu" # "tanh"#"elu" # ReLU keep predictions above zero loss = MeanSquaredError() - optimizer = tf.optimizers.Adam(learning_rate=0.001) + optimizer = tf.optimizers.Adam(learning_rate=0.01) batch_size = 1000 - epochs = 100 + epochs = 500 + hidden_regularization = None # l2(2e-10) + + preprocess_start_time = time.process_time() + # graph_creator(output_dir, hdf_path) # Load data - data = FlowDataset() + data = FlowDataset(output_dir=output_dir) data = data # [:10000] + preprocess_time = time.process_time() - preprocess_start_time + + learning_time_start = time.process_time() # data.a = conv_layer.preprocess(data.a) data.a = sp_matrix_to_sp_tensor(data.a) train_data_len = int(len(data) * 0.8) - train_data_len = 10000 + train_data_len = 2000 # 2000 # Train/valid/test split data_tr, data_te = data[:train_data_len], data[train_data_len:], @@ -214,9 +224,9 @@ def bootstrap_GNN(): val_data_len = int(len(data_tr) * 0.2) data_tr, data_va = data_tr[:-val_data_len], data_tr[-val_data_len:] - print("data_tr len ", len(data_tr)) - print("data_va len ", len(data_va)) - print("data_te len ", len(data_te)) + # print("data_tr len ", len(data_tr)) + # print("data_va len ", len(data_va)) + # print("data_te len ", len(data_te)) # We use a MixedLoader since the dataset is in mixed mode loader_tr = MixedLoader(data_tr, batch_size=batch_size, epochs=epochs) @@ -224,49 +234,116 @@ def bootstrap_GNN(): loader_te = MixedLoader(data_te, batch_size=batch_size) gnn = GNN(loss=loss, optimizer=optimizer, conv_layer=conv_layer, output_activation=abs_activation, - hidden_activation='relu', patience=10) - gnn.fit(loader_tr, loader_va, loader_te) + hidden_activation='relu', patience=50, hidden_reqularizer=hidden_regularization) + train_targets = gnn.fit(loader_tr, loader_va, loader_te) + + val_targets = gnn.val_targets targets, predictions = gnn.predict(loader_te) predictions = np.squeeze(predictions) + learning_time = time.process_time() - learning_time_start + print("learning time ", learning_time) + + #plot_loss(gnn._train_loss, gnn._val_loss) + #analyze_results(targets, predictions) + + predict_l_0_start_time = time.process_time() + l_0_targets, l_0_predictions = predict_level_zero(gnn, l_0_output_dir, l_0_hdf_path, batch_size) + predict_l_0_time = time.process_time() - predict_l_0_start_time + + #save_times(save_path, False, (preprocess_time, len(data)), learning_time, (predict_l_0_time, len(l_0_targets))) + #save_load_data(save_path, False, targets, predictions, train_targets, val_targets, l_0_targets, l_0_predictions) all_test_outputs.append(targets) - all_predictions(predictions) + all_predictions.append(predictions) train_losses.append(gnn._train_loss[-1]) val_losses.append(gnn._val_loss[-1]) + test_losses.append(gnn._test_loss[-1]) + learning_times.append(learning_time) + save_bootstrap(save_path, all_test_outputs, all_predictions, train_losses, val_losses, test_losses, learning_times) - plot_loss(train_losses, val_losses) - analyze_results(np.mean(all_test_outputs, axis=0), np.mean(all_predictions, axis=0)) - analyze_results(np.var(all_test_outputs, axis=0), np.var(all_predictions, axis=0)) + # plot_loss(train_losses, val_losses) + # analyze_results(np.mean(all_test_outputs, axis=0), np.mean(all_predictions, axis=0)) + # analyze_results(np.var(all_test_outputs, axis=0), np.var(all_predictions, axis=0)) + # + # estimate_density(np.mean(all_test_outputs, axis=0), title="Test outputs") + # estimate_density(np.mean(all_predictions, axis=0), title="Predictions") - estimate_density(np.mean(all_test_outputs, axis=0), title="Test outputs") - estimate_density(np.mean(all_predictions, axis=0), title="Predictions") +def save_bootstrap(path, all_test_outputs, all_predictions, train_losses, val_losses, test_losses, learning_times): + np.save(os.path.join(path, "all_test_outputs"), all_test_outputs) + np.save(os.path.join(path, "all_predictions"), all_predictions) + np.save(os.path.join(path, "train_losses"), train_losses) + np.save(os.path.join(path, "val_losses"), val_losses) + np.save(os.path.join(path, "test_losses"), test_losses) + np.save(os.path.join(path, "learning_times"), learning_times) -def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path): + +def load_bootstrap(path): + all_test_outputs = np.load(os.path.join(path, "all_test_outputs.npy")) + all_predictions = np.load(os.path.join(path, "all_predictions.npy")) + train_losses = np.load(os.path.join(path, "train_losses.npy")) + val_losses = np.load(os.path.join(path, "val_losses.npy")) + test_losses = np.load(os.path.join(path, "test_losses.npy")) + learning_times = np.load(os.path.join(path, "learning_times.npy")) + + return all_test_outputs, all_predictions, train_losses, val_losses, test_losses, learning_times + + +def analyze_bootstrap(save_path): + conv_layer = GCNConv + conv_layer = ChebConv # Seems better than GCNConv, good distribution of predictions + # conv_layer = GraphSageConv # Seems better than ChebConv, good loss but very narrow distribution of predictions + # # conv_layer = ARMAConv # Seems worse than GraphSageConv + # conv_layer = GATConv # Slow and not better than GraphSageConv + # # conv_layer = APPNPConv # Not bad but worse than GraphSageConv + # # conv_layer = GINConv # it is comparable to APPNPConv + # act_func = "relu" # "tanh"#"elu" # ReLU keep predictions above zero + save_path = os.path.join(save_path, conv_layer.__name__) + if not os.path.isdir(save_path): + print("dir exists") + exit() + + all_test_outputs, all_predictions, train_losses, val_losses, test_losses, learning_times = load_bootstrap(save_path) + + print("learning times ", learning_times) + print("test losses ", test_losses) + + for targets, predictions in zip(all_test_outputs, all_predictions): + print("mean targets ", np.mean(targets)) + print("mean predictions ", np.mean(predictions)) + + print("var targets ", np.var(targets)) + print("var predictions ", np.var(predictions)) + + +def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level): # Parameters conv_layer = GCNConv conv_layer = ChebConv # Seems better than GCNConv, good distribution of predictions - # # conv_layer = GraphSageConv # Seems better than ChebConv, good loss but very narrow distribution of predictions + #conv_layer = GraphSageConv # Seems better than ChebConv, good loss but very narrow distribution of predictions # # conv_layer = ARMAConv # Seems worse than GraphSageConv # conv_layer = GATConv # Slow and not better than GraphSageConv # # conv_layer = APPNPConv # Not bad but worse than GraphSageConv # # conv_layer = GINConv # it is comparable to APPNPConv # act_func = "relu" # "tanh"#"elu" # ReLU keep predictions above zero - loss = MeanSquaredError() + loss = MeanSquaredError()#var_loss_function# + loss = total_loss_function optimizer = tf.optimizers.Adam(learning_rate=0.01) batch_size = 1000 - epochs = 500 + epochs = 250 hidden_regularization = None#l2(2e-10) preprocess_start_time = time.process_time() - #graph_creator(output_dir, hdf_path) + #graph_creator(output_dir, hdf_path, mesh, level=level) # Load data - data = FlowDataset(output_dir=output_dir) - data = data # [:10000] + data = FlowDataset(output_dir=output_dir, level=level) + data = data#[:2000] + print("lev data ", len(data)) + preprocess_time = time.process_time() - preprocess_start_time learning_time_start = time.process_time() @@ -274,7 +351,8 @@ def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path): data.a = sp_matrix_to_sp_tensor(data.a) train_data_len = int(len(data) * 0.8) - train_data_len = 10000#2000 + train_data_len = 2000 + print("train data len ", train_data_len) # Train/valid/test split data_tr, data_te = data[:train_data_len], data[train_data_len:], @@ -293,7 +371,11 @@ def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path): loader_te = MixedLoader(data_te, batch_size=batch_size) gnn = GNN(loss=loss, optimizer=optimizer, conv_layer=conv_layer, output_activation=abs_activation, - hidden_activation='relu', patience=35, hidden_reqularizer=hidden_regularization) + hidden_activation='relu', patience=50, hidden_reqularizer=hidden_regularization) + + # batch_size 500, ideally 500 epochs, patience 35 + + #gnn.run_eagerly = True train_targets = gnn.fit(loader_tr, loader_va, loader_te) val_targets = gnn.val_targets @@ -309,12 +391,13 @@ def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path): # target_means, target_vars = estimate_density(targets, title="Test outputs") # pred_means, pred_vars = estimate_density(predictions, title="Predictions") - # + # # # print("target means ", target_means) # print("predic means ", pred_means) - # + # # # print("target vars ", target_vars) # print("predic vars ", pred_vars) + # # diff_means, diff_vars = estimate_density(targets - predictions, title="diff") # print("diff_means ", diff_means) @@ -323,15 +406,15 @@ def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path): #diff_moments(targets, predictions) predict_l_0_start_time = time.process_time() - l_0_targets, l_0_predictions = predict_level_zero(gnn, l_0_output_dir, l_0_hdf_path, batch_size) + l_0_targets, l_0_predictions = predict_level_zero(gnn, l_0_output_dir, l_0_hdf_path, mesh, batch_size) predict_l_0_time = time.process_time() - predict_l_0_start_time save_times(save_path, False, (preprocess_time, len(data)), learning_time, (predict_l_0_time, len(l_0_targets))) save_load_data(save_path, False, targets, predictions, train_targets, val_targets, l_0_targets, l_0_predictions) -def predict_level_zero(nn, output_dir, hdf_path, batch_size=1000): - #graph_creator(output_dir, hdf_path) +def predict_level_zero(nn, output_dir, hdf_path, mesh, batch_size=1000): + #graph_creator(output_dir, hdf_path, mesh, level=0) # Load data data = FlowDataset(output_dir=output_dir) @@ -407,7 +490,7 @@ def save_load_data(path, load=False, targets=None, predictions=None, train_targe np.save(os.path.join(path, "l_0_predictions"), l_0_predictions) -def process_results(hdf_path, sampling_info_path, ref_mlmc_file, save_path): +def process_results(hdf_path, sampling_info_path, ref_mlmc_file, save_path, nn_level, replace_level): targets, predictions, train_targets, val_targets, l_0_targets, l_0_predictions = save_load_data(save_path, load=True) preprocess_time, preprocess_n, learning_time, predict_l_0_time, predict_l_0_n = save_times(save_path, load=True) @@ -429,40 +512,111 @@ def process_results(hdf_path, sampling_info_path, ref_mlmc_file, save_path): print("len train targets ", len(train_targets)) print("len val targets ", len(val_targets)) - process_mlmc(hdf_path, sampling_info_path, ref_mlmc_file, targets, predictions, train_targets, val_targets, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time) + process_mlmc(hdf_path, sampling_info_path, ref_mlmc_file, targets, predictions, train_targets, val_targets, l_0_targets, + l_0_predictions, l1_sample_time, l0_sample_time, nn_level=nn_level, replace_level=replace_level) if __name__ == "__main__": - cl = "cl_0_3_s_4" - output_dir = "/home/martin/Documents/metamodels/data/1000_ele/{}/L5/test/01_cond_field/output/".format(cl) - hdf_path = "/home/martin/Documents/metamodels/data/1000_ele/{}/L5/mlmc_5.hdf5".format(cl) + case = 3 + + if case == 0: + cl = "cl_0_3_s_4" + nn_level = 0 + replace_level = False + mesh = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/l_step_0.055_common_files/mesh.msh" + #mesh = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L2/l_step_0.027624156655057155_common_files/mesh.msh" + output_dir = "/home/martin/Documents/metamodels/data/1000_ele/{}/L5/test/01_cond_field/output/".format(cl) + hdf_path = "/home/martin/Documents/metamodels/data/1000_ele/{}/L5/mlmc_5.hdf5".format(cl) + + save_path = "/home/martin/Documents/metamodels/data/1000_ele/{}".format(cl) + + l_0_output_dir = "/home/martin/Documents/metamodels/data/1000_ele/{}/L{}/test/01_cond_field/output/".format(cl, + nn_level+1) + l_0_hdf_path = "/home/martin/Documents/metamodels/data/1000_ele/{}/L{}/mlmc_{}.hdf5".format(cl, + nn_level+1, + nn_level+1) - save_path = "/home/martin/Documents/metamodels/data/1000_ele/{}".format(cl) + sampling_info_path = "/home/martin/Documents/metamodels/data/1000_ele/{}/sampling_info".format(cl) - l_0_output_dir = "/home/martin/Documents/metamodels/data/1000_ele/{}/L1/test/01_cond_field/output/".format(cl) - l_0_hdf_path = "/home/martin/Documents/metamodels/data/1000_ele/{}/L1/mlmc_1.hdf5".format(cl) + ref_mlmc_file = "/home/martin/Documents/metamodels/data/1000_ele/{}/L1_benchmark/mlmc_1.hdf5".format(cl) + elif case == 1: + cl = "cl_0_1_s_1" + nn_level = 1 + replace_level = False + #mesh = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/l_step_0.055_common_files/mesh.msh" + mesh = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L2/l_step_0.027624156655057155_common_files/mesh.msh" + output_dir = "/home/martin/Documents/metamodels/data/1000_ele/{}/L5/test/01_cond_field/output/".format(cl) + hdf_path = "/home/martin/Documents/metamodels/data/1000_ele/{}/L5/mlmc_5.hdf5".format(cl) + + save_path = "/home/martin/Documents/metamodels/data/1000_ele/{}".format(cl) + + l_0_output_dir = "/home/martin/Documents/metamodels/data/1000_ele/{}/L{}/test/01_cond_field/output/".format(cl, + nn_level + 1) + l_0_hdf_path = "/home/martin/Documents/metamodels/data/1000_ele/{}/L{}/mlmc_{}.hdf5".format(cl, nn_level + 1, + nn_level+1) + + sampling_info_path = "/home/martin/Documents/metamodels/data/1000_ele/{}/sampling_info".format(cl) + + ref_mlmc_file = "/home/martin/Documents/metamodels/data/1000_ele/{}/L1_benchmark/mlmc_1.hdf5".format(cl) + + elif case == 2: + cl = "cl_0_1_s_1" + nn_level = 3 + replace_level = False + # mesh = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/l_step_0.055_common_files/mesh.msh" + mesh = "/home/martin/Documents/metamodels/data/5_ele/cl_0_1_s_1/L5/l_step_0.020196309484414757_common_files/mesh.msh" + output_dir = "/home/martin/Documents/metamodels/data/5_ele/{}/L5/test/01_cond_field/output/".format(cl) + hdf_path = "/home/martin/Documents/metamodels/data/5_ele/{}/L5/mlmc_5.hdf5".format(cl) + + save_path = "/home/martin/Documents/metamodels/data/5_ele/{}".format(cl) + + l_0_output_dir = "/home/martin/Documents/metamodels/data/5_ele/{}/L1_{}/test/01_cond_field/output/".format(cl, + nn_level) + l_0_hdf_path = "/home/martin/Documents/metamodels/data/5_ele/{}/L1_{}/mlmc_1.hdf5".format(cl, nn_level) + + sampling_info_path = "/home/martin/Documents/metamodels/data/5_ele/{}/sampling_info".format(cl) + + ref_mlmc_file = "/home/martin/Documents/metamodels/data/5_ele/{}/L1_3/mlmc_1.hdf5".format(cl) + + elif case == 3: + cl = "cl_0_3_s_4" + nn_level = 3 + replace_level = False + # mesh = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/l_step_0.055_common_files/mesh.msh" + mesh = "/home/martin/Documents/metamodels/data/5_ele/cl_0_1_s_1/L5/l_step_0.020196309484414757_common_files/mesh.msh" + output_dir = "/home/martin/Documents/metamodels/data/5_ele/{}/L5/test/01_cond_field/output/".format(cl) + hdf_path = "/home/martin/Documents/metamodels/data/5_ele/{}/L5/mlmc_5.hdf5".format(cl) + + save_path = "/home/martin/Documents/metamodels/data/5_ele/{}".format(cl) + + l_0_output_dir = "/home/martin/Documents/metamodels/data/5_ele/{}/L1_{}/test/01_cond_field/output/".format(cl, + nn_level) + l_0_hdf_path = "/home/martin/Documents/metamodels/data/5_ele/{}/L1_{}/mlmc_1.hdf5".format(cl, nn_level) + + sampling_info_path = "/home/martin/Documents/metamodels/data/5_ele/{}/sampling_info".format(cl) + + ref_mlmc_file = "/home/martin/Documents/metamodels/data/5_ele/{}/L1_3/mlmc_1.hdf5".format(cl) - sampling_info_path = "/home/martin/Documents/metamodels/data/1000_ele/{}/sampling_info".format(cl) - ref_mlmc_file = "/home/martin/Documents/metamodels/data/1000_ele/{}/L1_benchmark/mlmc_1.hdf5".format(cl) # import cProfile # import pstats # pr = cProfile.Profile() # pr.enable() # - # my_result = run_GNN() + # my_result = run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level) # # pr.disable() # ps = pstats.Stats(pr).sort_stats('cumtime') # ps.print_stats() - run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path) - # - process_results(hdf_path, sampling_info_path, ref_mlmc_file, save_path) + run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level) + process_results(hdf_path, sampling_info_path, ref_mlmc_file, save_path, nn_level, replace_level) + + + # analyze_bootstrap(save_path) + # bootstrap_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path) #bootstrap_GNN() #run() - #run_CNN() - #bootstrap() \ No newline at end of file diff --git a/src/mlmc/metamodel/flow_dataset.py b/src/mlmc/metamodel/flow_dataset.py index e1c06c19..18755bcc 100644 --- a/src/mlmc/metamodel/flow_dataset.py +++ b/src/mlmc/metamodel/flow_dataset.py @@ -1,4 +1,5 @@ import os +import re import numpy as np import pandas as pd from mlmc.tool import gmsh_io @@ -17,15 +18,15 @@ class FlowDataset(Dataset): GRAPHS_FILE = "graphs" DATA_FILE = "data" - def __init__(self, output_dir=None, **kwargs): + def __init__(self, output_dir=None, level=0, **kwargs): self._output_dir = output_dir if self._output_dir is None: self._output_dir = OUTPUT_DIR + self.level = level self.adjacency_matrix = np.load(os.path.join(self._output_dir, "adjacency_matrix.npy"), allow_pickle=True) # adjacency matrix self.data = [] super().__init__(**kwargs) self.a = self.adjacency_matrix - self.dataset = pd.DataFrame(self.data) def read(self): @@ -38,13 +39,40 @@ def read(self): # # return graphs + i = 0 + graphs = [] for s_dir in os.listdir(self._output_dir): + try: + l = re.findall(r'L(\d+)_S', s_dir)[0] + if int(l) != self.level: + continue + except IndexError: + continue + + #print("s dir ", s_dir) + if os.path.isdir(os.path.join(self._output_dir, s_dir)): sample_dir = os.path.join(self._output_dir, s_dir) if os.path.exists(os.path.join(sample_dir, "nodes_features.npy")): features = np.load(os.path.join(sample_dir, "nodes_features.npy")) output = np.load(os.path.join(sample_dir, "output.npy")) + # if i < 10: + # print("output ", output) + # else: + # exit() + # i += 1 + maximum = np.max(features) + minimum = np.min(features) + + #features = (features - minimum) / (maximum - minimum) + # print("max ", maximum) + # print("max ", minimum) + # + # print("new featuers max ", np.max(new_features)) + # print("new featuers min ", np.min(new_features)) + # exit() + graphs.append(Graph(x=features, y=output))#, a=self.adjacency_matrix)) # Save data for pandas dataframe creation, not used with Graph neural network diff --git a/src/mlmc/metamodel/flow_task_GNN_2.py b/src/mlmc/metamodel/flow_task_GNN_2.py index 6c9d6855..640ace65 100644 --- a/src/mlmc/metamodel/flow_task_GNN_2.py +++ b/src/mlmc/metamodel/flow_task_GNN_2.py @@ -13,7 +13,7 @@ from spektral.layers import GCNConv, GlobalSumPool, ChebConv, GraphSageConv, ARMAConv, GATConv, APPNPConv, GINConv from spektral.layers.ops import sp_matrix_to_sp_tensor from tensorflow.keras.layers.experimental import preprocessing -from mlmc.metamodel.custom_methods import abs_activation +from mlmc.metamodel.custom_methods import abs_activation, var_loss_function from mlmc.metamodel.graph_models import Net1 @@ -40,12 +40,14 @@ def __init__(self, **kwargs): self._train_loss = [] self._val_loss = [] + self._test_loss = [] self.val_targets = [] self._model = Net1(conv_layer=self._conv_layer, hidden_activation=self._hidden_activation, output_activation=self._output_activation, - kernel_regularization=self._hidden_regularizer) + kernel_regularization=self._hidden_regularizer, + normalizer=self._normalizer) def fit(self, loader_tr, loader_va, loader_te): """ @@ -86,6 +88,7 @@ def fit(self, loader_tr, loader_va, loader_te): best_val_loss = results_va[0] current_patience = self._patience results_te = self.evaluate(loader_te) + self._test_loss.append(results_te[0]) else: current_patience -= 1 if current_patience == 0: @@ -115,7 +118,11 @@ def train_on_batch(self, inputs, target): predictions = self._model(inputs, training=True) # @TODO: try to add KLDivergence to loss # print(KLDivergence(target, predictions)) - loss = self._loss(target, predictions) + sum(self._model.losses) # + KLDivergence(target, predictions)#+ sum(model.losses) + # print("self._loss(target, predictions) ", self._loss(target, predictions)) + # print("sum(self._model.losses) ", sum(self._model.losses)) + # print("var_loss_function(target, predictions) ", var_loss_function(target, predictions)) + loss = self._loss(target, predictions) + sum(self._model.losses) #+ 5 * var_loss_function(target, predictions) + #loss = 100 * var_loss_function(target, predictions) acc = tf.reduce_mean(self._accuracy_func(target, predictions)) gradients = tape.gradient(loss, self._model.trainable_variables) diff --git a/src/mlmc/metamodel/graph_models.py b/src/mlmc/metamodel/graph_models.py index 7dc03ac5..6f271073 100644 --- a/src/mlmc/metamodel/graph_models.py +++ b/src/mlmc/metamodel/graph_models.py @@ -1,23 +1,71 @@ from tensorflow.keras import Model from tensorflow.keras.layers import Dense from spektral.layers import GlobalSumPool +import tensorflow as tf +from tensorflow import keras +from tensorflow.keras import layers +from mlmc.metamodel.custom_methods import abs_activation +from tensorflow.keras.layers.experimental import preprocessing # Build model class Net1(Model): - def __init__(self, conv_layer, hidden_activation, output_activation, kernel_regularization, **kwargs): + def __init__(self, conv_layer, hidden_activation, output_activation, kernel_regularization, normalizer, **kwargs): super().__init__(**kwargs) - self.conv1 = conv_layer(32, activation=hidden_activation, kernel_regularizer=kernel_regularization) - self.conv2 = conv_layer(32, activation=hidden_activation, kernel_regularizer=kernel_regularization) + #self.normalizer = normalizer + #self.norm_layer = tf.keras.layers.LayerNormalization(axis=1) + self.conv1 = conv_layer(32, K=1, activation=hidden_activation, kernel_regularizer=kernel_regularization) + #self.conv2 = conv_layer(32, activation=hidden_activation, kernel_regularizer=kernel_regularization) + #self.conv3 = conv_layer(64, activation=hidden_activation, kernel_regularizer=kernel_regularization) self.flatten = GlobalSumPool() - self.fc1 = Dense(512, activation=hidden_activation) + #self.fc1 = Dense(512, activation=hidden_activation) self.fc2 = Dense(1, activation=output_activation) # linear activation for output neuron def call(self, inputs): x, a = inputs + #print("x ", x) + #x = self.normalizer(x) + #x = self.norm_layer(x) + #print("normalized x ", x) + + #print("x[0,0,:] ", x[0, 0, :]) x = self.conv1([x, a]) - x = self.conv2([x, a]) + + #print("x[0,0,:] ", x[0,0,:]) + # print("x[0, 0, :] ", tf.make_ndarray(x[0,0,:].op.get_attr('net1/strided_slice_1:0'))) + # print("x.shape ", x.shape) + # x = self.conv2([x, a]) + # x = self.conv3([x, a]) output = self.flatten(x) - output = self.fc1(output) + #output = self.fc1(output) + output = self.fc2(output) - return output \ No newline at end of file + + #print("output ", output.shape) + + return output + + +def cnn_model(): + return keras.Sequential([ + #@TODO: Try normalization + #self._normalizer, # Seems worse results with normalization + layers.Conv1D(filters=64, kernel_size=3, activation='relu', input_shape=(958, 1)), + #layers.BatchNormalization(), + layers.MaxPooling1D(pool_size=2), + layers.Conv1D(filters=64, kernel_size=3, activation='relu'), + #layers.BatchNormalization(), + layers.MaxPooling1D(pool_size=2), + layers.Flatten(), + layers.Dense(64, activation='relu'), + layers.Dense(1, activation=abs_activation) + ]) + + +def dnn_model(): + return keras.Sequential([ + preprocessing.Normalization(), + layers.Dense(32, activation="relu"), + #layers.Dense(32, activation="relu"), + layers.Dense(1, activation=abs_activation) + ]) \ No newline at end of file diff --git a/src/mlmc/metamodel/postprocessing.py b/src/mlmc/metamodel/postprocessing.py index 0943345d..f193cb95 100644 --- a/src/mlmc/metamodel/postprocessing.py +++ b/src/mlmc/metamodel/postprocessing.py @@ -1,5 +1,6 @@ import os import random +import copy import matplotlib.pyplot as plt from scipy.stats import ks_2samp from mlmc.tool import plot @@ -14,6 +15,8 @@ from mlmc.quantity import make_root_quantity import mlmc.tool.simple_distribution +QUANTILE = 0.01 + def plot_loss(train_loss, val_loss): plt.plot(train_loss, label='loss') @@ -90,7 +93,7 @@ def estimate_density(values, title="Density"): location = time['0'] value_quantity = location[0] - quantile = 0.001 + quantile = QUANTILE true_domain = mlmc.estimator.Estimate.estimate_domain(value_quantity, sample_storage, quantile=quantile) moments_fn = Legendre(n_moments, true_domain) @@ -166,7 +169,7 @@ def diff_moments(target, predictions): n_moments = 25 quantity, target_sample_storage = create_quantity(target, predictions) - quantile = 0.001 + quantile = QUANTILE true_domain = mlmc.estimator.Estimate.estimate_domain(quantity, target_sample_storage, quantile=quantile) moments_fn = Legendre(n_moments, true_domain) @@ -185,22 +188,16 @@ def diff_moments(target, predictions): print("moments var ", moments_mean.var) -def create_quantity_mlmc(data, level_parameters=None): +def create_quantity_mlmc(data, level_parameters, num_ops=None): sample_storage = Memory() n_levels = len(data) result_format = [QuantitySpec(name="conductivity", unit="m", shape=(1, 1), times=[1], locations=['0'])] - - if level_parameters is None: - level_parameters = np.ones(n_levels) - sample_storage.save_global_data(result_format=result_format, level_parameters=level_parameters) successful_samples = {} failed_samples = {} n_ops = {} - n_successful = 15 - sizes = [] for l_id in range(n_levels): n_successful = data[l_id].shape[1] sizes = [] @@ -216,12 +213,23 @@ def create_quantity_mlmc(data, level_parameters=None): coarse_result = (np.zeros((np.sum(sizes),))) else: coarse_result = data[l_id][:, sample_id, 1] + successful_samples[l_id].append((str(sample_id), (fine_result, coarse_result))) + # if num_ops is not None: + # n_ops[l_id] = [num_ops[l_id], n_successful] + # else: n_ops[l_id] = [random.random(), n_successful] sample_storage.save_scheduled_samples(l_id, samples=["S{:07d}".format(i) for i in range(n_successful)]) + # + # print("successful samples ") + # print("l 0", successful_samples[0][:10]) + # print("l 1", successful_samples[1][:10]) + # print("l 2", successful_samples[2][:10]) + # print("l 3", successful_samples[3][:10]) + sample_storage.save_samples(successful_samples, failed_samples) sample_storage.save_n_ops(list(n_ops.items())) @@ -239,9 +247,8 @@ def estimate_moments(sample_storage, true_domain=None): q_value = location[0, 0] if true_domain is None: - quantile = 0.001 + quantile = QUANTILE true_domain = mlmc.estimator.Estimate.estimate_domain(q_value, sample_storage, quantile=quantile) - print("true domain ", true_domain) moments_fn = Legendre(n_moments, true_domain) estimator = mlmc.estimator.Estimate(quantity=q_value, sample_storage=sample_storage, moments_fn=moments_fn) @@ -268,19 +275,18 @@ def get_largest_domain(storages): q_value = location[0, 0] # @TODO: How to estimate true_domain? - quantile = 0.001 + quantile = QUANTILE domain = mlmc.estimator.Estimate.estimate_domain(q_value, storage, quantile=quantile) - print("domain ", domain) true_domains.append([domain[0], domain[1]]) true_domains = np.array(true_domains) - print("true domains ", true_domains) + #print("true domains ", true_domains) true_domain = [np.min(true_domains[:, 0]), np.max(true_domains[:, 1])] #true_domain = [np.max(true_domains[:, 0]), np.min(true_domains[:, 1])] #true_domain = [np.mean(true_domains[:, 0]), np.mean(true_domains[:, 1])] - print("true domain ", true_domain) + #true_domain = true_domain[-1] @@ -297,23 +303,26 @@ def compare_moments(original_q_estimator, predict_q_estimator, ref_estimator): ref_estimator.estimate_moments() ref_moments_mean = ref_estimator.moments_mean - print("ref moments mean ", ref_moments_mean.mean) + #print("ref moments mean ", ref_moments_mean.mean) print("orig moments mean ", orig_moments_mean.mean) print("predict moments mean ", predict_moments_mean.mean) - print("ref orig mean SSE ", np.sum((ref_moments_mean.mean - orig_moments_mean.mean)**2)) - print("ref predict mean SSE ", np.sum((ref_moments_mean.mean - predict_moments_mean.mean) ** 2)) - + # print("ref orig mean SSE ", np.sum((ref_moments_mean.mean - orig_moments_mean.mean)**2)) + # print("ref predict mean SSE ", np.sum((ref_moments_mean.mean - predict_moments_mean.mean) ** 2)) + # print("ref orig mean SE ", np.sum(np.abs((ref_moments_mean.mean - orig_moments_mean.mean)))) print("ref predict mean SE ", np.sum(np.abs((ref_moments_mean.mean - predict_moments_mean.mean)))) - print("ref moments var ", ref_moments_mean.var) + #print("ref moments var ", ref_moments_mean.var) print("orig moments var ", orig_moments_mean.var) print("predict moments var ", predict_moments_mean.var) - print("ref orig var SSE ", np.sum((ref_moments_mean.var - orig_moments_mean.var) ** 2)) - print("ref predict var SSE ", np.sum((ref_moments_mean.var - predict_moments_mean.var) ** 2)) + print("MAX orig moments var ", np.max(orig_moments_mean.l_vars, axis=1)) + print("MAX predict moments var ", np.max(predict_moments_mean.l_vars, axis=1)) + # print("ref orig var SSE ", np.sum((ref_moments_mean.var - orig_moments_mean.var) ** 2)) + # print("ref predict var SSE ", np.sum((ref_moments_mean.var - predict_moments_mean.var) ** 2)) + # print("ref orig var SE ", np.sum(np.abs((ref_moments_mean.var - orig_moments_mean.var)))) print("ref predict var SE ", np.sum(np.abs((ref_moments_mean.var - predict_moments_mean.var)))) @@ -361,7 +370,7 @@ def get_quantity_estimator(sample_storage, true_domain=None): quantity = location[0, 0] if true_domain is None: - quantile = 0.001 + quantile = QUANTILE true_domain = mlmc.estimator.Estimate.estimate_domain(quantity, sample_storage, quantile=quantile) moments_fn = Legendre(n_moments, true_domain) @@ -369,45 +378,275 @@ def get_quantity_estimator(sample_storage, true_domain=None): return mlmc.estimator.Estimate(quantity=quantity, sample_storage=sample_storage, moments_fn=moments_fn) +def get_n_estimated(sample_storage, estimator, n_ops=None): + target_var = 1e-5 + #moments, estimator, _, quantity = estimate_moments(sample_storage, true_domain=true_domain) + + n_level_samples = sample_storage.get_n_collected() + # New estimation according to already finished samples + + print("n level samples ", n_level_samples) + variances, n_samples = estimator.estimate_diff_vars() + #variances, est_n_ops = estimator.estimate_diff_vars_regression(n_level_samples) + + if n_ops is None: + n_ops = n_samples + print("get n estimated n ops ", n_ops) + n_estimated = mlmc.estimator.estimate_n_samples_for_target_variance(target_var, variances, n_ops, + n_levels=len(n_level_samples)) + return n_estimated, variances, n_samples + + +def get_storage_info(sample_storage): + moments, estimator, _, _ = estimate_moments(sample_storage) + print("n collected ", sample_storage.get_n_collected()) + print("moments.l_vars max ", np.max(moments.l_vars, axis=1)) + + +def cut_samples(data, sample_storage, new_n_collected): + new_data = [] + for d, n_est in zip(data, new_n_collected): + new_data.append(d[:, :n_est, :]) + sample_storage = create_quantity_mlmc(new_data, level_parameters=sample_storage.get_level_parameters()) + + return sample_storage + + def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predictions, train_targets, val_targets, l_0_targets=None, l_0_predictions=None, - l1_sample_time=None, l0_sample_time=None): - n_levels = 5 - #mlmc_file = "/home/martin/Documents/metamodels/data/cl_0_3_s_4/L5/mlmc_5.hdf5" - # mlmc_file = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L5/mlmc_5.hdf5" - # sampling_info_path = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/sampling_info" + l1_sample_time=None, l0_sample_time=None, nn_level=0, replace_level=False): + #level_zero = False - level_zero = False + print("nn_level ", nn_level) + print("replace level ", replace_level) + ####### + ### Create storage fromm original MLMC data + ###### sample_storage = SampleStorageHDF(file_path=mlmc_file) + n_levels = len(sample_storage.get_level_ids()) original_moments, estimator, original_true_domain, _ = estimate_moments(sample_storage) - # Test storage creation - data = [] + data_mlmc = [] for l_id in range(n_levels): level_samples = estimator.get_level_samples(level_id=l_id) - data.append(level_samples) - sample_storage_2 = create_quantity_mlmc(data) - moments_2, estimator_2, _, _ = estimate_moments(sample_storage_2) - assert np.allclose(original_moments.mean, moments_2.mean) - assert np.allclose(original_moments.var, moments_2.var) + data_mlmc.append(level_samples) + sample_storage = create_quantity_mlmc(data_mlmc, level_parameters=sample_storage.get_level_parameters()) + print("Original storage") + get_storage_info(sample_storage) + + ###### + ### Get n ops + ###### + n_ops, field_times, flow_times = get_sample_times(sampling_info_path) + print("n ops ", n_ops) - print("moments_2.l_vars max ", np.max(moments_2.l_vars, axis=1)) + if replace_level: + n_lev = n_levels + level_params = [*sample_storage.get_level_parameters()] + else: + n_lev = n_levels - nn_level + 1 + level_params = [sample_storage.get_level_parameters()[nn_level], *sample_storage.get_level_parameters()[nn_level:]] + + print("n lev ", n_lev) + print("level params ", level_params) + + # Use predicted data as zero level results and level one coarse results + data_nn = [] + n_ops_predict = [] + + for l_id in range(n_lev): + if l_id == 0: + level_samples = l_0_predictions.reshape(1, len(l_0_predictions), 1) + n_ops_predict.append(l0_sample_time + field_times[nn_level]) + #level_samples = level_samples[:, :50000, :] + else: + if replace_level: + level_id = l_id + level_samples = estimator.get_level_samples(level_id=level_id) + n_ops_predict.append(n_ops[level_id]) + else: + if l_id == 1: + print("l id replaced level ", l_id) + print("len(predictions ) ", len(predictions)) + coarse_level_samples = predictions.reshape(1, len(predictions), 1) + fine_level_samples = targets.reshape(1, len(targets), 1) + # coarse_level_samples = coarse_level_samples[:, :len(predictions)//2, :] + # fine_level_samples = fine_level_samples[:, :len(predictions)//2, :] + level_samples = np.concatenate((fine_level_samples, coarse_level_samples), axis=2) + n_ops_predict.append(n_ops[nn_level] + l1_sample_time) + else: + if replace_level: + level_id = l_id #- 1 + else: + level_id = l_id + nn_level - 1 + print("level id ", level_id) + level_samples = estimator.get_level_samples(level_id=level_id) + n_ops_predict.append(n_ops[level_id]) + data_nn.append(level_samples) + + sample_storage_predict = create_quantity_mlmc(data_nn, level_parameters=level_params) + print("n ops predict ", n_ops_predict) + + print("Storage predict info") + get_storage_info(sample_storage_predict) - n_ops, field_times, flow_times = get_sample_times(sampling_info_path) - n_collected_times = n_ops * np.array(sample_storage_2.get_n_collected()) - # print("n collected times ", n_collected_times) - # print("total time ", np.sum(n_collected_times)) - # print("n ops ", n_ops) - # print("field_times ", field_times) - # print("flow_times ", flow_times) - # exit() - n_ops_predict = [l0_sample_time + field_times[0], n_ops[0] + l1_sample_time, *n_ops[1:]] + ###### + ### Create estimators + ###### + ref_sample_storage = ref_storage(ref_mlmc_file) + domain = get_largest_domain([sample_storage, sample_storage_predict, ref_sample_storage]) + original_q_estimator = get_quantity_estimator(sample_storage, true_domain=domain) + predict_q_estimator = get_quantity_estimator(sample_storage_predict, true_domain=domain) + #ref_estimator = get_quantity_estimator(ref_sample_storage, true_domain=domain) + ####### + ### Calculate N estimated samples + ####### print("n ops ", n_ops) print("n ops predict ", n_ops_predict) + #### Original data + n_estimated_orig, l_vars_orig, n_samples_orig = get_n_estimated(sample_storage, original_q_estimator, n_ops=n_ops) + print("n estimated orig ", n_estimated_orig) + sample_storage = cut_samples(data_mlmc, sample_storage, n_estimated_orig) + # + # sample_storage_predict = cut_samples(data_nn, sample_storage_predict, [sample_storage_predict.get_n_collected()[0], + # *n_estimated_orig]) + # + # predict_q_estimator = get_quantity_estimator(sample_storage_predict, true_domain=domain) + # + # print("sample storage predict n collected ", sample_storage_predict.get_n_collected()) + # print("sample storage n collected ", sample_storage.get_n_collected()) + + n_ops_predict_orig = copy.deepcopy(n_ops_predict) + #n_ops_predict_orig[0] = n_ops_predict_orig[0] /5 + #n_ops_predict = np.array(n_ops_predict)**2 + n_ops_predict[0] = n_ops_predict[0] / 1000 + #print("n ops predict for estimate ", n_ops_predict) + # n_samples = [10000, 2000, 500, 150, 40, 11] + + n_estimated_nn, l_var_nn, n_samples_nn = get_n_estimated(sample_storage_predict, predict_q_estimator, + n_ops=n_ops_predict) + + #n_estimated_nn = [50000, 10000, 850] + sample_storage_predict = cut_samples(data_nn, sample_storage_predict, n_estimated_nn) + + # print("N estimated orig ", n_estimated_orig) + # print("n sampels nn ", n_samples_nn) + # print("N estimated nn ", n_estimated_nn) + # print("est time ", np.sum(n_ops_predict_orig * n_estimated_nn)) + # print("time ", np.sum(n_ops_predict_orig * n_samples_nn)) + # print("orig time ", np.sum(n_ops * n_estimated_orig)) + # exit() + #sample_storage_predict = cut_samples(data_nn, sample_storage_predict, n_estimated_nn) + + # print("l vars orig [-1] ", l_vars_orig[0]) + # print("l var nn[-1] ", l_var_nn[:2]) + + #print("l vars orig ", l_vars_orig) + # print("l vars nn ", l_var_nn[:2]) + # print("l vars orig ", np.max(l_vars_orig, axis=1)) + # print("l vars nn ", np.max(l_var_nn, axis=1)) + # + # orig_total_var = np.sum(l_vars_orig / n_samples_orig[:, None], axis=0) + # orig_total_var_max = np.max(np.sum(l_vars_orig / n_samples_orig[:, None], axis=0)) + # print("orig total var ", orig_total_var) + # print("orig total var max ", orig_total_var_max) + # nn_total_var = np.sum(l_var_nn / n_samples_nn[:, None], axis=0) + # nn_total_var_max = np.max(np.sum(l_var_nn / n_samples_nn[:, None], axis=0)) + # print("nn total var ", nn_total_var) + # print("nn total var max ", nn_total_var_max) + # exit() + # + # + # print("np.sum(l_var_nn[:2], axis=0) ", np.sum(l_var_nn[:2], axis=0)) + # print(" l_vars_orig[0] - np.sum(l_var_nn[:2], axis=0) ", l_vars_orig[0] - np.sum(l_var_nn[:2], axis=0)) + # + # print("l_vars orig - l_var_nn ", l_vars_orig[-1] - l_var_nn[-1]) + # print("l_vars orig - l_var_nn ", np.sum(l_vars_orig[1:] - l_var_nn[2:], axis=0)) + # + # print("MAX l vars orig [0] ", np.max(l_vars_orig[0])) + # print("MAX l var nn[:2] ", np.max(l_var_nn[:2], axis=1)) + # + # print("n samples orig ", n_samples_orig) + # print("n samples nn ", n_samples_nn) + # + # vars_n_orig = l_vars_orig / n_samples_orig[:, None] + # vars_n_nn = l_var_nn / n_samples_nn[:, None] + # + # print("vars n orig ", vars_n_orig) + # print("vars n nn ", vars_n_nn) + # + # # max_vars_n_orig = np.max(vars_n_orig, axis=1) + # max_vars_n_nn = np.max(vars_n_nn, axis=1) + # + # # print("vars vars n orig ", max_vars_n_orig) + # # print("vars vars n nn ", max_vars_n_nn) + # + # print("np.sum(l_var_nn[1:, ...] / n_samples_nn[1:, None], axis=0) ", np.sum(l_var_nn[1:, ...] / n_samples_nn[1:, None], axis=0)) + # n0_samples = l_var_nn[0] / np.abs(orig_total_var - np.sum(l_var_nn[1:, ...] / n_samples_nn[1:, None], axis=0)) + # + # print("n0_samples ", n0_samples) + # print("max n0_samples ", int(np.max(n0_samples[1:]))) + + # sample_storage_pred_cut = cut_samples(data_nn, sample_storage_predict, [int(np.max(n0_samples[1:])), *n_samples_nn[1:]]) + # predict_q_estimator_cut = get_quantity_estimator(sample_storage_pred_cut)#, true_domain=domain) + # n_estimated_nn, l_var_nn, n_samples_nn = get_n_estimated(sample_storage_pred_cut, predict_q_estimator_cut, + # n_ops=n_ops_predict) + # + # nn_total_var = np.max(np.sum(l_var_nn / n_samples_nn[:, None], axis=0)) + # print("nn total var ", nn_total_var) + # print("n samples nn ", n_samples_nn) + # exit() + + + # com_levels = np.max(np.sum(vars_n_orig[:2], axis=0))#(np.max(l_vars_orig[0])/n_samples_orig[0]) + (np.max(l_vars_orig[1])/n_samples_orig[1]) + # print("com levels ", com_levels) + # print("(np.max(l_var_nn[1])/n_samples_nn[1]) ", (np.max(l_var_nn[1])/n_samples_nn[1])) + # print("np.max(l_var_nn[0]) ", np.max(l_var_nn[0])) + # print("np.max(np.sum(vars_n_nn[1:2], axis=0)) ", np.max(np.sum(vars_n_nn[1:2], axis=0))) + # nn_l_0_n = np.max(l_var_nn[0]) / (com_levels - np.max(np.sum(vars_n_nn[1:2], axis=0))) #(np.max(l_var_nn[1])/n_samples_nn[1])) + # + # print("nn_l_0_n ", nn_l_0_n) + # + # print("MAX l vars orig ", np.max(l_vars_orig, axis=1)) + # print("MAX l var nn ", np.max(l_var_nn, axis=1)) + # + # nn_vars = np.max(l_var_nn[0])/nn_l_0_n + np.max(np.sum(vars_n_nn[1:2], axis=0)) + # + # + # print("nn vars ", nn_vars) + # print("rogi vars ", com_levels) + + + + + # orig_sample_l_5 = original_q_estimator.get_level_samples(4) + # nn_sample_l_5 = predict_q_estimator.get_level_samples(5) + # + # print("orig samples l 5 ", orig_sample_l_5) + # print("nn sample l 5 ", nn_sample_l_5) + # + # assert np.allclose(orig_sample_l_5, nn_sample_l_5) + + + + ####### + ## Estimate total time + ####### + print("NN estimated ", n_estimated_nn) + print("MLMC estimated ", n_estimated_orig) + NN_time_levels = n_ops_predict_orig * np.array(n_estimated_nn) + n_collected_times = n_ops * np.array(n_estimated_orig) + print("NN time levels ", NN_time_levels) + print("MLMC time levels", n_collected_times) + print("NN total time ", np.sum(NN_time_levels)) + print("MLMC total time ", np.sum(n_collected_times)) + + #original_moments, estimator, original_true_domain, _ = estimate_moments(sample_storage) + # # Use train and test data without validation data # data = [] # for l_id in range(n_levels): @@ -419,37 +658,7 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti # sample_storage_nn = create_quantity_mlmc(data) # moments_nn, estimator_nn, _, _ = estimate_moments(sample_storage_nn, true_domain=original_true_domain) - if level_zero is True: - n_lev = n_levels - n_ops_predict = [l0_sample_time + field_times[0], *n_ops[1:]] - level_params = sample_storage.get_level_parameters() - else: - n_lev = n_levels + 1 - level_params = [sample_storage.get_level_parameters()[0], *sample_storage.get_level_parameters()] - # Use predicted data as zero level results and level one coarse results - - if l_0_targets is not None and l_0_predictions is not None: - data = [] - for l_id in range(n_lev): - if l_id == 0: - level_samples = l_0_predictions.reshape(1, len(l_0_predictions), 1) - #level_samples = level_samples[:, :100000, :] - else: - if level_zero is True: - level_id = l_id - level_samples = estimator.get_level_samples(level_id=level_id) - else: - level_id = l_id - 1 - level_samples = estimator.get_level_samples(level_id=level_id) - if l_id == 1: - coarse_level_samples = predictions.reshape(1, len(predictions), 1) - fine_level_samples = targets.reshape(1, len(targets), 1) - level_samples = np.concatenate((fine_level_samples, coarse_level_samples), axis=2) - - - - data.append(level_samples) # n0 = 100 # nL = 10 @@ -462,45 +671,7 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti # print("data[i].shape ", data[i].shape) #level_params = [sample_storage.get_level_parameters()[0], *sample_storage.get_level_parameters()] - sample_storage_predict = create_quantity_mlmc(data, level_parameters=level_params) - - moments_predict, estimator_predict, _, quantity = estimate_moments(sample_storage_predict, true_domain=original_true_domain) - target_var = 1e-5 - - #n_level_samples = initial_n_samples #sample_storage_predict.get_n_collected() - n_level_samples = sample_storage_predict.get_n_collected() - # custom_n_ops = [sample_storage.get_n_ops()[0], *sample_storage.get_n_ops()] - # print("custom n ops ", custom_n_ops) - - print("n level samples ", n_level_samples) - - # @TODO: test - # New estimation according to already finished samples - variances, n_ops = estimator_predict.estimate_diff_vars_regression(n_level_samples) - #print("variances ", variances) - print("level max var ", np.max(variances, axis=1)) - print("n ops ", n_ops) - n_ops = n_ops_predict - n_estimated = mlmc.estimator.estimate_n_samples_for_target_variance(target_var, variances, n_ops, - n_levels=len(n_level_samples)) - print("n estimated ", n_estimated) - print("original collected ", sample_storage_2.get_n_collected()) - est_times = n_ops_predict * np.array(n_estimated) - - print("est times ", est_times) - print("original n collected times ", n_collected_times) - print("total est times ", np.sum(est_times)) - print("original total collected times ", np.sum(n_collected_times)) - - - ref_sample_storage = ref_storage(ref_mlmc_file) - domain = get_largest_domain([sample_storage, sample_storage_predict, ref_sample_storage]) - #domain = [0.001, 5] - - original_q_estimator = get_quantity_estimator(sample_storage, true_domain=domain) - predict_q_estimator = get_quantity_estimator(sample_storage_predict, true_domain=domain) - ref_estimator = get_quantity_estimator(ref_sample_storage, true_domain=domain) # print("means nn ", moments_nn.mean) # print("means_predict ", moments_predict.mean) @@ -522,10 +693,14 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti # for l_id, (l_mom, l_mom_pred) in enumerate(zip(moments_nn.l_means, moments_predict.l_means)): # print("L id: {}, mom diff: {}".format(l_id, l_mom - l_mom_pred)) - compare_moments(original_q_estimator, predict_q_estimator, ref_estimator) + original_q_estimator = get_quantity_estimator(sample_storage, true_domain=domain) + predict_q_estimator = get_quantity_estimator(sample_storage_predict, true_domain=domain) + ref_estimator = get_quantity_estimator(ref_sample_storage, true_domain=domain) + + compare_moments(original_q_estimator, predict_q_estimator, ref_estimator) - compare_densities(original_q_estimator, predict_q_estimator, ref_estimator, label_1="orig N: {}".format(moments_2.n_samples), - label_2="gnn N: {}".format(moments_predict.n_samples)) + compare_densities(original_q_estimator, predict_q_estimator, ref_estimator, label_1="orig N: {}".format(sample_storage.get_n_collected()), + label_2="gnn N: {}".format(sample_storage_predict.get_n_collected())) def analyze_mlmc_data(): @@ -559,9 +734,6 @@ def analyze_mlmc_data(): def get_sample_times(sampling_info_path): n_levels = [5] for nl in n_levels: - # sampling_info_path = "/home/martin/Documents/MLMC_article/data/sampling_info" - #sampling_info_path = "/home/martin/Documents/metamodels/data/1000_ele/sampling_info" - variances = [] n_ops = [] times = [] @@ -597,12 +769,21 @@ def time_for_sample_func(data): running_times = time_for_sample_func(running_times) flow_running_times = time_for_sample_func(flow_running_times) + field_times = np.mean(np.array(running_times) - np.array(flow_running_times) - np.array(flow_running_times), axis=0) flow_times = np.mean(np.array(flow_running_times), axis=0) n_ops = np.mean(n_ops, axis=0) + # print("n ops ", n_ops) + # print("running times ", np.mean(running_times, axis=0)) + # print("flow running times ", flow_times) + # exit() + + #n_ops = np.mean(running_times, axis=0) # CPU time of simulation (fields + flow for both coarse and fine sample) + + print("field times ", field_times) return n_ops, field_times, flow_times diff --git a/src/mlmc/metamodel/random_field_time.py b/src/mlmc/metamodel/random_field_time.py new file mode 100644 index 00000000..e69de29b From 3d737e5530e351fac99ad6eeb0279d6b07c6423f Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Wed, 17 Mar 2021 16:20:47 +0100 Subject: [PATCH 12/67] milestone 1 --- src/mlmc/metamodel/analyze_nn.py | 321 +++++++++++++------------- src/mlmc/metamodel/create_graph.py | 41 +++- src/mlmc/metamodel/custom_methods.py | 24 ++ src/mlmc/metamodel/flow_dataset.py | 38 ++- src/mlmc/metamodel/flow_task_GNN_2.py | 20 +- src/mlmc/metamodel/graph_models.py | 51 +++- src/mlmc/metamodel/postprocessing.py | 132 ++++++++++- 7 files changed, 439 insertions(+), 188 deletions(-) diff --git a/src/mlmc/metamodel/analyze_nn.py b/src/mlmc/metamodel/analyze_nn.py index 453e0e59..8fd3f22f 100644 --- a/src/mlmc/metamodel/analyze_nn.py +++ b/src/mlmc/metamodel/analyze_nn.py @@ -2,14 +2,14 @@ import numpy as np import time import random -#os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only +# os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only from mlmc.metamodel.create_graph import graph_creator from mlmc.metamodel.flow_dataset import FlowDataset from mlmc.metamodel.own_cheb_conv import OwnChebConv # Make numpy printouts easier to read. -#np.set_printoptions(precision=9, suppress=True) +# np.set_printoptions(precision=9, suppress=True) import tensorflow as tf from scipy.stats import ks_2samp import sklearn.model_selection @@ -17,14 +17,15 @@ from mlmc.metamodel.postprocessing import analyze_results, plot_loss, estimate_density, diff_moments, process_mlmc from mlmc.metamodel.flow_task_NN import DNN from mlmc.metamodel.flow_task_CNN import CNN - +from mlmc.metamodel.graph_models import NetGCN from mlmc.metamodel.flow_task_GNN_2 import GNN from spektral.layers import GCNConv, GlobalSumPool, ChebConv, GraphSageConv, ARMAConv, GATConv, APPNPConv, GINConv -from tensorflow.keras.losses import MeanSquaredError +from tensorflow.keras.losses import MeanSquaredError, KLDivergence from tensorflow.keras.regularizers import l2 from spektral.data import MixedLoader from spektral.layers.ops import sp_matrix_to_sp_tensor + print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU'))) epochs = 100 @@ -65,8 +66,8 @@ def run(): train_input = prepare_data(dataset.x) train_output = prepare_data(dataset.y) - #train_input, train_output, test__input, test_output = split_dataset(dataset) - #print("len test(output) ", len(test_output)) + # train_input, train_output, test__input, test_output = split_dataset(dataset) + # print("len test(output) ", len(test_output)) dnn = DNN(loss=loss, optimizer=optimizer, output_activation=abs_activation, hidden_activation='relu', epochs=150) dnn.fit(train_input, train_output) @@ -86,17 +87,19 @@ def run(): estimate_density(predictions) -def run_CNN(): +def run_CNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level): # Parameters loss = "mean_squared_error" - optimizer = tf.optimizers.Adam(learning_rate=0.001) - - data = FlowDataset() - dataset = data.dataset[:10000] + optimizer = tf.optimizers.Adam(learning_rate=0.01) + data = FlowDataset(output_dir=output_dir, level=level) + dataset = data.dataset[:] train_input, train_output, test_input, test_output = split_dataset(dataset) + train_input = train_input[:2000] + train_output = train_output[:2000] + print("len test(output) ", len(test_output)) train_input = np.expand_dims(train_input, axis=-1) @@ -108,8 +111,10 @@ def run_CNN(): dnn.fit(train_input, train_output) - test_dataset = data.dataset[10000:] + test_dataset = data.dataset[2000:] test_input = prepare_data(test_dataset.x) + test_input = np.expand_dims(test_input, axis=-1) + print("test input shape ", test_input.shape) test_output = prepare_data(test_dataset.y) predictions = dnn.predict(test_input) @@ -119,8 +124,8 @@ def run_CNN(): analyze_results(test_output, predictions) - estimate_density(test_output) - estimate_density(predictions) + # estimate_density(test_output) + # estimate_density(predictions) def bootstrap(): @@ -169,8 +174,8 @@ def bootstrap(): # estimate_density(np.mean(all_predictions, axis=0), title="Predictions") -def bootstrap_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path): - n_subsamples = 10 +def statistics(run_method, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level): + n_subsamples = 3 train_losses = [] val_losses = [] test_losses = [] @@ -179,7 +184,7 @@ def bootstrap_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path) learning_times = [] # Parameters - #conv_layer = GCNConv + # conv_layer = GCNConv conv_layer = ChebConv # Seems better than GCNConv, good distribution of predictions # conv_layer = GraphSageConv # Seems better than ChebConv, good loss but very narrow distribution of predictions # # conv_layer = ARMAConv # Seems worse than GraphSageConv @@ -192,67 +197,15 @@ def bootstrap_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path) if not os.path.isdir(save_path): os.makedirs(save_path) else: - print("dir exists") + print("dir exists {}".format(save_path)) exit() for i in range(n_subsamples): - loss = MeanSquaredError() - optimizer = tf.optimizers.Adam(learning_rate=0.01) - batch_size = 1000 - epochs = 500 - hidden_regularization = None # l2(2e-10) - - preprocess_start_time = time.process_time() - # graph_creator(output_dir, hdf_path) - - # Load data - data = FlowDataset(output_dir=output_dir) - data = data # [:10000] - preprocess_time = time.process_time() - preprocess_start_time - - learning_time_start = time.process_time() - # data.a = conv_layer.preprocess(data.a) - data.a = sp_matrix_to_sp_tensor(data.a) - - train_data_len = int(len(data) * 0.8) - train_data_len = 2000 # 2000 - - # Train/valid/test split - data_tr, data_te = data[:train_data_len], data[train_data_len:], - np.random.shuffle(data_tr) - - val_data_len = int(len(data_tr) * 0.2) - data_tr, data_va = data_tr[:-val_data_len], data_tr[-val_data_len:] - - # print("data_tr len ", len(data_tr)) - # print("data_va len ", len(data_va)) - # print("data_te len ", len(data_te)) - - # We use a MixedLoader since the dataset is in mixed mode - loader_tr = MixedLoader(data_tr, batch_size=batch_size, epochs=epochs) - loader_va = MixedLoader(data_va, batch_size=batch_size) - loader_te = MixedLoader(data_te, batch_size=batch_size) - - gnn = GNN(loss=loss, optimizer=optimizer, conv_layer=conv_layer, output_activation=abs_activation, - hidden_activation='relu', patience=50, hidden_reqularizer=hidden_regularization) - train_targets = gnn.fit(loader_tr, loader_va, loader_te) + gnn, targets, predictions, learning_time = run_method(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, + save_path, mesh, level=nn_level, stats=True) - val_targets = gnn.val_targets - - targets, predictions = gnn.predict(loader_te) - predictions = np.squeeze(predictions) - learning_time = time.process_time() - learning_time_start - print("learning time ", learning_time) - - #plot_loss(gnn._train_loss, gnn._val_loss) - #analyze_results(targets, predictions) - - predict_l_0_start_time = time.process_time() - l_0_targets, l_0_predictions = predict_level_zero(gnn, l_0_output_dir, l_0_hdf_path, batch_size) - predict_l_0_time = time.process_time() - predict_l_0_start_time - - #save_times(save_path, False, (preprocess_time, len(data)), learning_time, (predict_l_0_time, len(l_0_targets))) - #save_load_data(save_path, False, targets, predictions, train_targets, val_targets, l_0_targets, l_0_predictions) + # save_times(save_path, False, (preprocess_time, len(data)), learning_time, (predict_l_0_time, len(l_0_targets))) + # save_load_data(save_path, False, targets, predictions, train_targets, val_targets, l_0_targets, l_0_predictions) all_test_outputs.append(targets) all_predictions.append(predictions) @@ -262,7 +215,17 @@ def bootstrap_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path) test_losses.append(gnn._test_loss[-1]) learning_times.append(learning_time) - save_bootstrap(save_path, all_test_outputs, all_predictions, train_losses, val_losses, test_losses, learning_times) + for i in range(len(train_losses)): + print("train loss ", train_losses[i]) + print("test loss ", test_losses[i]) + analyze_results(all_test_outputs[i], all_predictions[i]) + print("learning time ", learning_times[i]) + print("##################################################") + + print("worse train loss ", np.max(train_losses)) + print("worse test loss ", np.max(test_losses)) + + save_statistics(save_path, all_test_outputs, all_predictions, train_losses, val_losses, test_losses, learning_times) # plot_loss(train_losses, val_losses) # analyze_results(np.mean(all_test_outputs, axis=0), np.mean(all_predictions, axis=0)) @@ -272,7 +235,7 @@ def bootstrap_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path) # estimate_density(np.mean(all_predictions, axis=0), title="Predictions") -def save_bootstrap(path, all_test_outputs, all_predictions, train_losses, val_losses, test_losses, learning_times): +def save_statistics(path, all_test_outputs, all_predictions, train_losses, val_losses, test_losses, learning_times): np.save(os.path.join(path, "all_test_outputs"), all_test_outputs) np.save(os.path.join(path, "all_predictions"), all_predictions) np.save(os.path.join(path, "train_losses"), train_losses) @@ -281,7 +244,7 @@ def save_bootstrap(path, all_test_outputs, all_predictions, train_losses, val_lo np.save(os.path.join(path, "learning_times"), learning_times) -def load_bootstrap(path): +def load_statistics(path): all_test_outputs = np.load(os.path.join(path, "all_test_outputs.npy")) all_predictions = np.load(os.path.join(path, "all_predictions.npy")) train_losses = np.load(os.path.join(path, "train_losses.npy")) @@ -292,7 +255,7 @@ def load_bootstrap(path): return all_test_outputs, all_predictions, train_losses, val_losses, test_losses, learning_times -def analyze_bootstrap(save_path): +def analyze_statistics(save_path): conv_layer = GCNConv conv_layer = ChebConv # Seems better than GCNConv, good distribution of predictions # conv_layer = GraphSageConv # Seems better than ChebConv, good loss but very narrow distribution of predictions @@ -319,29 +282,33 @@ def analyze_bootstrap(save_path): print("var predictions ", np.var(predictions)) -def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level): +def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level, conv_layer=None, stats=False, + gnn=None, model=None): # Parameters - conv_layer = GCNConv - conv_layer = ChebConv # Seems better than GCNConv, good distribution of predictions - #conv_layer = GraphSageConv # Seems better than ChebConv, good loss but very narrow distribution of predictions - # # conv_layer = ARMAConv # Seems worse than GraphSageConv - # conv_layer = GATConv # Slow and not better than GraphSageConv - # # conv_layer = APPNPConv # Not bad but worse than GraphSageConv - # # conv_layer = GINConv # it is comparable to APPNPConv - # act_func = "relu" # "tanh"#"elu" # ReLU keep predictions above zero - loss = MeanSquaredError()#var_loss_function# - loss = total_loss_function - optimizer = tf.optimizers.Adam(learning_rate=0.01) - batch_size = 1000 - epochs = 250 - hidden_regularization = None#l2(2e-10) + if conv_layer is None: + conv_layer = GCNConv + conv_layer = ChebConv # Seems better than GCNConv, good distribution of predictions + # conv_layer = GraphSageConv # Seems better than ChebConv, good loss but very narrow distribution of predictions + # # conv_layer = ARMAConv # Seems worse than GraphSageConv + # conv_layer = GATConv # Slow and not better than GraphSageConv + # # conv_layer = APPNPConv # Not bad but worse than GraphSageConv + # # conv_layer = GINConv # it is comparable to APPNPConv + # act_func = "relu" # "tanh"#"elu" # ReLU keep predictions above zero + + loss = MeanSquaredError() # var_loss_function# + #loss = KLDivergence() + # loss = total_loss_function + optimizer = tf.optimizers.Adam(learning_rate=0.001) + batch_size = 2000#2000 + epochs = 500 + hidden_regularization = None # l2(2e-10) preprocess_start_time = time.process_time() #graph_creator(output_dir, hdf_path, mesh, level=level) # Load data data = FlowDataset(output_dir=output_dir, level=level) - data = data#[:2000] + data = data#[:15000] print("lev data ", len(data)) preprocess_time = time.process_time() - preprocess_start_time @@ -361,22 +328,27 @@ def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, val_data_len = int(len(data_tr) * 0.2) data_tr, data_va = data_tr[:-val_data_len], data_tr[-val_data_len:] - # print("data_tr len ", len(data_tr)) - # print("data_va len ", len(data_va)) - # print("data_te len ", len(data_te)) + print("data_tr len ", len(data_tr)) + print("data_va len ", len(data_va)) + print("data_te len ", len(data_te)) # We use a MixedLoader since the dataset is in mixed mode loader_tr = MixedLoader(data_tr, batch_size=batch_size, epochs=epochs) loader_va = MixedLoader(data_va, batch_size=batch_size) loader_te = MixedLoader(data_te, batch_size=batch_size) - gnn = GNN(loss=loss, optimizer=optimizer, conv_layer=conv_layer, output_activation=abs_activation, - hidden_activation='relu', patience=50, hidden_reqularizer=hidden_regularization) + if gnn is None: + gnn = GNN(loss=loss, optimizer=optimizer, conv_layer=conv_layer, output_activation=abs_activation, + hidden_activation='relu', patience=150, hidden_reqularizer=hidden_regularization, + model=model) # tanh takes to much time + # ideally patience = 150 + # batch_size 500, ideally 500 epochs, patience 35 - # batch_size 500, ideally 500 epochs, patience 35 + # gnn.run_eagerly = True + train_targets = gnn.fit(loader_tr, loader_va, loader_te) - #gnn.run_eagerly = True - train_targets = gnn.fit(loader_tr, loader_va, loader_te) + train_targets, train_predictions = gnn.predict(loader_tr) + train_predictions = np.squeeze(train_predictions) val_targets = gnn.val_targets @@ -384,11 +356,32 @@ def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, predictions = np.squeeze(predictions) learning_time = time.process_time() - learning_time_start - #print("np.var(target-predictions) ", np.var(targets - predictions)) + # print("np.var(target-predictions) ", np.var(targets - predictions)) + + if stats is True: + return gnn, targets, predictions, learning_time plot_loss(gnn._train_loss, gnn._val_loss) analyze_results(targets, predictions) + # print("np.max(targets) ", np.max(targets)) + # print("np.min(targets) ", np.min(targets)) + # + # print("np.max(predictions) ", np.max(predictions)) + # print("np.min(predictions) ", np.min(predictions)) + # + # print("data.max_output ", data.max_output) + # print("data.min_output ", data.min_output) + + + # rescaled_targets = (data.max_output - data.min_output)/(np.max(targets) - np.min(targets))*(targets - np.max(targets)) + data.min_output + # + # + # rescaled_predictions = (data.max_output - data.min_output) / (np.max(predictions) - np.min(predictions)) * ( + # targets - np.max(predictions)) + data.min_output + # + # analyze_results(rescaled_targets, rescaled_predictions) + # target_means, target_vars = estimate_density(targets, title="Test outputs") # pred_means, pred_vars = estimate_density(predictions, title="Predictions") # # @@ -403,18 +396,19 @@ def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, # print("diff_means ", diff_means) # print("diff vars ", diff_vars) - #diff_moments(targets, predictions) + # diff_moments(targets, predictions) predict_l_0_start_time = time.process_time() l_0_targets, l_0_predictions = predict_level_zero(gnn, l_0_output_dir, l_0_hdf_path, mesh, batch_size) predict_l_0_time = time.process_time() - predict_l_0_start_time save_times(save_path, False, (preprocess_time, len(data)), learning_time, (predict_l_0_time, len(l_0_targets))) - save_load_data(save_path, False, targets, predictions, train_targets, val_targets, l_0_targets, l_0_predictions) + save_load_data(save_path, False, targets, predictions, train_targets, train_predictions, val_targets, l_0_targets, + l_0_predictions) -def predict_level_zero(nn, output_dir, hdf_path, mesh, batch_size=1000): - #graph_creator(output_dir, hdf_path, mesh, level=0) +def predict_level_zero(nn, output_dir, hdf_path, mesh, batch_size=1000, ): + # graph_creator(output_dir, hdf_path, mesh, level=0) # Load data data = FlowDataset(output_dir=output_dir) @@ -426,7 +420,7 @@ def predict_level_zero(nn, output_dir, hdf_path, mesh, batch_size=1000): targets, predictions = nn.predict(loader_te) predictions = np.squeeze(predictions) - #analyze_results(targets, predictions) + # analyze_results(targets, predictions) return targets, predictions @@ -458,9 +452,9 @@ def save_times(path, load=False, preprocess=None, learning_time=None, predict_l_ np.save(os.path.join(path, "predict_l_0_n"), predict_l_0[1]) -def save_load_data(path, load=False, targets=None, predictions=None, train_targets=None, val_targets=None, l_0_targets=None, +def save_load_data(path, load=False, targets=None, predictions=None, train_targets=None, train_predictions=None, + val_targets=None, l_0_targets=None, l_0_predictions=None): - if load: if os.path.exists(os.path.join(path, "targets.npy")): targets = np.load(os.path.join(path, "targets.npy")) @@ -468,13 +462,15 @@ def save_load_data(path, load=False, targets=None, predictions=None, train_targe predictions = np.load(os.path.join(path, "predictions.npy")) if os.path.exists(os.path.join(path, "train_targets.npy")): train_targets = np.load(os.path.join(path, "train_targets.npy")) + if os.path.exists(os.path.join(path, "train_predictions.npy")): + train_predictions = np.load(os.path.join(path, "train_predictions.npy")) if os.path.exists(os.path.join(path, "val_targets.npy")): val_targets = np.load(os.path.join(path, "val_targets.npy")) if os.path.exists(os.path.join(path, "l_0_targets.npy")): l_0_targets = np.load(os.path.join(path, "l_0_targets.npy")) if os.path.exists(os.path.join(path, "l_0_predictions.npy")): l_0_predictions = np.load(os.path.join(path, "l_0_predictions.npy")) - return targets, predictions, train_targets, val_targets, l_0_targets, l_0_predictions + return targets, predictions, train_targets, train_predictions, val_targets, l_0_targets, l_0_predictions else: if targets is not None: np.save(os.path.join(path, "targets"), targets) @@ -482,6 +478,8 @@ def save_load_data(path, load=False, targets=None, predictions=None, train_targe np.save(os.path.join(path, "predictions"), predictions) if train_targets is not None: np.save(os.path.join(path, "train_targets"), train_targets) + if train_predictions is not None: + np.save(os.path.join(path, "train_predictions"), train_predictions) if val_targets is not None: np.save(os.path.join(path, "val_targets"), val_targets) if l_0_targets is not None: @@ -491,7 +489,8 @@ def save_load_data(path, load=False, targets=None, predictions=None, train_targe def process_results(hdf_path, sampling_info_path, ref_mlmc_file, save_path, nn_level, replace_level): - targets, predictions, train_targets, val_targets, l_0_targets, l_0_predictions = save_load_data(save_path, load=True) + targets, predictions, train_targets, train_predictions, val_targets, l_0_targets, l_0_predictions = save_load_data( + save_path, load=True) preprocess_time, preprocess_n, learning_time, predict_l_0_time, predict_l_0_n = save_times(save_path, load=True) l1_sample_time = preprocess_time / preprocess_n + learning_time / preprocess_n @@ -512,51 +511,37 @@ def process_results(hdf_path, sampling_info_path, ref_mlmc_file, save_path, nn_l print("len train targets ", len(train_targets)) print("len val targets ", len(val_targets)) - process_mlmc(hdf_path, sampling_info_path, ref_mlmc_file, targets, predictions, train_targets, val_targets, l_0_targets, + process_mlmc(hdf_path, sampling_info_path, ref_mlmc_file, targets, predictions, train_targets, train_predictions, + val_targets, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, nn_level=nn_level, replace_level=replace_level) -if __name__ == "__main__": - case = 3 - +def get_config(case=0): if case == 0: cl = "cl_0_3_s_4" nn_level = 0 replace_level = False mesh = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/l_step_0.055_common_files/mesh.msh" - #mesh = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L2/l_step_0.027624156655057155_common_files/mesh.msh" + # mesh = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L2/l_step_0.027624156655057155_common_files/mesh.msh" output_dir = "/home/martin/Documents/metamodels/data/1000_ele/{}/L5/test/01_cond_field/output/".format(cl) hdf_path = "/home/martin/Documents/metamodels/data/1000_ele/{}/L5/mlmc_5.hdf5".format(cl) - save_path = "/home/martin/Documents/metamodels/data/1000_ele/{}".format(cl) - - l_0_output_dir = "/home/martin/Documents/metamodels/data/1000_ele/{}/L{}/test/01_cond_field/output/".format(cl, - nn_level+1) - l_0_hdf_path = "/home/martin/Documents/metamodels/data/1000_ele/{}/L{}/mlmc_{}.hdf5".format(cl, - nn_level+1, - nn_level+1) - + l_0_output_dir = "/home/martin/Documents/metamodels/data/1000_ele/{}/L{}/test/01_cond_field/output/".format(cl, nn_level + 1) + l_0_hdf_path = "/home/martin/Documents/metamodels/data/1000_ele/{}/L{}/mlmc_{}.hdf5".format(cl, nn_level + 1, nn_level + 1) sampling_info_path = "/home/martin/Documents/metamodels/data/1000_ele/{}/sampling_info".format(cl) - ref_mlmc_file = "/home/martin/Documents/metamodels/data/1000_ele/{}/L1_benchmark/mlmc_1.hdf5".format(cl) elif case == 1: - cl = "cl_0_1_s_1" + cl = "cl_0_3_s_4" nn_level = 1 replace_level = False - #mesh = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/l_step_0.055_common_files/mesh.msh" - mesh = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L2/l_step_0.027624156655057155_common_files/mesh.msh" + mesh = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/l_step_0.055_common_files/mesh.msh" + #mesh = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L2/l_step_0.027624156655057155_common_files/mesh.msh" output_dir = "/home/martin/Documents/metamodels/data/1000_ele/{}/L5/test/01_cond_field/output/".format(cl) hdf_path = "/home/martin/Documents/metamodels/data/1000_ele/{}/L5/mlmc_5.hdf5".format(cl) - save_path = "/home/martin/Documents/metamodels/data/1000_ele/{}".format(cl) - - l_0_output_dir = "/home/martin/Documents/metamodels/data/1000_ele/{}/L{}/test/01_cond_field/output/".format(cl, - nn_level + 1) - l_0_hdf_path = "/home/martin/Documents/metamodels/data/1000_ele/{}/L{}/mlmc_{}.hdf5".format(cl, nn_level + 1, - nn_level+1) - + l_0_output_dir = "/home/martin/Documents/metamodels/data/1000_ele/{}/L{}/test/01_cond_field/output/".format(cl, nn_level + 1) + l_0_hdf_path = "/home/martin/Documents/metamodels/data/1000_ele/{}/L{}/mlmc_{}.hdf5".format(cl, nn_level + 1, nn_level + 1) sampling_info_path = "/home/martin/Documents/metamodels/data/1000_ele/{}/sampling_info".format(cl) - ref_mlmc_file = "/home/martin/Documents/metamodels/data/1000_ele/{}/L1_benchmark/mlmc_1.hdf5".format(cl) elif case == 2: @@ -567,38 +552,50 @@ def process_results(hdf_path, sampling_info_path, ref_mlmc_file, save_path, nn_l mesh = "/home/martin/Documents/metamodels/data/5_ele/cl_0_1_s_1/L5/l_step_0.020196309484414757_common_files/mesh.msh" output_dir = "/home/martin/Documents/metamodels/data/5_ele/{}/L5/test/01_cond_field/output/".format(cl) hdf_path = "/home/martin/Documents/metamodels/data/5_ele/{}/L5/mlmc_5.hdf5".format(cl) - save_path = "/home/martin/Documents/metamodels/data/5_ele/{}".format(cl) - - l_0_output_dir = "/home/martin/Documents/metamodels/data/5_ele/{}/L1_{}/test/01_cond_field/output/".format(cl, - nn_level) + l_0_output_dir = "/home/martin/Documents/metamodels/data/5_ele/{}/L1_{}/test/01_cond_field/output/".format(cl, nn_level) l_0_hdf_path = "/home/martin/Documents/metamodels/data/5_ele/{}/L1_{}/mlmc_1.hdf5".format(cl, nn_level) - sampling_info_path = "/home/martin/Documents/metamodels/data/5_ele/{}/sampling_info".format(cl) - ref_mlmc_file = "/home/martin/Documents/metamodels/data/5_ele/{}/L1_3/mlmc_1.hdf5".format(cl) - elif case == 3: - cl = "cl_0_3_s_4" + elif case == 3 or case == 4: + cl = "cl_0_1_s_1" + if case == 4: + cl = "cl_0_3_s_4" nn_level = 3 replace_level = False # mesh = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/l_step_0.055_common_files/mesh.msh" mesh = "/home/martin/Documents/metamodels/data/5_ele/cl_0_1_s_1/L5/l_step_0.020196309484414757_common_files/mesh.msh" output_dir = "/home/martin/Documents/metamodels/data/5_ele/{}/L5/test/01_cond_field/output/".format(cl) hdf_path = "/home/martin/Documents/metamodels/data/5_ele/{}/L5/mlmc_5.hdf5".format(cl) - save_path = "/home/martin/Documents/metamodels/data/5_ele/{}".format(cl) - - l_0_output_dir = "/home/martin/Documents/metamodels/data/5_ele/{}/L1_{}/test/01_cond_field/output/".format(cl, - nn_level) + l_0_output_dir = "/home/martin/Documents/metamodels/data/5_ele/{}/L1_{}/test/01_cond_field/output/".format(cl,nn_level) l_0_hdf_path = "/home/martin/Documents/metamodels/data/5_ele/{}/L1_{}/mlmc_1.hdf5".format(cl, nn_level) - sampling_info_path = "/home/martin/Documents/metamodels/data/5_ele/{}/sampling_info".format(cl) + ref_mlmc_file = "/home/martin/Documents/metamodels/data/5_ele/{}/L1_3/mlmc_1.hdf5".format(cl) + elif case == 5: + cl = "cl_0_3_s_4" + nn_level = 0 + replace_level = False + # mesh = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/l_step_0.055_common_files/mesh.msh" + mesh = "/home/martin/Documents/metamodels/data/5_ele/cl_0_1_s_1/L5/l_step_0.020196309484414757_common_files/mesh.msh" + output_dir = "/home/martin/Documents/metamodels/data/5_ele/{}/L1_3/test/01_cond_field/output/".format(cl) + hdf_path = "/home/martin/Documents/metamodels/data/5_ele/{}/L1_3/mlmc_1.hdf5".format(cl) + save_path = "/home/martin/Documents/metamodels/data/5_ele/{}".format(cl) + l_0_output_dir = "/home/martin/Documents/metamodels/data/5_ele/{}/L1_{}/test/01_cond_field/output/".format(cl,nn_level) + l_0_hdf_path = "/home/martin/Documents/metamodels/data/5_ele/{}/L1_{}/mlmc_1.hdf5".format(cl, nn_level) + sampling_info_path = "/home/martin/Documents/metamodels/data/5_ele/{}/sampling_info".format(cl) ref_mlmc_file = "/home/martin/Documents/metamodels/data/5_ele/{}/L1_3/mlmc_1.hdf5".format(cl) + return output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, replace_level, nn_level, +if __name__ == "__main__": + case = 4 + output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, replace_level, nn_level = get_config( + case) + # import cProfile # import pstats # pr = cProfile.Profile() @@ -610,13 +607,23 @@ def process_results(hdf_path, sampling_info_path, ref_mlmc_file, save_path, nn_l # ps = pstats.Stats(pr).sort_stats('cumtime') # ps.print_stats() - run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level) - process_results(hdf_path, sampling_info_path, ref_mlmc_file, save_path, nn_level, replace_level) + # gnn, _, _, _ = run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, stats=True) + # + # case = 4 + # output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, replace_level, nn_level = get_config(case) + + #run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, conv_layer=GCNConv, model=NetGCN) # , gnn=gnn) + #run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level) # , gnn=gnn) + + # run_CNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level) + process_results(hdf_path, sampling_info_path, ref_mlmc_file, save_path, nn_level, replace_level) # analyze_bootstrap(save_path) - # bootstrap_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path) - #bootstrap_GNN() - #run() - #run_CNN() - #bootstrap() \ No newline at end of file + # statistics(run_GNN, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level) + # bootstrap_GNN() + # run() + # run_CNN() + # bootstrap() + + # gnn, targets, predictions, learning_time = run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, stats=True) diff --git a/src/mlmc/metamodel/create_graph.py b/src/mlmc/metamodel/create_graph.py index c0cd29a9..0a90a861 100644 --- a/src/mlmc/metamodel/create_graph.py +++ b/src/mlmc/metamodel/create_graph.py @@ -101,26 +101,44 @@ def plot_graph(adjacency_matrix): plt.show() -def graph_creator(output_dir, hdf_path): - adjacency_matrix = create_adjacency_matrix(extract_mesh_gmsh_io(MESH)) +def reject_outliers(data, m=2): + #print("abs(data - np.mean(data)) < m * np.std(data) ", abs(data - np.mean(data)) < m * np.std(data)) + #return data[abs(data - np.mean(data)) < m * np.std(data)] + return abs(data - np.mean(data)) < m * np.std(data) + + +def graph_creator(output_dir, hdf_path, mesh, level=0): + adjacency_matrix = create_adjacency_matrix(extract_mesh_gmsh_io(mesh)) np.save(os.path.join(output_dir, "adjacency_matrix"), adjacency_matrix, allow_pickle=True) loaded_adjacency_matrix = np.load(os.path.join(output_dir, "adjacency_matrix.npy"), allow_pickle=True) - plot_graph(loaded_adjacency_matrix) + #plot_graph(loaded_adjacency_matrix) + + hdf = HDF5(file_path=hdf_path, load_from_file=True) + level_group = hdf.add_level_group(level_id=str(level)) + + # print("collected ", level_group.collected()[:, 0, :]) + # indices = reject_outliers(np.squeeze(level_group.collected()[:, 0, :]), m=6) + # print("removed outliers ", np.count_nonzero(~indices)) + indices = np.ones(len(level_group.collected())) - hdf = HDF5(file_path=hdf_path, - load_from_file=True) - level_group = hdf.add_level_group(level_id=str(0)) collected = zip(level_group.get_collected_ids(), level_group.collected()) graphs = [] data = [] - - for sample_id, col_values in collected: + i = 0 + for keep, (sample_id, col_values) in zip(indices, collected): + if not keep: + continue output_value = col_values[0, 0] + sample_dir = os.path.join(output_dir, sample_id) field_mesh = os.path.join(sample_dir, FIELDS_SAMPLE) if os.path.exists(field_mesh): + # i += 1 + # if i > 150: + # break + features = get_node_features(field_mesh) np.save(os.path.join(sample_dir, "nodes_features"), features) np.save(os.path.join(sample_dir, "output"), output_value) @@ -137,12 +155,17 @@ def graph_creator(output_dir, hdf_path): if __name__ == "__main__": + + mesh = "/home/martin/Documents/metamodels/data/5_ele/cl_0_1_s_1/L5/l_step_0.020196309484414757_common_files/mesh.msh" + output_dir = "/home/martin/Documents/metamodels/data/5_ele/cl_0_3_s_4/L1_3/test/01_cond_field/output/" + hdf_path = "/home/martin/Documents/metamodels/data/5_ele/cl_0_1_s_1/L1_3/mlmc_1.hdf5" + import cProfile import pstats pr = cProfile.Profile() pr.enable() - my_result = graph_creator() + my_result = graph_creator(output_dir, hdf_path, mesh) pr.disable() ps = pstats.Stats(pr).sort_stats('cumtime') diff --git a/src/mlmc/metamodel/custom_methods.py b/src/mlmc/metamodel/custom_methods.py index f00a84b7..b2f2cf80 100644 --- a/src/mlmc/metamodel/custom_methods.py +++ b/src/mlmc/metamodel/custom_methods.py @@ -1,5 +1,29 @@ +import tensorflow as tf from tensorflow.keras import backend as K def abs_activation(x): return K.abs(x) + + +def var_loss_function(y_true, y_predict): + if tf.is_tensor(y_true): + y_true = float(y_true) + + # else: + # print("diff shape ", (y_true - K.squeeze(y_predict, axis=1)).shape) + + return K.var(y_true - K.squeeze(y_predict, axis=1)) + #return K.sum(K.abs(y_true - K.squeeze(y_predict, axis=1))) + + +def total_loss_function(y_true, y_predict): + if tf.is_tensor(y_true): + y_true = float(y_true) + + # else: + # print("diff shape ", (y_true - K.squeeze(y_predict, axis=1)).shape) + + #return K.var(K.abs(y_true - K.squeeze(y_predict, axis=1))) + + return K.mean((y_true - K.squeeze(y_predict, axis=1))**2) + K.var(K.abs(y_true - K.squeeze(y_predict, axis=1))) diff --git a/src/mlmc/metamodel/flow_dataset.py b/src/mlmc/metamodel/flow_dataset.py index 18755bcc..d50304c8 100644 --- a/src/mlmc/metamodel/flow_dataset.py +++ b/src/mlmc/metamodel/flow_dataset.py @@ -39,7 +39,37 @@ def read(self): # # return graphs - i = 0 + # i = 0 + all_outputs = [] + all_features = [] + # + # for s_dir in os.listdir(self._output_dir): + # try: + # l = re.findall(r'L(\d+)_S', s_dir)[0] + # if int(l) != self.level: + # continue + # except IndexError: + # continue + # if os.path.isdir(os.path.join(self._output_dir, s_dir)): + # sample_dir = os.path.join(self._output_dir, s_dir) + # if os.path.exists(os.path.join(sample_dir, "nodes_features.npy")): + # features = np.load(os.path.join(sample_dir, "nodes_features.npy")) + # output = np.load(os.path.join(sample_dir, "output.npy")) + # all_outputs.append(output) + # all_features.append(features) + # + # #print("all outputs ", np.array(all_outputs).shape) + # min_output = np.min(all_outputs) + # max_output = np.max(all_outputs) + # + # maximum = np.max(all_features) + # minimum = np.min(all_features) + # + # self.min_output = min_output + # self.max_output = max_output + # self.min_feature = minimum + # # self.max_output = maximum + graphs = [] for s_dir in os.listdir(self._output_dir): @@ -52,6 +82,7 @@ def read(self): #print("s dir ", s_dir) + if os.path.isdir(os.path.join(self._output_dir, s_dir)): sample_dir = os.path.join(self._output_dir, s_dir) if os.path.exists(os.path.join(sample_dir, "nodes_features.npy")): @@ -62,10 +93,10 @@ def read(self): # else: # exit() # i += 1 - maximum = np.max(features) - minimum = np.min(features) #features = (features - minimum) / (maximum - minimum) + # + # output = (output - min_output) / (max_output - min_output) # print("max ", maximum) # print("max ", minimum) # @@ -73,6 +104,7 @@ def read(self): # print("new featuers min ", np.min(new_features)) # exit() + graphs.append(Graph(x=features, y=output))#, a=self.adjacency_matrix)) # Save data for pandas dataframe creation, not used with Graph neural network diff --git a/src/mlmc/metamodel/flow_task_GNN_2.py b/src/mlmc/metamodel/flow_task_GNN_2.py index 640ace65..156594fa 100644 --- a/src/mlmc/metamodel/flow_task_GNN_2.py +++ b/src/mlmc/metamodel/flow_task_GNN_2.py @@ -44,10 +44,17 @@ def __init__(self, **kwargs): self.val_targets = [] - self._model = Net1(conv_layer=self._conv_layer, hidden_activation=self._hidden_activation, - output_activation=self._output_activation, - kernel_regularization=self._hidden_regularizer, - normalizer=self._normalizer) + model = kwargs.get('model') + if model is None: + self._model = Net1(conv_layer=self._conv_layer, hidden_activation=self._hidden_activation, + output_activation=self._output_activation, + kernel_regularization=self._hidden_regularizer, + normalizer=self._normalizer) + else: + self._model = model(conv_layer=self._conv_layer, hidden_activation=self._hidden_activation, + output_activation=self._output_activation, + kernel_regularization=self._hidden_regularizer, + normalizer=self._normalizer) def fit(self, loader_tr, loader_va, loader_te): """ @@ -159,16 +166,17 @@ def predict(self, loader): predictions = [] step = 0 for batch in loader: - step += 1 + step += 1 inputs, target = batch - targets.extend(target) predictions.extend(self._model(inputs, training=False)) if step == loader.steps_per_epoch: return targets, predictions + return targets, predictions + # if __name__ == "__main__": # # Parameters diff --git a/src/mlmc/metamodel/graph_models.py b/src/mlmc/metamodel/graph_models.py index 6f271073..75622922 100644 --- a/src/mlmc/metamodel/graph_models.py +++ b/src/mlmc/metamodel/graph_models.py @@ -1,6 +1,6 @@ from tensorflow.keras import Model from tensorflow.keras.layers import Dense -from spektral.layers import GlobalSumPool +from spektral.layers import GlobalSumPool, GlobalMaxPool, GlobalAvgPool import tensorflow as tf from tensorflow import keras from tensorflow.keras import layers @@ -18,7 +18,7 @@ def __init__(self, conv_layer, hidden_activation, output_activation, kernel_regu #self.conv2 = conv_layer(32, activation=hidden_activation, kernel_regularizer=kernel_regularization) #self.conv3 = conv_layer(64, activation=hidden_activation, kernel_regularizer=kernel_regularization) self.flatten = GlobalSumPool() - #self.fc1 = Dense(512, activation=hidden_activation) + #self.fc1 = Dense(32, activation=hidden_activation) self.fc2 = Dense(1, activation=output_activation) # linear activation for output neuron def call(self, inputs): @@ -38,7 +38,50 @@ def call(self, inputs): # x = self.conv3([x, a]) output = self.flatten(x) #output = self.fc1(output) + output = self.fc2(output) + + #print("output ", output.shape) + + return output + + +# Build model +class NetGCN(Model): + # Setup from https://arxiv.org/pdf/1901.06181.pdf + def __init__(self, conv_layer, hidden_activation, output_activation, kernel_regularization, normalizer, **kwargs): + super().__init__(**kwargs) + #self.normalizer = normalizer + #self.norm_layer = tf.keras.layers.LayerNormalization(axis=1) + self.conv1 = conv_layer(32, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv2 = conv_layer(8, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv3 = conv_layer(16, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv4 = conv_layer(16, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv5 = conv_layer(32, activation=hidden_activation, kernel_regularizer=kernel_regularization) + self.flatten = GlobalSumPool() + #self.fc1 = Dense(128, activation="linear") + self.fc2 = Dense(1)#, activation=output_activation) # linear activation for output neuron + + def call(self, inputs): + x, a = inputs + #print("x ", x) + #x = self.normalizer(x) + #x = self.norm_layer(x) + #print("normalized x ", x) + #print("x[0,0,:] ", x[0, 0, :]) + x = self.conv1([x, a]) + # x = self.conv2([x, a]) + # x = self.conv3([x, a]) + # x = self.conv4([x, a]) + # x = self.conv5([x, a]) + # + # #print("x[0,0,:] ", x[0,0,:]) + # print("x[0, 0, :] ", tf.make_ndarray(x[0,0,:].op.get_attr('net1/strided_slice_1:0'))) + # print("x.shape ", x.shape) + # x = self.conv2([x, a]) + # x = self.conv3([x, a]) + output = self.flatten(x) + #output = self.fc1(output) output = self.fc2(output) #print("output ", output.shape) @@ -50,10 +93,10 @@ def cnn_model(): return keras.Sequential([ #@TODO: Try normalization #self._normalizer, # Seems worse results with normalization - layers.Conv1D(filters=64, kernel_size=3, activation='relu', input_shape=(958, 1)), + layers.Conv1D(filters=256, kernel_size=3, activation='relu', input_shape=(6672, 1)),#input_shape=(958, 1)), #layers.BatchNormalization(), layers.MaxPooling1D(pool_size=2), - layers.Conv1D(filters=64, kernel_size=3, activation='relu'), + layers.Conv1D(filters=128, kernel_size=3, activation='relu'), #layers.BatchNormalization(), layers.MaxPooling1D(pool_size=2), layers.Flatten(), diff --git a/src/mlmc/metamodel/postprocessing.py b/src/mlmc/metamodel/postprocessing.py index f193cb95..ded4ea54 100644 --- a/src/mlmc/metamodel/postprocessing.py +++ b/src/mlmc/metamodel/postprocessing.py @@ -326,6 +326,23 @@ def compare_moments(original_q_estimator, predict_q_estimator, ref_estimator): print("ref orig var SE ", np.sum(np.abs((ref_moments_mean.var - orig_moments_mean.var)))) print("ref predict var SE ", np.sum(np.abs((ref_moments_mean.var - predict_moments_mean.var)))) + # l_0_samples = predict_q_estimator.get_level_samples(level_id=0) + # l_1_samples = predict_q_estimator.get_level_samples(level_id=1) + # l_2_samples = predict_q_estimator.get_level_samples(level_id=2) + # + # print("l 0 samples shape ", np.squeeze(l_0_samples).shape) + # print("l 1 samples shape ", np.squeeze(l_1_samples[..., 0]).shape) + # + # print("l_0_samples.var ", np.var(np.squeeze(l_0_samples))) + # print("l_1_samples ", l_1_samples) + # + # diff = l_1_samples[..., 0] - l_1_samples[..., 1] + # + # print("l1 diff ", diff) + # print("var l1 diff ", np.var(diff)) + # print("fine l_1_samples.var ", np.var(np.squeeze(l_1_samples[..., 0]))) + # print("fine l_2_samples.var ", np.var(np.squeeze(l_2_samples[..., 0]))) + def compare_densities(estimator_1, estimator_2, ref_estimator, label_1="", label_2=""): @@ -333,12 +350,15 @@ def compare_densities(estimator_1, estimator_2, ref_estimator, label_1="", label tol = 1e-10 reg_param = 0 + print("orig estimator") distr_obj_1, result, _, _ = estimator_1.construct_density(tol=tol, reg_param=reg_param) #distr_plot.add_distribution(distr_obj_1, label=label_1, color="blue") + print("predict estimator") distr_obj_2, result, _, _ = estimator_2.construct_density(tol=tol, reg_param=reg_param) #distr_plot.add_distribution(distr_obj_2, label=label_2, color="red", line_style="--") + print("Ref estimator") ref_distr_obj, result, _, _ = ref_estimator.construct_density(tol=tol, reg_param=reg_param) #distr_plot.add_distribution(ref_distr_obj, label="MC reference", color="black", line_style=":") @@ -403,22 +423,52 @@ def get_storage_info(sample_storage): print("moments.l_vars max ", np.max(moments.l_vars, axis=1)) -def cut_samples(data, sample_storage, new_n_collected): +def cut_samples(data, sample_storage, new_n_collected, new_l_0=0): new_data = [] - for d, n_est in zip(data, new_n_collected): - new_data.append(d[:, :n_est, :]) + for l_id, (d, n_est) in enumerate(zip(data, new_n_collected)): + if n_est > 0: + if l_id == new_l_0: + print(d.shape) + print("np.min(d.shape[1], n_est) ", np.min([d.shape[1], n_est])) + fine_samples = d[:, :np.min([d.shape[1], n_est]), 0].reshape(1, np.min([d.shape[1], n_est]), 1) + coarse_samples = np.zeros(fine_samples.shape) + new_data.append(np.concatenate((fine_samples, coarse_samples), axis=2)) + else: + new_data.append(d[:, :np.min([d.shape[1], n_est]), :]) sample_storage = create_quantity_mlmc(new_data, level_parameters=sample_storage.get_level_parameters()) return sample_storage -def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predictions, train_targets, val_targets, l_0_targets=None, l_0_predictions=None, +def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predictions, train_targets, train_predictions, + val_targets, l_0_targets=None, l_0_predictions=None, l1_sample_time=None, l0_sample_time=None, nn_level=0, replace_level=False): #level_zero = False print("nn_level ", nn_level) print("replace level ", replace_level) + print("targets ", targets) + print("predictions ", predictions) + plt.hist(targets, bins=50, alpha=0.5, label='target', density=True) + plt.hist(predictions, bins=50, alpha=0.5, label='predictions', density=True) + + #plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) + plt.legend(loc='upper right') + #plt.xlim(-0.5, 1000) + plt.yscale('log') + plt.show() + + # targets = np.exp(targets) + # predictions = np.exp(predictions) + + # # Creating plot + # plt.boxplot(targets) + # plt.boxplot(predictions) + # # show plot + # plt.show() + #exit() + ####### ### Create storage fromm original MLMC data ###### @@ -454,10 +504,22 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti data_nn = [] n_ops_predict = [] + + nn_lev_sim_time = 0 + for l_id in range(nn_level): + nn_lev_sim_time = n_ops[l_id] - nn_lev_sim_time + + print("nn lev sim time ", nn_lev_sim_time) + for l_id in range(n_lev): if l_id == 0: + print("np.mean(l 0 predictions) ", np.mean(l_0_predictions)) level_samples = l_0_predictions.reshape(1, len(l_0_predictions), 1) - n_ops_predict.append(l0_sample_time + field_times[nn_level]) + #level_samples = np.ones((1, len(l_0_predictions), 1)) + ft_index = nn_level + if nn_level > 0: + ft_index = nn_level - 1 + n_ops_predict.append(l0_sample_time + field_times[ft_index] / 2) #level_samples = level_samples[:, :50000, :] else: if replace_level: @@ -468,12 +530,23 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti if l_id == 1: print("l id replaced level ", l_id) print("len(predictions ) ", len(predictions)) + coarse_level_samples = predictions.reshape(1, len(predictions), 1) + #coarse_level_samples = np.ones((1, len(predictions), 1)) fine_level_samples = targets.reshape(1, len(targets), 1) + + # coarse_level_samples = np.concatenate((coarse_level_samples, + # train_predictions.reshape(1, len(train_predictions), 1)), axis=1) + # + # fine_level_samples = np.concatenate((fine_level_samples, + # train_targets.reshape(1, len(train_targets), 1)), axis=1) + # coarse_level_samples = coarse_level_samples[:, :len(predictions)//2, :] # fine_level_samples = fine_level_samples[:, :len(predictions)//2, :] level_samples = np.concatenate((fine_level_samples, coarse_level_samples), axis=2) - n_ops_predict.append(n_ops[nn_level] + l1_sample_time) + + n_ops_predict.append(n_ops[nn_level] - nn_lev_sim_time + + l1_sample_time) else: if replace_level: level_id = l_id #- 1 @@ -490,8 +563,6 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti print("Storage predict info") get_storage_info(sample_storage_predict) - - ###### ### Create estimators ###### @@ -507,8 +578,20 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti print("n ops ", n_ops) print("n ops predict ", n_ops_predict) + # remove levels + # #@TODO: remove asap + new_n_samples = [0, 0, sample_storage.get_n_collected()[-3], sample_storage.get_n_collected()[-2], sample_storage.get_n_collected()[-1]] + sample_storage = cut_samples(data_mlmc, sample_storage, new_n_samples, new_l_0=2) + print("Cut storage info") + get_storage_info(sample_storage) + original_q_estimator = get_quantity_estimator(sample_storage, true_domain=domain) + n_ops = n_ops[2:] + # ##### + #### Original data - n_estimated_orig, l_vars_orig, n_samples_orig = get_n_estimated(sample_storage, original_q_estimator, n_ops=n_ops) + n_ops_est = copy.deepcopy(n_ops) + n_ops_est[0] = n_ops_est[0] / 1000 + n_estimated_orig, l_vars_orig, n_samples_orig = get_n_estimated(sample_storage, original_q_estimator, n_ops=n_ops_est) print("n estimated orig ", n_estimated_orig) sample_storage = cut_samples(data_mlmc, sample_storage, n_estimated_orig) # @@ -699,10 +782,41 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti compare_moments(original_q_estimator, predict_q_estimator, ref_estimator) + plot_moments({"ref": ref_estimator, "orig": original_q_estimator, "nn": predict_q_estimator}) + compare_densities(original_q_estimator, predict_q_estimator, ref_estimator, label_1="orig N: {}".format(sample_storage.get_n_collected()), label_2="gnn N: {}".format(sample_storage_predict.get_n_collected())) +def plot_moments(mlmc_estimators): + n_moments = 25 + moments_plot = mlmc.tool.plot.MomentsPlots( + title="Legendre {} moments".format(n_moments)) + + # moments_plot = mlmc.tool.plot.PlotMoments( + # title="Monomial {} moments".format(self.n_moments), log_mean_y=False) + + for nl, estimator in mlmc_estimators.items(): + + moments_mean = qe.estimate_mean(qe.moments(estimator._quantity, estimator._moments_fn)) + est_moments = moments_mean.mean + est_vars = moments_mean.var + + n_collected = [str(n_c) for n_c in estimator._sample_storage.get_n_collected()] + moments_plot.add_moments((moments_mean.mean, moments_mean.var), label="#{} N:".format(nl) + ", ".join(n_collected)) + + # print("moments level means ", moments_mean.l_means) + # print("moments level vars ", moments_mean.l_vars) + # print("moments level max vars ", np.max(moments_mean.l_vars, axis=1)) + # print("est moments ", est_moments) + # print("est_vars ", est_vars) + # print("np.max(est_vars) ", np.max(est_vars)) + + moments_plot.show(None) + #moments_plot.show(file=os.path.join(self.work_dir, "{}_moments".format(self.n_moments))) + moments_plot.reset() + + def analyze_mlmc_data(): n_levels = 5 # mlmc_file = "/home/martin/Documents/metamodels/data/cl_0_3_s_4/L5/mlmc_5.hdf5" From f6dd96b0609932c0e039bd2db18ff14fd12464b9 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Fri, 26 Mar 2021 20:26:10 +0100 Subject: [PATCH 13/67] SVR --- src/mlmc/metamodel/analyze_nn.py | 207 ++++++++++++++++++++++++++++--- 1 file changed, 188 insertions(+), 19 deletions(-) diff --git a/src/mlmc/metamodel/analyze_nn.py b/src/mlmc/metamodel/analyze_nn.py index 8fd3f22f..a59c1e49 100644 --- a/src/mlmc/metamodel/analyze_nn.py +++ b/src/mlmc/metamodel/analyze_nn.py @@ -21,7 +21,7 @@ from mlmc.metamodel.flow_task_GNN_2 import GNN from spektral.layers import GCNConv, GlobalSumPool, ChebConv, GraphSageConv, ARMAConv, GATConv, APPNPConv, GINConv -from tensorflow.keras.losses import MeanSquaredError, KLDivergence +from tensorflow.keras.losses import MeanSquaredError, KLDivergence, MeanAbsoluteError, MeanSquaredLogarithmicError from tensorflow.keras.regularizers import l2 from spektral.data import MixedLoader from spektral.layers.ops import sp_matrix_to_sp_tensor @@ -174,6 +174,142 @@ def bootstrap(): # estimate_density(np.mean(all_predictions, axis=0), title="Predictions") +def run_SVR(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level, conv_layer=None, stats=False, + gnn=None, model=None, log=False): + from sklearn.svm import SVR + from sklearn.preprocessing import StandardScaler + from sklearn.model_selection import train_test_split + + batch_size = 2000 + epochs = 1000 + hidden_regularization = None # l2(2e-10) + + preprocess_start_time = time.process_time() + #graph_creator(output_dir, hdf_path, mesh, level=level) + + # Load data + data = FlowDataset(output_dir=output_dir, level=level, log=log) + dataset = data.dataset + + dataset = dataset.sample(frac=1) + + train = dataset[:2000] + test = dataset[2000:] + + train_input, train_output = train.x, train.y + test_input, test_output = test.x, test.y + + train_input = prepare_data(train_input) + train_output = prepare_data(train_output) + + test_input = prepare_data(test_input) + test_output = prepare_data(test_output) + + # sc_X = StandardScaler() + # sc_y = StandardScaler() + # train_input = sc_X.fit_transform(train_input) + # train_output = sc_y.fit_transform(train_output.reshape(-1,1)) + # test_input = sc_X.fit_transform(test_input) + # test_output = sc_y.fit_transform(test_output.reshape(-1,1)) + + + #train_input, train_output, test_input, test_output = split_dataset(dataset) + + + preprocess_time = time.process_time() - preprocess_start_time + learning_time_start = time.process_time() + + + print("train input ", train_input.shape) + print("train output ", train_output.shape) + + svr_rbf = SVR(kernel='rbf', verbose=True) # 'linear' kernel fitting is never-ending and 'poly' kernel gives very bad score (e.g. -2450), sigmoid gives also bad score (e.g. -125) + svr_rbf.fit(train_input, train_output) + train_error = svr_rbf.score(train_input, train_output) + + #test_input = sc_X.fit_transform(test_input) + test_error = svr_rbf.score(test_input, test_output) + + targets = test_output + + # test_y = sc_y.fit_transform(test.y.to_numpy().reshape(-1,1)) + + predictions = svr_rbf.predict(test_input) + + print("train error ", train_error) + print("test error ", test_error) + + train_predictions = svr_rbf.predict(train_input) + #train_predictions = np.squeeze(train_predictions) + + learning_time = time.process_time() - learning_time_start + + if stats is True: + return gnn, targets, predictions, learning_time + + if log: + targets = np.exp(targets) + predictions = np.exp(predictions) + + print("np.var(target-predictions) ", np.var(targets - predictions)) + + #plot_loss(gnn._train_loss, gnn._val_loss) + analyze_results(targets, predictions) + + import matplotlib.pyplot as plt + + # plt.hist(train_output, bins=50, alpha=0.5, label='train target', density=True) + # plt.hist(train_predictions, bins=50, alpha=0.5, label='train predictions', density=True) + # + # # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) + # plt.legend(loc='upper right') + # # plt.xlim(-0.5, 1000) + # plt.yscale('log') + # plt.show() + + plt.hist(targets, bins=50, alpha=0.5, label='target', density=True) + plt.hist(predictions, bins=50, alpha=0.5, label='predictions', density=True) + + # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) + plt.legend(loc='upper right') + # plt.xlim(-0.5, 1000) + plt.yscale('log') + plt.show() + + predict_l_0_start_time = time.process_time() + l_0_targets, l_0_predictions = predict_level_zero_SVR(svr_rbf, l_0_output_dir, l_0_hdf_path, mesh, batch_size, log) + predict_l_0_time = time.process_time() - predict_l_0_start_time + + save_times(save_path, False, (preprocess_time, len(data)), learning_time, (predict_l_0_time, len(l_0_targets))) + save_load_data(save_path, False, targets, predictions, train_output, train_predictions, test_output, l_0_targets, + l_0_predictions) + + +def predict_level_zero_SVR(nn, output_dir, hdf_path, mesh, batch_size=1000, log=False): + #graph_creator(output_dir, hdf_path, mesh, level=0) + + # Load data + data = FlowDataset(output_dir=output_dir, log=log) + dataset = data.dataset[:] + + test_input = prepare_data(dataset.x) + targets = prepare_data(dataset.y) + print("data prepared") + + predictions = [] + for i in range(0, len(test_input), batch_size): + predictions.extend(nn.predict(test_input[i:i + batch_size])) + predictions = np.array(predictions) + print("predictison shape ", predictions.shape) + predictions = np.squeeze(predictions) + + if log: + targets = np.exp(targets) + predictions = np.exp(predictions) + # analyze_results(targets, predictions) + return targets, predictions + + def statistics(run_method, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level): n_subsamples = 3 train_losses = [] @@ -283,33 +419,36 @@ def analyze_statistics(save_path): def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level, conv_layer=None, stats=False, - gnn=None, model=None): + gnn=None, model=None, log=False): # Parameters if conv_layer is None: - conv_layer = GCNConv + #conv_layer = GCNConv conv_layer = ChebConv # Seems better than GCNConv, good distribution of predictions - # conv_layer = GraphSageConv # Seems better than ChebConv, good loss but very narrow distribution of predictions + #conv_layer = OwnChebConv + #conv_layer = GraphSageConv # Seems better than ChebConv, good loss but very narrow distribution of predictions # # conv_layer = ARMAConv # Seems worse than GraphSageConv # conv_layer = GATConv # Slow and not better than GraphSageConv # # conv_layer = APPNPConv # Not bad but worse than GraphSageConv # # conv_layer = GINConv # it is comparable to APPNPConv - # act_func = "relu" # "tanh"#"elu" # ReLU keep predictions above zero + # act_func = "relu" # "tanh"#"elu" loss = MeanSquaredError() # var_loss_function# + # loss = MeanAbsoluteError() + # loss = MeanSquaredLogarithmicError() #loss = KLDivergence() # loss = total_loss_function optimizer = tf.optimizers.Adam(learning_rate=0.001) batch_size = 2000#2000 - epochs = 500 + epochs = 1000 hidden_regularization = None # l2(2e-10) preprocess_start_time = time.process_time() #graph_creator(output_dir, hdf_path, mesh, level=level) # Load data - data = FlowDataset(output_dir=output_dir, level=level) + data = FlowDataset(output_dir=output_dir, level=level, log=log) data = data#[:15000] - print("lev data ", len(data)) + print("len data ", len(data)) preprocess_time = time.process_time() - preprocess_start_time @@ -361,9 +500,34 @@ def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, if stats is True: return gnn, targets, predictions, learning_time + if log: + targets = np.exp(targets) + predictions = np.exp(predictions) + plot_loss(gnn._train_loss, gnn._val_loss) analyze_results(targets, predictions) + import matplotlib.pyplot as plt + + plt.hist(targets, bins=50, alpha=0.5, label='target', density=True) + plt.hist(predictions, bins=50, alpha=0.5, label='predictions', density=True) + + # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) + plt.legend(loc='upper right') + # plt.xlim(-0.5, 1000) + plt.yscale('log') + plt.show() + + + # plt.hist(np.exp(targets), bins=50, alpha=0.5, label='target', density=True) + # plt.hist(np.exp(predictions), bins=50, alpha=0.5, label='predictions', density=True) + # + # # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) + # plt.legend(loc='upper right') + # # plt.xlim(-0.5, 1000) + # plt.yscale('log') + # plt.show() + # print("np.max(targets) ", np.max(targets)) # print("np.min(targets) ", np.min(targets)) # @@ -399,7 +563,7 @@ def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, # diff_moments(targets, predictions) predict_l_0_start_time = time.process_time() - l_0_targets, l_0_predictions = predict_level_zero(gnn, l_0_output_dir, l_0_hdf_path, mesh, batch_size) + l_0_targets, l_0_predictions = predict_level_zero(gnn, l_0_output_dir, l_0_hdf_path, mesh, batch_size, log) predict_l_0_time = time.process_time() - predict_l_0_start_time save_times(save_path, False, (preprocess_time, len(data)), learning_time, (predict_l_0_time, len(l_0_targets))) @@ -407,11 +571,11 @@ def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, l_0_predictions) -def predict_level_zero(nn, output_dir, hdf_path, mesh, batch_size=1000, ): - # graph_creator(output_dir, hdf_path, mesh, level=0) +def predict_level_zero(nn, output_dir, hdf_path, mesh, batch_size=1000, log=False): + #graph_creator(output_dir, hdf_path, mesh, level=0) # Load data - data = FlowDataset(output_dir=output_dir) + data = FlowDataset(output_dir=output_dir, log=log) data = data # [:10000] # data.a = conv_layer.preprocess(data.a) data.a = sp_matrix_to_sp_tensor(data.a) @@ -420,6 +584,10 @@ def predict_level_zero(nn, output_dir, hdf_path, mesh, batch_size=1000, ): targets, predictions = nn.predict(loader_te) predictions = np.squeeze(predictions) + + if log: + targets = np.exp(targets) + predictions = np.exp(predictions) # analyze_results(targets, predictions) return targets, predictions @@ -518,7 +686,7 @@ def process_results(hdf_path, sampling_info_path, ref_mlmc_file, save_path, nn_l def get_config(case=0): if case == 0: - cl = "cl_0_3_s_4" + cl = "cl_0_1_s_1" nn_level = 0 replace_level = False mesh = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/l_step_0.055_common_files/mesh.msh" @@ -534,13 +702,13 @@ def get_config(case=0): cl = "cl_0_3_s_4" nn_level = 1 replace_level = False - mesh = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/l_step_0.055_common_files/mesh.msh" - #mesh = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L2/l_step_0.027624156655057155_common_files/mesh.msh" + #mesh = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/l_step_0.055_common_files/mesh.msh" + mesh = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L2/l_step_0.027624156655057155_common_files/mesh.msh" output_dir = "/home/martin/Documents/metamodels/data/1000_ele/{}/L5/test/01_cond_field/output/".format(cl) hdf_path = "/home/martin/Documents/metamodels/data/1000_ele/{}/L5/mlmc_5.hdf5".format(cl) save_path = "/home/martin/Documents/metamodels/data/1000_ele/{}".format(cl) l_0_output_dir = "/home/martin/Documents/metamodels/data/1000_ele/{}/L{}/test/01_cond_field/output/".format(cl, nn_level + 1) - l_0_hdf_path = "/home/martin/Documents/metamodels/data/1000_ele/{}/L{}/mlmc_{}.hdf5".format(cl, nn_level + 1, nn_level + 1) + l_0_hdf_path = "/home/martin/Documents/metamodels/data/1000_ele/{}/L{}/mlmc_{}.hdf5".format(cl, nn_level + 1, nn_level) sampling_info_path = "/home/martin/Documents/metamodels/data/1000_ele/{}/sampling_info".format(cl) ref_mlmc_file = "/home/martin/Documents/metamodels/data/1000_ele/{}/L1_benchmark/mlmc_1.hdf5".format(cl) @@ -592,7 +760,7 @@ def get_config(case=0): if __name__ == "__main__": - case = 4 + case = 1 output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, replace_level, nn_level = get_config( case) @@ -614,9 +782,10 @@ def get_config(case=0): #run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, conv_layer=GCNConv, model=NetGCN) # , gnn=gnn) - #run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level) # , gnn=gnn) + #run_SVR(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True) # , gnn=gnn) + #exit() - # run_CNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level) + run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True) # , gnn=gnn) process_results(hdf_path, sampling_info_path, ref_mlmc_file, save_path, nn_level, replace_level) # analyze_bootstrap(save_path) From cd9d739b1fd9e9e1eff3fad7de3b09035ec1ea0c Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Mon, 5 Apr 2021 10:35:56 +0200 Subject: [PATCH 14/67] GNN refactoring --- src/mlmc/estimator.py | 8 +- src/mlmc/metamodel/analyze_nn.py | 444 +++++++++++--------------- src/mlmc/metamodel/flow_dataset.py | 47 +-- src/mlmc/metamodel/flow_task_GNN_2.py | 39 ++- src/mlmc/metamodel/graph_models.py | 44 ++- src/mlmc/metamodel/postprocessing.py | 116 ++++--- src/mlmc/sample_storage.py | 15 +- test/metamodels/__init__.py | 0 test/metamodels/metamodel_test.py | 238 ++++++++++++++ test/metamodels/nn_config.py | 92 ++++++ 10 files changed, 699 insertions(+), 344 deletions(-) create mode 100644 test/metamodels/__init__.py create mode 100644 test/metamodels/metamodel_test.py create mode 100644 test/metamodels/nn_config.py diff --git a/src/mlmc/estimator.py b/src/mlmc/estimator.py index 643658e2..1e7f59f7 100644 --- a/src/mlmc/estimator.py +++ b/src/mlmc/estimator.py @@ -25,7 +25,6 @@ def estimate_n_samples_for_target_variance(target_variance, prescribe_vars, n_op # Limit maximal number of samples per level n_samples_estimate_safe = np.maximum( np.minimum(n_samples_estimate, vars * n_levels / target_variance), 2) - return np.max(n_samples_estimate_safe, axis=1).astype(int) @@ -78,6 +77,7 @@ def estimate_moments(self, moments_fn=None): moments_fn = self._moments_fn moments_mean = qe.estimate_mean(qe.moments(self._quantity, moments_fn)) + self.moments_mean = moments_mean return moments_mean.mean, moments_mean.var def estimate_covariance(self, moments_fn=None): @@ -106,6 +106,7 @@ def estimate_diff_vars_regression(self, n_created_samples, moments_fn=None, raw_ if moments_fn is None: moments_fn = self._moments_fn raw_vars, n_samples = self.estimate_diff_vars(moments_fn) + sim_steps = np.squeeze(self._sample_storage.get_level_parameters()) vars = self._all_moments_variance_regression(raw_vars, sim_steps) @@ -120,6 +121,8 @@ def estimate_diff_vars(self, moments_fn=None): diff_variance - shape LxR, variances of diffs of moments_fn n_samples - shape L, num samples for individual levels. """ + if moments_fn is None: + moments_fn = self._moments_fn moments_mean = qe.estimate_mean(qe.moments(self._quantity, moments_fn)) return moments_mean.l_vars, moments_mean.n_samples @@ -187,7 +190,6 @@ def _variance_of_variance(self, n_samples=None): ns, var_var = self._saved_var_var if np.sum(np.abs(np.array(ns) - np.array(n_samples))) == 0: return var_var - vars = [] for ns in n_samples: df = ns - 1 @@ -349,7 +351,7 @@ def construct_density(self, tol=1e-8, reg_param=0.0, orth_moments_tol=1e-4, exac Construct approximation of the density using given moment functions. """ cov_mean = qe.estimate_mean(qe.covariance(self._quantity, self._moments_fn)) - cov_mat = cov_mean() + cov_mat = cov_mean.mean moments_obj, info = mlmc.tool.simple_distribution.construct_ortogonal_moments(self._moments_fn, cov_mat, tol=orth_moments_tol) diff --git a/src/mlmc/metamodel/analyze_nn.py b/src/mlmc/metamodel/analyze_nn.py index a59c1e49..c63be63f 100644 --- a/src/mlmc/metamodel/analyze_nn.py +++ b/src/mlmc/metamodel/analyze_nn.py @@ -1,36 +1,30 @@ import os import numpy as np import time -import random -# os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only -from mlmc.metamodel.create_graph import graph_creator -from mlmc.metamodel.flow_dataset import FlowDataset +import glob -from mlmc.metamodel.own_cheb_conv import OwnChebConv +os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only +from mlmc.metamodel.flow_dataset import FlowDataset +import test.metamodels.nn_config # Make numpy printouts easier to read. # np.set_printoptions(precision=9, suppress=True) import tensorflow as tf from scipy.stats import ks_2samp import sklearn.model_selection -from mlmc.metamodel.custom_methods import abs_activation, var_loss_function, total_loss_function -from mlmc.metamodel.postprocessing import analyze_results, plot_loss, estimate_density, diff_moments, process_mlmc +from mlmc.metamodel.custom_methods import abs_activation +from mlmc.metamodel.postprocessing import analyze_results, plot_loss, estimate_density, process_mlmc from mlmc.metamodel.flow_task_NN import DNN from mlmc.metamodel.flow_task_CNN import CNN -from mlmc.metamodel.graph_models import NetGCN from mlmc.metamodel.flow_task_GNN_2 import GNN -from spektral.layers import GCNConv, GlobalSumPool, ChebConv, GraphSageConv, ARMAConv, GATConv, APPNPConv, GINConv -from tensorflow.keras.losses import MeanSquaredError, KLDivergence, MeanAbsoluteError, MeanSquaredLogarithmicError -from tensorflow.keras.regularizers import l2 +from tensorflow.keras.losses import MeanSquaredError from spektral.data import MixedLoader from spektral.layers.ops import sp_matrix_to_sp_tensor print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU'))) - epochs = 100 - def prepare_data(data): data = np.squeeze(np.stack(data.to_numpy(), axis=0)) return np.asarray(data).astype('float64') @@ -87,12 +81,12 @@ def run(): estimate_density(predictions) -def run_CNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level): +def run_CNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level, log): # Parameters loss = "mean_squared_error" optimizer = tf.optimizers.Adam(learning_rate=0.01) - data = FlowDataset(output_dir=output_dir, level=level) + data = FlowDataset(output_dir=output_dir, level=level, log=log) dataset = data.dataset[:] train_input, train_output, test_input, test_output = split_dataset(dataset) @@ -104,7 +98,6 @@ def run_CNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, train_input = np.expand_dims(train_input, axis=-1) test_input = np.expand_dims(test_input, axis=-1) - print("train input shape ", train_input.shape) dnn = CNN(loss=loss, optimizer=optimizer, output_activation=abs_activation, hidden_activation='relu') @@ -177,8 +170,6 @@ def bootstrap(): def run_SVR(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level, conv_layer=None, stats=False, gnn=None, model=None, log=False): from sklearn.svm import SVR - from sklearn.preprocessing import StandardScaler - from sklearn.model_selection import train_test_split batch_size = 2000 epochs = 1000 @@ -189,8 +180,9 @@ def run_SVR(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, # Load data data = FlowDataset(output_dir=output_dir, level=level, log=log) - dataset = data.dataset + data.shuffle() + dataset = data.dataset dataset = dataset.sample(frac=1) train = dataset[:2000] @@ -219,7 +211,6 @@ def run_SVR(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, preprocess_time = time.process_time() - preprocess_start_time learning_time_start = time.process_time() - print("train input ", train_input.shape) print("train output ", train_output.shape) @@ -245,7 +236,7 @@ def run_SVR(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, learning_time = time.process_time() - learning_time_start if stats is True: - return gnn, targets, predictions, learning_time + return targets, predictions, learning_time, train_output, train_predictions if log: targets = np.exp(targets) @@ -310,26 +301,22 @@ def predict_level_zero_SVR(nn, output_dir, hdf_path, mesh, batch_size=1000, log= return targets, predictions -def statistics(run_method, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level): - n_subsamples = 3 - train_losses = [] - val_losses = [] - test_losses = [] - all_test_outputs = [] - all_predictions = [] - learning_times = [] +def statistics(models, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level, conv_layer, gnn=None): + n_subsamples = 50 + + model_data = {} + for model_key in models.keys(): + model_data[model_key] = {} + model_data[model_key]["test_targets"] = [] + model_data[model_key]["test_predictions"] = [] + model_data[model_key]["train_targets"] = [] + model_data[model_key]["train_predictions"] = [] + model_data[model_key]["learning_times"] = [] + model_data[model_key]["orig_max_vars"] = [] + model_data[model_key]["predict_max_vars"] = [] + model_data[model_key]["total_steps"] = [] + model_data[model_key]["log"] = models[model_key][1] - # Parameters - # conv_layer = GCNConv - conv_layer = ChebConv # Seems better than GCNConv, good distribution of predictions - # conv_layer = GraphSageConv # Seems better than ChebConv, good loss but very narrow distribution of predictions - # # conv_layer = ARMAConv # Seems worse than GraphSageConv - # conv_layer = GATConv # Slow and not better than GraphSageConv - # # conv_layer = APPNPConv # Not bad but worse than GraphSageConv - # # conv_layer = GINConv # it is comparable to APPNPConv - # act_func = "relu" # "tanh"#"elu" # ReLU keep predictions above zero - - save_path = os.path.join(save_path, conv_layer.__name__) if not os.path.isdir(save_path): os.makedirs(save_path) else: @@ -337,31 +324,35 @@ def statistics(run_method, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, s exit() for i in range(n_subsamples): - gnn, targets, predictions, learning_time = run_method(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, - save_path, mesh, level=nn_level, stats=True) + for model_key, (model, log) in models.items(): - # save_times(save_path, False, (preprocess_time, len(data)), learning_time, (predict_l_0_time, len(l_0_targets))) - # save_load_data(save_path, False, targets, predictions, train_targets, val_targets, l_0_targets, l_0_predictions) + targets, predictions, learning_time, train_targets, train_predictions, orig_max_vars, predict_max_vars, total_steps = model(output_dir, + hdf_path, l_0_output_dir, l_0_hdf_path, + save_path, mesh, level=level, stats=True, log=log, conv_layer=conv_layer, gnn=gnn, train=True) - all_test_outputs.append(targets) - all_predictions.append(predictions) + model_data[model_key]["test_targets"].append(targets) + model_data[model_key]["test_predictions"].append(predictions) + model_data[model_key]["train_targets"].append(train_targets) + model_data[model_key]["train_predictions"].append(train_predictions) + model_data[model_key]["orig_max_vars"].append(orig_max_vars) + model_data[model_key]["predict_max_vars"].append(predict_max_vars) + model_data[model_key]["total_steps"].append(total_steps) + model_data[model_key]["learning_times"].append(learning_time) - train_losses.append(gnn._train_loss[-1]) - val_losses.append(gnn._val_loss[-1]) - test_losses.append(gnn._test_loss[-1]) - learning_times.append(learning_time) + # save_times(save_path, False, (preprocess_time, len(data)), learning_time, (predict_l_0_time, len(l_0_targets))) + # save_load_data(save_path, False, targets, predictions, train_targets, val_targets, l_0_targets, l_0_predictions) - for i in range(len(train_losses)): - print("train loss ", train_losses[i]) - print("test loss ", test_losses[i]) - analyze_results(all_test_outputs[i], all_predictions[i]) - print("learning time ", learning_times[i]) - print("##################################################") + # for i in range(len(train_losses)): + # print("train loss ", train_losses[i]) + # print("test loss ", test_losses[i]) + # analyze_results(all_targets[i], all_predictions[i]) + # print("learning time ", learning_times[i]) + # print("##################################################") - print("worse train loss ", np.max(train_losses)) - print("worse test loss ", np.max(test_losses)) - save_statistics(save_path, all_test_outputs, all_predictions, train_losses, val_losses, test_losses, learning_times) + save_statistics(save_path, model_data) + + analyze_statistics(save_path) # plot_loss(train_losses, val_losses) # analyze_results(np.mean(all_test_outputs, axis=0), np.mean(all_predictions, axis=0)) @@ -371,66 +362,108 @@ def statistics(run_method, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, s # estimate_density(np.mean(all_predictions, axis=0), title="Predictions") -def save_statistics(path, all_test_outputs, all_predictions, train_losses, val_losses, test_losses, learning_times): - np.save(os.path.join(path, "all_test_outputs"), all_test_outputs) - np.save(os.path.join(path, "all_predictions"), all_predictions) - np.save(os.path.join(path, "train_losses"), train_losses) - np.save(os.path.join(path, "val_losses"), val_losses) - np.save(os.path.join(path, "test_losses"), test_losses) - np.save(os.path.join(path, "learning_times"), learning_times) +def save_statistics(dir_path, model_data): + dirs = (os.path.split(dir_path)[-1]).split("_") + for data_dir in dirs: + if not os.path.isdir(os.path.join(dir_path, data_dir)): + os.makedirs(os.path.join(dir_path, data_dir)) + else: + print("dir exists {}".format(os.path.join(dir_path, data_dir))) + exit() + + print("dirs ", dirs) + for key, data_dict in model_data.items(): + path = os.path.join(dir_path, key) + for file_name, data in data_dict.items(): + np.save(os.path.join(path, file_name), data) -def load_statistics(path): - all_test_outputs = np.load(os.path.join(path, "all_test_outputs.npy")) - all_predictions = np.load(os.path.join(path, "all_predictions.npy")) - train_losses = np.load(os.path.join(path, "train_losses.npy")) - val_losses = np.load(os.path.join(path, "val_losses.npy")) - test_losses = np.load(os.path.join(path, "test_losses.npy")) - learning_times = np.load(os.path.join(path, "learning_times.npy")) +def load_statistics(dir_path): + models_data = {} - return all_test_outputs, all_predictions, train_losses, val_losses, test_losses, learning_times + dirs = (os.path.split(dir_path)[-1]).split("_") + + for data_dir in dirs: + models_data[data_dir] = {} + data_dir_path = os.path.join(dir_path, data_dir) + if not os.path.isdir(data_dir_path): + print("data dir not exists {}".format(data_dir_path)) + exit() + + for file in glob.glob(os.path.join(data_dir_path, "*.npy")): + file_name = os.path.split(file)[-1] + file_name = file_name.split(".")[0] + models_data[data_dir][file_name] = np.load(file) + + return models_data def analyze_statistics(save_path): - conv_layer = GCNConv - conv_layer = ChebConv # Seems better than GCNConv, good distribution of predictions - # conv_layer = GraphSageConv # Seems better than ChebConv, good loss but very narrow distribution of predictions - # # conv_layer = ARMAConv # Seems worse than GraphSageConv - # conv_layer = GATConv # Slow and not better than GraphSageConv - # # conv_layer = APPNPConv # Not bad but worse than GraphSageConv - # # conv_layer = GINConv # it is comparable to APPNPConv - # act_func = "relu" # "tanh"#"elu" # ReLU keep predictions above zero - save_path = os.path.join(save_path, conv_layer.__name__) + print("save path ", save_path) + if not os.path.isdir(save_path): - print("dir exists") + print("dir not exists") exit() - all_test_outputs, all_predictions, train_losses, val_losses, test_losses, learning_times = load_bootstrap(save_path) + models_data = load_statistics(save_path) - print("learning times ", learning_times) - print("test losses ", test_losses) + for key, data_dict in models_data.items(): + print("model: {}".format(key)) - for targets, predictions in zip(all_test_outputs, all_predictions): - print("mean targets ", np.mean(targets)) - print("mean predictions ", np.mean(predictions)) + if data_dict["log"]: + data_dict["test_targets"] = np.exp(data_dict["test_targets"]) + data_dict["test_predictions"] = np.exp(data_dict["test_predictions"]) + data_dict["train_targets"] = np.exp(data_dict["train_targets"]) + data_dict["train_predictions"] = np.exp(data_dict["train_predictions"]) - print("var targets ", np.var(targets)) - print("var predictions ", np.var(predictions)) + print("test targets ", data_dict["test_targets"]) + print("test predictions ", data_dict["test_predictions"]) + print("orig max vars ", data_dict["orig_max_vars"]) + print("predict max vars ", data_dict["predict_max_vars"]) -def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level, conv_layer=None, stats=False, - gnn=None, model=None, log=False): - # Parameters - if conv_layer is None: - #conv_layer = GCNConv - conv_layer = ChebConv # Seems better than GCNConv, good distribution of predictions - #conv_layer = OwnChebConv - #conv_layer = GraphSageConv # Seems better than ChebConv, good loss but very narrow distribution of predictions - # # conv_layer = ARMAConv # Seems worse than GraphSageConv - # conv_layer = GATConv # Slow and not better than GraphSageConv - # # conv_layer = APPNPConv # Not bad but worse than GraphSageConv - # # conv_layer = GINConv # it is comparable to APPNPConv - # act_func = "relu" # "tanh"#"elu" + mean_orig_vars = np.mean(data_dict["orig_max_vars"], axis=0) + mean_predict_vars = np.mean(data_dict["predict_max_vars"], axis=0) + total_steps = np.mean(data_dict["total_steps"]) + + print("mean orig vars ", mean_orig_vars) + print("mean predict vars ", mean_predict_vars) + print("total steps ", total_steps) + + test_MSE = np.mean((data_dict["test_predictions"] - data_dict["test_targets"])**2, axis=1) + test_RMSE = np.sqrt(test_MSE) + test_MAE = np.mean(np.abs(data_dict["test_predictions"] - data_dict["test_targets"]), axis=1) + + train_MSE = np.mean((data_dict["train_predictions"] - data_dict["train_targets"]) ** 2, axis=1) + train_RMSE = np.sqrt(train_MSE) + train_MAE = np.mean(np.abs( data_dict["train_predictions"] - data_dict["train_targets"]), axis=1) + learning_times = data_dict["learning_times"] + + # plot_data(test_MSE, label="test MSE") + # plot_data(test_MAE, label="test MAE") + + print("mean test MSE ", np.mean(test_MSE)) + print("mean test RMSE ", np.mean(test_RMSE)) + print("mean test MAE ", np.mean(test_MAE)) + print("max test MSE ", np.max(test_MSE)) + print("max test RMSE ", np.max(test_RMSE)) + print("max test MAE ", np.max(test_MAE)) + + print("mean train MSE ", np.mean(train_MSE)) + print("mean train RMSE ", np.mean(train_RMSE)) + print("mean train MAE ", np.mean(train_MAE)) + print("max train MSE ", np.max(train_MSE)) + print("max train RMSE ", np.max(train_RMSE)) + print("max train MAE ", np.max(train_MAE)) + + print("mean learning time ", np.mean(learning_times)) + print("max learning time ", np.max(learning_times)) + + print("######################################") + + +def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level, conv_layer, stats=False, + gnn=None, model=None, log=False, train=True): loss = MeanSquaredError() # var_loss_function# # loss = MeanAbsoluteError() @@ -438,8 +471,8 @@ def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, #loss = KLDivergence() # loss = total_loss_function optimizer = tf.optimizers.Adam(learning_rate=0.001) - batch_size = 2000#2000 - epochs = 1000 + batch_size = 200#2000 + epochs = 2000#1000 hidden_regularization = None # l2(2e-10) preprocess_start_time = time.process_time() @@ -447,29 +480,23 @@ def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, # Load data data = FlowDataset(output_dir=output_dir, level=level, log=log) - data = data#[:15000] - print("len data ", len(data)) - + data = data#[:10000] + data.shuffle() preprocess_time = time.process_time() - preprocess_start_time - learning_time_start = time.process_time() - # data.a = conv_layer.preprocess(data.a) + data.a = conv_layer.preprocess(data.a) data.a = sp_matrix_to_sp_tensor(data.a) - - train_data_len = int(len(data) * 0.8) + #train_data_len = int(len(data) * 0.8) train_data_len = 2000 - print("train data len ", train_data_len) - # Train/valid/test split data_tr, data_te = data[:train_data_len], data[train_data_len:], np.random.shuffle(data_tr) - val_data_len = int(len(data_tr) * 0.2) data_tr, data_va = data_tr[:-val_data_len], data_tr[-val_data_len:] - print("data_tr len ", len(data_tr)) - print("data_va len ", len(data_va)) - print("data_te len ", len(data_te)) + # print("data_tr len ", len(data_tr)) + # print("data_va len ", len(data_va)) + # print("data_te len ", len(data_te)) # We use a MixedLoader since the dataset is in mixed mode loader_tr = MixedLoader(data_tr, batch_size=batch_size, epochs=epochs) @@ -483,40 +510,55 @@ def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, # ideally patience = 150 # batch_size 500, ideally 500 epochs, patience 35 + if train: # gnn.run_eagerly = True train_targets = gnn.fit(loader_tr, loader_va, loader_te) + # states = gnn._states + # print("list(states.keys() ", list(states.keys())) + # min_key = np.min(list(states.keys())) + # print("min key ", min_key) + # print("states[min_key] ", states[min_key]) + # gnn = states[min_key] + train_targets, train_predictions = gnn.predict(loader_tr) train_predictions = np.squeeze(train_predictions) val_targets = gnn.val_targets + total_steps = gnn._total_n_steps + targets, predictions = gnn.predict(loader_te) predictions = np.squeeze(predictions) learning_time = time.process_time() - learning_time_start # print("np.var(target-predictions) ", np.var(targets - predictions)) + print("learning time ", learning_time) - if stats is True: - return gnn, targets, predictions, learning_time + orig_targets = targets + orig_predictions = predictions + + print("MSE ", np.mean((predictions-targets)**2)) if log: targets = np.exp(targets) predictions = np.exp(predictions) - plot_loss(gnn._train_loss, gnn._val_loss) - analyze_results(targets, predictions) + if not stats: + analyze_results(targets, predictions) + plot_loss(gnn._train_loss, gnn._val_loss) + analyze_results(targets, predictions) - import matplotlib.pyplot as plt + import matplotlib.pyplot as plt - plt.hist(targets, bins=50, alpha=0.5, label='target', density=True) - plt.hist(predictions, bins=50, alpha=0.5, label='predictions', density=True) + plt.hist(targets, bins=50, alpha=0.5, label='target', density=True) + plt.hist(predictions, bins=50, alpha=0.5, label='predictions', density=True) - # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) - plt.legend(loc='upper right') - # plt.xlim(-0.5, 1000) - plt.yscale('log') - plt.show() + # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) + plt.legend(loc='upper right') + # plt.xlim(-0.5, 1000) + plt.yscale('log') + plt.show() # plt.hist(np.exp(targets), bins=50, alpha=0.5, label='target', density=True) @@ -563,21 +605,33 @@ def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, # diff_moments(targets, predictions) predict_l_0_start_time = time.process_time() - l_0_targets, l_0_predictions = predict_level_zero(gnn, l_0_output_dir, l_0_hdf_path, mesh, batch_size, log) + l_0_targets, l_0_predictions = predict_level_zero(gnn, l_0_output_dir, l_0_hdf_path, mesh, conv_layer, batch_size, log, stats=stats) predict_l_0_time = time.process_time() - predict_l_0_start_time + if stats: + l1_sample_time = preprocess_time / len(data) + learning_time / len(data) + l0_sample_time = predict_l_0_time / len(l_0_targets) + + orig_max_vars, predict_max_vars = process_mlmc(hdf_path, sampling_info_path, ref_mlmc_file, targets, predictions, train_targets, + train_predictions, + val_targets, l_0_targets, + l_0_predictions, l1_sample_time, l0_sample_time, nn_level=nn_level, replace_level=replace_level, + stats=stats) + + return orig_targets, orig_predictions, learning_time, train_targets, train_predictions, orig_max_vars, predict_max_vars, total_steps + save_times(save_path, False, (preprocess_time, len(data)), learning_time, (predict_l_0_time, len(l_0_targets))) save_load_data(save_path, False, targets, predictions, train_targets, train_predictions, val_targets, l_0_targets, l_0_predictions) -def predict_level_zero(nn, output_dir, hdf_path, mesh, batch_size=1000, log=False): +def predict_level_zero(nn, output_dir, hdf_path, mesh, conv_layer, batch_size=1000, log=False, stats=False): #graph_creator(output_dir, hdf_path, mesh, level=0) # Load data data = FlowDataset(output_dir=output_dir, log=log) data = data # [:10000] - # data.a = conv_layer.preprocess(data.a) + data.a = conv_layer.preprocess(data.a) data.a = sp_matrix_to_sp_tensor(data.a) loader_te = MixedLoader(data, batch_size=batch_size) @@ -585,10 +639,14 @@ def predict_level_zero(nn, output_dir, hdf_path, mesh, batch_size=1000, log=Fals targets, predictions = nn.predict(loader_te) predictions = np.squeeze(predictions) + if not stats: + analyze_results(targets, predictions) + if log: targets = np.exp(targets) predictions = np.exp(predictions) - # analyze_results(targets, predictions) + if not stats: + analyze_results(targets, predictions) return targets, predictions @@ -682,117 +740,3 @@ def process_results(hdf_path, sampling_info_path, ref_mlmc_file, save_path, nn_l process_mlmc(hdf_path, sampling_info_path, ref_mlmc_file, targets, predictions, train_targets, train_predictions, val_targets, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, nn_level=nn_level, replace_level=replace_level) - - -def get_config(case=0): - if case == 0: - cl = "cl_0_1_s_1" - nn_level = 0 - replace_level = False - mesh = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/l_step_0.055_common_files/mesh.msh" - # mesh = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L2/l_step_0.027624156655057155_common_files/mesh.msh" - output_dir = "/home/martin/Documents/metamodels/data/1000_ele/{}/L5/test/01_cond_field/output/".format(cl) - hdf_path = "/home/martin/Documents/metamodels/data/1000_ele/{}/L5/mlmc_5.hdf5".format(cl) - save_path = "/home/martin/Documents/metamodels/data/1000_ele/{}".format(cl) - l_0_output_dir = "/home/martin/Documents/metamodels/data/1000_ele/{}/L{}/test/01_cond_field/output/".format(cl, nn_level + 1) - l_0_hdf_path = "/home/martin/Documents/metamodels/data/1000_ele/{}/L{}/mlmc_{}.hdf5".format(cl, nn_level + 1, nn_level + 1) - sampling_info_path = "/home/martin/Documents/metamodels/data/1000_ele/{}/sampling_info".format(cl) - ref_mlmc_file = "/home/martin/Documents/metamodels/data/1000_ele/{}/L1_benchmark/mlmc_1.hdf5".format(cl) - elif case == 1: - cl = "cl_0_3_s_4" - nn_level = 1 - replace_level = False - #mesh = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/l_step_0.055_common_files/mesh.msh" - mesh = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L2/l_step_0.027624156655057155_common_files/mesh.msh" - output_dir = "/home/martin/Documents/metamodels/data/1000_ele/{}/L5/test/01_cond_field/output/".format(cl) - hdf_path = "/home/martin/Documents/metamodels/data/1000_ele/{}/L5/mlmc_5.hdf5".format(cl) - save_path = "/home/martin/Documents/metamodels/data/1000_ele/{}".format(cl) - l_0_output_dir = "/home/martin/Documents/metamodels/data/1000_ele/{}/L{}/test/01_cond_field/output/".format(cl, nn_level + 1) - l_0_hdf_path = "/home/martin/Documents/metamodels/data/1000_ele/{}/L{}/mlmc_{}.hdf5".format(cl, nn_level + 1, nn_level) - sampling_info_path = "/home/martin/Documents/metamodels/data/1000_ele/{}/sampling_info".format(cl) - ref_mlmc_file = "/home/martin/Documents/metamodels/data/1000_ele/{}/L1_benchmark/mlmc_1.hdf5".format(cl) - - elif case == 2: - cl = "cl_0_1_s_1" - nn_level = 3 - replace_level = False - # mesh = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/l_step_0.055_common_files/mesh.msh" - mesh = "/home/martin/Documents/metamodels/data/5_ele/cl_0_1_s_1/L5/l_step_0.020196309484414757_common_files/mesh.msh" - output_dir = "/home/martin/Documents/metamodels/data/5_ele/{}/L5/test/01_cond_field/output/".format(cl) - hdf_path = "/home/martin/Documents/metamodels/data/5_ele/{}/L5/mlmc_5.hdf5".format(cl) - save_path = "/home/martin/Documents/metamodels/data/5_ele/{}".format(cl) - l_0_output_dir = "/home/martin/Documents/metamodels/data/5_ele/{}/L1_{}/test/01_cond_field/output/".format(cl, nn_level) - l_0_hdf_path = "/home/martin/Documents/metamodels/data/5_ele/{}/L1_{}/mlmc_1.hdf5".format(cl, nn_level) - sampling_info_path = "/home/martin/Documents/metamodels/data/5_ele/{}/sampling_info".format(cl) - ref_mlmc_file = "/home/martin/Documents/metamodels/data/5_ele/{}/L1_3/mlmc_1.hdf5".format(cl) - - elif case == 3 or case == 4: - cl = "cl_0_1_s_1" - if case == 4: - cl = "cl_0_3_s_4" - nn_level = 3 - replace_level = False - # mesh = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/l_step_0.055_common_files/mesh.msh" - mesh = "/home/martin/Documents/metamodels/data/5_ele/cl_0_1_s_1/L5/l_step_0.020196309484414757_common_files/mesh.msh" - output_dir = "/home/martin/Documents/metamodels/data/5_ele/{}/L5/test/01_cond_field/output/".format(cl) - hdf_path = "/home/martin/Documents/metamodels/data/5_ele/{}/L5/mlmc_5.hdf5".format(cl) - save_path = "/home/martin/Documents/metamodels/data/5_ele/{}".format(cl) - l_0_output_dir = "/home/martin/Documents/metamodels/data/5_ele/{}/L1_{}/test/01_cond_field/output/".format(cl,nn_level) - l_0_hdf_path = "/home/martin/Documents/metamodels/data/5_ele/{}/L1_{}/mlmc_1.hdf5".format(cl, nn_level) - sampling_info_path = "/home/martin/Documents/metamodels/data/5_ele/{}/sampling_info".format(cl) - ref_mlmc_file = "/home/martin/Documents/metamodels/data/5_ele/{}/L1_3/mlmc_1.hdf5".format(cl) - - elif case == 5: - cl = "cl_0_3_s_4" - nn_level = 0 - replace_level = False - # mesh = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/l_step_0.055_common_files/mesh.msh" - mesh = "/home/martin/Documents/metamodels/data/5_ele/cl_0_1_s_1/L5/l_step_0.020196309484414757_common_files/mesh.msh" - output_dir = "/home/martin/Documents/metamodels/data/5_ele/{}/L1_3/test/01_cond_field/output/".format(cl) - hdf_path = "/home/martin/Documents/metamodels/data/5_ele/{}/L1_3/mlmc_1.hdf5".format(cl) - save_path = "/home/martin/Documents/metamodels/data/5_ele/{}".format(cl) - l_0_output_dir = "/home/martin/Documents/metamodels/data/5_ele/{}/L1_{}/test/01_cond_field/output/".format(cl,nn_level) - l_0_hdf_path = "/home/martin/Documents/metamodels/data/5_ele/{}/L1_{}/mlmc_1.hdf5".format(cl, nn_level) - sampling_info_path = "/home/martin/Documents/metamodels/data/5_ele/{}/sampling_info".format(cl) - ref_mlmc_file = "/home/martin/Documents/metamodels/data/5_ele/{}/L1_3/mlmc_1.hdf5".format(cl) - - return output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, replace_level, nn_level, - - -if __name__ == "__main__": - case = 1 - output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, replace_level, nn_level = get_config( - case) - - # import cProfile - # import pstats - # pr = cProfile.Profile() - # pr.enable() - # - # my_result = run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level) - # - # pr.disable() - # ps = pstats.Stats(pr).sort_stats('cumtime') - # ps.print_stats() - - # gnn, _, _, _ = run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, stats=True) - # - # case = 4 - # output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, replace_level, nn_level = get_config(case) - - #run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, conv_layer=GCNConv, model=NetGCN) # , gnn=gnn) - - #run_SVR(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True) # , gnn=gnn) - #exit() - - run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True) # , gnn=gnn) - process_results(hdf_path, sampling_info_path, ref_mlmc_file, save_path, nn_level, replace_level) - - # analyze_bootstrap(save_path) - # statistics(run_GNN, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level) - # bootstrap_GNN() - # run() - # run_CNN() - # bootstrap() - - # gnn, targets, predictions, learning_time = run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, stats=True) diff --git a/src/mlmc/metamodel/flow_dataset.py b/src/mlmc/metamodel/flow_dataset.py index d50304c8..a6439120 100644 --- a/src/mlmc/metamodel/flow_dataset.py +++ b/src/mlmc/metamodel/flow_dataset.py @@ -1,32 +1,41 @@ import os import re +import random import numpy as np import pandas as pd from mlmc.tool import gmsh_io from spektral.data import Dataset, Graph import pickle -MESH = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/l_step_0.055_common_files/mesh.msh" +#MESH = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/l_step_0.055_common_files/mesh.msh" FIELDS_SAMPLE = "fine_fields_sample.msh" #OUTPUT_DIR = "/home/martin/Documents/metamodels/data/1000_ele/test/01_cond_field/output/" #OUTPUT_DIR = "/home/martin/Documents/metamodels/data/cl_0_3_s_4/L5/test/01_cond_field/output/" #OUTPUT_DIR = "/home/martin/Documents/metamodels/data/cl_0_1_s_1/L5/test/01_cond_field/output/" -OUTPUT_DIR = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L5/test/01_cond_field/output/" +#OUTPUT_DIR = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L5/test/01_cond_field/output/" class FlowDataset(Dataset): GRAPHS_FILE = "graphs" DATA_FILE = "data" - def __init__(self, output_dir=None, level=0, **kwargs): + def __init__(self, output_dir=None, level=0, log=False, **kwargs): self._output_dir = output_dir - if self._output_dir is None: - self._output_dir = OUTPUT_DIR + # if self._output_dir is None: + # self._output_dir = OUTPUT_DIR + self._log = log self.level = level self.adjacency_matrix = np.load(os.path.join(self._output_dir, "adjacency_matrix.npy"), allow_pickle=True) # adjacency matrix self.data = [] super().__init__(**kwargs) - self.a = self.adjacency_matrix + #self.a = self.adjacency_matrix + self.dataset = pd.DataFrame(self.data) + + def shuffle(self, seed=None): + if seed is not None: + random.seed(seed) + + random.shuffle(self.data) self.dataset = pd.DataFrame(self.data) def read(self): @@ -65,11 +74,14 @@ def read(self): # maximum = np.max(all_features) # minimum = np.min(all_features) # + # if self._log: + # minimum = np.log(minimum) + # maximum = np.log(maximum) + # # self.min_output = min_output # self.max_output = max_output # self.min_feature = minimum - # # self.max_output = maximum - + # self.max_feature = maximum graphs = [] for s_dir in os.listdir(self._output_dir): @@ -80,19 +92,11 @@ def read(self): except IndexError: continue - #print("s dir ", s_dir) - - if os.path.isdir(os.path.join(self._output_dir, s_dir)): sample_dir = os.path.join(self._output_dir, s_dir) if os.path.exists(os.path.join(sample_dir, "nodes_features.npy")): features = np.load(os.path.join(sample_dir, "nodes_features.npy")) output = np.load(os.path.join(sample_dir, "output.npy")) - # if i < 10: - # print("output ", output) - # else: - # exit() - # i += 1 #features = (features - minimum) / (maximum - minimum) # @@ -104,16 +108,23 @@ def read(self): # print("new featuers min ", np.min(new_features)) # exit() + if self._log: + features = np.log(features) + output = np.log(output) + + #features = (features - minimum) / (maximum - minimum) graphs.append(Graph(x=features, y=output))#, a=self.adjacency_matrix)) # Save data for pandas dataframe creation, not used with Graph neural network self.data.append({'x': features, 'y': output}) + + self.a = self.adjacency_matrix return graphs @staticmethod - def pickle_data(data, file_path): - with open(os.path.join(OUTPUT_DIR, file_path), 'wb') as writer: + def pickle_data(data, output_dir, file_path): + with open(os.path.join(output_dir, file_path), 'wb') as writer: pickle.dump(data, writer) diff --git a/src/mlmc/metamodel/flow_task_GNN_2.py b/src/mlmc/metamodel/flow_task_GNN_2.py index 156594fa..97df5775 100644 --- a/src/mlmc/metamodel/flow_task_GNN_2.py +++ b/src/mlmc/metamodel/flow_task_GNN_2.py @@ -37,24 +37,30 @@ def __init__(self, **kwargs): self._optimizer = kwargs.get('optimizer', tf.optimizers.Adam(learning_rate=0.001)) self._normalizer = kwargs.get('normalizer', preprocessing.Normalization()) self._patience = kwargs.get('patience', 20) + self._verbose = kwargs.get('verbose', True) self._train_loss = [] self._val_loss = [] self._test_loss = [] self.val_targets = [] + self._states = {} + self._total_n_steps = 0 model = kwargs.get('model') + if model is None: self._model = Net1(conv_layer=self._conv_layer, hidden_activation=self._hidden_activation, output_activation=self._output_activation, kernel_regularization=self._hidden_regularizer, normalizer=self._normalizer) else: - self._model = model(conv_layer=self._conv_layer, hidden_activation=self._hidden_activation, - output_activation=self._output_activation, - kernel_regularization=self._hidden_regularizer, - normalizer=self._normalizer) + self._model = model + # self._model = model(conv_layer=self._conv_layer, hidden_activation=self._hidden_activation, + # output_activation=self._output_activation, + # kernel_regularization=self._hidden_regularizer, + # normalizer=self._normalizer) + #self._model = model(n_labels=1, output_activation="relu") def fit(self, loader_tr, loader_va, loader_te): """ @@ -64,6 +70,7 @@ def fit(self, loader_tr, loader_va, loader_te): best_val_loss = np.inf current_patience = self._patience step = 0 + self._total_n_steps = 0 train_targets = True train_targets_list = [] @@ -72,13 +79,13 @@ def fit(self, loader_tr, loader_va, loader_te): results_tr = [] for batch in loader_tr: step += 1 + self._total_n_steps += 1 # Training step inputs, target = batch if train_targets: train_targets_list.extend(target) - print("len(train targets ", len(train_targets_list)) loss, acc = self.train_on_batch(inputs, target) self._train_loss.append(loss) @@ -91,13 +98,18 @@ def fit(self, loader_tr, loader_va, loader_te): train_targets = False # results_va = self.evaluate(loader_va) # self._val_loss.append(results_va[0]) + if results_va[0] < best_val_loss: best_val_loss = results_va[0] current_patience = self._patience + self._states = {} results_te = self.evaluate(loader_te) self._test_loss.append(results_te[0]) else: current_patience -= 1 + #results_tr_0 = np.array(results_tr) + # loss_tr = np.average(results_tr_0[:, :-1], 0, weights=results_tr_0[:, -1])[0] + # self._states[loss_tr] = self if current_patience == 0: print("Early stopping") break @@ -105,13 +117,14 @@ def fit(self, loader_tr, loader_va, loader_te): # Print results results_tr = np.array(results_tr) results_tr = np.average(results_tr[:, :-1], 0, weights=results_tr[:, -1]) - print( - "Train loss: {:.4f}, acc: {:.4f} | " - "Valid loss: {:.4f}, acc: {:.4f} | " - "Test loss: {:.4f}, acc: {:.4f}".format( - *results_tr, *results_va, *results_te + if self._verbose: + print( + "Train loss: {:.4f}, acc: {:.4f} | " + "Valid loss: {:.4f}, acc: {:.4f} | " + "Test loss: {:.4f}, acc: {:.4f}".format( + *results_tr, *results_va, *results_te + ) ) - ) # Reset epoch results_tr = [] step = 0 @@ -128,12 +141,14 @@ def train_on_batch(self, inputs, target): # print("self._loss(target, predictions) ", self._loss(target, predictions)) # print("sum(self._model.losses) ", sum(self._model.losses)) # print("var_loss_function(target, predictions) ", var_loss_function(target, predictions)) + loss = self._loss(target, predictions) + sum(self._model.losses) #+ 5 * var_loss_function(target, predictions) #loss = 100 * var_loss_function(target, predictions) acc = tf.reduce_mean(self._accuracy_func(target, predictions)) gradients = tape.gradient(loss, self._model.trainable_variables) self._optimizer.apply_gradients(zip(gradients, self._model.trainable_variables)) + return loss, acc def evaluate(self, loader): @@ -155,6 +170,8 @@ def evaluate(self, loader): predictions = self._model(inputs, training=False) loss = self._loss(target, predictions) + #print("target ", target) + #print("loss ", np.mean((target - predictions)**2)) acc = tf.reduce_mean(self._accuracy_func(target, predictions)) results.append((loss, acc, len(target))) # Keep track of batch size if step == loader.steps_per_epoch: diff --git a/src/mlmc/metamodel/graph_models.py b/src/mlmc/metamodel/graph_models.py index 75622922..beb7aea7 100644 --- a/src/mlmc/metamodel/graph_models.py +++ b/src/mlmc/metamodel/graph_models.py @@ -14,12 +14,18 @@ def __init__(self, conv_layer, hidden_activation, output_activation, kernel_regu super().__init__(**kwargs) #self.normalizer = normalizer #self.norm_layer = tf.keras.layers.LayerNormalization(axis=1) - self.conv1 = conv_layer(32, K=1, activation=hidden_activation, kernel_regularizer=kernel_regularization) - #self.conv2 = conv_layer(32, activation=hidden_activation, kernel_regularizer=kernel_regularization) + self.conv1 = conv_layer(256, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv2 = conv_layer(128, K=2, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv3 = conv_layer(16, K=2, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv4 = conv_layer(8, K=2,activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv2 = conv_layer(32, K=2, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv3 = conv_layer(16, K=2, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv3 = conv_layer(8, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv4 = conv_layer(4, activation=hidden_activation, kernel_regularizer=kernel_regularization) #self.conv3 = conv_layer(64, activation=hidden_activation, kernel_regularizer=kernel_regularization) self.flatten = GlobalSumPool() #self.fc1 = Dense(32, activation=hidden_activation) - self.fc2 = Dense(1, activation=output_activation) # linear activation for output neuron + self.fc2 = Dense(1)#, activation=output_activation) # linear activation for output neuron def call(self, inputs): x, a = inputs @@ -33,13 +39,19 @@ def call(self, inputs): #print("x[0,0,:] ", x[0,0,:]) # print("x[0, 0, :] ", tf.make_ndarray(x[0,0,:].op.get_attr('net1/strided_slice_1:0'))) - # print("x.shape ", x.shape) + #print("x.shape ", x.shape) # x = self.conv2([x, a]) + # # print("conv2 x shape", x.shape) # x = self.conv3([x, a]) - output = self.flatten(x) - #output = self.fc1(output) - output = self.fc2(output) - + # x = self.conv4([x, a]) + output1 = self.flatten(x) + #output2 = self.fc1(output1) + output = self.fc2(output1) + + # print("x1 " ,x1) + # print("output1 ", output1) + # print("output2 ", output2) + # print("output ", output) #print("output ", output.shape) return output @@ -50,15 +62,16 @@ class NetGCN(Model): # Setup from https://arxiv.org/pdf/1901.06181.pdf def __init__(self, conv_layer, hidden_activation, output_activation, kernel_regularization, normalizer, **kwargs): super().__init__(**kwargs) + #self.normalizer = normalizer #self.norm_layer = tf.keras.layers.LayerNormalization(axis=1) - self.conv1 = conv_layer(32, activation=hidden_activation, kernel_regularizer=kernel_regularization) - # self.conv2 = conv_layer(8, activation=hidden_activation, kernel_regularizer=kernel_regularization) + self.conv1 = conv_layer(256, activation=hidden_activation, kernel_regularizer=kernel_regularization) + #.conv2 = conv_layer(32, activation=hidden_activation, kernel_regularizer=kernel_regularization) # self.conv3 = conv_layer(16, activation=hidden_activation, kernel_regularizer=kernel_regularization) # self.conv4 = conv_layer(16, activation=hidden_activation, kernel_regularizer=kernel_regularization) # self.conv5 = conv_layer(32, activation=hidden_activation, kernel_regularizer=kernel_regularization) self.flatten = GlobalSumPool() - #self.fc1 = Dense(128, activation="linear") + #self.fc1 = Dense(16, activation=hidden_activation) self.fc2 = Dense(1)#, activation=output_activation) # linear activation for output neuron def call(self, inputs): @@ -70,7 +83,7 @@ def call(self, inputs): #print("x[0,0,:] ", x[0, 0, :]) x = self.conv1([x, a]) - # x = self.conv2([x, a]) + #x = self.conv2([x, a]) # x = self.conv3([x, a]) # x = self.conv4([x, a]) # x = self.conv5([x, a]) @@ -93,12 +106,13 @@ def cnn_model(): return keras.Sequential([ #@TODO: Try normalization #self._normalizer, # Seems worse results with normalization - layers.Conv1D(filters=256, kernel_size=3, activation='relu', input_shape=(6672, 1)),#input_shape=(958, 1)), + layers.Conv1D(filters=256, kernel_size=3, activation='relu', input_shape=(958, 1)),#input_shape=(958, 1)), #layers.BatchNormalization(), - layers.MaxPooling1D(pool_size=2), + layers.AveragePooling1D(pool_size=2), layers.Conv1D(filters=128, kernel_size=3, activation='relu'), #layers.BatchNormalization(), - layers.MaxPooling1D(pool_size=2), + layers.AveragePooling1D(pool_size=2), + layers.Flatten(), layers.Dense(64, activation='relu'), layers.Dense(1, activation=abs_activation) diff --git a/src/mlmc/metamodel/postprocessing.py b/src/mlmc/metamodel/postprocessing.py index ded4ea54..1ae62d1a 100644 --- a/src/mlmc/metamodel/postprocessing.py +++ b/src/mlmc/metamodel/postprocessing.py @@ -419,8 +419,11 @@ def get_n_estimated(sample_storage, estimator, n_ops=None): def get_storage_info(sample_storage): moments, estimator, _, _ = estimate_moments(sample_storage) - print("n collected ", sample_storage.get_n_collected()) - print("moments.l_vars max ", np.max(moments.l_vars, axis=1)) + n_collected = sample_storage.get_n_collected() + max_vars = np.max(moments.l_vars, axis=1) + print("n collected ", n_collected) + print("moments.l_vars max ", max_vars) + return n_collected, max_vars def cut_samples(data, sample_storage, new_n_collected, new_l_0=0): @@ -442,22 +445,32 @@ def cut_samples(data, sample_storage, new_n_collected, new_l_0=0): def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predictions, train_targets, train_predictions, val_targets, l_0_targets=None, l_0_predictions=None, - l1_sample_time=None, l0_sample_time=None, nn_level=0, replace_level=False): + l1_sample_time=None, l0_sample_time=None, nn_level=0, replace_level=False, stats=False): #level_zero = False - print("nn_level ", nn_level) - print("replace level ", replace_level) + if not stats: + print("nn_level ", nn_level) + print("replace level ", replace_level) - print("targets ", targets) - print("predictions ", predictions) - plt.hist(targets, bins=50, alpha=0.5, label='target', density=True) - plt.hist(predictions, bins=50, alpha=0.5, label='predictions', density=True) + # print("targets ", targets) + # print("predictions ", predictions) + plt.hist(targets, bins=50, alpha=0.5, label='target', density=True) + plt.hist(predictions, bins=50, alpha=0.5, label='predictions', density=True) - #plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) - plt.legend(loc='upper right') - #plt.xlim(-0.5, 1000) - plt.yscale('log') - plt.show() + #plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) + plt.legend(loc='upper right') + #plt.xlim(-0.5, 1000) + plt.yscale('log') + plt.show() + + plt.hist(l_0_targets, bins=50, alpha=0.5, label='l_0_target', density=True) + plt.hist(l_0_predictions, bins=50, alpha=0.5, label='l_0_predictions', density=True) + + # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) + plt.legend(loc='upper right') + # plt.xlim(-0.5, 1000) + plt.yscale('log') + plt.show() # targets = np.exp(targets) # predictions = np.exp(predictions) @@ -475,6 +488,16 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti sample_storage = SampleStorageHDF(file_path=mlmc_file) n_levels = len(sample_storage.get_level_ids()) original_moments, estimator, original_true_domain, _ = estimate_moments(sample_storage) + + ###### + ### Get n ops + ###### + n_ops, field_times, flow_times = get_sample_times(sampling_info_path) + + if n_ops is None: + n_ops = sample_storage.get_n_ops() + field_times = np.zeros(len(n_ops)) + flow_times = np.zeros(len(n_ops)) # Test storage creation data_mlmc = [] for l_id in range(n_levels): @@ -482,13 +505,7 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti data_mlmc.append(level_samples) sample_storage = create_quantity_mlmc(data_mlmc, level_parameters=sample_storage.get_level_parameters()) print("Original storage") - get_storage_info(sample_storage) - - ###### - ### Get n ops - ###### - n_ops, field_times, flow_times = get_sample_times(sampling_info_path) - print("n ops ", n_ops) + orig_storage_n_collected, orig_storage_max_vars = get_storage_info(sample_storage) if replace_level: n_lev = n_levels @@ -497,23 +514,16 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti n_lev = n_levels - nn_level + 1 level_params = [sample_storage.get_level_parameters()[nn_level], *sample_storage.get_level_parameters()[nn_level:]] - print("n lev ", n_lev) - print("level params ", level_params) - # Use predicted data as zero level results and level one coarse results data_nn = [] n_ops_predict = [] - nn_lev_sim_time = 0 for l_id in range(nn_level): nn_lev_sim_time = n_ops[l_id] - nn_lev_sim_time - print("nn lev sim time ", nn_lev_sim_time) - for l_id in range(n_lev): if l_id == 0: - print("np.mean(l 0 predictions) ", np.mean(l_0_predictions)) level_samples = l_0_predictions.reshape(1, len(l_0_predictions), 1) #level_samples = np.ones((1, len(l_0_predictions), 1)) ft_index = nn_level @@ -528,8 +538,8 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti n_ops_predict.append(n_ops[level_id]) else: if l_id == 1: - print("l id replaced level ", l_id) - print("len(predictions ) ", len(predictions)) + # print("l id replaced level ", l_id) + # print("len(predictions ) ", len(predictions)) coarse_level_samples = predictions.reshape(1, len(predictions), 1) #coarse_level_samples = np.ones((1, len(predictions), 1)) @@ -552,16 +562,18 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti level_id = l_id #- 1 else: level_id = l_id + nn_level - 1 - print("level id ", level_id) + #print("level id ", level_id) level_samples = estimator.get_level_samples(level_id=level_id) n_ops_predict.append(n_ops[level_id]) data_nn.append(level_samples) sample_storage_predict = create_quantity_mlmc(data_nn, level_parameters=level_params) print("n ops predict ", n_ops_predict) - print("Storage predict info") - get_storage_info(sample_storage_predict) + predict_storage_n_collected, predict_storage_max_vars = get_storage_info(sample_storage_predict) + + if stats: + return orig_storage_max_vars, predict_storage_max_vars ###### ### Create estimators @@ -578,15 +590,15 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti print("n ops ", n_ops) print("n ops predict ", n_ops_predict) - # remove levels - # #@TODO: remove asap - new_n_samples = [0, 0, sample_storage.get_n_collected()[-3], sample_storage.get_n_collected()[-2], sample_storage.get_n_collected()[-1]] - sample_storage = cut_samples(data_mlmc, sample_storage, new_n_samples, new_l_0=2) - print("Cut storage info") - get_storage_info(sample_storage) - original_q_estimator = get_quantity_estimator(sample_storage, true_domain=domain) - n_ops = n_ops[2:] - # ##### + # # remove levels + # # #@TODO: remove asap + # new_n_samples = [0, 0, sample_storage.get_n_collected()[-3], sample_storage.get_n_collected()[-2], sample_storage.get_n_collected()[-1]] + # sample_storage = cut_samples(data_mlmc, sample_storage, new_n_samples, new_l_0=2) + # print("Cut storage info") + # get_storage_info(sample_storage) + # original_q_estimator = get_quantity_estimator(sample_storage, true_domain=domain) + # n_ops = n_ops[2:] + # # ##### #### Original data n_ops_est = copy.deepcopy(n_ops) @@ -607,7 +619,7 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti #n_ops_predict_orig[0] = n_ops_predict_orig[0] /5 #n_ops_predict = np.array(n_ops_predict)**2 n_ops_predict[0] = n_ops_predict[0] / 1000 - #print("n ops predict for estimate ", n_ops_predict) + print("n ops predict for estimate ", n_ops_predict) # n_samples = [10000, 2000, 500, 150, 40, 11] n_estimated_nn, l_var_nn, n_samples_nn = get_n_estimated(sample_storage_predict, predict_q_estimator, @@ -898,8 +910,26 @@ def time_for_sample_func(data): #n_ops = np.mean(running_times, axis=0) # CPU time of simulation (fields + flow for both coarse and fine sample) print("field times ", field_times) + + print("n ops ", n_ops) + print("type n ops ", type(n_ops)) + + if np.isnan(np.all(n_ops)): + n_ops = None + return n_ops, field_times, flow_times +def plot_data(data, label): + plt.hist(data, alpha=0.5, label=label, density=True) + + #plt.hist(predictions, bins=50, alpha=0.5, label='predictions', density=True) + # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) + plt.legend(loc='upper right') + # plt.xlim(-0.5, 1000) + #plt.yscale('log') + plt.show() + + if __name__ == "__main__": analyze_mlmc_data() diff --git a/src/mlmc/sample_storage.py b/src/mlmc/sample_storage.py index bd9118f9..156e133b 100644 --- a/src/mlmc/sample_storage.py +++ b/src/mlmc/sample_storage.py @@ -242,6 +242,13 @@ def sample_pairs_level(self, chunk_spec): if chunk_spec.n_samples is not None and chunk_spec.n_samples < results.shape[0]\ else results.shape[0] + # Handle scalar simulation result + # @TODO: think it over again + if len(results.shape) != 3: + results = results.reshape(results.shape[0], results.shape[1], + 1 if np.prod(results.shape) == results.shape[0] * results.shape[1] else + int(np.prod(results.shape) / results.shape[0] * results.shape[1])) + # Remove auxiliary zeros from level zero sample pairs if chunk_spec.level_id == 0: results = results[:, :1, :] @@ -280,14 +287,14 @@ def unfinished_ids(self): def get_level_ids(self): return list(self._results.keys()) - def get_chunks_info(self, level_id, i_chunk): + def get_chunks_info(self, chunk_spec): """ The start and end index of a chunk from a whole dataset point of view :param level_id: level id :param i_chunk: chunk id :return: List[int, int] """ - return [0, len(self._results[level_id])-1] + return [0, len(self._results[chunk_spec.level_id])-1] def level_chunk_n_samples(self, level_id): """ @@ -303,8 +310,8 @@ def get_n_collected(self): :return: List """ n_collected = list(np.zeros(len(self._results))) - for level in self._results: - n_collected[int(level.level_id)] = level.collected_n_items() + for level_id, result in self._results.items(): + n_collected[int(level_id)] = len(result) return n_collected def get_n_levels(self): diff --git a/test/metamodels/__init__.py b/test/metamodels/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/metamodels/metamodel_test.py b/test/metamodels/metamodel_test.py new file mode 100644 index 00000000..a71cf6d5 --- /dev/null +++ b/test/metamodels/metamodel_test.py @@ -0,0 +1,238 @@ +import os + +#os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only +import sys +import subprocess +from mlmc.metamodel.analyze_nn import run_GNN, statistics, analyze_statistics +import tensorflow as tf + +from mlmc.metamodel.flow_task_GNN_2 import GNN +from spektral.layers import GCNConv, GlobalSumPool, ChebConv, GraphSageConv, ARMAConv, GATConv, APPNPConv, GINConv, GeneralConv +from tensorflow.keras.losses import MeanSquaredError, KLDivergence, MeanAbsoluteError +from tensorflow.keras import Model +from tensorflow.keras.layers import Dense +from spektral.layers import GlobalSumPool, GlobalMaxPool, GlobalAvgPool +import tensorflow as tf +from tensorflow.keras.layers.experimental import preprocessing + + +def get_gnn(): + # Parameters + # conv_layer = GCNConv + conv_layer = ChebConv # Seems better than GCNConv, good distribution of predictions + # conv_layer = OwnChebConv + # conv_layer = GraphSageConv # Seems better than ChebConv, good loss but very narrow distribution of predictions + # # conv_layer = ARMAConv # Seems worse than GraphSageConv + # conv_layer = GATConv # Slow and not better than GraphSageConv + # # conv_layer = APPNPConv # Not bad but worse than GraphSageConv + # # conv_layer = GINConv # it is comparable to APPNPConv + # act_func = "relu" # "tanh"#"elu" + + loss = MeanSquaredError() # var_loss_function# + # loss = MeanAbsoluteError() + # loss = MeanSquaredLogarithmicError() + # loss = KLDivergence() + # loss = total_loss_function + optimizer = tf.optimizers.Adam(learning_rate=0.001) + patience = 150 + hidden_regularization = None # l2(2e-10) + + model_config = { + "conv_layer": conv_layer, + "hidden_activation": 'relu', + "output_activation": 'linear', + "kernel_regularization": hidden_regularization, + "normalizer": preprocessing.Normalization() + } + + model = Net(**model_config) + + model_config = {"loss": loss, + "optimizer": optimizer, + "patience": patience, + "model": model, + "verbose": False} + + return GNN(**model_config), conv_layer + + +class Net(Model): + def __init__(self, conv_layer, hidden_activation, output_activation, kernel_regularization, normalizer, + **kwargs): + super().__init__(**kwargs) + # self.normalizer = normalizer + # self.norm_layer = tf.keras.layers.LayerNormalization(axis=1) + self.conv1 = conv_layer(32, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv2 = conv_layer(128, K=2, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv3 = conv_layer(16, K=2, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv4 = conv_layer(8, K=2,activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv2 = conv_layer(32, K=2, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv3 = conv_layer(16, K=2, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv3 = conv_layer(8, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv4 = conv_layer(4, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv3 = conv_layer(64, activation=hidden_activation, kernel_regularizer=kernel_regularization) + self.flatten = GlobalSumPool() + # self.fc1 = Dense(32, activation=hidden_activation) + self.fc2 = Dense(1) # , activation=output_activation) # linear activation for output neuron + + def call(self, inputs): + x, a = inputs + # print("x ", x) + # x = self.normalizer(x) + # x = self.norm_layer(x) + # print("normalized x ", x) + # print("x[0,0,:] ", x[0, 0, :]) + x = self.conv1([x, a]) + # print("x.shape ", x.shape) + # x = self.conv2([x, a]) + # # print("conv2 x shape", x.shape) + # x = self.conv3([x, a]) + # x = self.conv4([x, a]) + output1 = self.flatten(x) + # output2 = self.fc1(output1) + output = self.fc2(output1) + return output + + +def get_config(data_dir, case=0): + if case == 0: + cl = "cl_0_3_s_4" + nn_level = 0 + replace_level = False + mesh = os.path.join(data_dir, "l_step_0.055_common_files/mesh.msh") + output_dir = os.path.join(data_dir, "{}/L5/test/01_cond_field/output/".format(cl)) + hdf_path = os.path.join(data_dir, "{}/L5/mlmc_5.hdf5".format(cl)) + save_path = os.path.join(data_dir, "{}".format(cl)) + l_0_output_dir = os.path.join(data_dir, "{}/L{}/test/01_cond_field/output/".format(cl, nn_level + 1)) + l_0_hdf_path = os.path.join(data_dir, "{}/L{}/mlmc_{}.hdf5".format(cl, nn_level + 1, nn_level + 1)) + sampling_info_path = os.path.join(data_dir, "{}/sampling_info".format(cl)) + ref_mlmc_file = os.path.join(data_dir, "{}/L1_benchmark/mlmc_1.hdf5".format(cl)) + + elif case == 1: + cl = "cl_0_3_s_4" + nn_level = 1 + replace_level = False + mesh = os.path.join(data_dir, "/cl_0_1_s_1/L2/l_step_0.027624156655057155_common_files/mesh.msh") + output_dir = os.path.join(data_dir, "{}/L5/test/01_cond_field/output/".format(cl)) + hdf_path = os.path.join(data_dir, "{}/L5/mlmc_5.hdf5".format(cl)) + save_path = os.path.join(data_dir, "{}".format(cl)) + l_0_output_dir = os.path.join(data_dir, "{}/L{}/test/01_cond_field/output/".format(cl, nn_level + 1)) + l_0_hdf_path = os.path.join(data_dir, "{}/L{}/mlmc_{}.hdf5".format(cl, nn_level + 1, nn_level)) + sampling_info_path = os.path.join(data_dir, "{}/sampling_info".format(cl)) + ref_mlmc_file = os.path.join(data_dir, "{}/L1_benchmark/mlmc_1.hdf5".format(cl)) + + elif case == 2: + data_dir = "/home/martin/Documents/metamodels/data/5_ele/" + cl = "cl_0_1_s_1" + nn_level = 3 + replace_level = False + mesh = os.path.join(data_dir, "cl_0_1_s_1/L5/l_step_0.020196309484414757_common_files/mesh.msh") + output_dir = os.path.join(data_dir, "{}/L5/test/01_cond_field/output/".format(cl)) + hdf_path = os.path.join(data_dir, "{}/L5/mlmc_5.hdf5".format(cl)) + save_path = os.path.join(data_dir, "{}".format(cl)) + l_0_output_dir = os.path.join(data_dir, "{}/L1_{}/test/01_cond_field/output/".format(cl, nn_level)) + l_0_hdf_path = os.path.join(data_dir, "{}/L1_{}/mlmc_1.hdf5".format(cl, nn_level)) + sampling_info_path = os.path.join(data_dir, "{}/sampling_info".format(cl)) + ref_mlmc_file = os.path.join(data_dir, "{}/L1_3/mlmc_1.hdf5".format(cl)) + + elif case == 3 or case == 4: + data_dir = "/home/martin/Documents/metamodels/data/5_ele/" + cl = "cl_0_1_s_1" + if case == 4: + cl = "cl_0_3_s_4" + nn_level = 3 + replace_level = False + mesh = os.path.join(data_dir, "cl_0_1_s_1/L5/l_step_0.020196309484414757_common_files/mesh.msh") + output_dir = os.path.join(data_dir, "{}/L5/test/01_cond_field/output/".format(cl)) + hdf_path = os.path.join(data_dir, "{}/L5/mlmc_5.hdf5".format(cl)) + save_path = os.path.join(data_dir, "{}".format(cl)) + l_0_output_dir = os.path.join(data_dir, "{}/L1_{}/test/01_cond_field/output/".format(cl,nn_level)) + l_0_hdf_path = os.path.join(data_dir, "{}/L1_{}/mlmc_1.hdf5".format(cl, nn_level)) + sampling_info_path = os.path.join(data_dir, "{}/sampling_info".format(cl)) + ref_mlmc_file = os.path.join(data_dir, "{}/L1_3/mlmc_1.hdf5".format(cl)) + + elif case == 5: + data_dir = "/home/martin/Documents/metamodels/data/5_ele/" + cl = "cl_0_3_s_4" + nn_level = 0 + replace_level = False + mesh = os.path.join(data_dir, "cl_0_1_s_1/L5/l_step_0.020196309484414757_common_files/mesh.msh") + output_dir = os.path.join(data_dir,"{}/L1_3/test/01_cond_field/output/".format(cl)) + hdf_path = os.path.join(data_dir,"{}/L1_3/mlmc_1.hdf5".format(cl)) + save_path = os.path.join(data_dir,"{}".format(cl)) + l_0_output_dir = os.path.join(data_dir,"{}/L1_{}/test/01_cond_field/output/".format(cl,nn_level)) + l_0_hdf_path = os.path.join(data_dir,"{}/L1_{}/mlmc_1.hdf5".format(cl, nn_level)) + sampling_info_path = os.path.join(data_dir, "{}/sampling_info".format(cl)) + ref_mlmc_file = os.path.join(data_dir,"{}/L1_3/mlmc_1.hdf5".format(cl)) + # elif case == 6: + # nn_level = 0 + # replace_level = True + # mesh = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/l_step_0.055_common_files/mesh.msh" + # output_dir = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/output" + # hdf_path = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/mlmc_1.hdf5" + # + # save_path = "/home/martin/Documents/metamodels/data/L1/" + # l_0_output_dir = output_dir + # l_0_hdf_path = hdf_path + # sampling_info_path = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/" + # ref_mlmc_file = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L1_benchmark/mlmc_1.hdf5" + + return output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, replace_level, nn_level, + + +def get_arguments(arguments): + """ + Getting arguments from console + :param arguments: list of arguments + :return: namespace + """ + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('data_dir', help='data directory') + args = parser.parse_args(arguments) + return args + + +if __name__ == "__main__": + args = get_arguments(sys.argv[1:]) + data_dir = args.data_dir + case = 0 + #data_dir = "/home/martin/Documents/metamodels/data/1000_ele/" + output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, replace_level, nn_level = get_config( + data_dir, case) + + # import cProfile + # import pstats + # pr = cProfile.Profile() + # pr.enable() + # + # my_result = run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level) + # + # pr.disable() + # ps = pstats.Stats(pr).sort_stats('cumtime') + # ps.print_stats() + + #run_SVR(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True) # , gnn=gnn) + #run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True) # , gnn=gnn) + #run_CNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True) # , gnn=gnn) + #process_results(hdf_path, sampling_info_path, ref_mlmc_file, save_path, nn_level, replace_level) + + # gnn, conv_layer = mlmc.metamodel.nn_config.get_gnn() + # #run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, model=GCN, level=nn_level, log=True) # CGN model leads to constant value + # #run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True, gnn=gnn, conv_layer=conv_layer) + # run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True, conv_layer=conv_layer) + # process_results(hdf_path, sampling_info_path, ref_mlmc_file, save_path, nn_level, replace_level) + + gnn, conv_layer = get_gnn() + + # print("gnn ", gnn) + #print("conv layer ", conv_layer) + + #models = {"ChebConv": (run_GNN, False), "SVR": (run_SVR, False)} + models = {"ChebConvTEST": (run_GNN, True)} + save_path = os.path.join(save_path, "_".join(list(models.keys()))) + statistics(models, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, conv_layer=conv_layer, gnn=gnn) + analyze_statistics(save_path) + + # save_path = os.path.join(save_path, "SVR") + # statistics(run_SVR, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True) diff --git a/test/metamodels/nn_config.py b/test/metamodels/nn_config.py new file mode 100644 index 00000000..9b080a8e --- /dev/null +++ b/test/metamodels/nn_config.py @@ -0,0 +1,92 @@ +from mlmc.metamodel.flow_task_GNN_2 import GNN +from spektral.layers import GCNConv, GlobalSumPool, ChebConv, GraphSageConv, ARMAConv, GATConv, APPNPConv, GINConv, GeneralConv +from tensorflow.keras.losses import MeanSquaredError, KLDivergence, MeanAbsoluteError +from tensorflow.keras import Model +from tensorflow.keras.layers import Dense +from spektral.layers import GlobalSumPool, GlobalMaxPool, GlobalAvgPool +import tensorflow as tf +from tensorflow.keras.layers.experimental import preprocessing + + +def get_gnn(): + # Parameters + # conv_layer = GCNConv + conv_layer = ChebConv # Seems better than GCNConv, good distribution of predictions + # conv_layer = OwnChebConv + # conv_layer = GraphSageConv # Seems better than ChebConv, good loss but very narrow distribution of predictions + # # conv_layer = ARMAConv # Seems worse than GraphSageConv + # conv_layer = GATConv # Slow and not better than GraphSageConv + # # conv_layer = APPNPConv # Not bad but worse than GraphSageConv + # # conv_layer = GINConv # it is comparable to APPNPConv + # act_func = "relu" # "tanh"#"elu" + + loss = MeanSquaredError() # var_loss_function# + # loss = MeanAbsoluteError() + # loss = MeanSquaredLogarithmicError() + # loss = KLDivergence() + # loss = total_loss_function + optimizer = tf.optimizers.Adam(learning_rate=0.001) + patience = 150 + hidden_regularization = None # l2(2e-10) + + model_config = { + "conv_layer": conv_layer, + "hidden_activation": 'relu', + "output_activation": 'linear', + "kernel_regularization": hidden_regularization, + "normalizer": preprocessing.Normalization() + } + + model = Net(**model_config) + + model_config = {"loss": loss, + "optimizer": optimizer, + "patience": patience, + "model": model, + "verbose": False} + + print("get gnn") + + return GNN(**model_config), conv_layer + + +class Net(Model): + def __init__(self, conv_layer, hidden_activation, output_activation, kernel_regularization, normalizer, + **kwargs): + super().__init__(**kwargs) + # self.normalizer = normalizer + # self.norm_layer = tf.keras.layers.LayerNormalization(axis=1) + self.conv1 = conv_layer(32, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv2 = conv_layer(128, K=2, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv3 = conv_layer(16, K=2, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv4 = conv_layer(8, K=2,activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv2 = conv_layer(32, K=2, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv3 = conv_layer(16, K=2, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv3 = conv_layer(8, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv4 = conv_layer(4, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv3 = conv_layer(64, activation=hidden_activation, kernel_regularizer=kernel_regularization) + self.flatten = GlobalSumPool() + # self.fc1 = Dense(32, activation=hidden_activation) + self.fc2 = Dense(1) # , activation=output_activation) # linear activation for output neuron + + def call(self, inputs): + x, a = inputs + # print("x ", x) + # x = self.normalizer(x) + # x = self.norm_layer(x) + # print("normalized x ", x) + # print("x[0,0,:] ", x[0, 0, :]) + x = self.conv1([x, a]) + # print("x.shape ", x.shape) + # x = self.conv2([x, a]) + # # print("conv2 x shape", x.shape) + # x = self.conv3([x, a]) + # x = self.conv4([x, a]) + output1 = self.flatten(x) + # output2 = self.fc1(output1) + output = self.fc2(output1) + return output + + +if __name__ == "__main__": + get_gnn() From 6cb2b964324415ac9307ca94650175d4e8c624f7 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Mon, 5 Apr 2021 13:41:05 +0200 Subject: [PATCH 15/67] rm test import --- src/mlmc/metamodel/analyze_nn.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/mlmc/metamodel/analyze_nn.py b/src/mlmc/metamodel/analyze_nn.py index c63be63f..2a871276 100644 --- a/src/mlmc/metamodel/analyze_nn.py +++ b/src/mlmc/metamodel/analyze_nn.py @@ -5,7 +5,6 @@ os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only from mlmc.metamodel.flow_dataset import FlowDataset -import test.metamodels.nn_config # Make numpy printouts easier to read. # np.set_printoptions(precision=9, suppress=True) From c174b713776cccd22fb2ee22bb1c073a935eda25 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Mon, 5 Apr 2021 16:30:53 +0200 Subject: [PATCH 16/67] gnn stats --- src/mlmc/metamodel/analyze_nn.py | 16 +++++++++------- test/metamodels/metamodel_test.py | 12 +++++++----- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src/mlmc/metamodel/analyze_nn.py b/src/mlmc/metamodel/analyze_nn.py index 2a871276..9a7c72c0 100644 --- a/src/mlmc/metamodel/analyze_nn.py +++ b/src/mlmc/metamodel/analyze_nn.py @@ -300,8 +300,9 @@ def predict_level_zero_SVR(nn, output_dir, hdf_path, mesh, batch_size=1000, log= return targets, predictions -def statistics(models, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level, conv_layer, gnn=None): - n_subsamples = 50 +def statistics(models, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, + sampling_info_path, ref_mlmc_file, level, conv_layer, gnn=None, replace_level=False, ): + n_subsamples = 25 model_data = {} for model_key in models.keys(): @@ -327,7 +328,9 @@ def statistics(models, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_ targets, predictions, learning_time, train_targets, train_predictions, orig_max_vars, predict_max_vars, total_steps = model(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, - save_path, mesh, level=level, stats=True, log=log, conv_layer=conv_layer, gnn=gnn, train=True) + save_path, mesh, sampling_info_path, ref_mlmc_file, + level=level, stats=True, log=log, conv_layer=conv_layer, gnn=gnn, train=True, + replace_level=replace_level) model_data[model_key]["test_targets"].append(targets) model_data[model_key]["test_predictions"].append(predictions) @@ -350,7 +353,6 @@ def statistics(models, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_ save_statistics(save_path, model_data) - analyze_statistics(save_path) # plot_loss(train_losses, val_losses) @@ -461,8 +463,8 @@ def analyze_statistics(save_path): print("######################################") -def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level, conv_layer, stats=False, - gnn=None, model=None, log=False, train=True): +def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, level, conv_layer, stats=False, + gnn=None, model=None, log=False, train=True, replace_level=False): loss = MeanSquaredError() # var_loss_function# # loss = MeanAbsoluteError() @@ -614,7 +616,7 @@ def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, orig_max_vars, predict_max_vars = process_mlmc(hdf_path, sampling_info_path, ref_mlmc_file, targets, predictions, train_targets, train_predictions, val_targets, l_0_targets, - l_0_predictions, l1_sample_time, l0_sample_time, nn_level=nn_level, replace_level=replace_level, + l_0_predictions, l1_sample_time, l0_sample_time, nn_level=level, replace_level=replace_level, stats=stats) return orig_targets, orig_predictions, learning_time, train_targets, train_predictions, orig_max_vars, predict_max_vars, total_steps diff --git a/test/metamodels/metamodel_test.py b/test/metamodels/metamodel_test.py index a71cf6d5..7bb91df0 100644 --- a/test/metamodels/metamodel_test.py +++ b/test/metamodels/metamodel_test.py @@ -205,8 +205,9 @@ def get_arguments(arguments): # import pstats # pr = cProfile.Profile() # pr.enable() + # gnn, conv_layer = get_gnn() # - # my_result = run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level) + # my_result = run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True, conv_layer=conv_layer) # # pr.disable() # ps = pstats.Stats(pr).sort_stats('cumtime') @@ -217,9 +218,9 @@ def get_arguments(arguments): #run_CNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True) # , gnn=gnn) #process_results(hdf_path, sampling_info_path, ref_mlmc_file, save_path, nn_level, replace_level) - # gnn, conv_layer = mlmc.metamodel.nn_config.get_gnn() - # #run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, model=GCN, level=nn_level, log=True) # CGN model leads to constant value - # #run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True, gnn=gnn, conv_layer=conv_layer) + # gnn, conv_layer = get_gnn() + # # #run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, model=GCN, level=nn_level, log=True) # CGN model leads to constant value + # # #run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True, gnn=gnn, conv_layer=conv_layer) # run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True, conv_layer=conv_layer) # process_results(hdf_path, sampling_info_path, ref_mlmc_file, save_path, nn_level, replace_level) @@ -231,7 +232,8 @@ def get_arguments(arguments): #models = {"ChebConv": (run_GNN, False), "SVR": (run_SVR, False)} models = {"ChebConvTEST": (run_GNN, True)} save_path = os.path.join(save_path, "_".join(list(models.keys()))) - statistics(models, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, conv_layer=conv_layer, gnn=gnn) + statistics(models, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, level=nn_level, + conv_layer=conv_layer, gnn=gnn, replace_level=replace_level) analyze_statistics(save_path) # save_path = os.path.join(save_path, "SVR") From 6d734b2863625d9fa50e433b92959a0bb4f6f26d Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Tue, 6 Apr 2021 15:26:07 +0200 Subject: [PATCH 17/67] SVR refactoring --- src/mlmc/metamodel/analyze_nn.py | 87 +++++++++++++++++++------------ test/metamodels/metamodel_test.py | 33 ++++++------ 2 files changed, 71 insertions(+), 49 deletions(-) diff --git a/src/mlmc/metamodel/analyze_nn.py b/src/mlmc/metamodel/analyze_nn.py index 9a7c72c0..9c585ef0 100644 --- a/src/mlmc/metamodel/analyze_nn.py +++ b/src/mlmc/metamodel/analyze_nn.py @@ -5,6 +5,7 @@ os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only from mlmc.metamodel.flow_dataset import FlowDataset +from mlmc.metamodel.create_graph import graph_creator # Make numpy printouts easier to read. # np.set_printoptions(precision=9, suppress=True) @@ -166,16 +167,16 @@ def bootstrap(): # estimate_density(np.mean(all_predictions, axis=0), title="Predictions") -def run_SVR(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level, conv_layer=None, stats=False, - gnn=None, model=None, log=False): +def run_SVR(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, level, conv_layer, stats=False, + gnn=None, model=None, log=False, train=True, replace_level=False): from sklearn.svm import SVR - batch_size = 2000 + batch_size = 200 epochs = 1000 hidden_regularization = None # l2(2e-10) preprocess_start_time = time.process_time() - #graph_creator(output_dir, hdf_path, mesh, level=level) + graph_creator(output_dir, hdf_path, mesh, level=level) # Load data data = FlowDataset(output_dir=output_dir, level=level, log=log) @@ -202,8 +203,6 @@ def run_SVR(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, # train_output = sc_y.fit_transform(train_output.reshape(-1,1)) # test_input = sc_X.fit_transform(test_input) # test_output = sc_y.fit_transform(test_output.reshape(-1,1)) - - #train_input, train_output, test_input, test_output = split_dataset(dataset) @@ -217,10 +216,14 @@ def run_SVR(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, svr_rbf.fit(train_input, train_output) train_error = svr_rbf.score(train_input, train_output) + #print("svr_rbf.get_params() ", svr_rbf.get_params()) + total_steps = 0 + #test_input = sc_X.fit_transform(test_input) test_error = svr_rbf.score(test_input, test_output) targets = test_output + train_targets = train_output # test_y = sc_y.fit_transform(test.y.to_numpy().reshape(-1,1)) @@ -233,9 +236,13 @@ def run_SVR(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, #train_predictions = np.squeeze(train_predictions) learning_time = time.process_time() - learning_time_start + print("learning time ", learning_time) + + val_targets = [] - if stats is True: - return targets, predictions, learning_time, train_output, train_predictions + orig_targets = targets + orig_predictions = predictions + print("MSE ", np.mean((predictions - targets) ** 2)) if log: targets = np.exp(targets) @@ -243,40 +250,54 @@ def run_SVR(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, print("np.var(target-predictions) ", np.var(targets - predictions)) - #plot_loss(gnn._train_loss, gnn._val_loss) - analyze_results(targets, predictions) + if not stats: + #plot_loss(gnn._train_loss, gnn._val_loss) + analyze_results(targets, predictions) - import matplotlib.pyplot as plt + import matplotlib.pyplot as plt - # plt.hist(train_output, bins=50, alpha=0.5, label='train target', density=True) - # plt.hist(train_predictions, bins=50, alpha=0.5, label='train predictions', density=True) - # - # # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) - # plt.legend(loc='upper right') - # # plt.xlim(-0.5, 1000) - # plt.yscale('log') - # plt.show() + # plt.hist(train_output, bins=50, alpha=0.5, label='train target', density=True) + # plt.hist(train_predictions, bins=50, alpha=0.5, label='train predictions', density=True) + # + # # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) + # plt.legend(loc='upper right') + # # plt.xlim(-0.5, 1000) + # plt.yscale('log') + # plt.show() - plt.hist(targets, bins=50, alpha=0.5, label='target', density=True) - plt.hist(predictions, bins=50, alpha=0.5, label='predictions', density=True) + plt.hist(targets, bins=50, alpha=0.5, label='target', density=True) + plt.hist(predictions, bins=50, alpha=0.5, label='predictions', density=True) - # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) - plt.legend(loc='upper right') - # plt.xlim(-0.5, 1000) - plt.yscale('log') - plt.show() + # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) + plt.legend(loc='upper right') + # plt.xlim(-0.5, 1000) + plt.yscale('log') + plt.show() predict_l_0_start_time = time.process_time() - l_0_targets, l_0_predictions = predict_level_zero_SVR(svr_rbf, l_0_output_dir, l_0_hdf_path, mesh, batch_size, log) + l_0_targets, l_0_predictions = predict_level_zero_SVR(svr_rbf, l_0_output_dir, l_0_hdf_path, mesh, batch_size, log, stats=stats) predict_l_0_time = time.process_time() - predict_l_0_start_time + if stats: + l1_sample_time = preprocess_time / len(data) + learning_time / len(data) + l0_sample_time = predict_l_0_time / len(l_0_targets) + + orig_max_vars, predict_max_vars = process_mlmc(hdf_path, sampling_info_path, ref_mlmc_file, targets, + predictions, train_targets, + train_predictions, + val_targets, l_0_targets, + l_0_predictions, l1_sample_time, l0_sample_time, nn_level=level, + replace_level=replace_level, + stats=stats) + + return orig_targets, orig_predictions, learning_time, train_targets, train_predictions, orig_max_vars, predict_max_vars, total_steps + save_times(save_path, False, (preprocess_time, len(data)), learning_time, (predict_l_0_time, len(l_0_targets))) - save_load_data(save_path, False, targets, predictions, train_output, train_predictions, test_output, l_0_targets, + save_load_data(save_path, False, targets, predictions, train_targets, train_predictions, val_targets, l_0_targets, l_0_predictions) - -def predict_level_zero_SVR(nn, output_dir, hdf_path, mesh, batch_size=1000, log=False): - #graph_creator(output_dir, hdf_path, mesh, level=0) +def predict_level_zero_SVR(nn, output_dir, hdf_path, mesh, batch_size=1000, log=False, stats=False): + graph_creator(output_dir, hdf_path, mesh, level=0) # Load data data = FlowDataset(output_dir=output_dir, log=log) @@ -284,13 +305,13 @@ def predict_level_zero_SVR(nn, output_dir, hdf_path, mesh, batch_size=1000, log= test_input = prepare_data(dataset.x) targets = prepare_data(dataset.y) - print("data prepared") + #print("data prepared") predictions = [] for i in range(0, len(test_input), batch_size): predictions.extend(nn.predict(test_input[i:i + batch_size])) predictions = np.array(predictions) - print("predictison shape ", predictions.shape) + #print("predictison shape ", predictions.shape) predictions = np.squeeze(predictions) if log: diff --git a/test/metamodels/metamodel_test.py b/test/metamodels/metamodel_test.py index 7bb91df0..e76b4566 100644 --- a/test/metamodels/metamodel_test.py +++ b/test/metamodels/metamodel_test.py @@ -3,7 +3,7 @@ #os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only import sys import subprocess -from mlmc.metamodel.analyze_nn import run_GNN, statistics, analyze_statistics +from mlmc.metamodel.analyze_nn import run_GNN, run_SVR, statistics, analyze_statistics, process_results import tensorflow as tf from mlmc.metamodel.flow_task_GNN_2 import GNN @@ -205,7 +205,7 @@ def get_arguments(arguments): # import pstats # pr = cProfile.Profile() # pr.enable() - # gnn, conv_layer = get_gnn() + gnn, conv_layer = get_gnn() # # my_result = run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True, conv_layer=conv_layer) # @@ -213,28 +213,29 @@ def get_arguments(arguments): # ps = pstats.Stats(pr).sort_stats('cumtime') # ps.print_stats() - #run_SVR(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True) # , gnn=gnn) + run_SVR(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, level=nn_level, log=True, conv_layer=conv_layer) # , gnn=gnn) + #run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True) # , gnn=gnn) #run_CNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True) # , gnn=gnn) - #process_results(hdf_path, sampling_info_path, ref_mlmc_file, save_path, nn_level, replace_level) + process_results(hdf_path, sampling_info_path, ref_mlmc_file, save_path, nn_level, replace_level) # gnn, conv_layer = get_gnn() # # #run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, model=GCN, level=nn_level, log=True) # CGN model leads to constant value # # #run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True, gnn=gnn, conv_layer=conv_layer) # run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True, conv_layer=conv_layer) # process_results(hdf_path, sampling_info_path, ref_mlmc_file, save_path, nn_level, replace_level) - - gnn, conv_layer = get_gnn() - - # print("gnn ", gnn) - #print("conv layer ", conv_layer) - - #models = {"ChebConv": (run_GNN, False), "SVR": (run_SVR, False)} - models = {"ChebConvTEST": (run_GNN, True)} - save_path = os.path.join(save_path, "_".join(list(models.keys()))) - statistics(models, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, level=nn_level, - conv_layer=conv_layer, gnn=gnn, replace_level=replace_level) - analyze_statistics(save_path) + # + # gnn, conv_layer = get_gnn() + # + # # print("gnn ", gnn) + # #print("conv layer ", conv_layer) + # + # #models = {"ChebConv": (run_GNN, False), "SVR": (run_SVR, False)} + # models = {"SVRTEST": (run_SVR, True)} + # save_path = os.path.join(save_path, "_".join(list(models.keys()))) + # statistics(models, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, level=nn_level, + # conv_layer=conv_layer, gnn=gnn, replace_level=replace_level) + # analyze_statistics(save_path) # save_path = os.path.join(save_path, "SVR") # statistics(run_SVR, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True) From 3057eb713fc8cedbde4281e6d8f34d0cc293f489 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Sat, 10 Apr 2021 11:41:14 +0200 Subject: [PATCH 18/67] stats --- src/mlmc/metamodel/analyze_nn.py | 353 +++++++++++++++--------- src/mlmc/metamodel/create_graph.py | 6 +- src/mlmc/metamodel/custom_methods.py | 13 + src/mlmc/metamodel/flow_dataset.py | 47 +++- src/mlmc/metamodel/flow_task_GNN_2.py | 11 +- src/mlmc/metamodel/postprocessing.py | 11 +- src/mlmc/metamodel/random_field_time.py | 93 +++++++ src/mlmc/moments.py | 114 ++++++++ test/metamodels/compare_models.py | 0 test/metamodels/metamodel_test.py | 89 +++--- 10 files changed, 551 insertions(+), 186 deletions(-) create mode 100644 test/metamodels/compare_models.py diff --git a/src/mlmc/metamodel/analyze_nn.py b/src/mlmc/metamodel/analyze_nn.py index 9c585ef0..b67d73a1 100644 --- a/src/mlmc/metamodel/analyze_nn.py +++ b/src/mlmc/metamodel/analyze_nn.py @@ -2,17 +2,21 @@ import numpy as np import time import glob +import pickle os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only from mlmc.metamodel.flow_dataset import FlowDataset from mlmc.metamodel.create_graph import graph_creator +from mlmc.moments import Legendre_tf, Monomial +from mlmc.metamodel.random_field_time import corr_field_sample_time # Make numpy printouts easier to read. # np.set_printoptions(precision=9, suppress=True) import tensorflow as tf +from tensorflow import keras from scipy.stats import ks_2samp import sklearn.model_selection -from mlmc.metamodel.custom_methods import abs_activation +from mlmc.metamodel.custom_methods import abs_activation, MSE_moments from mlmc.metamodel.postprocessing import analyze_results, plot_loss, estimate_density, process_mlmc from mlmc.metamodel.flow_task_NN import DNN from mlmc.metamodel.flow_task_CNN import CNN @@ -168,7 +172,7 @@ def bootstrap(): def run_SVR(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, level, conv_layer, stats=False, - gnn=None, model=None, log=False, train=True, replace_level=False): + gnn=None, model=None, log=False, train=True, replace_level=False, corr_field_config=None): from sklearn.svm import SVR batch_size = 200 @@ -274,9 +278,11 @@ def run_SVR(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, plt.yscale('log') plt.show() - predict_l_0_start_time = time.process_time() - l_0_targets, l_0_predictions = predict_level_zero_SVR(svr_rbf, l_0_output_dir, l_0_hdf_path, mesh, batch_size, log, stats=stats) - predict_l_0_time = time.process_time() - predict_l_0_start_time + #predict_l_0_start_time = time.process_time() + l_0_targets, l_0_predictions, predict_l_0_time = predict_level_zero_SVR(svr_rbf, l_0_output_dir, l_0_hdf_path, mesh, + batch_size, log, stats=stats, corr_field_config=corr_field_config) + #predict_l_0_time = time.process_time() - predict_l_0_start_time + if stats: l1_sample_time = preprocess_time / len(data) + learning_time / len(data) @@ -296,13 +302,18 @@ def run_SVR(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, save_load_data(save_path, False, targets, predictions, train_targets, train_predictions, val_targets, l_0_targets, l_0_predictions) -def predict_level_zero_SVR(nn, output_dir, hdf_path, mesh, batch_size=1000, log=False, stats=False): - graph_creator(output_dir, hdf_path, mesh, level=0) + +def predict_level_zero_SVR(nn, output_dir, hdf_path, mesh, batch_size=1000, log=False, stats=False, corr_field_config=None): + #graph_creator(output_dir, hdf_path, mesh, level=0) + sample_time = 0 + if corr_field_config: + sample_time = corr_field_sample_time(mesh, corr_field_config) # Load data data = FlowDataset(output_dir=output_dir, log=log) dataset = data.dataset[:] + predict_time_start = time.process_time() test_input = prepare_data(dataset.x) targets = prepare_data(dataset.y) #print("data prepared") @@ -318,25 +329,18 @@ def predict_level_zero_SVR(nn, output_dir, hdf_path, mesh, batch_size=1000, log= targets = np.exp(targets) predictions = np.exp(predictions) # analyze_results(targets, predictions) - return targets, predictions + predict_time = time.process_time() - predict_time_start + return targets, predictions, predict_time + sample_time * len(data) -def statistics(models, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, - sampling_info_path, ref_mlmc_file, level, conv_layer, gnn=None, replace_level=False, ): +def statistics(machine_learning_model, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, + sampling_info_path, ref_mlmc_file, level, conv_layer, gnn=None, replace_level=False, + corr_field_config=None, graph_creation_time=0): n_subsamples = 25 + model_title, mch_l_model, log = machine_learning_model model_data = {} - for model_key in models.keys(): - model_data[model_key] = {} - model_data[model_key]["test_targets"] = [] - model_data[model_key]["test_predictions"] = [] - model_data[model_key]["train_targets"] = [] - model_data[model_key]["train_predictions"] = [] - model_data[model_key]["learning_times"] = [] - model_data[model_key]["orig_max_vars"] = [] - model_data[model_key]["predict_max_vars"] = [] - model_data[model_key]["total_steps"] = [] - model_data[model_key]["log"] = models[model_key][1] + model_data["log"] = log if not os.path.isdir(save_path): os.makedirs(save_path) @@ -345,22 +349,31 @@ def statistics(models, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_ exit() for i in range(n_subsamples): - for model_key, (model, log) in models.items(): - - targets, predictions, learning_time, train_targets, train_predictions, orig_max_vars, predict_max_vars, total_steps = model(output_dir, - hdf_path, l_0_output_dir, l_0_hdf_path, - save_path, mesh, sampling_info_path, ref_mlmc_file, - level=level, stats=True, log=log, conv_layer=conv_layer, gnn=gnn, train=True, - replace_level=replace_level) - - model_data[model_key]["test_targets"].append(targets) - model_data[model_key]["test_predictions"].append(predictions) - model_data[model_key]["train_targets"].append(train_targets) - model_data[model_key]["train_predictions"].append(train_predictions) - model_data[model_key]["orig_max_vars"].append(orig_max_vars) - model_data[model_key]["predict_max_vars"].append(predict_max_vars) - model_data[model_key]["total_steps"].append(total_steps) - model_data[model_key]["learning_times"].append(learning_time) + iter_dir = os.path.join(save_path, "{}".format(i)) + if not os.path.isdir(iter_dir): + os.makedirs(iter_dir) + + model, targets, predictions, learning_time, train_targets, train_predictions, \ + val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, total_steps = \ + mch_l_model(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, + level=level, stats=True, log=log, conv_layer=conv_layer, gnn=gnn, train=True, + replace_level=replace_level, corr_field_config=corr_field_config, + graph_creation_time=graph_creation_time) + + model_data["model"] = model + model_data["test_targets"] = targets + model_data["test_predictions"] = predictions + model_data["train_targets"] = train_targets + model_data["train_predictions"] = train_predictions + model_data["val_targets"] = val_targets + model_data["val_predictions"] = val_predictions + model_data["l_0_targets"] = l_0_targets + model_data["l_0_predictions"] = l_0_predictions + model_data["l1_sample_time"] = l1_sample_time + model_data["l0_sample_time"] = l0_sample_time + model_data["total_steps"] = total_steps + model_data["learning_times"] = learning_time + save_statistics(iter_dir, model_data) # save_times(save_path, False, (preprocess_time, len(data)), learning_time, (predict_l_0_time, len(l_0_targets))) # save_load_data(save_path, False, targets, predictions, train_targets, val_targets, l_0_targets, l_0_predictions) @@ -373,8 +386,8 @@ def statistics(models, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_ # print("##################################################") - save_statistics(save_path, model_data) - analyze_statistics(save_path) + analyze_statistics(machine_learning_model, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, + sampling_info_path, ref_mlmc_file, level, conv_layer, gnn=None, replace_level=False, corr_field_config=None) # plot_loss(train_losses, val_losses) # analyze_results(np.mean(all_test_outputs, axis=0), np.mean(all_predictions, axis=0)) @@ -384,110 +397,148 @@ def statistics(models, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_ # estimate_density(np.mean(all_predictions, axis=0), title="Predictions") -def save_statistics(dir_path, model_data): - dirs = (os.path.split(dir_path)[-1]).split("_") - for data_dir in dirs: - if not os.path.isdir(os.path.join(dir_path, data_dir)): - os.makedirs(os.path.join(dir_path, data_dir)) +def save_statistics(save_dir_path, model_data): + for file_name, data in model_data.items(): + if file_name == "model": + data.save(os.path.join(save_dir_path, file_name)) else: - print("dir exists {}".format(os.path.join(dir_path, data_dir))) - exit() - - print("dirs ", dirs) - for key, data_dict in model_data.items(): - path = os.path.join(dir_path, key) - for file_name, data in data_dict.items(): - np.save(os.path.join(path, file_name), data) + np.save(os.path.join(save_dir_path, file_name), data) def load_statistics(dir_path): models_data = {} - - dirs = (os.path.split(dir_path)[-1]).split("_") - - for data_dir in dirs: - models_data[data_dir] = {} - data_dir_path = os.path.join(dir_path, data_dir) + models_data["model"] = [] + models_data["test_targets"] = [] + models_data["test_predictions"] = [] + models_data["train_targets"] = [] + models_data["train_predictions"] = [] + models_data["val_targets"] = [] + models_data["val_predictions"] = [] + models_data["l_0_targets"] = [] + models_data["l_0_predictions"] = [] + models_data["l1_sample_time"] = [] + models_data["l0_sample_time"] = [] + models_data["total_steps"] = [] + models_data["learning_times"] = [] + models_data["log"] = [] + + #dirs = (os.path.split(dir_path)[-1]).split("_") + n_iters = 25 + for i in range(n_iters): + data_dir_path = os.path.join(dir_path, str(i)) if not os.path.isdir(data_dir_path): print("data dir not exists {}".format(data_dir_path)) - exit() + break + models_data['model'].append(keras.models.load_model(os.path.join(data_dir_path,'model'))) for file in glob.glob(os.path.join(data_dir_path, "*.npy")): file_name = os.path.split(file)[-1] file_name = file_name.split(".")[0] - models_data[data_dir][file_name] = np.load(file) + models_data[file_name].append(np.load(file)) return models_data -def analyze_statistics(save_path): - print("save path ", save_path) +def analyze_statistics(machine_learning_model, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, + sampling_info_path, ref_mlmc_file, level, conv_layer, gnn=None, replace_level=False, corr_field_config=None): + #print("save path ", save_path) if not os.path.isdir(save_path): print("dir not exists") exit() - models_data = load_statistics(save_path) + data_dict = load_statistics(save_path) - for key, data_dict in models_data.items(): - print("model: {}".format(key)) + #print("data dict ", data_dict) - if data_dict["log"]: - data_dict["test_targets"] = np.exp(data_dict["test_targets"]) - data_dict["test_predictions"] = np.exp(data_dict["test_predictions"]) - data_dict["train_targets"] = np.exp(data_dict["train_targets"]) - data_dict["train_predictions"] = np.exp(data_dict["train_predictions"]) + # for key, data_dict in models_data.items(): + # print("model: {}".format(key)) - print("test targets ", data_dict["test_targets"]) - print("test predictions ", data_dict["test_predictions"]) + # for i in range(len(data_dict["test_targets"])): + # predictions = data_dict["test_predictions"][i] + # targets = data_dict["test_targets"][i] + # train_predictions = data_dict["train_predictions"][i] + # train_targets = data_dict["train_targets"][i] + # val_predictions = data_dict["val_predictions"][i] + # val_targets = data_dict["val_targets"][i] + # l_0_predictions = data_dict["l_0_predictions"][i] + # l_0_targets = data_dict["l_0_targets"][i] + # l1_sample_time = data_dict["l1_sample_time"][i] + # l0_sample_time = data_dict["l0_sample_time"][i] + # + # process_mlmc(hdf_path, sampling_info_path, ref_mlmc_file, targets, + # predictions, train_targets, + # train_predictions, + # val_targets, l_0_targets, + # l_0_predictions, l1_sample_time, l0_sample_time, + # nn_level=level, + # replace_level=replace_level, + # stats=False) + + + data_dict["test_targets"] = np.array(data_dict["test_targets"]) + data_dict["test_predictions"] = np.array(data_dict["test_predictions"]) + data_dict["train_targets"] = np.array(data_dict["train_targets"]) + data_dict["train_predictions"] = np.array(data_dict["train_predictions"]) + if data_dict["log"]: + data_dict["test_targets"] = np.exp(data_dict["test_targets"]) + data_dict["test_predictions"] = np.exp(data_dict["test_predictions"]) + data_dict["train_targets"] = np.exp(data_dict["train_targets"]) + data_dict["train_predictions"] = np.exp(data_dict["train_predictions"]) + + # print("test targets ", data_dict["test_targets"]) + # print("test predictions ", data_dict["test_predictions"]) + # + # print("orig max vars ", data_dict["orig_max_vars"]) + # print("predict max vars ", data_dict["predict_max_vars"]) - print("orig max vars ", data_dict["orig_max_vars"]) - print("predict max vars ", data_dict["predict_max_vars"]) + # mean_orig_vars = np.mean(data_dict["orig_max_vars"], axis=0) + # mean_predict_vars = np.mean(data_dict["predict_max_vars"], axis=0) + total_steps = np.mean(data_dict["total_steps"]) - mean_orig_vars = np.mean(data_dict["orig_max_vars"], axis=0) - mean_predict_vars = np.mean(data_dict["predict_max_vars"], axis=0) - total_steps = np.mean(data_dict["total_steps"]) + # print("mean orig vars ", mean_orig_vars) + # print("mean predict vars ", mean_predict_vars) + print("total steps ", total_steps) - print("mean orig vars ", mean_orig_vars) - print("mean predict vars ", mean_predict_vars) - print("total steps ", total_steps) + test_MSE = np.mean((data_dict["test_predictions"] - data_dict["test_targets"])**2, axis=1) + test_RMSE = np.sqrt(test_MSE) + test_MAE = np.mean(np.abs(data_dict["test_predictions"] - data_dict["test_targets"]), axis=1) - test_MSE = np.mean((data_dict["test_predictions"] - data_dict["test_targets"])**2, axis=1) - test_RMSE = np.sqrt(test_MSE) - test_MAE = np.mean(np.abs(data_dict["test_predictions"] - data_dict["test_targets"]), axis=1) + train_MSE = np.mean((data_dict["train_predictions"] - data_dict["train_targets"]) ** 2, axis=1) + train_RMSE = np.sqrt(train_MSE) + train_MAE = np.mean(np.abs( data_dict["train_predictions"] - data_dict["train_targets"]), axis=1) + learning_times = data_dict["learning_times"] - train_MSE = np.mean((data_dict["train_predictions"] - data_dict["train_targets"]) ** 2, axis=1) - train_RMSE = np.sqrt(train_MSE) - train_MAE = np.mean(np.abs( data_dict["train_predictions"] - data_dict["train_targets"]), axis=1) - learning_times = data_dict["learning_times"] + # plot_data(test_MSE, label="test MSE") + # plot_data(test_MAE, label="test MAE") - # plot_data(test_MSE, label="test MSE") - # plot_data(test_MAE, label="test MAE") + print("test_MSE ", test_MSE) - print("mean test MSE ", np.mean(test_MSE)) - print("mean test RMSE ", np.mean(test_RMSE)) - print("mean test MAE ", np.mean(test_MAE)) - print("max test MSE ", np.max(test_MSE)) - print("max test RMSE ", np.max(test_RMSE)) - print("max test MAE ", np.max(test_MAE)) + print("mean test MSE ", np.mean(test_MSE)) + print("mean test RMSE ", np.mean(test_RMSE)) + print("mean test MAE ", np.mean(test_MAE)) + print("max test MSE ", np.max(test_MSE)) + print("max test RMSE ", np.max(test_RMSE)) + print("max test MAE ", np.max(test_MAE)) - print("mean train MSE ", np.mean(train_MSE)) - print("mean train RMSE ", np.mean(train_RMSE)) - print("mean train MAE ", np.mean(train_MAE)) - print("max train MSE ", np.max(train_MSE)) - print("max train RMSE ", np.max(train_RMSE)) - print("max train MAE ", np.max(train_MAE)) + print("mean train MSE ", np.mean(train_MSE)) + print("mean train RMSE ", np.mean(train_RMSE)) + print("mean train MAE ", np.mean(train_MAE)) + print("max train MSE ", np.max(train_MSE)) + print("max train RMSE ", np.max(train_RMSE)) + print("max train MAE ", np.max(train_MAE)) - print("mean learning time ", np.mean(learning_times)) - print("max learning time ", np.max(learning_times)) + print("mean learning time ", np.mean(learning_times)) + print("max learning time ", np.max(learning_times)) - print("######################################") + print("######################################") def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, level, conv_layer, stats=False, - gnn=None, model=None, log=False, train=True, replace_level=False): + gnn=None, model=None, log=False, train=True, replace_level=False, corr_field_config=None, graph_creation_time=0): loss = MeanSquaredError() # var_loss_function# + accuracy_func = MSE_moments # loss = MeanAbsoluteError() # loss = MeanSquaredLogarithmicError() #loss = KLDivergence() @@ -497,21 +548,42 @@ def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, epochs = 2000#1000 hidden_regularization = None # l2(2e-10) - preprocess_start_time = time.process_time() - #graph_creator(output_dir, hdf_path, mesh, level=level) + if graph_creation_time == 0: + graph_creator_preproces_time = time.process_time() + graph_creator(output_dir, hdf_path, mesh, level=level) + graph_creation_time = time.process_time() - graph_creator_preproces_time + print("graph creation time ", graph_creation_time) + exit() + preprocess_start_time = time.process_time() # Load data data = FlowDataset(output_dir=output_dir, level=level, log=log) data = data#[:10000] + + #print("len data ", len(data)) data.shuffle() preprocess_time = time.process_time() - preprocess_start_time + #print("preproces time ", preprocess_time) + preprocess_time = preprocess_time + graph_creation_time + print("total preprocess time ", preprocess_time) + learning_time_start = time.process_time() data.a = conv_layer.preprocess(data.a) data.a = sp_matrix_to_sp_tensor(data.a) #train_data_len = int(len(data) * 0.8) train_data_len = 2000 # Train/valid/test split - data_tr, data_te = data[:train_data_len], data[train_data_len:], + data_tr, data_te = data[:train_data_len], data[train_data_len:] + + if hasattr(gnn._loss,'__name__') and gnn._loss.__name__ == "MSE_moments": + tr_output = [g.y for g in data_tr] + n_moments = 3 + quantile = 0.0001 + domain = np.percentile(tr_output, [100 * quantile, 100 * (1 - quantile)]) + moments_fn = Legendre_tf(n_moments, domain) + #accuracy_func = MSE_moments(moments_fn=moments_fn) + gnn._loss = MSE_moments(moments_fn=moments_fn) + np.random.shuffle(data_tr) val_data_len = int(len(data_tr) * 0.2) data_tr, data_va = data_tr[:-val_data_len], data_tr[-val_data_len:] @@ -528,7 +600,7 @@ def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, if gnn is None: gnn = GNN(loss=loss, optimizer=optimizer, conv_layer=conv_layer, output_activation=abs_activation, hidden_activation='relu', patience=150, hidden_reqularizer=hidden_regularization, - model=model) # tanh takes to much time + model=model, accuracy_func=accuracy_func) # tanh takes to much time # ideally patience = 150 # batch_size 500, ideally 500 epochs, patience 35 @@ -537,30 +609,28 @@ def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, train_targets = gnn.fit(loader_tr, loader_va, loader_te) # states = gnn._states - # print("list(states.keys() ", list(states.keys())) - # min_key = np.min(list(states.keys())) - # print("min key ", min_key) - # print("states[min_key] ", states[min_key]) - # gnn = states[min_key] + # if len(states) > 0: + # min_key = np.min(list(states.keys())) + # gnn = states[min_key] train_targets, train_predictions = gnn.predict(loader_tr) train_predictions = np.squeeze(train_predictions) - val_targets = gnn.val_targets + val_targets, val_predictions = gnn.predict(loader_va) + val_predictions = np.squeeze(val_predictions) + #val_targets = gnn.val_targets total_steps = gnn._total_n_steps targets, predictions = gnn.predict(loader_te) predictions = np.squeeze(predictions) learning_time = time.process_time() - learning_time_start + #print("learning time ", learning_time) - # print("np.var(target-predictions) ", np.var(targets - predictions)) - print("learning time ", learning_time) - - orig_targets = targets - orig_predictions = predictions + targets = np.array(targets) + predictions = np.array(predictions) - print("MSE ", np.mean((predictions-targets)**2)) + #print("MSE ", np.mean((predictions-targets)**2)) if log: targets = np.exp(targets) @@ -626,33 +696,43 @@ def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, # diff_moments(targets, predictions) - predict_l_0_start_time = time.process_time() - l_0_targets, l_0_predictions = predict_level_zero(gnn, l_0_output_dir, l_0_hdf_path, mesh, conv_layer, batch_size, log, stats=stats) - predict_l_0_time = time.process_time() - predict_l_0_start_time + #predict_l_0_start_time = time.process_time() + l_0_targets, l_0_predictions, predict_l_0_time = predict_level_zero(gnn, l_0_output_dir, l_0_hdf_path, mesh, conv_layer, batch_size, log, stats=stats, corr_field_config=corr_field_config) + #predict_l_0_time = time.process_time() - predict_l_0_start_time if stats: l1_sample_time = preprocess_time / len(data) + learning_time / len(data) l0_sample_time = predict_l_0_time / len(l_0_targets) - orig_max_vars, predict_max_vars = process_mlmc(hdf_path, sampling_info_path, ref_mlmc_file, targets, predictions, train_targets, - train_predictions, - val_targets, l_0_targets, - l_0_predictions, l1_sample_time, l0_sample_time, nn_level=level, replace_level=replace_level, - stats=stats) + # print("targets ", targets) + # print("predictions ", predictions) - return orig_targets, orig_predictions, learning_time, train_targets, train_predictions, orig_max_vars, predict_max_vars, total_steps + # orig_max_vars, predict_max_vars = process_mlmc(hdf_path, sampling_info_path, ref_mlmc_file, targets, predictions, train_targets, + # train_predictions, + # val_targets, l_0_targets, + # l_0_predictions, l1_sample_time, l0_sample_time, nn_level=level, replace_level=replace_level, + # stats=stats) + + return gnn._model, targets, predictions, learning_time, train_targets, train_predictions,\ + val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, total_steps save_times(save_path, False, (preprocess_time, len(data)), learning_time, (predict_l_0_time, len(l_0_targets))) save_load_data(save_path, False, targets, predictions, train_targets, train_predictions, val_targets, l_0_targets, l_0_predictions) -def predict_level_zero(nn, output_dir, hdf_path, mesh, conv_layer, batch_size=1000, log=False, stats=False): +def predict_level_zero(nn, output_dir, hdf_path, mesh, conv_layer, batch_size=1000, log=False, stats=False, corr_field_config=None): #graph_creator(output_dir, hdf_path, mesh, level=0) - # Load data - data = FlowDataset(output_dir=output_dir, log=log) - data = data # [:10000] + + sample_time = 0 + if corr_field_config: + sample_time = corr_field_sample_time(mesh, corr_field_config) + + data = FlowDataset(output_dir=output_dir, log=log)#, mesh=mesh, corr_field_config=corr_field_config) + #data = data # [:10000] + + predict_time_start = time.process_time() data.a = conv_layer.preprocess(data.a) data.a = sp_matrix_to_sp_tensor(data.a) @@ -669,7 +749,10 @@ def predict_level_zero(nn, output_dir, hdf_path, mesh, conv_layer, batch_size=10 predictions = np.exp(predictions) if not stats: analyze_results(targets, predictions) - return targets, predictions + + predict_time = time.process_time() - predict_time_start + + return targets, predictions, predict_time + sample_time*len(data) def save_times(path, load=False, preprocess=None, learning_time=None, predict_l_0=None): diff --git a/src/mlmc/metamodel/create_graph.py b/src/mlmc/metamodel/create_graph.py index 0a90a861..2c2eecbb 100644 --- a/src/mlmc/metamodel/create_graph.py +++ b/src/mlmc/metamodel/create_graph.py @@ -156,9 +156,9 @@ def graph_creator(output_dir, hdf_path, mesh, level=0): if __name__ == "__main__": - mesh = "/home/martin/Documents/metamodels/data/5_ele/cl_0_1_s_1/L5/l_step_0.020196309484414757_common_files/mesh.msh" - output_dir = "/home/martin/Documents/metamodels/data/5_ele/cl_0_3_s_4/L1_3/test/01_cond_field/output/" - hdf_path = "/home/martin/Documents/metamodels/data/5_ele/cl_0_1_s_1/L1_3/mlmc_1.hdf5" + # mesh = "/home/martin/Documents/metamodels/data/5_ele/cl_0_1_s_1/L5/l_step_0.020196309484414757_common_files/mesh.msh" + # output_dir = "/home/martin/Documents/metamodels/data/5_ele/cl_0_3_s_4/L1_3/test/01_cond_field/output/" + # hdf_path = "/home/martin/Documents/metamodels/data/5_ele/cl_0_1_s_1/L1_3/mlmc_1.hdf5" import cProfile import pstats diff --git a/src/mlmc/metamodel/custom_methods.py b/src/mlmc/metamodel/custom_methods.py index b2f2cf80..a76d1680 100644 --- a/src/mlmc/metamodel/custom_methods.py +++ b/src/mlmc/metamodel/custom_methods.py @@ -1,5 +1,9 @@ +import numpy as np import tensorflow as tf from tensorflow.keras import backend as K +import tensorflow.experimental.numpy as tnp +tnp.experimental_enable_numpy_behavior() +from mlmc.moments import Monomial, Legendre def abs_activation(x): @@ -27,3 +31,12 @@ def total_loss_function(y_true, y_predict): #return K.var(K.abs(y_true - K.squeeze(y_predict, axis=1))) return K.mean((y_true - K.squeeze(y_predict, axis=1))**2) + K.var(K.abs(y_true - K.squeeze(y_predict, axis=1))) + + +def MSE_moments(moments_fn=None): + if moments_fn is None: + raise ValueError + + def calc_err(y_true, y_predict): + return K.mean(K.sum((moments_fn.eval_all(y_true) - moments_fn.eval_all(y_predict))**2, axis=1)) + return calc_err \ No newline at end of file diff --git a/src/mlmc/metamodel/flow_dataset.py b/src/mlmc/metamodel/flow_dataset.py index a6439120..eb22a3cd 100644 --- a/src/mlmc/metamodel/flow_dataset.py +++ b/src/mlmc/metamodel/flow_dataset.py @@ -4,6 +4,7 @@ import numpy as np import pandas as pd from mlmc.tool import gmsh_io +from mlmc.tool.flow_mc import FlowSim, create_corr_field from spektral.data import Dataset, Graph import pickle @@ -19,12 +20,14 @@ class FlowDataset(Dataset): GRAPHS_FILE = "graphs" DATA_FILE = "data" - def __init__(self, output_dir=None, level=0, log=False, **kwargs): + def __init__(self, output_dir=None, level=0, log=False, mesh=None, corr_field_config=None, **kwargs): self._output_dir = output_dir # if self._output_dir is None: # self._output_dir = OUTPUT_DIR self._log = log self.level = level + self._mesh = mesh + self._corr_field_config = corr_field_config self.adjacency_matrix = np.load(os.path.join(self._output_dir, "adjacency_matrix.npy"), allow_pickle=True) # adjacency matrix self.data = [] super().__init__(**kwargs) @@ -38,7 +41,49 @@ def shuffle(self, seed=None): random.shuffle(self.data) self.dataset = pd.DataFrame(self.data) + # def generate_data(self): + # n_samples = 10**5 + # graphs = [] + # mesh_data = FlowSim.extract_mesh(self._mesh) + # fields = create_corr_field(model="exp", dim=2, + # sigma=self._corr_field_config['sigma'], + # corr_length=self._corr_field_config['corr_length'], + # log=self._corr_field_config['log']) + # + # # # Create fields both fine and coarse + # fields = FlowSim.make_fields(fields, mesh_data, None) + # + # for i in range(n_samples): + # fine_input_sample, coarse_input_sample = FlowSim.generate_random_sample(fields, coarse_step=0, + # n_fine_elements=len( + # mesh_data['points'])) + # # print("len fine input sample ", len(fine_input_sample["conductivity"])) + # # print("fine input sample ", fine_input_sample["conductivity"]) + # + # features = fine_input_sample["conductivity"] + # output = 1 + # + # # gmsh_io.GmshIO().write_fields('fields_sample.msh', mesh_data['ele_ids'], fine_input_sample) + # # + # # mesh = gmsh_io.GmshIO('fields_sample.msh') + # # element_data = mesh.current_elem_data + # # features = list(element_data.values()) + # + # if self._log: + # features = np.log(features) + # output = np.log(output) + # # features = (features - minimum) / (maximum - minimum) + # graphs.append(Graph(x=features, y=output)) # , a=self.adjacency_matrix)) + # # Save data for pandas dataframe creation, not used with Graph neural network + # self.data.append({'x': features, 'y': output}) + # + # self.a = self.adjacency_matrix + # return graphs + def read(self): + # if self._mesh is not None: + # return self.generate_data() + # with open(os.path.join(OUTPUT_DIR, FlowDataset.GRAPHS_FILE), 'rb') as reader: # graphs = pickle.loads(reader) # diff --git a/src/mlmc/metamodel/flow_task_GNN_2.py b/src/mlmc/metamodel/flow_task_GNN_2.py index 97df5775..34af162a 100644 --- a/src/mlmc/metamodel/flow_task_GNN_2.py +++ b/src/mlmc/metamodel/flow_task_GNN_2.py @@ -98,6 +98,7 @@ def fit(self, loader_tr, loader_va, loader_te): train_targets = False # results_va = self.evaluate(loader_va) # self._val_loss.append(results_va[0]) + #print("results_va[0] ", results_va[0]) if results_va[0] < best_val_loss: best_val_loss = results_va[0] @@ -108,8 +109,8 @@ def fit(self, loader_tr, loader_va, loader_te): else: current_patience -= 1 #results_tr_0 = np.array(results_tr) - # loss_tr = np.average(results_tr_0[:, :-1], 0, weights=results_tr_0[:, -1])[0] - # self._states[loss_tr] = self + loss_tr = results_va[0] + self._states[loss_tr] = self if current_patience == 0: print("Early stopping") break @@ -136,12 +137,6 @@ def fit(self, loader_tr, loader_va, loader_te): def train_on_batch(self, inputs, target): with tf.GradientTape() as tape: predictions = self._model(inputs, training=True) - # @TODO: try to add KLDivergence to loss - # print(KLDivergence(target, predictions)) - # print("self._loss(target, predictions) ", self._loss(target, predictions)) - # print("sum(self._model.losses) ", sum(self._model.losses)) - # print("var_loss_function(target, predictions) ", var_loss_function(target, predictions)) - loss = self._loss(target, predictions) + sum(self._model.losses) #+ 5 * var_loss_function(target, predictions) #loss = 100 * var_loss_function(target, predictions) acc = tf.reduce_mean(self._accuracy_func(target, predictions)) diff --git a/src/mlmc/metamodel/postprocessing.py b/src/mlmc/metamodel/postprocessing.py index 1ae62d1a..a1f2a64f 100644 --- a/src/mlmc/metamodel/postprocessing.py +++ b/src/mlmc/metamodel/postprocessing.py @@ -399,7 +399,7 @@ def get_quantity_estimator(sample_storage, true_domain=None): def get_n_estimated(sample_storage, estimator, n_ops=None): - target_var = 1e-5 + target_var = 1e-4 #moments, estimator, _, quantity = estimate_moments(sample_storage, true_domain=true_domain) n_level_samples = sample_storage.get_n_collected() @@ -529,7 +529,10 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti ft_index = nn_level if nn_level > 0: ft_index = nn_level - 1 - n_ops_predict.append(l0_sample_time + field_times[ft_index] / 2) + # print("l0_sample_time ", l0_sample_time) + # print("fields times ", field_times[ft_index] / 2) + + n_ops_predict.append(l0_sample_time)# + field_times[ft_index] / 2) #level_samples = level_samples[:, :50000, :] else: if replace_level: @@ -602,7 +605,7 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti #### Original data n_ops_est = copy.deepcopy(n_ops) - n_ops_est[0] = n_ops_est[0] / 1000 + #n_ops_est[0] = n_ops_est[0] / 1000 n_estimated_orig, l_vars_orig, n_samples_orig = get_n_estimated(sample_storage, original_q_estimator, n_ops=n_ops_est) print("n estimated orig ", n_estimated_orig) sample_storage = cut_samples(data_mlmc, sample_storage, n_estimated_orig) @@ -618,7 +621,7 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti n_ops_predict_orig = copy.deepcopy(n_ops_predict) #n_ops_predict_orig[0] = n_ops_predict_orig[0] /5 #n_ops_predict = np.array(n_ops_predict)**2 - n_ops_predict[0] = n_ops_predict[0] / 1000 + #n_ops_predict[0] = n_ops_predict[0] / 1000 print("n ops predict for estimate ", n_ops_predict) # n_samples = [10000, 2000, 500, 150, 40, 11] diff --git a/src/mlmc/metamodel/random_field_time.py b/src/mlmc/metamodel/random_field_time.py index e69de29b..da931af6 100644 --- a/src/mlmc/metamodel/random_field_time.py +++ b/src/mlmc/metamodel/random_field_time.py @@ -0,0 +1,93 @@ +import time +import numpy as np +from mlmc.tool import gmsh_io +from mlmc.tool.flow_mc import FlowSim, create_corr_field + + +def corr_field_sample_time(mesh_file=None, corr_length_config=None): + import matplotlib + from matplotlib import ticker, cm + #matplotlib.rcParams.update({'font.size': 22}) + dim = 2 + log = True + cl = 0.1 + s = 1 + + if mesh_file is None: + #mesh_file = "/home/martin/Sync/Documents/flow123d_results/flow_experiments/Exponential/corr_length_0_01/l_step_0.0055_common_files/mesh.msh" + #mesh_file = "/home/martin/Documents/metamodels/data/5_ele/cl_0_1_s_1/L5/l_step_0.020196309484414757_common_files/mesh.msh" + mesh_file = "/home/martin/Documents/metamodels/data/1000_ele/l_step_0.055_common_files/mesh.msh" + + + start_time = time.process_time() + mesh_data = FlowSim.extract_mesh(mesh_file) + if corr_length_config is not None: + fields = create_corr_field(model="exp", dim=dim, + sigma=corr_length_config['sigma'], + corr_length=corr_length_config['corr_length'], + log=corr_length_config['log']) + else: + fields = create_corr_field(model="exp", dim=dim, + sigma=s, + corr_length=cl, + log=log) + # # Create fields both fine and coarse + fields = FlowSim.make_fields(fields, mesh_data, None) + + n_samples = 200 + for i in range(n_samples): + + fine_input_sample, coarse_input_sample = FlowSim.generate_random_sample(fields, coarse_step=0, + n_fine_elements=len( + mesh_data['points'])) + + len(fine_input_sample["conductivity"]) + features_log = np.log(fine_input_sample["conductivity"]) + output = 1 + # + # print("fine input sample ", fine_input_sample["conductivity"].shape) + # + # gmsh_io.GmshIO().write_fields('fields_sample.msh', mesh_data['ele_ids'], fine_input_sample) + # + # mesh = gmsh_io.GmshIO('fields_sample.msh') + # element_data = mesh.current_elem_data + # features = list(element_data.values()) + # print("features ", np.array(features).shape) + + rnd_time = time.process_time() - start_time + print("rnd_time / n_samples ", rnd_time / n_samples) + return rnd_time / n_samples + + #Xfinal, Yfinal = fields.fields[0].correlated_field.points[:, 0], fields.fields[0].correlated_field.points[:, 1] + + # cont = ax.tricontourf(Xfinal, + # Yfinal, + # fine_input_sample['conductivity'].ravel())#, locator=ticker.LogLocator()) + + # fig.colorbar(cont) + # fig.savefig("cl_{}_var_{}.pdf".format(cl, s ** 2)) + # plt.show() + + # print("fields ", fields) + # model = gs.Exponential(dim=2, len_scale=cl) + # srf = gs.SRF(model, mesh_type="unstructed", seed=20170519, mode_no=1000, generator='RandMeth') + # print("model.var ", model.var) + # field = srf( + # (fields.fields[0].correlated_field.points[:, 0], fields.fields[0].correlated_field.points[:, 1])) + # srf.vtk_export("field") + # ax = srf.plot() + # ax.set_aspect("equal") + + +if __name__ == "__main__": + import cProfile + import pstats + pr = cProfile.Profile() + pr.enable() + + my_result = corr_field_sample_time() + + pr.disable() + ps = pstats.Stats(pr).sort_stats('cumtime') + ps.print_stats() + diff --git a/src/mlmc/moments.py b/src/mlmc/moments.py index accebb9e..58919409 100644 --- a/src/mlmc/moments.py +++ b/src/mlmc/moments.py @@ -1,5 +1,13 @@ import numpy as np import numpy.ma as ma +import tensorflow as tf +import tensorflow.experimental.numpy as tnp +tnp.experimental_enable_numpy_behavior() +import tensorflow_graphics as tfg +import tensorflow_graphics.math.spherical_harmonics as tfgms + + + from scipy.interpolate import BSpline @@ -108,6 +116,23 @@ def eval_diff2(self, value, size=None): return self._eval_diff2(value, size) +class Moments_tf(Moments): + + def clip(self, value): + """ + Remove outliers and replace them with NaN + :param value: array of numbers + :return: masked_array, out + """ + #print("ref domain ", self.ref_domain) + #print("value ", value) + # Masked array + return value + # out = tnp.ma.masked_outside(value, self.ref_domain[0], self.ref_domain[1]) + # # Replace outliers with NaN + # return tnp.ma.filled(out, np.nan) + + class Monomial(Moments): def __init__(self, size, domain=(0, 1), ref_domain=None, log=False, safe_eval=True): if ref_domain is not None: @@ -220,6 +245,95 @@ def _eval_diff2(self, value, size): return P_n @ self.diff2_mat +class Legendre_tf(Moments_tf): + + def __init__(self, size, domain, ref_domain=None, log=False, safe_eval=True): + if ref_domain is not None: + self.ref_domain = ref_domain + else: + self.ref_domain = (-1, 1) + + self.diff_mat = np.zeros((size, size)) + for n in range(size - 1): + self.diff_mat[n, n + 1::2] = 2 * n + 1 + self.diff2_mat = self.diff_mat @ self.diff_mat + + super().__init__(size, domain, log, safe_eval) + + def _eval_value(self, x, size): + return tnp.polynomial.legendre.legvander(x, deg=size-1) + + def _eval_all(self, value, size): + value = self.transform(tnp.atleast_1d(value)) + #print("value ", value.shape) + + + # out = tfgms.evaluate_legendre_polynomial(degree_l=size-1, order_m=1, x=1) + # + # + + # return out + leg_poly = [] + for i in range(1, size): + out = Legendre_tf.P(i, value) + + # print("type out ", type(out)) + # print("out shape ", out.shape) + out = tf.convert_to_tensor(tf.squeeze(out)) + out = tf.dtypes.cast(out, tf.float64) + + #print("out shape ", out.shape) + + leg_poly.append(out) + + leg_poly = tf.stack(leg_poly, 0) + # print("leg poly shape ", leg_poly.shape) + # print("leg poly T shape ", tf.transpose(leg_poly).shape) + + return tf.transpose(leg_poly) + + @staticmethod + def P(n, x): + if (n == 0): + return np.ones(x.shape) # P0 = 1 + elif (n == 1): + return x # P1 = x + else: + return (((2 * n) - 1) * x * Legendre_tf.P(n - 1, x) - (n - 1) * Legendre_tf.P(n - 2, x)) / float(n) + + def _eval_all_der(self, value, size, degree=1): + """ + Derivative of Legendre polynomials + :param value: values to evaluate + :param size: number of moments + :param degree: degree of derivative + :return: + """ + value = self.transform(np.atleast_1d(value)) + eval_values = np.empty((value.shape + (size,))) + + for s in range(size): + if s == 0: + coef = [1] + else: + coef = np.zeros(s+1) + coef[-1] = 1 + + coef = np.polynomial.legendre.legder(coef, degree) + eval_values[:, s] = np.polynomial.legendre.legval(value, coef) + return eval_values + + def _eval_diff(self, value, size): + t = self.transform(np.atleast_1d(value)) + P_n = np.polynomial.legendre.legvander(t, deg=size - 1) + return P_n @ self.diff_mat + + def _eval_diff2(self, value, size): + t = self.transform(np.atleast_1d(value)) + P_n = np.polynomial.legendre.legvander(t, deg=size - 1) + return P_n @ self.diff2_mat + + class TransformedMoments(Moments): def __init__(self, other_moments, matrix): """ diff --git a/test/metamodels/compare_models.py b/test/metamodels/compare_models.py new file mode 100644 index 00000000..e69de29b diff --git a/test/metamodels/metamodel_test.py b/test/metamodels/metamodel_test.py index e76b4566..0cb62e27 100644 --- a/test/metamodels/metamodel_test.py +++ b/test/metamodels/metamodel_test.py @@ -4,11 +4,12 @@ import sys import subprocess from mlmc.metamodel.analyze_nn import run_GNN, run_SVR, statistics, analyze_statistics, process_results -import tensorflow as tf +from mlmc.moments import Legendre_tf, Monomial from mlmc.metamodel.flow_task_GNN_2 import GNN from spektral.layers import GCNConv, GlobalSumPool, ChebConv, GraphSageConv, ARMAConv, GATConv, APPNPConv, GINConv, GeneralConv from tensorflow.keras.losses import MeanSquaredError, KLDivergence, MeanAbsoluteError +from mlmc.metamodel.custom_methods import abs_activation, MSE_moments from tensorflow.keras import Model from tensorflow.keras.layers import Dense from spektral.layers import GlobalSumPool, GlobalMaxPool, GlobalAvgPool @@ -29,6 +30,7 @@ def get_gnn(): # act_func = "relu" # "tanh"#"elu" loss = MeanSquaredError() # var_loss_function# + #loss = MSE_moments # loss = MeanAbsoluteError() # loss = MeanSquaredLogarithmicError() # loss = KLDivergence() @@ -40,7 +42,8 @@ def get_gnn(): model_config = { "conv_layer": conv_layer, "hidden_activation": 'relu', - "output_activation": 'linear', + "output_activation": abs_activation, + #"output_activation": 'linear', "kernel_regularization": hidden_regularization, "normalizer": preprocessing.Normalization() } @@ -53,7 +56,9 @@ def get_gnn(): "model": model, "verbose": False} - return GNN(**model_config), conv_layer + corr_field_config = {'corr_length': 0.1, 'sigma': 1, 'log': True} + + return GNN(**model_config), conv_layer, corr_field_config class Net(Model): @@ -62,18 +67,18 @@ def __init__(self, conv_layer, hidden_activation, output_activation, kernel_regu super().__init__(**kwargs) # self.normalizer = normalizer # self.norm_layer = tf.keras.layers.LayerNormalization(axis=1) - self.conv1 = conv_layer(32, activation=hidden_activation, kernel_regularizer=kernel_regularization) - # self.conv2 = conv_layer(128, K=2, activation=hidden_activation, kernel_regularizer=kernel_regularization) - # self.conv3 = conv_layer(16, K=2, activation=hidden_activation, kernel_regularizer=kernel_regularization) - # self.conv4 = conv_layer(8, K=2,activation=hidden_activation, kernel_regularizer=kernel_regularization) - # self.conv2 = conv_layer(32, K=2, activation=hidden_activation, kernel_regularizer=kernel_regularization) + self.conv1 = conv_layer(32, K=1, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv2 = conv_layer(64, K=1, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv3 = conv_layer(32, K=1, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv4 = conv_layer(32, K=1, activation=hidden_activation, kernel_regularizer=kernel_regularization) + #self.conv2 = conv_layer(32, K=2, activation=hidden_activation, kernel_regularizer=kernel_regularization) # self.conv3 = conv_layer(16, K=2, activation=hidden_activation, kernel_regularizer=kernel_regularization) # self.conv3 = conv_layer(8, activation=hidden_activation, kernel_regularizer=kernel_regularization) # self.conv4 = conv_layer(4, activation=hidden_activation, kernel_regularizer=kernel_regularization) # self.conv3 = conv_layer(64, activation=hidden_activation, kernel_regularizer=kernel_regularization) self.flatten = GlobalSumPool() # self.fc1 = Dense(32, activation=hidden_activation) - self.fc2 = Dense(1) # , activation=output_activation) # linear activation for output neuron + self.fc2 = Dense(1)#, activation=output_activation) # linear activation for output neuron def call(self, inputs): x, a = inputs @@ -85,7 +90,7 @@ def call(self, inputs): x = self.conv1([x, a]) # print("x.shape ", x.shape) # x = self.conv2([x, a]) - # # print("conv2 x shape", x.shape) + # # # print("conv2 x shape", x.shape) # x = self.conv3([x, a]) # x = self.conv4([x, a]) output1 = self.flatten(x) @@ -109,10 +114,10 @@ def get_config(data_dir, case=0): ref_mlmc_file = os.path.join(data_dir, "{}/L1_benchmark/mlmc_1.hdf5".format(cl)) elif case == 1: - cl = "cl_0_3_s_4" + cl = "cl_0_1_s_1" nn_level = 1 replace_level = False - mesh = os.path.join(data_dir, "/cl_0_1_s_1/L2/l_step_0.027624156655057155_common_files/mesh.msh") + mesh = os.path.join(data_dir, "l_step_0.027624156655057155_common_files/mesh.msh") output_dir = os.path.join(data_dir, "{}/L5/test/01_cond_field/output/".format(cl)) hdf_path = os.path.join(data_dir, "{}/L5/mlmc_5.hdf5".format(cl)) save_path = os.path.join(data_dir, "{}".format(cl)) @@ -140,19 +145,20 @@ def get_config(data_dir, case=0): cl = "cl_0_1_s_1" if case == 4: cl = "cl_0_3_s_4" - nn_level = 3 + nn_level = 2 replace_level = False - mesh = os.path.join(data_dir, "cl_0_1_s_1/L5/l_step_0.020196309484414757_common_files/mesh.msh") + # mesh = os.path.join(data_dir, "{}/L5/l_step_0.020196309484414757_common_files/mesh.msh".format(cl)) + mesh = os.path.join(data_dir, "l_step_0.07416198487095663_common_files/mesh.msh".format(cl)) output_dir = os.path.join(data_dir, "{}/L5/test/01_cond_field/output/".format(cl)) hdf_path = os.path.join(data_dir, "{}/L5/mlmc_5.hdf5".format(cl)) save_path = os.path.join(data_dir, "{}".format(cl)) l_0_output_dir = os.path.join(data_dir, "{}/L1_{}/test/01_cond_field/output/".format(cl,nn_level)) l_0_hdf_path = os.path.join(data_dir, "{}/L1_{}/mlmc_1.hdf5".format(cl, nn_level)) sampling_info_path = os.path.join(data_dir, "{}/sampling_info".format(cl)) - ref_mlmc_file = os.path.join(data_dir, "{}/L1_3/mlmc_1.hdf5".format(cl)) + ref_mlmc_file = os.path.join(data_dir, "{}/L1_benchmark/mlmc_1.hdf5".format(cl)) elif case == 5: - data_dir = "/home/martin/Documents/metamodels/data/5_ele/" + #data_dir = "/home/martin/Documents/metamodels/data/5_ele/" cl = "cl_0_3_s_4" nn_level = 0 replace_level = False @@ -189,6 +195,7 @@ def get_arguments(arguments): import argparse parser = argparse.ArgumentParser() parser.add_argument('data_dir', help='data directory') + parser.add_argument('work_dir', help='work directory') args = parser.parse_args(arguments) return args @@ -196,16 +203,22 @@ def get_arguments(arguments): if __name__ == "__main__": args = get_arguments(sys.argv[1:]) data_dir = args.data_dir - case = 0 + work_dir = args.work_dir + case = 3 #data_dir = "/home/martin/Documents/metamodels/data/1000_ele/" output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, replace_level, nn_level = get_config( data_dir, case) + if os.path.exists(os.path.join(work_dir, "mlmc_{}.hdf5".format(nn_level + 1))): + l_0_hdf_path = os.path.join(work_dir, "mlmc_{}.hdf5".format(nn_level + 1)) + hdf_path = os.path.join(work_dir, "mlmc_5.hdf5") + ref_mlmc_file = os.path.join(work_dir, "benchmark_mlmc_1.hdf5") + # import cProfile # import pstats # pr = cProfile.Profile() # pr.enable() - gnn, conv_layer = get_gnn() + #gnn, conv_layer, corr_field_config = get_gnn() # # my_result = run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True, conv_layer=conv_layer) # @@ -213,29 +226,35 @@ def get_arguments(arguments): # ps = pstats.Stats(pr).sort_stats('cumtime') # ps.print_stats() - run_SVR(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, level=nn_level, log=True, conv_layer=conv_layer) # , gnn=gnn) + #run_SVR(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, level=nn_level, log=True, conv_layer=conv_layer) # , gnn=gnn) #run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True) # , gnn=gnn) #run_CNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True) # , gnn=gnn) - process_results(hdf_path, sampling_info_path, ref_mlmc_file, save_path, nn_level, replace_level) + #process_results(hdf_path, sampling_info_path, ref_mlmc_file, save_path, nn_level, replace_level) + - # gnn, conv_layer = get_gnn() - # # #run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, model=GCN, level=nn_level, log=True) # CGN model leads to constant value - # # #run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True, gnn=gnn, conv_layer=conv_layer) - # run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True, conv_layer=conv_layer) + # Graph creation time for cl_0_1_s_1 case 1 = 100 s + + # gnn, conv_layer, corr_field_config = get_gnn() + # # # #run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, model=GCN, level=nn_level, log=True) # CGN model leads to constant value + # # # #run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True, gnn=gnn, conv_layer=conv_layer) + # run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, + # level=nn_level, log=True, conv_layer=conv_layer, gnn=gnn, corr_field_config=corr_field_config, graph_creation_time=100) # process_results(hdf_path, sampling_info_path, ref_mlmc_file, save_path, nn_level, replace_level) # - # gnn, conv_layer = get_gnn() - # - # # print("gnn ", gnn) - # #print("conv layer ", conv_layer) - # - # #models = {"ChebConv": (run_GNN, False), "SVR": (run_SVR, False)} - # models = {"SVRTEST": (run_SVR, True)} - # save_path = os.path.join(save_path, "_".join(list(models.keys()))) - # statistics(models, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, level=nn_level, - # conv_layer=conv_layer, gnn=gnn, replace_level=replace_level) - # analyze_statistics(save_path) + gnn, conv_layer, corr_field_config = get_gnn() + + # print("gnn ", gnn) + #print("conv layer ", conv_layer) + + #models = {"ChebConv": (run_GNN, False), "SVR": (run_SVR, False)} + machine_learning_model = ("GNNTEST_case_3", run_GNN, False) + save_path = os.path.join(save_path, machine_learning_model[0]) + statistics(machine_learning_model, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, level=nn_level, + conv_layer=conv_layer, gnn=gnn, replace_level=replace_level, corr_field_config=corr_field_config, graph_creation_time=31) + + # analyze_statistics(save_path, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, level=nn_level, + # conv_layer=conv_layer, gnn=gnn, replace_level=replace_level, corr_field_config=corr_field_config) # save_path = os.path.join(save_path, "SVR") # statistics(run_SVR, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True) From 9b070e7e69f5acadf17a315ee01473d4d6cd40d8 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Sat, 10 Apr 2021 12:06:12 +0200 Subject: [PATCH 19/67] tf moments --- src/mlmc/moments.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/mlmc/moments.py b/src/mlmc/moments.py index 58919409..076ecb15 100644 --- a/src/mlmc/moments.py +++ b/src/mlmc/moments.py @@ -1,8 +1,8 @@ import numpy as np import numpy.ma as ma import tensorflow as tf -import tensorflow.experimental.numpy as tnp -tnp.experimental_enable_numpy_behavior() +# import tensorflow.experimental.numpy as tnp +# tnp.experimental_enable_numpy_behavior() import tensorflow_graphics as tfg import tensorflow_graphics.math.spherical_harmonics as tfgms @@ -260,11 +260,11 @@ def __init__(self, size, domain, ref_domain=None, log=False, safe_eval=True): super().__init__(size, domain, log, safe_eval) - def _eval_value(self, x, size): - return tnp.polynomial.legendre.legvander(x, deg=size-1) + # def _eval_value(self, x, size): + # return tnp.polynomial.legendre.legvander(x, deg=size-1) def _eval_all(self, value, size): - value = self.transform(tnp.atleast_1d(value)) + value = self.transform(value) #print("value ", value.shape) From 9b66083590811fb866997a6171816d5e9a69b6c1 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Sat, 10 Apr 2021 12:15:53 +0200 Subject: [PATCH 20/67] work dir --- test/metamodels/metamodel_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/metamodels/metamodel_test.py b/test/metamodels/metamodel_test.py index 0cb62e27..1005ed52 100644 --- a/test/metamodels/metamodel_test.py +++ b/test/metamodels/metamodel_test.py @@ -194,8 +194,8 @@ def get_arguments(arguments): """ import argparse parser = argparse.ArgumentParser() - parser.add_argument('data_dir', help='data directory') parser.add_argument('work_dir', help='work directory') + parser.add_argument('data_dir', help='data directory') args = parser.parse_args(arguments) return args From 429085c5752fa7c39e4b4037e25cb736ae4ab6de Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Sat, 10 Apr 2021 12:46:10 +0200 Subject: [PATCH 21/67] tfg moments --- src/mlmc/moments.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/mlmc/moments.py b/src/mlmc/moments.py index 076ecb15..1c17a6ba 100644 --- a/src/mlmc/moments.py +++ b/src/mlmc/moments.py @@ -1,13 +1,6 @@ import numpy as np import numpy.ma as ma import tensorflow as tf -# import tensorflow.experimental.numpy as tnp -# tnp.experimental_enable_numpy_behavior() -import tensorflow_graphics as tfg -import tensorflow_graphics.math.spherical_harmonics as tfgms - - - from scipy.interpolate import BSpline From 43944fb7f933e348b5c23ad94181e5caa6674471 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Sat, 10 Apr 2021 13:29:40 +0200 Subject: [PATCH 22/67] moments rm tnp --- src/mlmc/metamodel/custom_methods.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlmc/metamodel/custom_methods.py b/src/mlmc/metamodel/custom_methods.py index a76d1680..3a66ce0c 100644 --- a/src/mlmc/metamodel/custom_methods.py +++ b/src/mlmc/metamodel/custom_methods.py @@ -1,8 +1,8 @@ import numpy as np import tensorflow as tf from tensorflow.keras import backend as K -import tensorflow.experimental.numpy as tnp -tnp.experimental_enable_numpy_behavior() +# import tensorflow.experimental.numpy as tnp +# tnp.experimental_enable_numpy_behavior() from mlmc.moments import Monomial, Legendre From 0a51c0e04922b6fac633e5403370ccf9a4aa294c Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Wed, 14 Apr 2021 15:17:13 +0200 Subject: [PATCH 23/67] metamodels stats --- src/mlmc/metamodel/analyze_nn.py | 319 ++++++++++++++++----------- src/mlmc/metamodel/postprocessing.py | 203 +++++++---------- test/metamodels/metamodel_test.py | 70 ++++-- 3 files changed, 325 insertions(+), 267 deletions(-) diff --git a/src/mlmc/metamodel/analyze_nn.py b/src/mlmc/metamodel/analyze_nn.py index b67d73a1..92fbc673 100644 --- a/src/mlmc/metamodel/analyze_nn.py +++ b/src/mlmc/metamodel/analyze_nn.py @@ -9,6 +9,8 @@ from mlmc.metamodel.create_graph import graph_creator from mlmc.moments import Legendre_tf, Monomial from mlmc.metamodel.random_field_time import corr_field_sample_time +from mlmc.tool import plot +import matplotlib.pyplot as plt # Make numpy printouts easier to read. # np.set_printoptions(precision=9, suppress=True) @@ -171,26 +173,30 @@ def bootstrap(): # estimate_density(np.mean(all_predictions, axis=0), title="Predictions") -def run_SVR(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, level, conv_layer, stats=False, - gnn=None, model=None, log=False, train=True, replace_level=False, corr_field_config=None): +def run_SVR(config, stats=True, train=True, log=False): from sklearn.svm import SVR batch_size = 200 epochs = 1000 hidden_regularization = None # l2(2e-10) + graph_creation_time = config['graph_creation_time'] + if graph_creation_time == 0: + graph_creator_preproces_time = time.process_time() + graph_creator(config['output_dir'], config['hdf_path'], config['mesh'], level=config['level']) + graph_creation_time = time.process_time() - graph_creator_preproces_time + print("graph creation time ", graph_creation_time) + exit() preprocess_start_time = time.process_time() - graph_creator(output_dir, hdf_path, mesh, level=level) - # Load data - data = FlowDataset(output_dir=output_dir, level=level, log=log) + data = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log) data.shuffle() dataset = data.dataset dataset = dataset.sample(frac=1) - train = dataset[:2000] - test = dataset[2000:] + train = dataset[:config['n_train_samples']] + test = dataset[config['n_train_samples']:] train_input, train_output = train.x, train.y test_input, test_output = test.x, test.y @@ -209,8 +215,8 @@ def run_SVR(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, # test_output = sc_y.fit_transform(test_output.reshape(-1,1)) #train_input, train_output, test_input, test_output = split_dataset(dataset) - preprocess_time = time.process_time() - preprocess_start_time + preprocess_time = preprocess_time + graph_creation_time learning_time_start = time.process_time() print("train input ", train_input.shape) @@ -279,27 +285,32 @@ def run_SVR(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, plt.show() #predict_l_0_start_time = time.process_time() - l_0_targets, l_0_predictions, predict_l_0_time = predict_level_zero_SVR(svr_rbf, l_0_output_dir, l_0_hdf_path, mesh, - batch_size, log, stats=stats, corr_field_config=corr_field_config) - #predict_l_0_time = time.process_time() - predict_l_0_start_time + l_0_targets, l_0_predictions, predict_l_0_time = predict_level_zero_SVR(svr_rbf, config['l_0_output_dir'], + config['l_0_hdf_path'], + config['mesh'], batch_size, log, + stats=stats, + corr_field_config=config['corr_field_config']) + val_predictions = [] if stats: l1_sample_time = preprocess_time / len(data) + learning_time / len(data) l0_sample_time = predict_l_0_time / len(l_0_targets) - orig_max_vars, predict_max_vars = process_mlmc(hdf_path, sampling_info_path, ref_mlmc_file, targets, - predictions, train_targets, - train_predictions, - val_targets, l_0_targets, - l_0_predictions, l1_sample_time, l0_sample_time, nn_level=level, - replace_level=replace_level, - stats=stats) + # print("targets ", targets) + # print("predictions ", predictions) - return orig_targets, orig_predictions, learning_time, train_targets, train_predictions, orig_max_vars, predict_max_vars, total_steps + # orig_max_vars, predict_max_vars = process_mlmc(hdf_path, sampling_info_path, ref_mlmc_file, targets, predictions, train_targets, + # train_predictions, + # val_targets, l_0_targets, + # l_0_predictions, l1_sample_time, l0_sample_time, nn_level=level, replace_level=replace_level, + # stats=stats) + + return svr_rbf, targets, predictions, learning_time, train_targets, train_predictions, \ + val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, total_steps - save_times(save_path, False, (preprocess_time, len(data)), learning_time, (predict_l_0_time, len(l_0_targets))) - save_load_data(save_path, False, targets, predictions, train_targets, train_predictions, val_targets, l_0_targets, + save_times(config['save_path'], False, (preprocess_time, len(data)), learning_time, (predict_l_0_time, len(l_0_targets))) + save_load_data(config['save_path'], False, targets, predictions, train_targets, train_predictions, val_targets, l_0_targets, l_0_predictions) @@ -333,34 +344,30 @@ def predict_level_zero_SVR(nn, output_dir, hdf_path, mesh, batch_size=1000, log= return targets, predictions, predict_time + sample_time * len(data) -def statistics(machine_learning_model, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, - sampling_info_path, ref_mlmc_file, level, conv_layer, gnn=None, replace_level=False, - corr_field_config=None, graph_creation_time=0): - n_subsamples = 25 +def statistics(config): + n_subsamples = 2 - model_title, mch_l_model, log = machine_learning_model + model_title, mch_l_model, log = config['machine_learning_model'] model_data = {} model_data["log"] = log - if not os.path.isdir(save_path): - os.makedirs(save_path) + if not os.path.isdir(config['save_path']): + os.makedirs(config['save_path']) else: - print("dir exists {}".format(save_path)) + print("dir exists {}".format(config['save_path'])) exit() for i in range(n_subsamples): - iter_dir = os.path.join(save_path, "{}".format(i)) + iter_dir = os.path.join(config['save_path'], "{}".format(i)) if not os.path.isdir(iter_dir): os.makedirs(iter_dir) model, targets, predictions, learning_time, train_targets, train_predictions, \ val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, total_steps = \ - mch_l_model(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, - level=level, stats=True, log=log, conv_layer=conv_layer, gnn=gnn, train=True, - replace_level=replace_level, corr_field_config=corr_field_config, - graph_creation_time=graph_creation_time) + mch_l_model(config, stats=True, train=True, log=log) - model_data["model"] = model + if config['save_model']: + model_data["model"] = model model_data["test_targets"] = targets model_data["test_predictions"] = predictions model_data["train_targets"] = train_targets @@ -386,8 +393,7 @@ def statistics(machine_learning_model, output_dir, hdf_path, l_0_output_dir, l_0 # print("##################################################") - analyze_statistics(machine_learning_model, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, - sampling_info_path, ref_mlmc_file, level, conv_layer, gnn=None, replace_level=False, corr_field_config=None) + analyze_statistics(config) # plot_loss(train_losses, val_losses) # analyze_results(np.mean(all_test_outputs, axis=0), np.mean(all_predictions, axis=0)) @@ -399,7 +405,7 @@ def statistics(machine_learning_model, output_dir, hdf_path, l_0_output_dir, l_0 def save_statistics(save_dir_path, model_data): for file_name, data in model_data.items(): - if file_name == "model": + if file_name == "model" and data is not None: data.save(os.path.join(save_dir_path, file_name)) else: np.save(os.path.join(save_dir_path, file_name), data) @@ -429,8 +435,8 @@ def load_statistics(dir_path): if not os.path.isdir(data_dir_path): print("data dir not exists {}".format(data_dir_path)) break - - models_data['model'].append(keras.models.load_model(os.path.join(data_dir_path,'model'))) + if os.path.exists(os.path.join(data_dir_path,'model')): + models_data['model'].append(keras.models.load_model(os.path.join(data_dir_path, 'model'))) for file in glob.glob(os.path.join(data_dir_path, "*.npy")): file_name = os.path.split(file)[-1] file_name = file_name.split(".")[0] @@ -439,48 +445,136 @@ def load_statistics(dir_path): return models_data -def analyze_statistics(machine_learning_model, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, - sampling_info_path, ref_mlmc_file, level, conv_layer, gnn=None, replace_level=False, corr_field_config=None): - #print("save path ", save_path) +def plot_sse(data_nn, data_mlmc, x_label="ith moment", y_label="MSE", title=""): + import matplotlib + #matplotlib.rcParams.update({'font.size': 38}) + matplotlib.rcParams.update({'lines.markersize': 14}) + fig, axes = plt.subplots(1, 1, figsize=(22, 10)) + data = np.array(data_nn) + x = range(data.shape[1]) + axes.set_title(title) + axes.set_xlabel(x_label) + axes.set_ylabel(y_label) + axes.errorbar(x, np.mean(data_nn, axis=0), yerr=np.sqrt(np.var(data_nn, axis=0)), fmt='o', label="NN MLMC", color="red") + axes.errorbar(x, np.mean(data_mlmc, axis=0), yerr=np.sqrt(np.var(data_mlmc, axis=0)), fmt='o', label="MLMC", color="blue") + fig.legend() + fig.savefig("{}.pdf".format(title)) + fig.show() + + +def analyze_statistics(config): - if not os.path.isdir(save_path): + if not os.path.isdir(config['save_path']): print("dir not exists") exit() - data_dict = load_statistics(save_path) + data_dict = load_statistics(config['save_path']) #print("data dict ", data_dict) # for key, data_dict in models_data.items(): # print("model: {}".format(key)) - # for i in range(len(data_dict["test_targets"])): - # predictions = data_dict["test_predictions"][i] - # targets = data_dict["test_targets"][i] - # train_predictions = data_dict["train_predictions"][i] - # train_targets = data_dict["train_targets"][i] - # val_predictions = data_dict["val_predictions"][i] - # val_targets = data_dict["val_targets"][i] - # l_0_predictions = data_dict["l_0_predictions"][i] - # l_0_targets = data_dict["l_0_targets"][i] - # l1_sample_time = data_dict["l1_sample_time"][i] - # l0_sample_time = data_dict["l0_sample_time"][i] - # - # process_mlmc(hdf_path, sampling_info_path, ref_mlmc_file, targets, - # predictions, train_targets, - # train_predictions, - # val_targets, l_0_targets, - # l_0_predictions, l1_sample_time, l0_sample_time, - # nn_level=level, - # replace_level=replace_level, - # stats=False) + mlmc_n_collected_all = [] + nn_n_collected_all = [] + n_ops_all = [] + n_ops_predict_all = [] + + mlmc_times = [] + nn_times = [] + + mlmc_l_vars = [] + nn_l_vars = [] + mlmc_vars_mse = [] + nn_vars_mse = [] + mlmc_means_mse = [] + nn_means_mse = [] + for i in range(len(data_dict["test_targets"])): + if i == 2: + break + predictions = data_dict["test_predictions"][i] + targets = data_dict["test_targets"][i] + train_predictions = data_dict["train_predictions"][i] + train_targets = data_dict["train_targets"][i] + val_predictions = data_dict["val_predictions"][i] + val_targets = data_dict["val_targets"][i] + l_0_predictions = data_dict["l_0_predictions"][i] + l_0_targets = data_dict["l_0_targets"][i] + l1_sample_time = data_dict["l1_sample_time"][i] + l0_sample_time = data_dict["l0_sample_time"][i] + + mlmc_n_collected, nn_mlmc_n_collected, n_ops, n_ops_predict, orig_moments_mean, predict_moments_mean,\ + ref_moments_mean, orig_level_params, nn_level_params = process_mlmc(config['hdf_path'], + config['sampling_info_path'], + config['ref_mlmc_file'], targets, + predictions, train_targets, + train_predictions, + val_targets, l_0_targets, + l_0_predictions, l1_sample_time, l0_sample_time, + nn_level=config['level'], + replace_level=config['replace_level'], + stats=True) + + mlmc_n_collected_all.append(mlmc_n_collected) + nn_n_collected_all.append(nn_mlmc_n_collected) + n_ops_all.append(n_ops) + n_ops_predict_all.append(n_ops_predict) + + mlmc_times.append(np.sum(np.array(mlmc_n_collected) * np.array(n_ops))) + nn_times.append(np.sum(np.array(nn_mlmc_n_collected) * np.array(n_ops_predict))) + + mlmc_l_vars.append(orig_moments_mean.l_vars) + nn_l_vars.append(predict_moments_mean.l_vars) + + mlmc_vars_mse.append((ref_moments_mean.var - orig_moments_mean.var) ** 2) + nn_vars_mse.append((ref_moments_mean.var - predict_moments_mean.var) ** 2) + + mlmc_means_mse.append((ref_moments_mean.mean - orig_moments_mean.mean) ** 2) + nn_means_mse.append((ref_moments_mean.mean - predict_moments_mean.mean) ** 2) + + mlmc_total_time = np.mean(mlmc_times) + nn_total_time = np.mean(nn_times) + print("mlmc total time ", mlmc_total_time) + print("nn total time ", nn_total_time) + + n_ops_mlmc_mean = np.mean(n_ops_all, axis=0) + n_ops_nn_mean = np.mean(n_ops_predict_all, axis=0) + + print("n ops mlmc mean ", n_ops_mlmc_mean) + print("n ops nn mean ", n_ops_nn_mean) + + mlmc_n_collected = np.mean(mlmc_n_collected_all, axis=0) + nn_n_collected = np.mean(nn_n_collected_all, axis=0) + + print("mlmc n collected ", mlmc_n_collected_all) + print("nn n collected all ", nn_n_collected_all) + print("mlmc n collected ", mlmc_n_collected) + print("nn n collected ", nn_n_collected) + + plt_var = plot.Variance() + l_vars = np.mean(mlmc_l_vars, axis=0) + plt_var.add_level_variances(np.squeeze(orig_level_params), l_vars) + plt_var.show("mlmc_vars") + + plt_var = plot.Variance() + l_vars = np.mean(nn_l_vars, axis=0) + print("l vars shape ", l_vars.shape) + print("nn level parsm ", nn_level_params) + level_params = np.squeeze(nn_level_params) + level_params[0] /= 2 + plt_var.add_level_variances(level_params, l_vars) + plt_var.show("nn_vars") + + plot_sse(nn_vars_mse, mlmc_vars_mse, title="moments_var") + plot_sse(nn_means_mse, mlmc_means_mse, title="moments_mean") data_dict["test_targets"] = np.array(data_dict["test_targets"]) data_dict["test_predictions"] = np.array(data_dict["test_predictions"]) data_dict["train_targets"] = np.array(data_dict["train_targets"]) data_dict["train_predictions"] = np.array(data_dict["train_predictions"]) - if data_dict["log"]: + + if data_dict["log"][0]: data_dict["test_targets"] = np.exp(data_dict["test_targets"]) data_dict["test_predictions"] = np.exp(data_dict["test_predictions"]) data_dict["train_targets"] = np.exp(data_dict["train_targets"]) @@ -500,6 +594,11 @@ def analyze_statistics(machine_learning_model, output_dir, hdf_path, l_0_output_ # print("mean predict vars ", mean_predict_vars) print("total steps ", total_steps) + print("test targets ", data_dict["test_targets"]) + print("test predictions ", data_dict["test_predictions"]) + print("test diff ", data_dict["test_predictions"] - data_dict["test_targets"]) + print("test diff squared ", (data_dict["test_predictions"] - data_dict["test_targets"])**2) + test_MSE = np.mean((data_dict["test_predictions"] - data_dict["test_targets"])**2, axis=1) test_RMSE = np.sqrt(test_MSE) test_MAE = np.mean(np.abs(data_dict["test_predictions"] - data_dict["test_targets"]), axis=1) @@ -514,6 +613,8 @@ def analyze_statistics(machine_learning_model, output_dir, hdf_path, l_0_output_ print("test_MSE ", test_MSE) + print("NN moments MSE sum ", np.sum(np.mean(nn_means_mse, axis=0))) + print("mean test MSE ", np.mean(test_MSE)) print("mean test RMSE ", np.mean(test_RMSE)) print("mean test MAE ", np.mean(test_MAE)) @@ -534,8 +635,7 @@ def analyze_statistics(machine_learning_model, output_dir, hdf_path, l_0_output_ print("######################################") -def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, level, conv_layer, stats=False, - gnn=None, model=None, log=False, train=True, replace_level=False, corr_field_config=None, graph_creation_time=0): +def run_GNN(config, stats=True, train=True, log=False): loss = MeanSquaredError() # var_loss_function# accuracy_func = MSE_moments @@ -543,21 +643,22 @@ def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, # loss = MeanSquaredLogarithmicError() #loss = KLDivergence() # loss = total_loss_function - optimizer = tf.optimizers.Adam(learning_rate=0.001) - batch_size = 200#2000 - epochs = 2000#1000 + optimizer = tf.optimizers.Adam(learning_rate=config['learning_rate']) + batch_size = config['batch_size']#2000 + epochs = config['epochs']#1000 hidden_regularization = None # l2(2e-10) + graph_creation_time = config['graph_creation_time'] if graph_creation_time == 0: graph_creator_preproces_time = time.process_time() - graph_creator(output_dir, hdf_path, mesh, level=level) + graph_creator(config['output_dir'], config['hdf_path'], config['mesh'], level=config['level']) graph_creation_time = time.process_time() - graph_creator_preproces_time print("graph creation time ", graph_creation_time) exit() preprocess_start_time = time.process_time() # Load data - data = FlowDataset(output_dir=output_dir, level=level, log=log) + data = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log) data = data#[:10000] #print("len data ", len(data)) @@ -568,24 +669,26 @@ def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, print("total preprocess time ", preprocess_time) learning_time_start = time.process_time() - data.a = conv_layer.preprocess(data.a) + data.a = config['conv_layer'].preprocess(data.a) data.a = sp_matrix_to_sp_tensor(data.a) #train_data_len = int(len(data) * 0.8) - train_data_len = 2000 + train_data_len = config['n_train_samples'] # Train/valid/test split data_tr, data_te = data[:train_data_len], data[train_data_len:] + gnn = config['gnn'] + if hasattr(gnn._loss,'__name__') and gnn._loss.__name__ == "MSE_moments": tr_output = [g.y for g in data_tr] n_moments = 3 - quantile = 0.0001 + quantile = 0.001 domain = np.percentile(tr_output, [100 * quantile, 100 * (1 - quantile)]) moments_fn = Legendre_tf(n_moments, domain) #accuracy_func = MSE_moments(moments_fn=moments_fn) gnn._loss = MSE_moments(moments_fn=moments_fn) np.random.shuffle(data_tr) - val_data_len = int(len(data_tr) * 0.2) + val_data_len = int(len(data_tr) * config['val_samples_ratio']) data_tr, data_va = data_tr[:-val_data_len], data_tr[-val_data_len:] # print("data_tr len ", len(data_tr)) @@ -596,11 +699,11 @@ def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, loader_tr = MixedLoader(data_tr, batch_size=batch_size, epochs=epochs) loader_va = MixedLoader(data_va, batch_size=batch_size) loader_te = MixedLoader(data_te, batch_size=batch_size) - + # if gnn is None: - gnn = GNN(loss=loss, optimizer=optimizer, conv_layer=conv_layer, output_activation=abs_activation, + gnn = GNN(loss=loss, optimizer=optimizer, conv_layer=config['conv_layer'], output_activation=abs_activation, hidden_activation='relu', patience=150, hidden_reqularizer=hidden_regularization, - model=model, accuracy_func=accuracy_func) # tanh takes to much time + model=config['model'], accuracy_func=accuracy_func) # tanh takes to much time # ideally patience = 150 # batch_size 500, ideally 500 epochs, patience 35 @@ -652,52 +755,12 @@ def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, plt.yscale('log') plt.show() - - # plt.hist(np.exp(targets), bins=50, alpha=0.5, label='target', density=True) - # plt.hist(np.exp(predictions), bins=50, alpha=0.5, label='predictions', density=True) - # - # # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) - # plt.legend(loc='upper right') - # # plt.xlim(-0.5, 1000) - # plt.yscale('log') - # plt.show() - - # print("np.max(targets) ", np.max(targets)) - # print("np.min(targets) ", np.min(targets)) - # - # print("np.max(predictions) ", np.max(predictions)) - # print("np.min(predictions) ", np.min(predictions)) - # - # print("data.max_output ", data.max_output) - # print("data.min_output ", data.min_output) - - - # rescaled_targets = (data.max_output - data.min_output)/(np.max(targets) - np.min(targets))*(targets - np.max(targets)) + data.min_output - # - # - # rescaled_predictions = (data.max_output - data.min_output) / (np.max(predictions) - np.min(predictions)) * ( - # targets - np.max(predictions)) + data.min_output - # - # analyze_results(rescaled_targets, rescaled_predictions) - - # target_means, target_vars = estimate_density(targets, title="Test outputs") - # pred_means, pred_vars = estimate_density(predictions, title="Predictions") - # # - # print("target means ", target_means) - # print("predic means ", pred_means) - # # - # print("target vars ", target_vars) - # print("predic vars ", pred_vars) - - # - # diff_means, diff_vars = estimate_density(targets - predictions, title="diff") - # print("diff_means ", diff_means) - # print("diff vars ", diff_vars) - - # diff_moments(targets, predictions) - #predict_l_0_start_time = time.process_time() - l_0_targets, l_0_predictions, predict_l_0_time = predict_level_zero(gnn, l_0_output_dir, l_0_hdf_path, mesh, conv_layer, batch_size, log, stats=stats, corr_field_config=corr_field_config) + l_0_targets, l_0_predictions, predict_l_0_time = predict_level_zero(gnn, config['l_0_output_dir'], + config['l_0_hdf_path'], config['mesh'], + config['conv_layer'], batch_size, log, + stats=stats, + corr_field_config=config['corr_field_config']) #predict_l_0_time = time.process_time() - predict_l_0_start_time if stats: @@ -716,13 +779,13 @@ def run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, return gnn._model, targets, predictions, learning_time, train_targets, train_predictions,\ val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, total_steps - save_times(save_path, False, (preprocess_time, len(data)), learning_time, (predict_l_0_time, len(l_0_targets))) - save_load_data(save_path, False, targets, predictions, train_targets, train_predictions, val_targets, l_0_targets, + save_times(config['save_path'], False, (preprocess_time, len(data)), learning_time, (predict_l_0_time, len(l_0_targets))) + save_load_data(config['save_path'], False, targets, predictions, train_targets, train_predictions, val_targets, l_0_targets, l_0_predictions) def predict_level_zero(nn, output_dir, hdf_path, mesh, conv_layer, batch_size=1000, log=False, stats=False, corr_field_config=None): - #graph_creator(output_dir, hdf_path, mesh, level=0) + graph_creator(output_dir, hdf_path, mesh, level=0) # Load data sample_time = 0 diff --git a/src/mlmc/metamodel/postprocessing.py b/src/mlmc/metamodel/postprocessing.py index a1f2a64f..ec554741 100644 --- a/src/mlmc/metamodel/postprocessing.py +++ b/src/mlmc/metamodel/postprocessing.py @@ -237,7 +237,7 @@ def create_quantity_mlmc(data, level_parameters, num_ops=None): def estimate_moments(sample_storage, true_domain=None): - n_moments = 25 + n_moments = 15 result_format = sample_storage.load_result_format() root_quantity = make_root_quantity(sample_storage, result_format) @@ -307,8 +307,8 @@ def compare_moments(original_q_estimator, predict_q_estimator, ref_estimator): print("orig moments mean ", orig_moments_mean.mean) print("predict moments mean ", predict_moments_mean.mean) - # print("ref orig mean SSE ", np.sum((ref_moments_mean.mean - orig_moments_mean.mean)**2)) - # print("ref predict mean SSE ", np.sum((ref_moments_mean.mean - predict_moments_mean.mean) ** 2)) + print("ref orig mean SSE ", np.sum((ref_moments_mean.mean - orig_moments_mean.mean)**2)) + print("ref predict mean SSE ", np.sum((ref_moments_mean.mean - predict_moments_mean.mean) ** 2)) # print("ref orig mean SE ", np.sum(np.abs((ref_moments_mean.mean - orig_moments_mean.mean)))) print("ref predict mean SE ", np.sum(np.abs((ref_moments_mean.mean - predict_moments_mean.mean)))) @@ -317,15 +317,17 @@ def compare_moments(original_q_estimator, predict_q_estimator, ref_estimator): print("orig moments var ", orig_moments_mean.var) print("predict moments var ", predict_moments_mean.var) - print("MAX orig moments var ", np.max(orig_moments_mean.l_vars, axis=1)) - print("MAX predict moments var ", np.max(predict_moments_mean.l_vars, axis=1)) + # print("MAX orig moments var ", np.max(orig_moments_mean.l_vars, axis=1)) + # print("MAX predict moments var ", np.max(predict_moments_mean.l_vars, axis=1)) - # print("ref orig var SSE ", np.sum((ref_moments_mean.var - orig_moments_mean.var) ** 2)) - # print("ref predict var SSE ", np.sum((ref_moments_mean.var - predict_moments_mean.var) ** 2)) + print("ref orig var SSE ", np.sum((ref_moments_mean.var - orig_moments_mean.var) ** 2)) + print("ref predict var SSE ", np.sum((ref_moments_mean.var - predict_moments_mean.var) ** 2)) # print("ref orig var SE ", np.sum(np.abs((ref_moments_mean.var - orig_moments_mean.var)))) print("ref predict var SE ", np.sum(np.abs((ref_moments_mean.var - predict_moments_mean.var)))) + return orig_moments_mean, predict_moments_mean, ref_moments_mean + # l_0_samples = predict_q_estimator.get_level_samples(level_id=0) # l_1_samples = predict_q_estimator.get_level_samples(level_id=1) # l_2_samples = predict_q_estimator.get_level_samples(level_id=2) @@ -399,7 +401,7 @@ def get_quantity_estimator(sample_storage, true_domain=None): def get_n_estimated(sample_storage, estimator, n_ops=None): - target_var = 1e-4 + target_var = 5e-5 #moments, estimator, _, quantity = estimate_moments(sample_storage, true_domain=true_domain) n_level_samples = sample_storage.get_n_collected() @@ -420,7 +422,7 @@ def get_n_estimated(sample_storage, estimator, n_ops=None): def get_storage_info(sample_storage): moments, estimator, _, _ = estimate_moments(sample_storage) n_collected = sample_storage.get_n_collected() - max_vars = np.max(moments.l_vars, axis=1) + max_vars = np.max(np.array(moments.l_vars) / np.array(sample_storage.get_n_collected())[:, np.newaxis], axis=1) print("n collected ", n_collected) print("moments.l_vars max ", max_vars) return n_collected, max_vars @@ -486,13 +488,18 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti ### Create storage fromm original MLMC data ###### sample_storage = SampleStorageHDF(file_path=mlmc_file) + print("get n collected ", sample_storage.get_n_collected()) n_levels = len(sample_storage.get_level_ids()) original_moments, estimator, original_true_domain, _ = estimate_moments(sample_storage) + orig_max_vars = np.max(original_moments.l_vars, axis=1) + print("orig max vars ", orig_max_vars) + ###### ### Get n ops ###### - n_ops, field_times, flow_times = get_sample_times(sampling_info_path) + n_ops, field_times, coarse_flow, fine_flow = get_sample_times_mlmc(mlmc_file) + #n_ops_2, _, _ = get_sample_times(sampling_info_path) if n_ops is None: n_ops = sample_storage.get_n_ops() @@ -503,6 +510,8 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti for l_id in range(n_levels): level_samples = estimator.get_level_samples(level_id=l_id) data_mlmc.append(level_samples) + + print("original level params" , sample_storage.get_level_parameters()) sample_storage = create_quantity_mlmc(data_mlmc, level_parameters=sample_storage.get_level_parameters()) print("Original storage") orig_storage_n_collected, orig_storage_max_vars = get_storage_info(sample_storage) @@ -529,9 +538,6 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti ft_index = nn_level if nn_level > 0: ft_index = nn_level - 1 - # print("l0_sample_time ", l0_sample_time) - # print("fields times ", field_times[ft_index] / 2) - n_ops_predict.append(l0_sample_time)# + field_times[ft_index] / 2) #level_samples = level_samples[:, :50000, :] else: @@ -570,13 +576,14 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti n_ops_predict.append(n_ops[level_id]) data_nn.append(level_samples) + print("level params ", level_params) sample_storage_predict = create_quantity_mlmc(data_nn, level_parameters=level_params) print("n ops predict ", n_ops_predict) print("Storage predict info") predict_storage_n_collected, predict_storage_max_vars = get_storage_info(sample_storage_predict) - if stats: - return orig_storage_max_vars, predict_storage_max_vars + # if stats: + # return orig_storage_max_vars, predict_storage_max_vars ###### ### Create estimators @@ -593,21 +600,23 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti print("n ops ", n_ops) print("n ops predict ", n_ops_predict) - # # remove levels - # # #@TODO: remove asap + # # # remove levels + # # # #@TODO: remove asap # new_n_samples = [0, 0, sample_storage.get_n_collected()[-3], sample_storage.get_n_collected()[-2], sample_storage.get_n_collected()[-1]] # sample_storage = cut_samples(data_mlmc, sample_storage, new_n_samples, new_l_0=2) # print("Cut storage info") # get_storage_info(sample_storage) # original_q_estimator = get_quantity_estimator(sample_storage, true_domain=domain) # n_ops = n_ops[2:] - # # ##### + # # # ##### #### Original data n_ops_est = copy.deepcopy(n_ops) #n_ops_est[0] = n_ops_est[0] / 1000 n_estimated_orig, l_vars_orig, n_samples_orig = get_n_estimated(sample_storage, original_q_estimator, n_ops=n_ops_est) print("n estimated orig ", n_estimated_orig) + #print("l vars orig ", np.array(l_vars_orig) / np.array(sample_storage.get_n_collected())[:, np.newaxis]) + sample_storage = cut_samples(data_mlmc, sample_storage, n_estimated_orig) # # sample_storage_predict = cut_samples(data_nn, sample_storage_predict, [sample_storage_predict.get_n_collected()[0], @@ -627,110 +636,9 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti n_estimated_nn, l_var_nn, n_samples_nn = get_n_estimated(sample_storage_predict, predict_q_estimator, n_ops=n_ops_predict) - #n_estimated_nn = [50000, 10000, 850] sample_storage_predict = cut_samples(data_nn, sample_storage_predict, n_estimated_nn) - # print("N estimated orig ", n_estimated_orig) - # print("n sampels nn ", n_samples_nn) - # print("N estimated nn ", n_estimated_nn) - # print("est time ", np.sum(n_ops_predict_orig * n_estimated_nn)) - # print("time ", np.sum(n_ops_predict_orig * n_samples_nn)) - # print("orig time ", np.sum(n_ops * n_estimated_orig)) - # exit() - #sample_storage_predict = cut_samples(data_nn, sample_storage_predict, n_estimated_nn) - - # print("l vars orig [-1] ", l_vars_orig[0]) - # print("l var nn[-1] ", l_var_nn[:2]) - - #print("l vars orig ", l_vars_orig) - # print("l vars nn ", l_var_nn[:2]) - # print("l vars orig ", np.max(l_vars_orig, axis=1)) - # print("l vars nn ", np.max(l_var_nn, axis=1)) - # - # orig_total_var = np.sum(l_vars_orig / n_samples_orig[:, None], axis=0) - # orig_total_var_max = np.max(np.sum(l_vars_orig / n_samples_orig[:, None], axis=0)) - # print("orig total var ", orig_total_var) - # print("orig total var max ", orig_total_var_max) - # nn_total_var = np.sum(l_var_nn / n_samples_nn[:, None], axis=0) - # nn_total_var_max = np.max(np.sum(l_var_nn / n_samples_nn[:, None], axis=0)) - # print("nn total var ", nn_total_var) - # print("nn total var max ", nn_total_var_max) - # exit() - # - # - # print("np.sum(l_var_nn[:2], axis=0) ", np.sum(l_var_nn[:2], axis=0)) - # print(" l_vars_orig[0] - np.sum(l_var_nn[:2], axis=0) ", l_vars_orig[0] - np.sum(l_var_nn[:2], axis=0)) - # - # print("l_vars orig - l_var_nn ", l_vars_orig[-1] - l_var_nn[-1]) - # print("l_vars orig - l_var_nn ", np.sum(l_vars_orig[1:] - l_var_nn[2:], axis=0)) - # - # print("MAX l vars orig [0] ", np.max(l_vars_orig[0])) - # print("MAX l var nn[:2] ", np.max(l_var_nn[:2], axis=1)) - # - # print("n samples orig ", n_samples_orig) - # print("n samples nn ", n_samples_nn) - # - # vars_n_orig = l_vars_orig / n_samples_orig[:, None] - # vars_n_nn = l_var_nn / n_samples_nn[:, None] - # - # print("vars n orig ", vars_n_orig) - # print("vars n nn ", vars_n_nn) - # - # # max_vars_n_orig = np.max(vars_n_orig, axis=1) - # max_vars_n_nn = np.max(vars_n_nn, axis=1) - # - # # print("vars vars n orig ", max_vars_n_orig) - # # print("vars vars n nn ", max_vars_n_nn) - # - # print("np.sum(l_var_nn[1:, ...] / n_samples_nn[1:, None], axis=0) ", np.sum(l_var_nn[1:, ...] / n_samples_nn[1:, None], axis=0)) - # n0_samples = l_var_nn[0] / np.abs(orig_total_var - np.sum(l_var_nn[1:, ...] / n_samples_nn[1:, None], axis=0)) - # - # print("n0_samples ", n0_samples) - # print("max n0_samples ", int(np.max(n0_samples[1:]))) - - # sample_storage_pred_cut = cut_samples(data_nn, sample_storage_predict, [int(np.max(n0_samples[1:])), *n_samples_nn[1:]]) - # predict_q_estimator_cut = get_quantity_estimator(sample_storage_pred_cut)#, true_domain=domain) - # n_estimated_nn, l_var_nn, n_samples_nn = get_n_estimated(sample_storage_pred_cut, predict_q_estimator_cut, - # n_ops=n_ops_predict) - # - # nn_total_var = np.max(np.sum(l_var_nn / n_samples_nn[:, None], axis=0)) - # print("nn total var ", nn_total_var) - # print("n samples nn ", n_samples_nn) - # exit() - - - # com_levels = np.max(np.sum(vars_n_orig[:2], axis=0))#(np.max(l_vars_orig[0])/n_samples_orig[0]) + (np.max(l_vars_orig[1])/n_samples_orig[1]) - # print("com levels ", com_levels) - # print("(np.max(l_var_nn[1])/n_samples_nn[1]) ", (np.max(l_var_nn[1])/n_samples_nn[1])) - # print("np.max(l_var_nn[0]) ", np.max(l_var_nn[0])) - # print("np.max(np.sum(vars_n_nn[1:2], axis=0)) ", np.max(np.sum(vars_n_nn[1:2], axis=0))) - # nn_l_0_n = np.max(l_var_nn[0]) / (com_levels - np.max(np.sum(vars_n_nn[1:2], axis=0))) #(np.max(l_var_nn[1])/n_samples_nn[1])) - # - # print("nn_l_0_n ", nn_l_0_n) - # - # print("MAX l vars orig ", np.max(l_vars_orig, axis=1)) - # print("MAX l var nn ", np.max(l_var_nn, axis=1)) - # - # nn_vars = np.max(l_var_nn[0])/nn_l_0_n + np.max(np.sum(vars_n_nn[1:2], axis=0)) - # - # - # print("nn vars ", nn_vars) - # print("rogi vars ", com_levels) - - - - - # orig_sample_l_5 = original_q_estimator.get_level_samples(4) - # nn_sample_l_5 = predict_q_estimator.get_level_samples(5) - # - # print("orig samples l 5 ", orig_sample_l_5) - # print("nn sample l 5 ", nn_sample_l_5) - # - # assert np.allclose(orig_sample_l_5, nn_sample_l_5) - - - ####### ## Estimate total time ####### @@ -740,8 +648,10 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti n_collected_times = n_ops * np.array(n_estimated_orig) print("NN time levels ", NN_time_levels) print("MLMC time levels", n_collected_times) - print("NN total time ", np.sum(NN_time_levels)) - print("MLMC total time ", np.sum(n_collected_times)) + nn_total_time = np.sum(NN_time_levels) + print("NN total time ", nn_total_time) + mlmc_total_time = np.sum(n_collected_times) + print("MLMC total time ", mlmc_total_time) #original_moments, estimator, original_true_domain, _ = estimate_moments(sample_storage) @@ -791,16 +701,23 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti # for l_id, (l_mom, l_mom_pred) in enumerate(zip(moments_nn.l_means, moments_predict.l_means)): # print("L id: {}, mom diff: {}".format(l_id, l_mom - l_mom_pred)) + domain = get_largest_domain([sample_storage, sample_storage_predict, ref_sample_storage]) original_q_estimator = get_quantity_estimator(sample_storage, true_domain=domain) predict_q_estimator = get_quantity_estimator(sample_storage_predict, true_domain=domain) ref_estimator = get_quantity_estimator(ref_sample_storage, true_domain=domain) - compare_moments(original_q_estimator, predict_q_estimator, ref_estimator) + orig_moments_mean, predict_moments_mean, ref_moments_mean = compare_moments(original_q_estimator, predict_q_estimator, ref_estimator) + + # compare_densities(original_q_estimator, predict_q_estimator, ref_estimator, + # label_1="orig N: {}".format(sample_storage.get_n_collected()), + # label_2="gnn N: {}".format(sample_storage_predict.get_n_collected())) + + if stats: + return sample_storage.get_n_collected(), sample_storage_predict.get_n_collected(), n_ops, n_ops_predict, orig_moments_mean,\ + predict_moments_mean, ref_moments_mean, sample_storage.get_level_parameters(), sample_storage_predict.get_level_parameters() plot_moments({"ref": ref_estimator, "orig": original_q_estimator, "nn": predict_q_estimator}) - compare_densities(original_q_estimator, predict_q_estimator, ref_estimator, label_1="orig N: {}".format(sample_storage.get_n_collected()), - label_2="gnn N: {}".format(sample_storage_predict.get_n_collected())) def plot_moments(mlmc_estimators): @@ -844,8 +761,6 @@ def analyze_mlmc_data(): data = [] for l_id in range(n_levels): level_samples = estimator.get_level_samples(level_id=l_id) - - l_fine = np.squeeze(level_samples[..., 0]) print("mean l_fine ", np.mean(l_fine)) @@ -860,6 +775,42 @@ def analyze_mlmc_data(): assert np.allclose(original_moments.var, moments_2.var) +def get_sample_times_mlmc(mlmc_file): + sample_storage = SampleStorageHDF(file_path=mlmc_file) + + n_ops = sample_storage.get_n_ops() + generate_rnd = sample_storage.get_generate_rnd_times() + extract_mesh = sample_storage.get_extract_mesh_times() + make_fields = sample_storage.get_make_field_times() + coarse_flow = sample_storage.get_coarse_flow_times() + fine_flow = sample_storage.get_fine_flow_times() + + def time_for_sample_func(data): + new_n_ops = [] + for nop in data: + nop = np.squeeze(nop) + if len(nop) > 0: + new_n_ops.append(nop[0] / nop[1]) + return new_n_ops + + print("generated rnd ", generate_rnd) + + generate_rnd = time_for_sample_func(generate_rnd) + extract_mesh = time_for_sample_func(extract_mesh) + make_fields = time_for_sample_func(make_fields) + coarse_flow = time_for_sample_func(coarse_flow) + fine_flow = time_for_sample_func(fine_flow) + + field_times = generate_rnd + extract_mesh + make_fields + + print("n ops ", n_ops) + print("field times ", field_times) + print("coarse flow ", coarse_flow) + print("fine flow ", fine_flow) + + return n_ops, field_times, coarse_flow, fine_flow + + def get_sample_times(sampling_info_path): n_levels = [5] for nl in n_levels: diff --git a/test/metamodels/metamodel_test.py b/test/metamodels/metamodel_test.py index 1005ed52..ae0762e7 100644 --- a/test/metamodels/metamodel_test.py +++ b/test/metamodels/metamodel_test.py @@ -1,5 +1,5 @@ import os - +import warnings #os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only import sys import subprocess @@ -15,6 +15,7 @@ from spektral.layers import GlobalSumPool, GlobalMaxPool, GlobalAvgPool import tensorflow as tf from tensorflow.keras.layers.experimental import preprocessing +warnings.filterwarnings("ignore", category=DeprecationWarning) def get_gnn(): @@ -141,11 +142,11 @@ def get_config(data_dir, case=0): ref_mlmc_file = os.path.join(data_dir, "{}/L1_3/mlmc_1.hdf5".format(cl)) elif case == 3 or case == 4: - data_dir = "/home/martin/Documents/metamodels/data/5_ele/" + #data_dir = "/home/martin/Documents/metamodels/data/5_ele/" cl = "cl_0_1_s_1" if case == 4: cl = "cl_0_3_s_4" - nn_level = 2 + nn_level = 0 replace_level = False # mesh = os.path.join(data_dir, "{}/L5/l_step_0.020196309484414757_common_files/mesh.msh".format(cl)) mesh = os.path.join(data_dir, "l_step_0.07416198487095663_common_files/mesh.msh".format(cl)) @@ -170,6 +171,20 @@ def get_config(data_dir, case=0): l_0_hdf_path = os.path.join(data_dir,"{}/L1_{}/mlmc_1.hdf5".format(cl, nn_level)) sampling_info_path = os.path.join(data_dir, "{}/sampling_info".format(cl)) ref_mlmc_file = os.path.join(data_dir,"{}/L1_3/mlmc_1.hdf5".format(cl)) + + elif case == 6: # mesh size comparison + cl = "cl_0_1_s_1" + nn_level = 0 + replace_level = False + mesh = os.path.join(data_dir, "l_step_0.07416198487095663_common_files/mesh.msh".format(cl)) + output_dir = os.path.join(data_dir, "{}/L1/test/01_cond_field/output/".format(cl)) + hdf_path = os.path.join(data_dir, "{}/L1/mlmc_1.hdf5".format(cl)) + save_path = os.path.join(data_dir, "{}".format(cl)) + l_0_output_dir = os.path.join(data_dir, "{}/L1_{}/test/01_cond_field/output/".format(cl, nn_level)) + l_0_hdf_path = os.path.join(data_dir, "{}/L1_{}/mlmc_1.hdf5".format(cl, nn_level)) + sampling_info_path = os.path.join(data_dir, "{}/sampling_info".format(cl)) + ref_mlmc_file = os.path.join(data_dir, "{}/L1_benchmark/mlmc_1.hdf5".format(cl)) + # elif case == 6: # nn_level = 0 # replace_level = True @@ -236,25 +251,54 @@ def get_arguments(arguments): # Graph creation time for cl_0_1_s_1 case 1 = 100 s # gnn, conv_layer, corr_field_config = get_gnn() - # # # #run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, model=GCN, level=nn_level, log=True) # CGN model leads to constant value - # # # #run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True, gnn=gnn, conv_layer=conv_layer) + # # # # #run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, model=GCN, level=nn_level, log=True) # CGN model leads to constant value + # # # # #run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True, gnn=gnn, conv_layer=conv_layer) # run_GNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, - # level=nn_level, log=True, conv_layer=conv_layer, gnn=gnn, corr_field_config=corr_field_config, graph_creation_time=100) + # level=nn_level, log=True, conv_layer=conv_layer, gnn=gnn, corr_field_config=corr_field_config, graph_creation_time=100) # process_results(hdf_path, sampling_info_path, ref_mlmc_file, save_path, nn_level, replace_level) # + gnn, conv_layer, corr_field_config = get_gnn() # print("gnn ", gnn) #print("conv layer ", conv_layer) - #models = {"ChebConv": (run_GNN, False), "SVR": (run_SVR, False)} - machine_learning_model = ("GNNTEST_case_3", run_GNN, False) + # #models = {"ChebConv": (run_GNN, False), "SVR": (run_SVR, False)} + machine_learning_model = ("5eleChebConvL3_2", run_GNN, False) + #machine_learning_model = ("5eleChebConvK2", run_GNN, False) + # # machine_learning_model = ("5eleChebConvK3", run_GNN, False) + #machine_learning_model = ("5eleChebConv32abs", run_GNN, False) + #machine_learning_model = ("5eleChebConv32msemom", run_GNN, False) save_path = os.path.join(save_path, machine_learning_model[0]) - statistics(machine_learning_model, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, level=nn_level, - conv_layer=conv_layer, gnn=gnn, replace_level=replace_level, corr_field_config=corr_field_config, graph_creation_time=31) - - # analyze_statistics(save_path, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, level=nn_level, - # conv_layer=conv_layer, gnn=gnn, replace_level=replace_level, corr_field_config=corr_field_config) + graph_creation_time = 0#66 + + config = {'machine_learning_model': machine_learning_model, + 'save_path': save_path, + 'sampling_info_path': sampling_info_path, + 'output_dir': output_dir, + 'hdf_path': hdf_path, + 'mesh': mesh, + 'l_0_output_dir': l_0_output_dir, + 'l_0_hdf_path': l_0_hdf_path, + 'sampling_info_path': sampling_info_path, + 'ref_mlmc_file': ref_mlmc_file, + 'level': nn_level, + 'conv_layer': conv_layer, + 'gnn': gnn, + 'replace_level': replace_level, + 'corr_field_config': corr_field_config, + 'n_train_samples': 2000, + 'val_samples_ratio': 0.2, + 'batch_size': 200, + 'epochs': 2000, + 'learning_rate': 0.001, + 'graph_creation_time': graph_creation_time, + 'save_model': False + } + + statistics(config) + + analyze_statistics(config) # save_path = os.path.join(save_path, "SVR") # statistics(run_SVR, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True) From 65898d6d314e55e81b6f84b4ffcd402df48ac05c Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Wed, 20 Oct 2021 09:41:42 +0200 Subject: [PATCH 24/67] metamodel moments changes --- src/mlmc/estimator.py | 16 ++ src/mlmc/metamodel/GCN.py | 102 ++++++++++++ src/mlmc/metamodel/flow_task_CNN.py | 59 +------ src/mlmc/metamodel/flow_task_GNN.py | 69 +++----- src/mlmc/metamodel/flow_task_NN.py | 232 ++++++++++++++++++++++++++- src/mlmc/metamodel/main.py | 63 ++++++-- src/mlmc/metamodel/mnist.py | 167 +++++++++++++++++++ src/mlmc/metamodel/postprocessing.py | 13 +- src/mlmc/tool/plot.py | 126 +++++++++++++-- test/01_cond_field/process_simple.py | 73 ++++++--- test/metamodels/compare_models.py | 33 ++++ test/metamodels/metamodel_test.py | 219 +++++++++++++++++++++++-- test/metamodels/nn_config.py | 83 ---------- test/test_quantity_concept.py | 5 +- 14 files changed, 1011 insertions(+), 249 deletions(-) diff --git a/src/mlmc/estimator.py b/src/mlmc/estimator.py index 1e7f59f7..7aad8f6d 100644 --- a/src/mlmc/estimator.py +++ b/src/mlmc/estimator.py @@ -18,13 +18,29 @@ def estimate_n_samples_for_target_variance(target_variance, prescribe_vars, n_op :param n_levels: number of levels :return: np.array with number of optimal samples for individual levels and moments_fn, array (LxR) """ + # print("vars ", prescribe_vars) + # print("n ops ", n_ops) + vars = prescribe_vars + #print("vars.T * n_ops ", vars.T * n_ops) + sqrt_var_n = np.sqrt(vars.T * n_ops) # moments_fn in rows, levels in cols total = np.sum(sqrt_var_n, axis=1) # sum over levels + #print("total ", total) + n_samples_estimate = np.round((sqrt_var_n / n_ops).T * total / target_variance).astype(int) # moments_fn in cols # Limit maximal number of samples per level n_samples_estimate_safe = np.maximum( np.minimum(n_samples_estimate, vars * n_levels / target_variance), 2) + + #print("vars ", vars) + + + # return np.max(n_samples_estimate, axis=1).astype(int) + # + # print("n samples estimate ", n_samples_estimate) + # print("n samples estaimte safe ", n_samples_estimate_safe) + #print("N: ", np.max(n_samples_estimate_safe, axis=1).astype(int)) return np.max(n_samples_estimate_safe, axis=1).astype(int) diff --git a/src/mlmc/metamodel/GCN.py b/src/mlmc/metamodel/GCN.py index e69de29b..d859c065 100644 --- a/src/mlmc/metamodel/GCN.py +++ b/src/mlmc/metamodel/GCN.py @@ -0,0 +1,102 @@ +""" +This example shows how to perform regression of molecular properties with the +QM9 database, using a simple GNN in disjoint mode. +""" + +import numpy as np +import tensorflow as tf +from tensorflow.keras.layers import Dense, Input +from tensorflow.keras.losses import MeanSquaredError +from tensorflow.keras.models import Model +from tensorflow.keras.optimizers import Adam + +from spektral.data import DisjointLoader +from spektral.datasets import QM9 +from spektral.layers import ECCConv, GlobalSumPool + +################################################################################ +# PARAMETERS +################################################################################ +learning_rate = 1e-3 # Learning rate +epochs = 10 # Number of training epochs +batch_size = 32 # Batch size + +################################################################################ +# LOAD DATA +################################################################################ +dataset = QM9(amount=1000) # Set amount=None to train on whole dataset + +# Parameters +F = dataset.n_node_features # Dimension of node features +S = dataset.n_edge_features # Dimension of edge features +n_out = dataset.n_labels # Dimension of the target + +# Train/test split +idxs = np.random.permutation(len(dataset)) +split = int(0.9 * len(dataset)) +idx_tr, idx_te = np.split(idxs, [split]) +dataset_tr, dataset_te = dataset[idx_tr], dataset[idx_te] + +loader_tr = DisjointLoader(dataset_tr, batch_size=batch_size, epochs=epochs) +loader_te = DisjointLoader(dataset_te, batch_size=batch_size, epochs=1) + +################################################################################ +# BUILD MODEL +################################################################################ +X_in = Input(shape=(F,), name="X_in") +A_in = Input(shape=(None,), sparse=True, name="A_in") +E_in = Input(shape=(S,), name="E_in") +I_in = Input(shape=(), name="segment_ids_in", dtype=tf.int32) + +X_1 = ECCConv(32, activation="relu")([X_in, A_in, E_in]) +X_2 = ECCConv(32, activation="relu")([X_1, A_in, E_in]) +X_3 = GlobalSumPool()([X_2, I_in]) +output = Dense(n_out)(X_3) + +# Build model +model = Model(inputs=[X_in, A_in, E_in, I_in], outputs=output) +opt = Adam(lr=learning_rate) +loss_fn = MeanSquaredError() + + +################################################################################ +# FIT MODEL +################################################################################ +@tf.function(input_signature=loader_tr.tf_signature(), experimental_relax_shapes=True) +def train_step(inputs, target): + with tf.GradientTape() as tape: + predictions = model(inputs, training=True) + loss = loss_fn(target, predictions) + loss += sum(model.losses) + gradients = tape.gradient(loss, model.trainable_variables) + opt.apply_gradients(zip(gradients, model.trainable_variables)) + return loss + + +print("Fitting model") +current_batch = 0 +model_loss = 0 +for batch in loader_tr: + outs = train_step(*batch) + + model_loss += outs + current_batch += 1 + if current_batch == loader_tr.steps_per_epoch: + print("Loss: {}".format(model_loss / loader_tr.steps_per_epoch)) + model_loss = 0 + current_batch = 0 + +################################################################################ +# EVALUATE MODEL +################################################################################ +print("Testing model") +model_loss = 0 +for batch in loader_te: + inputs, target = batch + predictions = model(inputs, training=False) + + print("target ", target) + print("prediction ", predictions) + model_loss += loss_fn(target, predictions) +model_loss /= loader_te.steps_per_epoch +print("Done. Test loss: {}".format(model_loss)) diff --git a/src/mlmc/metamodel/flow_task_CNN.py b/src/mlmc/metamodel/flow_task_CNN.py index e4d5805b..0fab3395 100644 --- a/src/mlmc/metamodel/flow_task_CNN.py +++ b/src/mlmc/metamodel/flow_task_CNN.py @@ -1,8 +1,5 @@ -import numpy as np -from mlmc.metamodel.flow_dataset import FlowDataset import tensorflow as tf -from tensorflow import keras -from tensorflow.keras import layers +from mlmc.metamodel.graph_models import cnn_model from tensorflow.keras.layers.experimental import preprocessing # Following 3 lines prevent "Failed to get convolution algorithm. This is probably because cuDNN failed to initialize" @@ -10,25 +7,6 @@ config.gpu_options.allow_growth = True session = tf.compat.v1.InteractiveSession(config=config) -import os -import numpy as np -#os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only -import tensorflow as tf -from tensorflow.keras.losses import MeanSquaredError, SparseCategoricalCrossentropy, KLDivergence -from tensorflow.keras.metrics import mean_squared_error, kl_divergence -from tensorflow.keras.callbacks import History -from tensorflow.keras.optimizers import Adam -from tensorflow.keras.regularizers import l2 -from mlmc.metamodel.postprocessing import analyze_results, plot_loss -from spektral.data import MixedLoader -from mlmc.metamodel.flow_dataset import FlowDataset -from spektral.layers import GCNConv, GlobalSumPool, ChebConv, GraphSageConv, ARMAConv, GATConv, APPNPConv, GINConv -from spektral.layers.ops import sp_matrix_to_sp_tensor -from tensorflow.keras.layers.experimental import preprocessing -from mlmc.metamodel.custom_methods import abs_activation - -from mlmc.metamodel.graph_models import Net1 - ################################## # Convolutional neural network # @@ -40,45 +18,12 @@ def __init__(self, **kwargs): self._val_split = kwargs.get('var_split', 0.2) self._verbose = kwargs.get('verbose', False) - self._hidden_activation = kwargs.get('hidden_activation', 'relu') - self._hidden_regularizer = kwargs.get('hidden_reqularizer', None) - self._output_activation = kwargs.get('output_activation', 'linear') - self._n_hidden_layers = kwargs.get('n_hidden_layers', 1) - self._n_hidden_neurons = kwargs.get('n_hidden_neurons', [64]) # Number of hidden neurons for each hidden layer - self._loss = kwargs.get('loss', 'mean_squared_error') self._optimizer = kwargs.get('optimizer', tf.optimizers.Adam(learning_rate=0.001)) self._normalizer = kwargs.get('normalizer', preprocessing.Normalization()) self.history = None # Set in fit method - self._create_model() - - def _create_model(self): - hidden_layers = [] - for i in range(self._n_hidden_layers): - if self._hidden_regularizer is not None: - hidden_layers.append( - layers.Dense(self._n_hidden_neurons[i], - kernel_regularizer=self._hidden_regularizer, - activation=self._hidden_activation)) - else: - hidden_layers.append( - layers.Dense(self._n_hidden_neurons[i],activation=self._hidden_activation)) - - self._model = keras.Sequential([ - #@TODO: Try normalization - #self._normalizer, # Seems worse results with normalization - layers.Conv1D(filters=64, kernel_size=3, activation='relu', input_shape=(958, 1)), - #layers.BatchNormalization(), - layers.MaxPooling1D(pool_size=2), - layers.Conv1D(filters=64, kernel_size=3, activation='relu'), - #layers.BatchNormalization(), - layers.MaxPooling1D(pool_size=2), - layers.Flatten(), - layers.Dense(64, activation='relu'), - layers.Dense(1, activation=self._output_activation) - ]) - + self._model = cnn_model() self._model.compile(loss=self._loss, optimizer=self._optimizer) def fit(self, train_input, train_output): diff --git a/src/mlmc/metamodel/flow_task_GNN.py b/src/mlmc/metamodel/flow_task_GNN.py index 83bb63c3..27ed5014 100644 --- a/src/mlmc/metamodel/flow_task_GNN.py +++ b/src/mlmc/metamodel/flow_task_GNN.py @@ -3,17 +3,21 @@ import matplotlib.pyplot as plt #os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only import tensorflow as tf -from tensorflow.keras import Model -from tensorflow.keras.layers import Dense +from tensorflow.keras.regularizers import l2 + from tensorflow.keras.losses import MeanSquaredError, SparseCategoricalCrossentropy, KLDivergence from tensorflow.keras.metrics import mean_squared_error, kl_divergence from tensorflow.keras.optimizers import Adam -from tensorflow.keras.regularizers import l2 +from mlmc.metamodel.postprocessing import analyze_results, plot_loss +from mlmc.metamodel.custom_methods import abs_activation from spektral.data import MixedLoader from mlmc.metamodel.flow_dataset import FlowDataset from spektral.layers import GCNConv, GlobalSumPool, ChebConv, GraphSageConv, ARMAConv, GATConv, APPNPConv, GINConv from spektral.layers.ops import sp_matrix_to_sp_tensor +from spektral.models import GeneralGNN + +from mlmc.metamodel.graph_models import Net1 @@ -31,17 +35,25 @@ #loss_fn = KLDivergence() acc_fn = mean_squared_error -acc_fn = kl_divergence +#acc_fn = kl_divergence print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU'))) # Parameters -batch_size = 1000 # Batch size +batch_size = 10000 # Batch size epochs = 1000 # Number of training epochs -patience = 30 # Patience for early stopping +patience = 10 # Patience for early stopping l2_reg = 0#5e-4 # Regularization rate for l2 +kernel_regularization = l2(l2_reg) + +# Create model +model = Net1(conv_layer=conv_layer, hidden_activation='relu', output_activation=abs_activation, + kernel_regularization=kernel_regularization) +#model = GeneralGNN(output=1, activation=abs_activation) + + # Load data data = FlowDataset() @@ -60,29 +72,9 @@ loader_va = MixedLoader(data_va, batch_size=batch_size) loader_te = MixedLoader(data_te, batch_size=batch_size) -# Build model -class Net(Model): - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.conv1 = conv_layer(32, activation=act_func, kernel_regularizer=l2(l2_reg)) - self.conv2 = conv_layer(32, activation=act_func, kernel_regularizer=l2(l2_reg)) - self.flatten = GlobalSumPool() - self.fc1 = Dense(512, activation="relu") - self.fc2 = Dense(1, activation="linear") # linear activation for output neuron - - def call(self, inputs): - x, a = inputs - x = self.conv1([x, a]) - x = self.conv2([x, a]) - output = self.flatten(x) - output = self.fc1(output) - output = self.fc2(output) - return output -# Create model -model = Net() @@ -119,26 +111,12 @@ def evaluate(loader): results.append((loss, acc, len(target))) # Keep track of batch size if step == loader.steps_per_epoch: results = np.array(results) + print("np.average(results[:, :-1], axis=0, weights=results[:, -1]) ", + np.average(results[:, :-1], axis=0, weights=results[:, -1])) + exit() return np.average(results[:, :-1], axis=0, weights=results[:, -1]), target, predictions - -def analyze_results(target, predictions): - from scipy.stats import ks_2samp - statistics, pvalue = ks_2samp(target, predictions) - print("KS statistics: {}, pvalue: {}".format(statistics, pvalue)) - # The closer KS statistic is to 0 the more likely it is that the two samples were drawn from the same distribution - - print("len(target) ", len(target)) - print("len(predictions) ", len(predictions)) - - plt.hist(target, bins=10, alpha=0.5, label='target', density=True) - plt.hist(predictions, bins=10, alpha=0.5, label='predictions', density=True) - plt.legend(loc='upper right') - plt.show() - - - # Setup training best_val_loss = np.inf current_patience = patience @@ -159,6 +137,9 @@ def analyze_results(target, predictions): if step == loader_tr.steps_per_epoch: results_va, target, predictions = evaluate(loader_va) + + print("results_va[0] ", results_va[0]) + exit() if results_va[0] < best_val_loss: best_val_loss = results_va[0] current_patience = patience @@ -171,7 +152,6 @@ def analyze_results(target, predictions): analyze_results(target, np.squeeze(predictions.numpy())) - else: current_patience -= 1 if current_patience == 0: @@ -193,7 +173,6 @@ def analyze_results(target, predictions): results_tr = [] step = 0 -analyze_results(target, np.squeeze(predictions.numpy())) diff --git a/src/mlmc/metamodel/flow_task_NN.py b/src/mlmc/metamodel/flow_task_NN.py index e565f862..98c80686 100644 --- a/src/mlmc/metamodel/flow_task_NN.py +++ b/src/mlmc/metamodel/flow_task_NN.py @@ -1,6 +1,8 @@ -import numpy as np -from mlmc.metamodel.flow_dataset import FlowDataset import tensorflow as tf +import numpy as np +from tensorflow.keras.metrics import mean_squared_error, kl_divergence +from tensorflow.keras.layers.experimental import preprocessing +from mlmc.metamodel.graph_models import dnn_model from tensorflow import keras from tensorflow.keras import layers from tensorflow.keras.layers.experimental import preprocessing @@ -71,6 +73,7 @@ # Deep neural network # ######################### + class DNN: def __init__(self, **kwargs): self._epochs = kwargs.get('epochs', 100) @@ -124,6 +127,231 @@ def summary(self): return self._model.summary() +class DNN_2: + def __init__(self, **kwargs): + print("######## Create GNN #########") + + self._epochs = kwargs.get('epochs', 100) + #self._val_split = kwargs.get('var_split', 0.2) + #self._verbose = kwargs.get('verbose', False) + + self._hidden_activation = kwargs.get('hidden_activation', 'relu') + self._hidden_regularizer = kwargs.get('hidden_reqularizer', None) + self._output_activation = kwargs.get('output_activation', 'linear') + self._conv_layer = kwargs.get('conv_layer', None) + #self._n_hidden_layers = kwargs.get('n_hidden_layers', 1) + #self._n_hidden_neurons = kwargs.get('n_hidden_neurons', [64]) # Number of hidden neurons for each hidden layer + + self._loss = kwargs.get('loss', mean_squared_error) + self._final_loss = kwargs.get('final_loss', mean_squared_error) + self._accuracy_func = kwargs.get('accuracy_func', mean_squared_error) + self._optimizer = kwargs.get('optimizer', tf.optimizers.Adam(learning_rate=0.001)) + self._normalizer = kwargs.get('normalizer', preprocessing.Normalization()) + self._patience = kwargs.get('patience', 20) + self._verbose = kwargs.get('verbose', True) + #self._loss_changed = False + + self._train_loss = [] + self._val_loss = [] + self._test_loss = [] + + self._loss_params = {} + self._n_moments = 3 + + self.val_targets = [] + self._states = {} + self._total_n_steps = 0 + + if 'model_class' in kwargs: + model_class = kwargs.get('model_class') + net_model_config = kwargs.get('net_model_config') + model = model_class(**net_model_config) + print("model class model ", model) + else: + model = kwargs.get('model') + + if model is None: + self._model = DNNNet(conv_layer=self._conv_layer, hidden_activation=self._hidden_activation, + output_activation=self._output_activation, + kernel_regularization=self._hidden_regularizer, + normalizer=self._normalizer) + else: + self._model = model + # self._model = model(conv_layer=self._conv_layer, hidden_activation=self._hidden_activation, + # output_activation=self._output_activation, + # kernel_regularization=self._hidden_regularizer, + # normalizer=self._normalizer) + #self._model = model(n_labels=1, output_activation="relu") + + def fit(self, train, validation, test): + """ + Training procedure + """ + print("fit init loss ", self._loss) + # Setup training + self._best_val_loss = np.inf + self._current_patience = self._patience + step = 0 + self._total_n_steps = 0 + + train_targets = True + train_targets_list = [] + + # Training loop + results_tr = [] + + step += 1 + self._total_n_steps += 1 + + # Training step + inputs, target = train.x, train.y + + if train_targets: + train_targets_list.extend(target) + + loss, acc = self.train_on_batch(inputs, target) + results_tr.append((loss, acc, len(target))) + + results_va = self.evaluate(validation) + self._val_loss.append(results_va[0]) + + + train_targets = False + # results_va = self.evaluate(loader_va) + # self._val_loss.append(results_va[0]) + #print("results_va[0] ", results_va[0]) + + #print("self best val loss ", self._best_val_loss) + + if (results_va[0] + results_tr[-1][0]) < self._best_val_loss:# or (self._val_loss[-1] < self._val_loss[-2] and self._val_loss[-2] < self._val_loss[-3]): # Continue to learn if validation loss is decreasing + self._best_val_loss = (results_va[0] + results_tr[-1][0])#results_va[0] + self._current_patience = self._patience + self._states = {} + results_te = self.evaluate(test) + self._test_loss.append(results_te[0]) + else: + self._current_patience -= 1 + #results_tr_0 = np.array(results_tr) + loss_tr = results_va[0] + self._states[loss_tr] = self + + if self._current_patience == 0: + #if self._update_loss(patience=True): + print("Early stopping") + return + + # Print results + results_tr = np.array(results_tr) + results_tr = np.average(results_tr[:, :-1], 0, weights=results_tr[:, -1]) + + self._train_loss.append(results_tr[0]) + if self._verbose: + print( + "Train loss: {:.4f}, acc: {:.4f} | " + "Valid loss: {:.4f}, acc: {:.4f} | " + "Test loss: {:.4f}, acc: {:.4f}".format( + *results_tr, *results_va, *results_te + ) + ) + #self._update_loss() + # Reset epoch + results_tr = [] + step = 0 + + return train_targets_list + + def _update_loss(self, patience=False): + condition_max_loss = self._loss_params["loss_max"] #/ self._n_moments + #condition_max_loss = self._loss_params["loss_max"] + # print("self.train_loss ", self._train_loss) + m_increment = 1 + + if patience and self._n_moments <= self._loss_params["max_moments"]: + self._n_moments += m_increment + moments_fn = self._loss_params['moments_class'](self._n_moments, self._loss_params["domain"]) + self._loss = self._final_loss(moments_fn=moments_fn) + # self._loss = MSE_moments_2(moments_fn=moments_fn) + + self._best_val_loss = np.inf + print("self._loss ", self._loss) + elif patience: + return True + + # if self._train_loss[-1] > 1e10: + # moments_fn = self._loss_params['moments_class'](self._n_moments, self._loss_params["domain"]) + # #self._loss = self._final_loss(moments_fn=moments_fn) + # self._loss = MSE_moments_2(moments_fn=moments_fn) + # else: + # moments_fn = self._loss_params['moments_class'](self._n_moments, self._loss_params["domain"]) + # self._loss = self._final_loss(moments_fn=moments_fn) + + if self._n_moments <= self._loss_params["max_moments"] and len(self._train_loss) > 0\ + and self._train_loss[-1] < condition_max_loss and self._val_loss[-1] < condition_max_loss: + # print("self._train_loss ", self._train_loss) + # print("change loss, n_moments {}, last train loss: {}".format(self._n_moments, self._train_loss[-1])) + #self._n_moments = self._loss_params["max_moments"] + + print("self._n_moments ", self._n_moments) + self._n_moments += m_increment + moments_fn = self._loss_params['moments_class'](self._n_moments, self._loss_params["domain"]) + self._loss = self._final_loss(moments_fn=moments_fn) + #self._loss = MSE_moments_2(moments_fn=moments_fn) + + self._best_val_loss = np.inf + print("self._loss ", self._loss) + + # Training function + #@tf.function + def train_on_batch(self, inputs, target): + with tf.GradientTape() as tape: + predictions = self._model(inputs, training=True) + loss = self._loss(target, predictions) + sum(self._model.losses) #+ 5 * var_loss_function(target, predictions) + #loss = 100 * var_loss_function(target, predictions) + acc = tf.reduce_mean(self._accuracy_func(target, predictions)) + + gradients = tape.gradient(loss, self._model.trainable_variables) + self._optimizer.apply_gradients(zip(gradients, self._model.trainable_variables)) + + return loss, acc + + def evaluate(self, data): + step = 0 + results = [] + + if len(self.val_targets) > 0: + val_targets = False + else: + val_targets = True + + step += 1 + inputs, target = data.x, data.y + + if val_targets: + self.val_targets.extend(target) + + predictions = self._model(inputs, training=False) + + #print("evaluate loss function ", self._loss) + + loss = self._loss(target, predictions) + #print("target ", target) + #print("loss ", np.mean((target - predictions)**2)) + acc = tf.reduce_mean(self._accuracy_func(target, predictions)) + results.append((loss, acc, len(target))) # Keep track of batch size + results = np.array(results) + return np.average(results[:, :-1], axis=0, weights=results[:, -1]) + + def predict(self, data): + targets = [] + predictions = [] + + inputs, target = data.x, data.y + targets.extend(target) + predictions.extend(self._model(inputs, training=False)) + + return targets, predictions + + # def build_and_compile_model(normalizer): # model = keras.Sequential([ # normalizer, diff --git a/src/mlmc/metamodel/main.py b/src/mlmc/metamodel/main.py index 74db545c..464fd9ae 100644 --- a/src/mlmc/metamodel/main.py +++ b/src/mlmc/metamodel/main.py @@ -62,26 +62,65 @@ def support_vector_regression(df): from sklearn.svm import SVR from sklearn.preprocessing import StandardScaler print("df. info ", df.info) - train, test = train_test_split(df, test_size=0.2) - + train, test = train_test_split(df[:8000], test_size=0.2) print("train describe", train.describe()) print("test describe ", test.describe()) - - - # sc_X = StandardScaler() - # sc_y = StandardScaler() - # X = sc_X.fit_transform(train.x) - # y = sc_y.fit_transform(train.y) - x = np.stack(train.x.to_numpy(), axis=0) y = train.y.to_numpy() - svr_rbf = SVR(kernel='rbf', gamma='auto') # 'linear' kernel fitting is never-ending and 'poly' kernel gives very bad score (e.g. -2450), sigmoid gives also bad score (e.g. -125) + sc_X = StandardScaler() + sc_y = StandardScaler() + x = sc_X.fit_transform(x) + # print("y.shape ", y.shape) + # print("y.reshape(-1, 1) ", y.reshape(-1, 1).shape) + y = sc_y.fit_transform(y.reshape(-1, 1)) + + test_x = sc_X.fit_transform(np.stack(test.x.to_numpy(), axis=0)) + test_y = test.y.to_numpy().reshape(-1, 1) + test_y = sc_y.fit_transform(test.y.to_numpy().reshape(-1, 1)) + + # plt.hist(y, bins=50, alpha=0.5, label='train', density=True) + # plt.hist(test_y, bins=50, alpha=0.5, label='test', density=True) + # plt.legend(loc='upper right') + # # plt.xlim(-0.5, 1000) + # plt.yscale('log') + # plt.show() + # exit() + + # print("x.shape ", x.shape) + # print("y.shape ", y.shape) + # exit() + + #svr_rbf = SVR(kernel='rbf', verbose=True) # 'linear' kernel fitting is never-ending and 'poly' kernel gives very bad score (e.g. -2450), sigmoid gives also bad score (e.g. -125) + svr_rbf = SVR(kernel='poly', degree=50, verbose=True) svr_rbf.fit(x, y) - train_error = svr_rbf.score(np.stack(train.x.to_numpy(), axis=0), train.y.to_numpy()) - test_error = svr_rbf.score(np.stack(test.x.to_numpy(), axis=0), test.y.to_numpy()) + train_error = svr_rbf.score(x, y) + + test_error = svr_rbf.score(test_x, test_y) + + predictions = svr_rbf.predict(test_x) + + plt.hist(test_y, bins=50, alpha=0.5, label='target', density=True) + plt.hist(predictions, bins=50, alpha=0.5, label='predictions', density=True) + + # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) + plt.legend(loc='upper right') + # plt.xlim(-0.5, 1000) + plt.yscale('log') + plt.show() + + plt.hist(test_y, bins=50, alpha=0.5, label='target', density=True) + plt.hist(predictions, bins=50, alpha=0.5, label='predictions', density=True) + + # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) + plt.legend(loc='upper right') + # plt.xlim(-0.5, 1000) + #plt.yscale('log') + plt.show() + + exit() print("train error ", train_error) print("test error ", test_error) diff --git a/src/mlmc/metamodel/mnist.py b/src/mlmc/metamodel/mnist.py index e69de29b..59a2158c 100644 --- a/src/mlmc/metamodel/mnist.py +++ b/src/mlmc/metamodel/mnist.py @@ -0,0 +1,167 @@ +import numpy as np +import tensorflow as tf +from tensorflow.keras import Model +from tensorflow.keras.layers import Dense +from tensorflow.keras.losses import SparseCategoricalCrossentropy +from tensorflow.keras.metrics import sparse_categorical_accuracy +from tensorflow.keras.optimizers import Adam +from tensorflow.keras.regularizers import l2 + +from spektral.data import MixedLoader +from spektral.datasets.mnist import MNIST +from spektral.layers import GCNConv, GlobalSumPool +from spektral.layers.ops import sp_matrix_to_sp_tensor + + + +print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU'))) + + +# Parameters +batch_size = 32 # Batch size +epochs = 1000 # Number of training epochs +patience = 10 # Patience for early stopping +l2_reg = 5e-4 # Regularization rate for l2 + +# Load data +data = MNIST() + +print("data.a ", data.a) + + +# The adjacency matrix is stored as an attribute of the dataset. +# Create filter for GCN and convert to sparse tensor. +data.a = GCNConv.preprocess(data.a) +data.a = sp_matrix_to_sp_tensor(data.a) + + +# Train/valid/test split +data_tr, data_te = data[:-10000], data[-10000:] +np.random.shuffle(data_tr) +data_tr, data_va = data_tr[:-10000], data_tr[-10000:] + + + +# for tr in data_tr[:10]: +# #print(tr.x) +# print(tr.y) +# exit() + +print("data_tr[0] ", data_tr[0].n_node_features) + + +# We use a MixedLoader since the dataset is in mixed mode +loader_tr = MixedLoader(data_tr, batch_size=batch_size, epochs=epochs) +loader_va = MixedLoader(data_va, batch_size=batch_size) +loader_te = MixedLoader(data_te, batch_size=batch_size) + + +# Build model +class Net(Model): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.conv1 = GCNConv(32, activation="elu", kernel_regularizer=l2(l2_reg)) + self.conv2 = GCNConv(32, activation="elu", kernel_regularizer=l2(l2_reg)) + self.flatten = GlobalSumPool() + self.fc1 = Dense(512, activation="relu") + self.fc2 = Dense(10, activation="softmax") # MNIST has 10 classes + + def call(self, inputs): + x, a = inputs + x = self.conv1([x, a]) + x = self.conv2([x, a]) + output = self.flatten(x) + output = self.fc1(output) + output = self.fc2(output) + + return output + + +# Create model +model = Net() +optimizer = Adam() +loss_fn = SparseCategoricalCrossentropy() + + +# Training function +@tf.function +def train_on_batch(inputs, target): + with tf.GradientTape() as tape: + predictions = model(inputs, training=True) + + print("predictions ", predictions) + print("target" , target) + loss = loss_fn(target, predictions) + sum(model.losses) + acc = tf.reduce_mean(sparse_categorical_accuracy(target, predictions)) + + gradients = tape.gradient(loss, model.trainable_variables) + optimizer.apply_gradients(zip(gradients, model.trainable_variables)) + return loss, acc + + +# Evaluation function +def evaluate(loader): + step = 0 + results = [] + for batch in loader: + step += 1 + inputs, target = batch + predictions = model(inputs, training=False) + loss = loss_fn(target, predictions) + acc = tf.reduce_mean(sparse_categorical_accuracy(target, predictions)) + results.append((loss, acc, len(target))) # Keep track of batch size + if step == loader.steps_per_epoch: + results = np.array(results) + return np.average(results[:, :-1], 0, weights=results[:, -1]) + + +# Setup training +best_val_loss = 99999 +current_patience = patience +step = 0 + +# Training loop +results_tr = [] +for batch in loader_tr: + step += 1 + + # Training step + inputs, target = batch + + # print("inputs ", inputs) + # print("target ", target) + # + # print("len(inputs) ", len(inputs)) + # print("len(target) ", len(target)) + # print("inputs shape ", np.array(inputs).shape) + # exit() + + loss, acc = train_on_batch(inputs, target) + results_tr.append((loss, acc, len(target))) + + if step == loader_tr.steps_per_epoch: + results_va = evaluate(loader_va) + if results_va[0] < best_val_loss: + best_val_loss = results_va[0] + current_patience = patience + results_te = evaluate(loader_te) + else: + current_patience -= 1 + if current_patience == 0: + print("Early stopping") + break + + # Print results + results_tr = np.array(results_tr) + results_tr = np.average(results_tr[:, :-1], 0, weights=results_tr[:, -1]) + print( + "Train loss: {:.4f}, acc: {:.4f} | " + "Valid loss: {:.4f}, acc: {:.4f} | " + "Test loss: {:.4f}, acc: {:.4f}".format( + *results_tr, *results_va, *results_te + ) + ) + + # Reset epoch + results_tr = [] + step = 0 diff --git a/src/mlmc/metamodel/postprocessing.py b/src/mlmc/metamodel/postprocessing.py index ec554741..a3c0fc8b 100644 --- a/src/mlmc/metamodel/postprocessing.py +++ b/src/mlmc/metamodel/postprocessing.py @@ -454,6 +454,15 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti print("nn_level ", nn_level) print("replace level ", replace_level) + if not stats: + # print("nn_level ", nn_level) + # print("replace level ", replace_level) + + # targets = np.exp(targets) + # predictions = np.exp(predictions) + # l_0_predictions = np.exp(l_0_predictions) + # l_0_targets = np.exp(l_0_targets) + # print("targets ", targets) # print("predictions ", predictions) plt.hist(targets, bins=50, alpha=0.5, label='target', density=True) @@ -574,6 +583,7 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti #print("level id ", level_id) level_samples = estimator.get_level_samples(level_id=level_id) n_ops_predict.append(n_ops[level_id]) + data_nn.append(level_samples) print("level params ", level_params) @@ -708,7 +718,8 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti orig_moments_mean, predict_moments_mean, ref_moments_mean = compare_moments(original_q_estimator, predict_q_estimator, ref_estimator) - # compare_densities(original_q_estimator, predict_q_estimator, ref_estimator, + kl_mlmc, kl_nn = -1, -1 + # kl_mlmc, kl_nn = compare_densities(original_q_estimator, predict_q_estimator, ref_estimator, # label_1="orig N: {}".format(sample_storage.get_n_collected()), # label_2="gnn N: {}".format(sample_storage_predict.get_n_collected())) diff --git a/src/mlmc/tool/plot.py b/src/mlmc/tool/plot.py index a33ec603..91ace4e4 100644 --- a/src/mlmc/tool/plot.py +++ b/src/mlmc/tool/plot.py @@ -8,7 +8,7 @@ import matplotlib.pyplot as plt -def create_color_bar(range, label, ax = None): +def create_color_bar(range, label, ax=None, colormap=None): """ Create colorbar for a variable with given range and add it to given axes. :param range: single value as high bound or tuple (low bound, high bound) @@ -16,8 +16,10 @@ def create_color_bar(range, label, ax = None): :param ax: :return: Function to map values to colors. (normalize + cmap) """ + # Create colorbar - colormap = plt.cm.gist_ncar + if colormap is None: + colormap = plt.cm.gist_ncar try: min_r, max_r = range except TypeError: @@ -36,6 +38,7 @@ def create_color_bar(range, label, ax = None): clb.set_label(label) return lambda v: colormap(normalize(v)) + def moments_subset(n_moments, moments=None): """ Return subset of range(n_moments) for ploting. @@ -630,6 +633,86 @@ def show(self, file=""): _show_and_save(self.fig, file, self.title) +class CorrLength: + """ + Plot level variances, i.e. Var X^l as a function of the mesh step. + Selected moments are plotted. + """ + def __init__(self, moments=None): + """ + :param moments: Size or type of moments subset, see moments_subset function. + """ + matplotlib.rcParams.update({'font.size': 26}) + matplotlib.rcParams.update({'lines.markersize': 8}) + self.fig = plt.figure(figsize=(15, 8)) + self.title = "" + self.fig.suptitle(self.title) + + self.ax = self.fig.add_subplot(1, 1, 1) + self.ax.set_xlabel("$\lambda$ - correlation length") + self.ax.set_ylabel("$MSE$") + self.ax.set_xscale('log') + self.ax.set_yscale('log') + + self.n_moments = None + self.subset_type = moments + self.min_step = 1e300 + self.max_step = 0 + self.data = {} + + self.nn_min_step = 1e300 + self.nn_max_step = 0 + self.nn_data = {} + + self._mse_train = {} + self._mse_test = {} + + #self._colormap = plt.cm.tab20 + + def add_mse_train(self, mse): + """ + Add variances for single MLMC instance. + :param steps, variances : as returned by Estimate.estimate_level_vars + :param n_levels: + """ + self._mse_train = mse + + def add_mse_test(self, mse): + """ + Add variances for single MLMC instance. + :param steps, variances : as returned by Estimate.estimate_level_vars + :param n_levels: + """ + self._mse_test = mse + + def show(self, file=""): + #self._colormap = create_color_bar(range=[1, self.n_moments], label=r'$M_i$', ax=self.ax, colormap=plt.cm.tab20) + res = 5 + step_range = self.max_step / self.min_step + log_scale = step_range ** 0.001 - 1 + #rv = st.lognorm(scale=1, s=log_scale) + + # if m == 5: + # break + col = "blue" + label = "MSE train" + + print("mse train ", self._mse_train) + print("mse test ", self._mse_test) + + print("list(self._mse_train.keys()) ", list(self._mse_train.keys())) + print("self._mse_train.items() ", self._mse_train.items()) + + self.ax.scatter(list(self._mse_train.keys()), list(self._mse_train.values()), color=col, label=label) + + col = "red" + label = "MSE test" + self.ax.scatter(list(self._mse_test.keys()), list(self._mse_test.values()), color=col, label=label) + + self.fig.legend() + _show_and_save(self.fig, file, self.title) + + class Variance: """ Plot level variances, i.e. Var X^l as a function of the mesh step. @@ -639,6 +722,8 @@ def __init__(self, moments=None): """ :param moments: Size or type of moments subset, see moments_subset function. """ + matplotlib.rcParams.update({'font.size': 26}) + matplotlib.rcParams.update({'lines.markersize': 8}) self.fig = plt.figure(figsize=(15, 8)) self.title = "Level variances" self.fig.suptitle(self.title) @@ -656,6 +741,8 @@ def __init__(self, moments=None): self.data = {} + self._colormap = plt.cm.tab20 + def add_level_variances(self, steps, variances): """ Add variances for single MLMC instance. @@ -678,34 +765,43 @@ def add_level_variances(self, steps, variances): Y.extend(vars.tolist()) self.data[m] = (X, Y) - - - # def add_diff_variances(self, step, variances): # pass def show(self, file=""): + self._colormap = create_color_bar(range=[1, self.n_moments], label=r'$M_i$', ax=self.ax, colormap=plt.cm.tab20) + res = 5 step_range = self.max_step / self.min_step log_scale = step_range ** 0.001 - 1 rv = st.lognorm(scale=1, s=log_scale) for m, (X, Y) in self.data.items(): - col = plt.cm.tab20(m) + # if m == 5: + # break + col = self._colormap(m) label = "M{}".format(self.moments_subset[m]) XX = np.array(X) * rv.rvs(size=len(X)) self.ax.scatter(XX, Y, color=col, label=label) - #f = interpolate.interp1d(X, Y, kind='cubic') XX, YY = make_monotone(X, Y) - #f = interpolate.PchipInterpolator(XX[1:], YY[1:]) - m = len(XX)-1 - spl = interpolate.splrep(XX[1:], YY[1:], k=3, s=m - np.sqrt(2*m)) - xf = np.geomspace(self.min_step, self.max_step, 100) - yf = interpolate.splev(xf, spl) - self.ax.plot(xf, yf, color=col) - self.fig.legend() - _show_and_save(self.fig, file, self.title) + # step_range = self.nn_max_step / self.nn_min_step + # log_scale = step_range ** 0.01 - 1 + # rv = st.lognorm(scale=1, s=log_scale) + for m, (X, Y) in self.nn_data.items(): + # if m == 5: + # break + col = self._colormap(m) + label = "M{}".format(self.moments_subset[m]) + XX = np.array(X) * 0.9#rv.rvs(size=len(X)) + + # print("XX ", XX) + # print("Y ", Y) + self.ax.scatter(XX, Y, color=col, label=label, marker='v') + XX, YY = make_monotone(X, Y) + + #self.fig.legend() + _show_and_save(self.fig, file, self.title) class BSplots: def __init__(self, n_samples, bs_n_samples, n_moments, ref_level_var): diff --git a/test/01_cond_field/process_simple.py b/test/01_cond_field/process_simple.py index 21eb4b3d..6786cca5 100644 --- a/test/01_cond_field/process_simple.py +++ b/test/01_cond_field/process_simple.py @@ -11,7 +11,7 @@ from mlmc.moments import Legendre, Monomial from mlmc.tool.process_base import ProcessBase from mlmc.random import correlated_field as cf -#from mlmc.quantity_estimate import QuantityEstimate +# from mlmc.quantity_estimate import QuantityEstimate from mlmc.quantity import make_root_quantity from mlmc.quantity_estimate import estimate_mean, moment, moments, covariance from mlmc import estimator @@ -30,13 +30,13 @@ def __init__(self): # Remove HDF5 file, start from scratch self.debug = args.debug # 'Debug' mode is on - keep sample directories - self.use_pbs = True + self.use_pbs = False # Use PBS sampling pool - self.n_levels = 1 + self.n_levels = 5 self.n_moments = 25 # Number of MLMC levels - step_range = [1, 0.005] + step_range = [1, 0.055] # step - elements # 0.1 - 262 # 0.08 - 478 @@ -77,30 +77,63 @@ def process(self): # @TODO: How to estimate true_domain? quantile = 0.001 true_domain = mlmc.estimator.Estimate.estimate_domain(q_value, sample_storage, quantile=quantile) + print("true domain ", true_domain) moments_fn = Legendre(self.n_moments, true_domain) + n_ops = np.array(sample_storage.get_n_ops()) + print("n ops ", n_ops[:, 0] / n_ops[:, 1]) + + print("sample storage n collected ", sample_storage.get_n_collected()) + estimator = mlmc.estimator.Estimate(quantity=q_value, sample_storage=sample_storage, moments_fn=moments_fn) means, vars = estimator.estimate_moments(moments_fn) + l_0_samples = estimator.get_level_samples(level_id=0) + l_1_samples = estimator.get_level_samples(level_id=1) + l_2_samples = estimator.get_level_samples(level_id=2) + l_3_samples = estimator.get_level_samples(level_id=3) + l_4_samples = estimator.get_level_samples(level_id=4) + + print("l 0 samples shape ", np.squeeze(l_0_samples).shape) + print("l 1 samples shape ", np.squeeze(l_1_samples[..., 0]).shape) + + print("l_0_samples.var ", np.var(np.squeeze(l_0_samples)[:10000])) + print("fine l_1_samples.var ", np.var(np.squeeze(l_1_samples[..., 0]))) + print("fine l_2_samples.var ", np.var(np.squeeze(l_2_samples[..., 0]))) + print("fine l_3_samples.var ", np.var(np.squeeze(l_3_samples[..., 0]))) + print("fine l_4_samples.var ", np.var(np.squeeze(l_4_samples[..., 0]))) + + exit() + moments_quantity = moments(root_quantity, moments_fn=moments_fn, mom_at_bottom=True) moments_mean = estimate_mean(moments_quantity) conductivity_mean = moments_mean['conductivity'] time_mean = conductivity_mean[1] # times: [1] location_mean = time_mean['0'] # locations: ['0'] - print("location_mean().shape ", location_mean().shape) - values_mean = location_mean[0, 0] # result shape: (1, 1) + values_mean = location_mean[0] # result shape: (1,) + + print("values_mean.n_samples ", values_mean.n_samples) + print("values_mean.l_means ", values_mean.l_means) + + print("l_means ", values_mean.l_means) + print("values mean. l_vars ", values_mean.l_vars) + + print("np.max values mean l vars ", np.max(values_mean.l_vars, axis=1)) + + print("values_mean mean ", values_mean.mean) + print("values_mean var ", values_mean.var) + value_mean = values_mean[0] - print("value_mean ", value_mean()) - assert value_mean() == 1 + assert value_mean.mean == 1 # true_domain = [-10, 10] # keep all values on the original domain # central_moments = Monomial(self.n_moments, true_domain, ref_domain=true_domain, mean=means()) # central_moments_quantity = moments(root_quantity, moments_fn=central_moments, mom_at_bottom=True) # central_moments_mean = estimate_mean(central_moments_quantity) - #estimator.sub_subselect(sample_vector=[10000]) + # estimator.sub_subselect(sample_vector=[10000]) - #self.process_target_var(estimator) + # self.process_target_var(estimator) self.construct_density(estimator, tol=1e-8) self.data_plots(estimator) @@ -111,7 +144,8 @@ def process_target_var(self, estimator): n0, nL = 100, 3 n_samples = np.round(np.exp2(np.linspace(np.log2(n0), np.log2(nL), self.n_levels))).astype(int) - n_estimated = estimator.bs_target_var_n_estimated(target_var=1e-5, sample_vec=n_samples) # number of estimated sampels for given target variance + n_estimated = estimator.bs_target_var_n_estimated(target_var=1e-5, + sample_vec=n_samples) # number of estimated sampels for given target variance estimator.plot_variances(sample_vec=n_estimated) estimator.plot_bs_var_log(sample_vec=n_estimated) @@ -125,8 +159,9 @@ def construct_density(self, estimator, tol=1.95, reg_param=0.0): :return: None """ distr_obj, result, _, _ = estimator.construct_density(tol=tol, reg_param=reg_param) - #distr_plot = mlmc.tool.plot.Distribution(title="{} levels, {} moments".format(self.n_levels, self.n_moments)) - distr_plot = mlmc.tool.plot.ArticleDistribution(title="{} levels, {} moments".format(self.n_levels, self.n_moments)) + # distr_plot = mlmc.tool.plot.Distribution(title="{} levels, {} moments".format(self.n_levels, self.n_moments)) + distr_plot = mlmc.tool.plot.ArticleDistribution( + title="{} levels, {} moments".format(self.n_levels, self.n_moments)) distr_plot.add_distribution(distr_obj, label="#{}".format(self.n_moments)) @@ -155,7 +190,7 @@ def run(self, renew=False): sampler = self.setup_config(clean=True) # Schedule samples self.generate_jobs(sampler, n_samples=None, renew=renew, target_var=1e-5) - #self.generate_jobs(sampler, n_samples=[500, 500], renew=renew, target_var=1e-5) + # self.generate_jobs(sampler, n_samples=[500, 500], renew=renew, target_var=1e-5) self.all_collect(sampler) # Check if all samples are finished self.calculate_moments(sampler) # Simple moment check @@ -186,7 +221,7 @@ def setup_config(self, clean): # Create HDF sample storage sample_storage = SampleStorageHDF( file_path=os.path.join(self.work_dir, "mlmc_{}.hdf5".format(self.n_levels)), - #append=self.append + # append=self.append ) # Create sampler, it manages sample scheduling and so on @@ -291,7 +326,7 @@ def generate_jobs(self, sampler, n_samples=None, renew=False, target_var=None): # New estimation according to already finished samples variances, n_ops = q_estimator.estimate_diff_vars_regression(sampler._n_scheduled_samples) n_estimated = estimator.estimate_n_samples_for_target_variance(target_var, variances, n_ops, - n_levels=sampler.n_levels) + n_levels=sampler.n_levels) # Loop until number of estimated samples is greater than the number of scheduled samples while not sampler.process_adding_samples(n_estimated, sleep, add_coef): @@ -303,13 +338,13 @@ def generate_jobs(self, sampler, n_samples=None, renew=False, target_var=None): n_ops_str = ",".join([str(n_o) for n_o in n_ops]) writer.write("{}; {}; {}; {}; {}; {}\n".format(n_target_str, n_scheduled_str, - n_estimated_str, variances_str, - n_ops_str, str(time.time() - start_time))) + n_estimated_str, variances_str, + n_ops_str, str(time.time() - start_time))) # New estimation according to already finished samples variances, n_ops = q_estimator.estimate_diff_vars_regression(sampler._n_scheduled_samples) n_estimated = estimator.estimate_n_samples_for_target_variance(target_var, variances, n_ops, - n_levels=sampler.n_levels) + n_levels=sampler.n_levels) def all_collect(self, sampler): """ diff --git a/test/metamodels/compare_models.py b/test/metamodels/compare_models.py index e69de29b..c48a98b3 100644 --- a/test/metamodels/compare_models.py +++ b/test/metamodels/compare_models.py @@ -0,0 +1,33 @@ +import os + +#os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only +import sys +import subprocess +from mlmc.metamodel.analyze_nn import run_GNN, run_SVR, statistics, analyze_statistics, process_results +import tensorflow as tf + +from mlmc.metamodel.flow_task_GNN_2 import GNN +from spektral.layers import GCNConv, GlobalSumPool, ChebConv, GraphSageConv, ARMAConv, GATConv, APPNPConv, GINConv, GeneralConv +from tensorflow.keras.losses import MeanSquaredError, KLDivergence, MeanAbsoluteError +from tensorflow.keras import Model +from tensorflow.keras.layers import Dense +from spektral.layers import GlobalSumPool, GlobalMaxPool, GlobalAvgPool +import tensorflow as tf +from tensorflow.keras.layers.experimental import preprocessing + + +def compare_channels(): + data_path = "/home/martin/Documents/metamodels/data/comparison/ChebConv_channels" + channels = [8, 16, 32, 64, 128, 256] + channels = [2, 4, 8, 16, 32, 128] + + for channel in channels: + channel_path = os.path.join(data_path, "{0}/ChebConv{0}".format(channel)) + print("channel path ", channel_path) + analyze_statistics(channel_path) + + + + +if __name__ == "__main__": + compare_channels() diff --git a/test/metamodels/metamodel_test.py b/test/metamodels/metamodel_test.py index ae0762e7..9e03b2a5 100644 --- a/test/metamodels/metamodel_test.py +++ b/test/metamodels/metamodel_test.py @@ -1,9 +1,10 @@ import os +import numpy as np import warnings #os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only import sys import subprocess -from mlmc.metamodel.analyze_nn import run_GNN, run_SVR, statistics, analyze_statistics, process_results +from mlmc.metamodel.analyze_nn import run_GNN, run_SVR, statistics, analyze_statistics, process_results, run_DNN from mlmc.moments import Legendre_tf, Monomial from mlmc.metamodel.flow_task_GNN_2 import GNN @@ -14,6 +15,7 @@ from tensorflow.keras.layers import Dense from spektral.layers import GlobalSumPool, GlobalMaxPool, GlobalAvgPool import tensorflow as tf +from mlmc.tool import plot from tensorflow.keras.layers.experimental import preprocessing warnings.filterwarnings("ignore", category=DeprecationWarning) @@ -40,7 +42,7 @@ def get_gnn(): patience = 150 hidden_regularization = None # l2(2e-10) - model_config = { + net_model_config = { "conv_layer": conv_layer, "hidden_activation": 'relu', "output_activation": abs_activation, @@ -49,13 +51,15 @@ def get_gnn(): "normalizer": preprocessing.Normalization() } - model = Net(**model_config) + #model = Net(**net_model_config) model_config = {"loss": loss, "optimizer": optimizer, "patience": patience, - "model": model, - "verbose": False} + "model_class": Net, + "net_model_config": net_model_config, + "verbose": True} + corr_field_config = {'corr_length': 0.1, 'sigma': 1, 'log': True} @@ -68,7 +72,7 @@ def __init__(self, conv_layer, hidden_activation, output_activation, kernel_regu super().__init__(**kwargs) # self.normalizer = normalizer # self.norm_layer = tf.keras.layers.LayerNormalization(axis=1) - self.conv1 = conv_layer(32, K=1, activation=hidden_activation, kernel_regularizer=kernel_regularization) + self.conv1 = conv_layer(8, K=1, activation=hidden_activation, kernel_regularizer=kernel_regularization) # self.conv2 = conv_layer(64, K=1, activation=hidden_activation, kernel_regularizer=kernel_regularization) # self.conv3 = conv_layer(32, K=1, activation=hidden_activation, kernel_regularizer=kernel_regularization) # self.conv4 = conv_layer(32, K=1, activation=hidden_activation, kernel_regularizer=kernel_regularization) @@ -185,6 +189,62 @@ def get_config(data_dir, case=0): sampling_info_path = os.path.join(data_dir, "{}/sampling_info".format(cl)) ref_mlmc_file = os.path.join(data_dir, "{}/L1_benchmark/mlmc_1.hdf5".format(cl)) + elif case == 7: # mesh size comparison + data_dir = "/home/martin/Documents/metamodels/data/mesh_size/" + cl = "cl_0_1_s_1" + level = 2 + nn_level = 0 + replace_level = False + #mesh = os.path.join(data_dir, "l_step_1.0_common_files/mesh.msh".format(cl)) #L1, 7s + mesh = os.path.join(data_dir, "l_step_0.27232698153315_common_files/mesh.msh".format(cl)) #L2 10.5 s + #mesh = os.path.join(data_dir, "l_step_0.07416198487095663_common_files/mesh.msh".format(cl)) #L3 12s + #mesh = os.path.join(data_dir, "l_step_0.020196309484414757_common_files/mesh.msh".format(cl)) #L4 22s + #mesh = os.path.join(data_dir, "l_step_0.0055_common_files/mesh.msh".format(cl)) #L5 + output_dir = os.path.join(data_dir, "{}/L1_{}/test/01_cond_field/output/".format(cl, level)) + hdf_path = os.path.join(data_dir, "{}/L1_{}/mlmc_1.hdf5".format(cl, level)) + mlmc_hdf_path = os.path.join(data_dir, "{}/mlmc_hdf/L1_{}/mlmc_1.hdf5".format(cl, level)) + save_path = os.path.join(data_dir, "{}".format(cl)) + l_0_output_dir = os.path.join(data_dir, "{}/L0_MC/L1_{}/test/01_cond_field/output/".format(cl, level)) + l_0_hdf_path = os.path.join(data_dir, "{}/L0_MC/L1_{}/mlmc_1.hdf5".format(cl, level)) + sampling_info_path = os.path.join(data_dir, "{}/sampling_info".format(cl)) + ref_mlmc_file = os.path.join(data_dir, "{}/L1_benchmark/mlmc_1.hdf5".format(cl)) + + elif case == 8 or case == 9: + data_dir = "/home/martin/Documents/metamodels/data/5_ele/" + cl = "cl_0_1_s_1" + if case == 9: + cl = "cl_0_3_s_4" + nn_level = 0 + replace_level = False + # mesh = os.path.join(data_dir, "{}/L5/l_step_0.020196309484414757_common_files/mesh.msh".format(cl)) + mesh = os.path.join(data_dir, "l_step_0.07416198487095663_common_files/mesh.msh".format(cl)) + output_dir = os.path.join(data_dir, "{}/L3_1/test/01_cond_field/output/".format(cl)) + hdf_path = os.path.join(data_dir, "{}/L3_1/mlmc_3.hdf5".format(cl)) + mlmc_hdf_path = os.path.join(data_dir, "{}/L3_2/mlmc_3.hdf5".format(cl)) + save_path = os.path.join(data_dir, "{}".format(cl)) + l_0_output_dir = os.path.join(data_dir, "{}/L1_2/test/01_cond_field/output/".format(cl,nn_level)) + l_0_hdf_path = os.path.join(data_dir, "{}/L1_2/mlmc_1.hdf5".format(cl, nn_level)) + sampling_info_path = os.path.join(data_dir, "{}/sampling_info".format(cl)) + ref_mlmc_file = os.path.join(data_dir, "{}/L1_benchmark/mlmc_1.hdf5".format(cl)) + + elif case == 10 or case == 11: + data_dir = "/home/martin/Documents/metamodels/data/5_ele/" + cl = "cl_0_1_s_1" + if case == 11: + cl = "cl_0_3_s_4" + nn_level = 0 + replace_level = False + # mesh = os.path.join(data_dir, "{}/L5/l_step_0.020196309484414757_common_files/mesh.msh".format(cl)) + mesh = os.path.join(data_dir, "l_step_0.07416198487095663_common_files/mesh.msh".format(cl)) + output_dir = os.path.join(data_dir, "{}/L2_1/test/01_cond_field/output/".format(cl)) + hdf_path = os.path.join(data_dir, "{}/L2_1/mlmc_2.hdf5".format(cl)) + mlmc_hdf_path = os.path.join(data_dir, "{}/L2_2/mlmc_2.hdf5".format(cl)) + save_path = os.path.join(data_dir, "{}".format(cl)) + l_0_output_dir = os.path.join(data_dir, "{}/L1_2/test/01_cond_field/output/".format(cl,nn_level)) + l_0_hdf_path = os.path.join(data_dir, "{}/L1_2/mlmc_1.hdf5".format(cl, nn_level)) + sampling_info_path = os.path.join(data_dir, "{}/sampling_info".format(cl)) + ref_mlmc_file = os.path.join(data_dir, "{}/L1_benchmark/mlmc_1.hdf5".format(cl)) + # elif case == 6: # nn_level = 0 # replace_level = True @@ -201,6 +261,95 @@ def get_config(data_dir, case=0): return output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, replace_level, nn_level, +def plot_results_corr_length(): + cl_all = {"cl_0_001_s_1": 0.001, "cl_0_01_s_1": 0.01, "cl_0_1_s_1": 0.1, "cl_1_s_1": 1, "cl_10_s_1": 10} + + # cl_all = {"cl_0_001_s_1": 0.001, "cl_10_s_1": 1} + # + cl_all = {"cl_0_001_s_1": 0.001} + + tr_MSE = {} + te_MSE = {} + tr_RSE = {} + te_RSE = {} + + for cl_dir, cl in cl_all.items(): + data_dir = "/home/martin/Documents/metamodels/data/mesh_size/" + level = 3 + nn_level = 0 + replace_level = False + # mesh = os.path.join(data_dir, "l_step_1.0_common_files/mesh.msh".format(cl)) #L1, 7s + # mesh = os.path.join(data_dir, "l_step_0.27232698153315_common_files/mesh.msh".format(cl)) #L2 10.5 s + mesh = os.path.join(data_dir, "l_step_0.07416198487095663_common_files/mesh.msh".format(cl_dir)) # L3 12s + # mesh = os.path.join(data_dir, "l_step_0.020196309484414757_common_files/mesh.msh".format(cl)) #L4 22s + # mesh = os.path.join(data_dir, "l_step_0.0055_common_files/mesh.msh".format(cl)) #L5 + output_dir = os.path.join(data_dir, "{}/L1_{}/test/01_cond_field/output/".format(cl_dir, level)) + hdf_path = os.path.join(data_dir, "{}/L1_{}/mlmc_1.hdf5".format(cl_dir, level)) + mlmc_hdf_path = os.path.join(data_dir, "{}/mlmc_hdf/L1_{}/mlmc_1.hdf5".format(cl_dir, level)) + save_path = os.path.join(data_dir, "{}".format(cl_dir)) + l_0_output_dir = os.path.join(data_dir, "{}/L0_MC/L1_{}/test/01_cond_field/output/".format(cl_dir, level)) + l_0_hdf_path = os.path.join(data_dir, "{}/L0_MC/L1_{}/mlmc_1.hdf5".format(cl_dir, level)) + sampling_info_path = os.path.join(data_dir, "{}/sampling_info".format(cl_dir)) + ref_mlmc_file = os.path.join(data_dir, "{}/L1_benchmark/mlmc_1.hdf5".format(cl_dir)) + + machine_learning_model = ("mesh_L3_log_15k", run_GNN, False) + + gnn, conv_layer, corr_field_config, model_config = get_gnn() + + save_path = os.path.join(save_path, machine_learning_model[0]) + + print("save path ", save_path) + graph_creation_time = 28 # 22#159#0#159#66 + + config = {'machine_learning_model': machine_learning_model, + 'save_path': save_path, + 'sampling_info_path': sampling_info_path, + 'output_dir': output_dir, + 'nn_hdf_path': hdf_path, + 'mlmc_hdf_path': mlmc_hdf_path, + 'mesh': mesh, + 'l_0_output_dir': l_0_output_dir, + 'l_0_hdf_path': l_0_hdf_path, + 'ref_mlmc_file': ref_mlmc_file, + 'level': nn_level, + 'conv_layer': conv_layer, + 'gnn': gnn, + 'model_config': model_config, + 'replace_level': replace_level, + 'corr_field_config': corr_field_config, + 'n_train_samples': 2000, + 'val_samples_ratio': 0.3, + 'batch_size': 200, + 'epochs': 2000, + 'learning_rate': 0.01, + 'graph_creation_time': graph_creation_time, + 'save_model': False, + 'loss_params': {'moments_class': Legendre_tf, "max_moments": 20, 'loss_max': 0.5, 'quantile': 1e-3} + } + + train_MSE, test_MSE, train_RSE, test_RSE = analyze_statistics(config) + + tr_MSE[cl] = np.mean(train_MSE) + te_MSE[cl] = np.mean(test_MSE) + tr_RSE[cl] = np.mean(train_RSE) + te_RSE[cl] = np.mean(test_RSE) + + + plt_cl = plot.CorrLength() + plt_cl.add_mse_test(te_MSE) + plt_cl.add_mse_train(tr_MSE) + + plt_cl.show(None) + plt_cl.show("corr_length_mse") + + plt_cl = plot.CorrLength() + plt_cl.add_mse_test(te_RSE) + plt_cl.add_mse_train(tr_RSE) + + plt_cl.show(None) + plt_cl.show("corr_length_mse") + + def get_arguments(arguments): """ Getting arguments from console @@ -224,10 +373,11 @@ def get_arguments(arguments): output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, replace_level, nn_level = get_config( data_dir, case) - if os.path.exists(os.path.join(work_dir, "mlmc_{}.hdf5".format(nn_level + 1))): - l_0_hdf_path = os.path.join(work_dir, "mlmc_{}.hdf5".format(nn_level + 1)) - hdf_path = os.path.join(work_dir, "mlmc_5.hdf5") - ref_mlmc_file = os.path.join(work_dir, "benchmark_mlmc_1.hdf5") + # if os.path.exists(os.path.join(work_dir, "mlmc_{}.hdf5".format(nn_level + 1))): + # l_0_hdf_path = os.path.join(work_dir, "mlmc_{}.hdf5".format(nn_level + 1)) + # hdf_path = os.path.join(work_dir, "mlmc_5.hdf5") + # ref_mlmc_file = os.path.join(work_dir, "benchmark_mlmc_1.hdf5") + # import cProfile # import pstats @@ -247,7 +397,6 @@ def get_arguments(arguments): #run_CNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True) # , gnn=gnn) #process_results(hdf_path, sampling_info_path, ref_mlmc_file, save_path, nn_level, replace_level) - # Graph creation time for cl_0_1_s_1 case 1 = 100 s # gnn, conv_layer, corr_field_config = get_gnn() @@ -269,8 +418,49 @@ def get_arguments(arguments): # # machine_learning_model = ("5eleChebConvK3", run_GNN, False) #machine_learning_model = ("5eleChebConv32abs", run_GNN, False) #machine_learning_model = ("5eleChebConv32msemom", run_GNN, False) + + #################### + ### Compare number of training samples ### + # machine_learning_model = ("mesh_L3", run_GNN, False) + # machine_learning_model = ("mesh_L3_t_5k", run_GNN, False) + # machine_learning_model = ("mesh_L3_t_10k", run_GNN, False) + # #machine_learning_model = ("mesh_L3_t_15k", run_GNN, False) + # ################ + # ################ + # + # #################### + # ### Compare number of training samples LOG ### + # machine_learning_model = ("mesh_L3_t_100_log", run_GNN, False) + # machine_learning_model = ("mesh_L3_t_500_log", run_GNN, False) + # machine_learning_model = ("mesh_L3_t_1000_log", run_GNN, False) + # machine_learning_model = ("mesh_L3_t_1500_log", run_GNN, False) + # machine_learning_model = ("mesh_L3_t_2000_log", run_GNN, False) + # machine_learning_model = ("mesh_L3_t_4000_log", run_GNN, False) + # machine_learning_model = ("mesh_L3_t_8000_log", run_GNN, False) + # # # machine_learning_model = ("mesh_L3_t_15k", run_GNN, False) + # ################ + # ################ + # + # ######## + # # Test different correlation lengths + # machine_learning_model = ("mesh_L3_log", run_GNN, False) + # #machine_learning_model = ("SVR_mesh_L3_log", run_GNN, False) + # ####### + # + #machine_learning_model = ("DNN_mesh_L3_log_deep", run_DNN, True) + # + machine_learning_model = ("DNN_mesh_L3_6", run_DNN, True) + machine_learning_model = ("GCN_mesh_L3_log_16", run_GNN, True) + machine_learning_model = ("mesh_L3_log", run_GNN, True) + + #machine_learning_model = ("mesh_L3_seed", run_GNN, False) + + #machine_learning_model = ("mesh_L3_log_sigmoid", run_GNN, False) # ReLU is much better + save_path = os.path.join(save_path, machine_learning_model[0]) - graph_creation_time = 0#66 + + print("save path ", save_path) + graph_creation_time = 11#22#159#0#159#66 config = {'machine_learning_model': machine_learning_model, 'save_path': save_path, @@ -290,15 +480,16 @@ def get_arguments(arguments): 'n_train_samples': 2000, 'val_samples_ratio': 0.2, 'batch_size': 200, - 'epochs': 2000, + 'epochs': 5, 'learning_rate': 0.001, 'graph_creation_time': graph_creation_time, 'save_model': False } - statistics(config) + #statistics(config) analyze_statistics(config) # save_path = os.path.join(save_path, "SVR") # statistics(run_SVR, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True) + diff --git a/test/metamodels/nn_config.py b/test/metamodels/nn_config.py index 9b080a8e..272dfc23 100644 --- a/test/metamodels/nn_config.py +++ b/test/metamodels/nn_config.py @@ -7,86 +7,3 @@ import tensorflow as tf from tensorflow.keras.layers.experimental import preprocessing - -def get_gnn(): - # Parameters - # conv_layer = GCNConv - conv_layer = ChebConv # Seems better than GCNConv, good distribution of predictions - # conv_layer = OwnChebConv - # conv_layer = GraphSageConv # Seems better than ChebConv, good loss but very narrow distribution of predictions - # # conv_layer = ARMAConv # Seems worse than GraphSageConv - # conv_layer = GATConv # Slow and not better than GraphSageConv - # # conv_layer = APPNPConv # Not bad but worse than GraphSageConv - # # conv_layer = GINConv # it is comparable to APPNPConv - # act_func = "relu" # "tanh"#"elu" - - loss = MeanSquaredError() # var_loss_function# - # loss = MeanAbsoluteError() - # loss = MeanSquaredLogarithmicError() - # loss = KLDivergence() - # loss = total_loss_function - optimizer = tf.optimizers.Adam(learning_rate=0.001) - patience = 150 - hidden_regularization = None # l2(2e-10) - - model_config = { - "conv_layer": conv_layer, - "hidden_activation": 'relu', - "output_activation": 'linear', - "kernel_regularization": hidden_regularization, - "normalizer": preprocessing.Normalization() - } - - model = Net(**model_config) - - model_config = {"loss": loss, - "optimizer": optimizer, - "patience": patience, - "model": model, - "verbose": False} - - print("get gnn") - - return GNN(**model_config), conv_layer - - -class Net(Model): - def __init__(self, conv_layer, hidden_activation, output_activation, kernel_regularization, normalizer, - **kwargs): - super().__init__(**kwargs) - # self.normalizer = normalizer - # self.norm_layer = tf.keras.layers.LayerNormalization(axis=1) - self.conv1 = conv_layer(32, activation=hidden_activation, kernel_regularizer=kernel_regularization) - # self.conv2 = conv_layer(128, K=2, activation=hidden_activation, kernel_regularizer=kernel_regularization) - # self.conv3 = conv_layer(16, K=2, activation=hidden_activation, kernel_regularizer=kernel_regularization) - # self.conv4 = conv_layer(8, K=2,activation=hidden_activation, kernel_regularizer=kernel_regularization) - # self.conv2 = conv_layer(32, K=2, activation=hidden_activation, kernel_regularizer=kernel_regularization) - # self.conv3 = conv_layer(16, K=2, activation=hidden_activation, kernel_regularizer=kernel_regularization) - # self.conv3 = conv_layer(8, activation=hidden_activation, kernel_regularizer=kernel_regularization) - # self.conv4 = conv_layer(4, activation=hidden_activation, kernel_regularizer=kernel_regularization) - # self.conv3 = conv_layer(64, activation=hidden_activation, kernel_regularizer=kernel_regularization) - self.flatten = GlobalSumPool() - # self.fc1 = Dense(32, activation=hidden_activation) - self.fc2 = Dense(1) # , activation=output_activation) # linear activation for output neuron - - def call(self, inputs): - x, a = inputs - # print("x ", x) - # x = self.normalizer(x) - # x = self.norm_layer(x) - # print("normalized x ", x) - # print("x[0,0,:] ", x[0, 0, :]) - x = self.conv1([x, a]) - # print("x.shape ", x.shape) - # x = self.conv2([x, a]) - # # print("conv2 x shape", x.shape) - # x = self.conv3([x, a]) - # x = self.conv4([x, a]) - output1 = self.flatten(x) - # output2 = self.fc1(output1) - output = self.fc2(output1) - return output - - -if __name__ == "__main__": - get_gnn() diff --git a/test/test_quantity_concept.py b/test/test_quantity_concept.py index b3e4c86f..8145e9a1 100644 --- a/test/test_quantity_concept.py +++ b/test/test_quantity_concept.py @@ -467,6 +467,9 @@ def fill_sample_storage(self, sample_storage, chunk_size=512000000): fine_result = np.random.randint(5 + 5*sample_id, high=5+5*(1+sample_id), size=(np.sum(sizes),)) + print("fine results shape ", fine_result.shape) + exit() + if l_id == 0: coarse_result = (np.zeros((np.sum(sizes),))) else: @@ -655,5 +658,5 @@ def dev_memory_usage_test(self): if __name__ == '__main__': qt = QuantityTests() - qt.test_moments() + qt.test_basics() #unittest.main() From b95b49de8ddc18d9827adfb7675e1c8b7569101b Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Wed, 20 Oct 2021 09:45:26 +0200 Subject: [PATCH 25/67] sparse matrix --- src/mlmc/metamodel/analyze_nn.py | 2 +- src/mlmc/metamodel/flow_task_GNN_2.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlmc/metamodel/analyze_nn.py b/src/mlmc/metamodel/analyze_nn.py index 92fbc673..4267fed6 100644 --- a/src/mlmc/metamodel/analyze_nn.py +++ b/src/mlmc/metamodel/analyze_nn.py @@ -26,7 +26,7 @@ from mlmc.metamodel.flow_task_GNN_2 import GNN from tensorflow.keras.losses import MeanSquaredError from spektral.data import MixedLoader -from spektral.layers.ops import sp_matrix_to_sp_tensor +from spektral.utils.sparse import sp_matrix_to_sp_tensor print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU'))) epochs = 100 diff --git a/src/mlmc/metamodel/flow_task_GNN_2.py b/src/mlmc/metamodel/flow_task_GNN_2.py index 34af162a..85bcfce7 100644 --- a/src/mlmc/metamodel/flow_task_GNN_2.py +++ b/src/mlmc/metamodel/flow_task_GNN_2.py @@ -11,7 +11,7 @@ from spektral.data import MixedLoader from mlmc.metamodel.flow_dataset import FlowDataset from spektral.layers import GCNConv, GlobalSumPool, ChebConv, GraphSageConv, ARMAConv, GATConv, APPNPConv, GINConv -from spektral.layers.ops import sp_matrix_to_sp_tensor +from spektral.utils.sparse import sp_matrix_to_sp_tensor from tensorflow.keras.layers.experimental import preprocessing from mlmc.metamodel.custom_methods import abs_activation, var_loss_function From 479167288abf7af7dc66bad00d1c0a8d3c1062d7 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Wed, 20 Oct 2021 15:53:33 +0200 Subject: [PATCH 26/67] metamodel small refactoring --- requirements.txt | 1 + src/mlmc/metamodel/analyze_nn.py | 68 ++++++--- src/mlmc/metamodel/flow_dataset.py | 8 ++ src/mlmc/metamodel/flow_task_GNN_2.py | 10 +- src/mlmc/sample_storage_hdf.py | 65 +++++++-- src/mlmc/tool/hdf5.py | 136 +++++++++++++++++- src/mlmc/tool/plot.py | 194 +++++++++++++++++++++++--- test/metamodels/metamodel_test.py | 138 ++++++++++++++++-- 8 files changed, 556 insertions(+), 64 deletions(-) diff --git a/requirements.txt b/requirements.txt index 2aaffe70..423e7e91 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ ruamel.yaml attrs gstools memoization +tensorflow diff --git a/src/mlmc/metamodel/analyze_nn.py b/src/mlmc/metamodel/analyze_nn.py index 4267fed6..86767a04 100644 --- a/src/mlmc/metamodel/analyze_nn.py +++ b/src/mlmc/metamodel/analyze_nn.py @@ -345,12 +345,16 @@ def predict_level_zero_SVR(nn, output_dir, hdf_path, mesh, batch_size=1000, log= def statistics(config): - n_subsamples = 2 + n_subsamples = 5 model_title, mch_l_model, log = config['machine_learning_model'] model_data = {} model_data["log"] = log + # seeds = [] + # for i in range(n_subsamples): + # seeds.append(i * 125) + if not os.path.isdir(config['save_path']): os.makedirs(config['save_path']) else: @@ -364,7 +368,7 @@ def statistics(config): model, targets, predictions, learning_time, train_targets, train_predictions, \ val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, total_steps = \ - mch_l_model(config, stats=True, train=True, log=log) + mch_l_model(config, stats=True, train=True, log=log, seed=i) if config['save_model']: model_data["model"] = model @@ -554,7 +558,8 @@ def analyze_statistics(config): plt_var = plot.Variance() l_vars = np.mean(mlmc_l_vars, axis=0) - plt_var.add_level_variances(np.squeeze(orig_level_params), l_vars) + orig_level_params = orig_level_params # np.squeeze(orig_level_params) + plt_var.add_level_variances(orig_level_params, l_vars) plt_var.show("mlmc_vars") plt_var = plot.Variance() @@ -635,7 +640,8 @@ def analyze_statistics(config): print("######################################") -def run_GNN(config, stats=True, train=True, log=False): +def run_GNN(config, stats=True, train=True, log=False, seed=0): + print("seed ", seed) loss = MeanSquaredError() # var_loss_function# accuracy_func = MSE_moments @@ -662,11 +668,11 @@ def run_GNN(config, stats=True, train=True, log=False): data = data#[:10000] #print("len data ", len(data)) - data.shuffle() + #data.shuffle(seed=seed) preprocess_time = time.process_time() - preprocess_start_time #print("preproces time ", preprocess_time) preprocess_time = preprocess_time + graph_creation_time - print("total preprocess time ", preprocess_time) + #print("total preprocess time ", preprocess_time) learning_time_start = time.process_time() data.a = config['conv_layer'].preprocess(data.a) @@ -674,18 +680,24 @@ def run_GNN(config, stats=True, train=True, log=False): #train_data_len = int(len(data) * 0.8) train_data_len = config['n_train_samples'] # Train/valid/test split - data_tr, data_te = data[:train_data_len], data[train_data_len:] - gnn = config['gnn'] + data_tr = data[seed*train_data_len: seed*train_data_len + train_data_len] + data_te = data.get_test_data(seed, train_data_len) + #data_tr, data_te = data[:train_data_len], data[train_data_len:] + - if hasattr(gnn._loss,'__name__') and gnn._loss.__name__ == "MSE_moments": - tr_output = [g.y for g in data_tr] - n_moments = 3 - quantile = 0.001 - domain = np.percentile(tr_output, [100 * quantile, 100 * (1 - quantile)]) - moments_fn = Legendre_tf(n_moments, domain) - #accuracy_func = MSE_moments(moments_fn=moments_fn) - gnn._loss = MSE_moments(moments_fn=moments_fn) + print("config['model_config'] ", config['model_config']) + + gnn = config['gnn'](**config['model_config']) + + # if hasattr(gnn._loss,'__name__') and gnn._loss.__name__ == "MSE_moments": + # tr_output = [g.y for g in data_tr] + # n_moments = 3 + # quantile = 0.001 + # domain = np.percentile(tr_output, [100 * quantile, 100 * (1 - quantile)]) + # moments_fn = Legendre_tf(n_moments, domain) + # #accuracy_func = MSE_moments(moments_fn=moments_fn) + # gnn._loss = MSE_moments(moments_fn=moments_fn) np.random.shuffle(data_tr) val_data_len = int(len(data_tr) * config['val_samples_ratio']) @@ -708,9 +720,12 @@ def run_GNN(config, stats=True, train=True, log=False): # batch_size 500, ideally 500 epochs, patience 35 if train: + print("gnn ", gnn) # gnn.run_eagerly = True train_targets = gnn.fit(loader_tr, loader_va, loader_te) + learning_time = time.process_time() - learning_time_start + # states = gnn._states # if len(states) > 0: # min_key = np.min(list(states.keys())) @@ -727,7 +742,7 @@ def run_GNN(config, stats=True, train=True, log=False): targets, predictions = gnn.predict(loader_te) predictions = np.squeeze(predictions) - learning_time = time.process_time() - learning_time_start + #print("learning time ", learning_time) targets = np.array(targets) @@ -760,7 +775,8 @@ def run_GNN(config, stats=True, train=True, log=False): config['l_0_hdf_path'], config['mesh'], config['conv_layer'], batch_size, log, stats=stats, - corr_field_config=config['corr_field_config']) + corr_field_config=config['corr_field_config'], + seed=seed) #predict_l_0_time = time.process_time() - predict_l_0_start_time if stats: @@ -784,16 +800,22 @@ def run_GNN(config, stats=True, train=True, log=False): l_0_predictions) -def predict_level_zero(nn, output_dir, hdf_path, mesh, conv_layer, batch_size=1000, log=False, stats=False, corr_field_config=None): - graph_creator(output_dir, hdf_path, mesh, level=0) +def predict_level_zero(nn, output_dir, hdf_path, mesh, conv_layer, batch_size=1000, log=False, stats=False, + corr_field_config=None, seed=1234): + # graph_creator(output_dir, hdf_path, mesh, level=0) # Load data sample_time = 0 if corr_field_config: sample_time = corr_field_sample_time(mesh, corr_field_config) - data = FlowDataset(output_dir=output_dir, log=log)#, mesh=mesh, corr_field_config=corr_field_config) - #data = data # [:10000] + data = FlowDataset(output_dir=output_dir, log=log) # , mesh=mesh, corr_field_config=corr_field_config) + # data = data # [:10000] + data.shuffle(seed=seed) + + print("output_dir ", output_dir) + print("len(data) ", len(data)) + print("data[0] ", data[0]) predict_time_start = time.process_time() data.a = conv_layer.preprocess(data.a) @@ -815,7 +837,7 @@ def predict_level_zero(nn, output_dir, hdf_path, mesh, conv_layer, batch_size=10 predict_time = time.process_time() - predict_time_start - return targets, predictions, predict_time + sample_time*len(data) + return targets, predictions, predict_time + sample_time * len(data) def save_times(path, load=False, preprocess=None, learning_time=None, predict_l_0=None): diff --git a/src/mlmc/metamodel/flow_dataset.py b/src/mlmc/metamodel/flow_dataset.py index eb22a3cd..ed025700 100644 --- a/src/mlmc/metamodel/flow_dataset.py +++ b/src/mlmc/metamodel/flow_dataset.py @@ -1,5 +1,6 @@ import os import re +import copy import random import numpy as np import pandas as pd @@ -34,6 +35,13 @@ def __init__(self, output_dir=None, level=0, log=False, mesh=None, corr_field_co #self.a = self.adjacency_matrix self.dataset = pd.DataFrame(self.data) + def get_test_data(self, index, length): + new_dataset = self.dataset[0:index * length] + self.dataset[index * length + length:] + new_obj = copy.deepcopy(self) + new_obj.dataset = new_dataset + + return new_obj + def shuffle(self, seed=None): if seed is not None: random.seed(seed) diff --git a/src/mlmc/metamodel/flow_task_GNN_2.py b/src/mlmc/metamodel/flow_task_GNN_2.py index 85bcfce7..35f0dcb7 100644 --- a/src/mlmc/metamodel/flow_task_GNN_2.py +++ b/src/mlmc/metamodel/flow_task_GNN_2.py @@ -32,7 +32,7 @@ def __init__(self, **kwargs): #self._n_hidden_layers = kwargs.get('n_hidden_layers', 1) #self._n_hidden_neurons = kwargs.get('n_hidden_neurons', [64]) # Number of hidden neurons for each hidden layer - self._loss = kwargs.get('loss', mean_squared_error) + self._loss = kwargs.get('loss', MeanSquaredError) self._accuracy_func = kwargs.get('accuracy_func', mean_squared_error) self._optimizer = kwargs.get('optimizer', tf.optimizers.Adam(learning_rate=0.001)) self._normalizer = kwargs.get('normalizer', preprocessing.Normalization()) @@ -47,7 +47,13 @@ def __init__(self, **kwargs): self._states = {} self._total_n_steps = 0 - model = kwargs.get('model') + if 'model_class' in kwargs: + model_class = kwargs.get('model_class') + net_model_config = kwargs.get('net_model_config') + model = model_class(**net_model_config) + print("model class model ", model) + else: + model = kwargs.get('model') if model is None: self._model = Net1(conv_layer=self._conv_layer, hidden_activation=self._hidden_activation, diff --git a/src/mlmc/sample_storage_hdf.py b/src/mlmc/sample_storage_hdf.py index 06b11a3d..c36376db 100644 --- a/src/mlmc/sample_storage_hdf.py +++ b/src/mlmc/sample_storage_hdf.py @@ -228,29 +228,71 @@ def get_n_ops(self): Get number of estimated operations on each level :return: List """ + # n_ops = list(np.zeros(len(self._level_groups))) + # for level in self._level_groups: + # if level.running_times[1] > 0: + # n_ops[int(level.level_id)] = level.running_times[0] / level.running_times[1] + # else: + # n_ops[int(level.level_id)] = 0 + # return n_ops + + n_ops = list(np.zeros(len(self._level_groups))) + for level in self._level_groups: + if level.n_ops_estimate[1] > 0: + n_ops[int(level.level_id)] = level.n_ops_estimate[0] / level.n_ops_estimate[1] + else: + n_ops[int(level.level_id)] = 0 + return n_ops + + def get_original_n_ops(self): n_ops = list(np.zeros(len(self._level_groups))) for level in self._level_groups: n_ops[int(level.level_id)] = level.n_ops_estimate return n_ops + def get_running_times(self): + n_ops = list(np.zeros(len(self._level_groups))) + for level in self._level_groups: + n_ops[int(level.level_id)] = level.running_times + return n_ops + + def get_extract_mesh_times(self): + n_ops = list(np.zeros(len(self._level_groups))) + for level in self._level_groups: + n_ops[int(level.level_id)] = level.extract_mesh_times + return n_ops + + def get_make_field_times(self): + n_ops = list(np.zeros(len(self._level_groups))) + for level in self._level_groups: + n_ops[int(level.level_id)] = level.make_field_times + return n_ops + + def get_generate_rnd_times(self): + n_ops = list(np.zeros(len(self._level_groups))) + for level in self._level_groups: + n_ops[int(level.level_id)] = level.generate_rnd_times + return n_ops + + def get_fine_flow_times(self): + n_ops = list(np.zeros(len(self._level_groups))) + for level in self._level_groups: + n_ops[int(level.level_id)] = level.fine_flow_times + return n_ops + + def get_coarse_flow_times(self): + n_ops = list(np.zeros(len(self._level_groups))) + for level in self._level_groups: + n_ops[int(level.level_id)] = level.coarse_flow_times + return n_ops + def get_level_ids(self): return [int(level.level_id) for level in self._level_groups] def get_level_parameters(self): return self._hdf_object.load_level_parameters() - def level_chunk_n_samples(self, level_id): - return self._level_groups[level_id].n_items_in_chunk - - def get_chunks_info(self, chunk_spec): - """ - The start and end index of a chunk from a whole dataset point of view - :param chunk_spec: ChunkSpec instance - :return: List[int, int] - """ - return self._level_groups[chunk_spec.level_id].get_chunks_info(chunk_spec) - def get_n_collected(self): """ Get number of collected samples at each level @@ -267,3 +309,4 @@ def get_n_levels(self): :return: int """ return len(self._level_groups) + diff --git a/src/mlmc/tool/hdf5.py b/src/mlmc/tool/hdf5.py index 620c9972..0c81dc2b 100644 --- a/src/mlmc/tool/hdf5.py +++ b/src/mlmc/tool/hdf5.py @@ -457,6 +457,138 @@ def n_ops_estimate(self): if 'n_ops_estimate' in hdf_file[self.level_group_path].attrs: return hdf_file[self.level_group_path].attrs['n_ops_estimate'] + @property + def running_times(self): + """ + Get number of operations estimate + :return: float + """ + with h5py.File(self.file_name, 'r') as hdf_file: + if 'running_times' in hdf_file[self.level_group_path].attrs: + return hdf_file[self.level_group_path].attrs['running_times'] + + @running_times.setter + def running_times(self, n_ops_estimate): + """ + Set property n_ops_estimate + :param n_ops_estimate: number of operations (time) per samples + :return: None + """ + with h5py.File(self.file_name, 'a') as hdf_file: + if 'running_times' not in hdf_file[self.level_group_path].attrs: + hdf_file[self.level_group_path].attrs['running_times'] = [0., 0.] + hdf_file[self.level_group_path].attrs['running_times'] = n_ops_estimate + + @property + def extract_mesh_times(self): + """ + Get number of operations estimate + :return: float + """ + with h5py.File(self.file_name, 'r') as hdf_file: + if 'extract_mesh_times' in hdf_file[self.level_group_path].attrs: + return hdf_file[self.level_group_path].attrs['extract_mesh_times'] + + @extract_mesh_times.setter + def extract_mesh_times(self, n_ops_estimate): + """ + Set property n_ops_estimate + :param n_ops_estimate: number of operations (time) per samples + :return: None + """ + with h5py.File(self.file_name, 'a') as hdf_file: + if 'extract_mesh_times' not in hdf_file[self.level_group_path].attrs: + hdf_file[self.level_group_path].attrs['extract_mesh_times'] = [0., 0.] + hdf_file[self.level_group_path].attrs['extract_mesh_times'] = n_ops_estimate + + @property + def make_field_times(self): + """ + Get number of operations estimate + :return: float + """ + with h5py.File(self.file_name, 'r') as hdf_file: + if 'make_field_times' in hdf_file[self.level_group_path].attrs: + return hdf_file[self.level_group_path].attrs['make_field_times'] + + @make_field_times.setter + def make_field_times(self, n_ops_estimate): + """ + Set property n_ops_estimate + :param n_ops_estimate: number of operations (time) per samples + :return: None + """ + with h5py.File(self.file_name, 'a') as hdf_file: + if 'make_field_times' not in hdf_file[self.level_group_path].attrs: + hdf_file[self.level_group_path].attrs['make_field_times'] = [0., 0.] + hdf_file[self.level_group_path].attrs['make_field_times'] = n_ops_estimate + + @property + def generate_rnd_times(self): + """ + Get number of operations estimate + :return: float + """ + with h5py.File(self.file_name, 'r') as hdf_file: + if 'generate_rnd_times' in hdf_file[self.level_group_path].attrs: + return hdf_file[self.level_group_path].attrs['generate_rnd_times'] + + @generate_rnd_times.setter + def generate_rnd_times(self, n_ops_estimate): + """ + Set property n_ops_estimate + :param n_ops_estimate: number of operations (time) per samples + :return: None + """ + with h5py.File(self.file_name, 'a') as hdf_file: + if 'generate_rnd_times' not in hdf_file[self.level_group_path].attrs: + hdf_file[self.level_group_path].attrs['generate_rnd_times'] = [0., 0.] + hdf_file[self.level_group_path].attrs['generate_rnd_times'] = n_ops_estimate + + @property + def fine_flow_times(self): + """ + Get number of operations estimate + :return: float + """ + with h5py.File(self.file_name, 'r') as hdf_file: + if 'fine_flow_times' in hdf_file[self.level_group_path].attrs: + return hdf_file[self.level_group_path].attrs['fine_flow_times'] + + @fine_flow_times.setter + def fine_flow_times(self, n_ops_estimate): + """ + Set property n_ops_estimate + :param n_ops_estimate: number of operations (time) per samples + :return: None + """ + with h5py.File(self.file_name, 'a') as hdf_file: + if 'fine_flow_times' not in hdf_file[self.level_group_path].attrs: + hdf_file[self.level_group_path].attrs['fine_flow_times'] = [0., 0.] + hdf_file[self.level_group_path].attrs['fine_flow_times'] = n_ops_estimate + + @property + def coarse_flow_times(self): + """ + Get number of operations estimate + :return: float + """ + with h5py.File(self.file_name, 'r') as hdf_file: + if 'coarse_flow_times' in hdf_file[self.level_group_path].attrs: + return hdf_file[self.level_group_path].attrs['coarse_flow_times'] + + @coarse_flow_times.setter + def coarse_flow_times(self, n_ops_estimate): + """ + Set property n_ops_estimate + :param n_ops_estimate: number of operations (time) per samples + :return: None + """ + with h5py.File(self.file_name, 'a') as hdf_file: + if 'coarse_flow_times' not in hdf_file[self.level_group_path].attrs: + hdf_file[self.level_group_path].attrs['coarse_flow_times'] = [0., 0.] + hdf_file[self.level_group_path].attrs['coarse_flow_times'] = n_ops_estimate + @n_ops_estimate.setter def n_ops_estimate(self, n_ops_estimate): """ @@ -466,8 +598,8 @@ def n_ops_estimate(self, n_ops_estimate): """ with h5py.File(self.file_name, 'a') as hdf_file: if 'n_ops_estimate' not in hdf_file[self.level_group_path].attrs: - hdf_file[self.level_group_path].attrs['n_ops_estimate'] = 0 - hdf_file[self.level_group_path].attrs['n_ops_estimate'] += n_ops_estimate + hdf_file[self.level_group_path].attrs['n_ops_estimate'] = [0., 0.] + hdf_file[self.level_group_path].attrs['n_ops_estimate'] = n_ops_estimate @property def n_items_in_chunk(self): diff --git a/src/mlmc/tool/plot.py b/src/mlmc/tool/plot.py index 91ace4e4..7c2172cd 100644 --- a/src/mlmc/tool/plot.py +++ b/src/mlmc/tool/plot.py @@ -725,23 +725,120 @@ def __init__(self, moments=None): matplotlib.rcParams.update({'font.size': 26}) matplotlib.rcParams.update({'lines.markersize': 8}) self.fig = plt.figure(figsize=(15, 8)) - self.title = "Level variances" + self.title = "" + self.fig.suptitle(self.title) + + self.ax = self.fig.add_subplot(1, 1, 1) + self.ax.set_xlabel("$\lambda$ - correlation length") + self.ax.set_ylabel("$MSE$") + self.ax.set_xscale('log') + self.ax.set_yscale('log') + + self.n_moments = None + self.subset_type = moments + self.min_step = 1e300 + self.max_step = 0 + self.data = {} + + self.nn_min_step = 1e300 + self.nn_max_step = 0 + self.nn_data = {} + + self._mse_train = {} + self._mse_test = {} + + #self._colormap = plt.cm.tab20 + + def add_mse_train(self, mse): + """ + Add variances for single MLMC instance. + :param steps, variances : as returned by Estimate.estimate_level_vars + :param n_levels: + """ + self._mse_train = mse + + def add_mse_test(self, mse): + """ + Add variances for single MLMC instance. + :param steps, variances : as returned by Estimate.estimate_level_vars + :param n_levels: + """ + self._mse_test = mse + + def show(self, file=""): + #self._colormap = create_color_bar(range=[1, self.n_moments], label=r'$M_i$', ax=self.ax, colormap=plt.cm.tab20) + res = 5 + step_range = self.max_step / self.min_step + log_scale = step_range ** 0.001 - 1 + #rv = st.lognorm(scale=1, s=log_scale) + + # if m == 5: + # break + col = "blue" + label = "MSE train" + + print("mse train ", self._mse_train) + print("mse test ", self._mse_test) + + print("list(self._mse_train.keys()) ", list(self._mse_train.keys())) + print("self._mse_train.items() ", self._mse_train.items()) + + self.ax.scatter(list(self._mse_train.keys()), list(self._mse_train.values()), color=col, label=label) + + col = "red" + label = "MSE test" + self.ax.scatter(list(self._mse_test.keys()), list(self._mse_test.values()), color=col, label=label) + + self.fig.legend() + _show_and_save(self.fig, file, self.title) + + +class Variance: + """ + Plot level variances, i.e. Var X^l as a function of the mesh step. + Selected moments are plotted. + """ + def __init__(self, moments=None): + """ + :param moments: Size or type of moments subset, see moments_subset function. + """ + matplotlib.rcParams.update({'font.size': 26}) + matplotlib.rcParams.update({'lines.markersize': 7}) + self.fig = plt.figure(figsize=(15, 8)) + + matplotlib.rcParams.update({'font.size': 16}) + matplotlib.rcParams.update({'lines.markersize': 8}) + # fig, axes = plt.subplots(1, 1, figsize=(22, 10)) + self.fig = plt.figure(figsize=(8, 5)) + + self.title = ""#"Level variances" self.fig.suptitle(self.title) self.ax = self.fig.add_subplot(1, 1, 1) self.ax.set_xlabel("$h$ - mesh step") - self.ax.set_ylabel("Var $X^h$") + #self.ax.set_ylabel("Var $X^h$") + self.ax.set_ylabel("$\hat{V}^r_l$") self.ax.set_xscale('log') self.ax.set_yscale('log') + #self.ax.set_xlim([1e-3, 1e0]) + self.n_moments = None self.subset_type = moments self.min_step = 1e300 self.max_step = 0 self.data = {} + self.nn_min_step = 1e300 + self.nn_max_step = 0 + self.nn_data = {} self._colormap = plt.cm.tab20 + self._n_ops = None + + def set_n_ops(self, n_ops): + print("n ops ", n_ops) + self._n_ops = n_ops def add_level_variances(self, steps, variances): """ @@ -765,42 +862,108 @@ def add_level_variances(self, steps, variances): Y.extend(vars.tolist()) self.data[m] = (X, Y) - # def add_diff_variances(self, step, variances): - # pass + def add_level_variances_nn(self, steps, variances): + """ + Add variances for single MLMC instance. + :param steps, variances : as returned by Estimate.estimate_level_vars + :param n_levels: + """ + n_levels, n_moments = variances.shape + if self.n_moments is None: + self.n_moments = n_moments + self.moments_subset = moments_subset(n_moments, self.subset_type) + else: + assert self.n_moments == n_moments + + variances = variances[:, self.moments_subset] + self.nn_min_step = min(self.min_step, steps[-1]) + self.nn_max_step = max(self.max_step, steps[0]) + for m, vars in enumerate(variances.T): + X, Y = self.nn_data.get(m, ([], [])) + X.extend(steps.tolist()) + Y.extend(vars.tolist()) + self.nn_data[m] = (X, Y) def show(self, file=""): - self._colormap = create_color_bar(range=[1, self.n_moments], label=r'$M_i$', ax=self.ax, colormap=plt.cm.tab20) + self._colormap = create_color_bar(range=[1, self.n_moments], label=r'$r$', ax=self.ax, colormap=plt.cm.tab20) res = 5 step_range = self.max_step / self.min_step log_scale = step_range ** 0.001 - 1 - rv = st.lognorm(scale=1, s=log_scale) + #rv = st.lognorm(scale=1, s=log_scale) for m, (X, Y) in self.data.items(): # if m == 5: # break col = self._colormap(m) label = "M{}".format(self.moments_subset[m]) - XX = np.array(X) * rv.rvs(size=len(X)) - self.ax.scatter(XX, Y, color=col, label=label) - + label = "MLMC" + # print("X ", X) + # print("len(X) ", len(X)) + # #print("rv.rvs(size=len(X)) ", rv.rvs(size=len(X))) + # print("Y ", Y) + XX = np.array(X) #* rv.rvs(size=len(X)) + # print("XX ", X) + self.ax.scatter(XX, Y, color=col) XX, YY = make_monotone(X, Y) # step_range = self.nn_max_step / self.nn_min_step # log_scale = step_range ** 0.01 - 1 # rv = st.lognorm(scale=1, s=log_scale) + levels = {} + + print("nn data ", self.nn_data) for m, (X, Y) in self.nn_data.items(): # if m == 5: # break - col = self._colormap(m) + col = plt.cm.tab20(m) label = "M{}".format(self.moments_subset[m]) - XX = np.array(X) * 0.9#rv.rvs(size=len(X)) + label = "MLMC meta" + XX = np.array(X) * 0.84 # rv.rvs(size=len(X)) + XY = np.array(X) * 1.16 - # print("XX ", XX) - # print("Y ", Y) + self.ax.scatter(XX, Y, color=col, marker='v') + # XX, YY = make_monotone(X, Y) - self.ax.scatter(XX, Y, color=col, label=label, marker='v') - XX, YY = make_monotone(X, Y) + for x, y in zip(X, Y): + if x not in levels: + levels[x] = [] + levels[x].append(y) + + print("XX ", XX) + print("np.max(Y) + np.max(Y)*0.3) ", np.max(Y) + np.max(Y) * 0.3) + if self._n_ops is not None: + for index, n_ops in enumerate(self._n_ops): + self.ax.annotate("{:0.3g}".format(n_ops), (XY[index], np.max(levels[X[index]]))) #self.fig.legend() + + legend = self.ax.legend() + ax = legend.axes + + from matplotlib.lines import Line2D + from matplotlib.patches import Rectangle, RegularPolygon, FancyBboxPatch + + handles, labels = ax.get_legend_handles_labels() + + mlmc_marker = Line2D([], [], color='black', marker='o', linestyle='None', + markersize=8, markeredgewidth=1.7, + label='MLMC') # Line2D([], [], color='black', marker='|') + + handles.append(mlmc_marker) + labels.append("MLMC") + + if self.nn_data: + mlmc_marker_meta = Line2D([], [], color='black', marker='v', linestyle='None', + markersize=8, markeredgewidth=1.7, + label='r"$MLMC_{meta}:$"') # Line2D([], [], color='black', marker='|') + + handles.append(mlmc_marker_meta) + labels.append("MLMC meta") + + legend._legend_box = None + legend._init_legend_box(handles, labels) + legend._set_loc(legend._loc) + legend.set_title(legend.get_title().get_text()) + _show_and_save(self.fig, file, self.title) class BSplots: @@ -1235,7 +1398,6 @@ def plot_means_and_vars(self, moments_mean, moments_var, n_levels, exact_moments plt.legend() plt.show() - def plot_var_regression(self, i_moments = None): """ Plot total and level variances and their regression and errors of regression. diff --git a/test/metamodels/metamodel_test.py b/test/metamodels/metamodel_test.py index 9e03b2a5..94a0138c 100644 --- a/test/metamodels/metamodel_test.py +++ b/test/metamodels/metamodel_test.py @@ -3,12 +3,14 @@ import warnings #os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only import sys +import shutil import subprocess -from mlmc.metamodel.analyze_nn import run_GNN, run_SVR, statistics, analyze_statistics, process_results, run_DNN +from mlmc.metamodel.analyze_nn import run_GNN, run_SVR, statistics, analyze_statistics, process_results from mlmc.moments import Legendre_tf, Monomial from mlmc.metamodel.flow_task_GNN_2 import GNN from spektral.layers import GCNConv, GlobalSumPool, ChebConv, GraphSageConv, ARMAConv, GATConv, APPNPConv, GINConv, GeneralConv +from mlmc.metamodel.own_cheb_conv import OwnChebConv from tensorflow.keras.losses import MeanSquaredError, KLDivergence, MeanAbsoluteError from mlmc.metamodel.custom_methods import abs_activation, MSE_moments from tensorflow.keras import Model @@ -51,7 +53,7 @@ def get_gnn(): "normalizer": preprocessing.Normalization() } - #model = Net(**net_model_config) + model = Net(**net_model_config) model_config = {"loss": loss, "optimizer": optimizer, @@ -60,10 +62,9 @@ def get_gnn(): "net_model_config": net_model_config, "verbose": True} - corr_field_config = {'corr_length': 0.1, 'sigma': 1, 'log': True} - return GNN(**model_config), conv_layer, corr_field_config + return GNN, conv_layer, corr_field_config, model_config class Net(Model): @@ -350,6 +351,95 @@ def plot_results_corr_length(): plt_cl.show("corr_length_mse") +def plot_results_corr_length(): + cl_all = {"cl_0_001_s_1": 0.001, "cl_0_01_s_1": 0.01, "cl_0_1_s_1": 0.1, "cl_1_s_1": 1, "cl_10_s_1": 10} + + # cl_all = {"cl_0_001_s_1": 0.001, "cl_10_s_1": 1} + # + cl_all = {"cl_0_001_s_1": 0.001} + + tr_MSE = {} + te_MSE = {} + tr_RSE = {} + te_RSE = {} + + for cl_dir, cl in cl_all.items(): + data_dir = "/home/martin/Documents/metamodels/data/mesh_size/" + level = 3 + nn_level = 0 + replace_level = False + # mesh = os.path.join(data_dir, "l_step_1.0_common_files/mesh.msh".format(cl)) #L1, 7s + # mesh = os.path.join(data_dir, "l_step_0.27232698153315_common_files/mesh.msh".format(cl)) #L2 10.5 s + mesh = os.path.join(data_dir, "l_step_0.07416198487095663_common_files/mesh.msh".format(cl_dir)) # L3 12s + # mesh = os.path.join(data_dir, "l_step_0.020196309484414757_common_files/mesh.msh".format(cl)) #L4 22s + # mesh = os.path.join(data_dir, "l_step_0.0055_common_files/mesh.msh".format(cl)) #L5 + output_dir = os.path.join(data_dir, "{}/L1_{}/test/01_cond_field/output/".format(cl_dir, level)) + hdf_path = os.path.join(data_dir, "{}/L1_{}/mlmc_1.hdf5".format(cl_dir, level)) + mlmc_hdf_path = os.path.join(data_dir, "{}/mlmc_hdf/L1_{}/mlmc_1.hdf5".format(cl_dir, level)) + save_path = os.path.join(data_dir, "{}".format(cl_dir)) + l_0_output_dir = os.path.join(data_dir, "{}/L0_MC/L1_{}/test/01_cond_field/output/".format(cl_dir, level)) + l_0_hdf_path = os.path.join(data_dir, "{}/L0_MC/L1_{}/mlmc_1.hdf5".format(cl_dir, level)) + sampling_info_path = os.path.join(data_dir, "{}/sampling_info".format(cl_dir)) + ref_mlmc_file = os.path.join(data_dir, "{}/L1_benchmark/mlmc_1.hdf5".format(cl_dir)) + + machine_learning_model = ("mesh_L3_log_15k", run_GNN, False) + + gnn, conv_layer, corr_field_config, model_config = get_gnn() + + save_path = os.path.join(save_path, machine_learning_model[0]) + + print("save path ", save_path) + graph_creation_time = 25 # 22#159#0#159#66 + + config = {'machine_learning_model': machine_learning_model, + 'save_path': save_path, + 'sampling_info_path': sampling_info_path, + 'output_dir': output_dir, + 'nn_hdf_path': hdf_path, + 'mlmc_hdf_path': mlmc_hdf_path, + 'mesh': mesh, + 'l_0_output_dir': l_0_output_dir, + 'l_0_hdf_path': l_0_hdf_path, + 'ref_mlmc_file': ref_mlmc_file, + 'level': nn_level, + 'conv_layer': conv_layer, + 'gnn': gnn, + 'model_config': model_config, + 'replace_level': replace_level, + 'corr_field_config': corr_field_config, + 'n_train_samples': 2000, + 'val_samples_ratio': 0.3, + 'batch_size': 200, + 'epochs': 2000, + 'learning_rate': 0.01, + 'graph_creation_time': graph_creation_time, + 'save_model': False, + 'loss_params': {'moments_class': Legendre_tf, "max_moments": 20, 'loss_max': 0.5, 'quantile': 1e-3} + } + + train_MSE, test_MSE, train_RSE, test_RSE = analyze_statistics(config) + + tr_MSE[cl] = np.mean(train_MSE) + te_MSE[cl] = np.mean(test_MSE) + tr_RSE[cl] = np.mean(train_RSE) + te_RSE[cl] = np.mean(test_RSE) + + + plt_cl = plot.CorrLength() + plt_cl.add_mse_test(te_MSE) + plt_cl.add_mse_train(tr_MSE) + + plt_cl.show(None) + plt_cl.show("corr_length_mse") + + plt_cl = plot.CorrLength() + plt_cl.add_mse_test(te_RSE) + plt_cl.add_mse_train(tr_RSE) + + plt_cl.show(None) + plt_cl.show("corr_length_mse") + + def get_arguments(arguments): """ Getting arguments from console @@ -368,11 +458,14 @@ def get_arguments(arguments): args = get_arguments(sys.argv[1:]) data_dir = args.data_dir work_dir = args.work_dir - case = 3 + case = 7 #data_dir = "/home/martin/Documents/metamodels/data/1000_ele/" output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, replace_level, nn_level = get_config( data_dir, case) + # plot_results_corr_length() + # exit() + # if os.path.exists(os.path.join(work_dir, "mlmc_{}.hdf5".format(nn_level + 1))): # l_0_hdf_path = os.path.join(work_dir, "mlmc_{}.hdf5".format(nn_level + 1)) # hdf_path = os.path.join(work_dir, "mlmc_5.hdf5") @@ -407,11 +500,33 @@ def get_arguments(arguments): # process_results(hdf_path, sampling_info_path, ref_mlmc_file, save_path, nn_level, replace_level) # - gnn, conv_layer, corr_field_config = get_gnn() + gnn, conv_layer, corr_field_config, model_config = get_gnn() # print("gnn ", gnn) #print("conv layer ", conv_layer) + #machine_learning_model = ("L2_test", run_GNN, False) + + machine_learning_model = ("ChC8L3_log", run_GNN, False) + #machine_learning_model = ("ChC8L2_log", run_GNN, False) + #machine_learning_model = ("SVR_L3_log", run_GNN, False) + # machine_learning_model = ("ChC32L3T25000", run_GNN, False) + # + # machine_learning_model = ("ChC32Loss2_adding_moments_2", run_GNN, False) + #machine_learning_model = ("ChC32Loss2_add_mom", run_GNN, True) + # + #machine_learning_model = ("ChC32L3M10_test", run_GNN, False) + + #machine_learning_model = ("mesh_L3", run_GNN, False) + + #machine_learning_model = ("SVR_mesh_L3_log", run_GNN, False) + + + machine_learning_model = ("GCN_mesh_L3_log", run_GNN, False) + # + #machine_learning_model = ("mesh_moments_test_2", run_GNN, True) + #machine_learning_model = ("mesh_L3_test_m", run_GNN, False) + # #models = {"ChebConv": (run_GNN, False), "SVR": (run_SVR, False)} machine_learning_model = ("5eleChebConvL3_2", run_GNN, False) #machine_learning_model = ("5eleChebConvK2", run_GNN, False) @@ -449,9 +564,9 @@ def get_arguments(arguments): # #machine_learning_model = ("DNN_mesh_L3_log_deep", run_DNN, True) # - machine_learning_model = ("DNN_mesh_L3_6", run_DNN, True) + #machine_learning_model = ("DNN_mesh_L3_6", run_DNN, True) machine_learning_model = ("GCN_mesh_L3_log_16", run_GNN, True) - machine_learning_model = ("mesh_L3_log", run_GNN, True) + machine_learning_model = ("mesh_L3_log_test", run_GNN, True) #machine_learning_model = ("mesh_L3_seed", run_GNN, False) @@ -459,12 +574,14 @@ def get_arguments(arguments): save_path = os.path.join(save_path, machine_learning_model[0]) + # if os.path.exists(save_path): + # shutil.rmtree(save_path) + print("save path ", save_path) - graph_creation_time = 11#22#159#0#159#66 + graph_creation_time = 25#11#22#159#0#159#66 config = {'machine_learning_model': machine_learning_model, 'save_path': save_path, - 'sampling_info_path': sampling_info_path, 'output_dir': output_dir, 'hdf_path': hdf_path, 'mesh': mesh, @@ -475,6 +592,7 @@ def get_arguments(arguments): 'level': nn_level, 'conv_layer': conv_layer, 'gnn': gnn, + 'model_config': model_config, 'replace_level': replace_level, 'corr_field_config': corr_field_config, 'n_train_samples': 2000, From d73413796f669277a4d6dbfae26f2a53e9e9aa0d Mon Sep 17 00:00:00 2001 From: Martin Spetlik Date: Thu, 21 Oct 2021 13:18:42 +0200 Subject: [PATCH 27/67] charon metamodel --- src/mlmc/metamodel/analyze_nn.py | 59 ++++++++++++++++--------- src/mlmc/metamodel/create_graph.py | 2 +- src/mlmc/metamodel/flow_task_CNN.py | 2 +- src/mlmc/metamodel/flow_task_GNN.py | 2 +- src/mlmc/metamodel/flow_task_GNN_2.py | 12 +++-- src/mlmc/metamodel/random_field_time.py | 4 +- 6 files changed, 53 insertions(+), 28 deletions(-) diff --git a/src/mlmc/metamodel/analyze_nn.py b/src/mlmc/metamodel/analyze_nn.py index 92fbc673..718f8821 100644 --- a/src/mlmc/metamodel/analyze_nn.py +++ b/src/mlmc/metamodel/analyze_nn.py @@ -9,8 +9,8 @@ from mlmc.metamodel.create_graph import graph_creator from mlmc.moments import Legendre_tf, Monomial from mlmc.metamodel.random_field_time import corr_field_sample_time -from mlmc.tool import plot -import matplotlib.pyplot as plt +#from mlmc.tool import plot +#import matplotlib.pyplot as plt # Make numpy printouts easier to read. # np.set_printoptions(precision=9, suppress=True) @@ -26,7 +26,7 @@ from mlmc.metamodel.flow_task_GNN_2 import GNN from tensorflow.keras.losses import MeanSquaredError from spektral.data import MixedLoader -from spektral.layers.ops import sp_matrix_to_sp_tensor +from spektral.utils.sparse import sp_matrix_to_sp_tensor print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU'))) epochs = 100 @@ -173,8 +173,9 @@ def bootstrap(): # estimate_density(np.mean(all_predictions, axis=0), title="Predictions") -def run_SVR(config, stats=True, train=True, log=False): +def run_SVR(config, stats=True, train=True, log=False, seed=1234): from sklearn.svm import SVR + print("seed ", seed) batch_size = 200 epochs = 1000 @@ -190,7 +191,7 @@ def run_SVR(config, stats=True, train=True, log=False): preprocess_start_time = time.process_time() # Load data data = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log) - data.shuffle() + data.shuffle(seed=seed) dataset = data.dataset dataset = dataset.sample(frac=1) @@ -289,7 +290,8 @@ def run_SVR(config, stats=True, train=True, log=False): config['l_0_hdf_path'], config['mesh'], batch_size, log, stats=stats, - corr_field_config=config['corr_field_config']) + corr_field_config=config['corr_field_config'], + seed=seed) val_predictions = [] @@ -314,7 +316,7 @@ def run_SVR(config, stats=True, train=True, log=False): l_0_predictions) -def predict_level_zero_SVR(nn, output_dir, hdf_path, mesh, batch_size=1000, log=False, stats=False, corr_field_config=None): +def predict_level_zero_SVR(nn, output_dir, hdf_path, mesh, batch_size=1000, log=False, stats=False, corr_field_config=None, seed=1234): #graph_creator(output_dir, hdf_path, mesh, level=0) sample_time = 0 if corr_field_config: @@ -322,6 +324,7 @@ def predict_level_zero_SVR(nn, output_dir, hdf_path, mesh, batch_size=1000, log= # Load data data = FlowDataset(output_dir=output_dir, log=log) + data.shuffle(seed=seed) dataset = data.dataset[:] predict_time_start = time.process_time() @@ -345,12 +348,16 @@ def predict_level_zero_SVR(nn, output_dir, hdf_path, mesh, batch_size=1000, log= def statistics(config): - n_subsamples = 2 + n_subsamples = 25 model_title, mch_l_model, log = config['machine_learning_model'] model_data = {} model_data["log"] = log + # seeds = [] + # for i in range(n_subsamples): + # seeds.append(i * 125) + if not os.path.isdir(config['save_path']): os.makedirs(config['save_path']) else: @@ -364,7 +371,7 @@ def statistics(config): model, targets, predictions, learning_time, train_targets, train_predictions, \ val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, total_steps = \ - mch_l_model(config, stats=True, train=True, log=log) + mch_l_model(config, stats=True, train=True, log=log, seed=i) if config['save_model']: model_data["model"] = model @@ -631,11 +638,11 @@ def analyze_statistics(config): print("mean learning time ", np.mean(learning_times)) print("max learning time ", np.max(learning_times)) - print("######################################") -def run_GNN(config, stats=True, train=True, log=False): +def run_GNN(config, stats=True, train=True, log=False, seed=0): + print("seed ", seed) loss = MeanSquaredError() # var_loss_function# accuracy_func = MSE_moments @@ -662,11 +669,11 @@ def run_GNN(config, stats=True, train=True, log=False): data = data#[:10000] #print("len data ", len(data)) - data.shuffle() + #data.shuffle(seed=seed) preprocess_time = time.process_time() - preprocess_start_time #print("preproces time ", preprocess_time) preprocess_time = preprocess_time + graph_creation_time - print("total preprocess time ", preprocess_time) + #print("total preprocess time ", preprocess_time) learning_time_start = time.process_time() data.a = config['conv_layer'].preprocess(data.a) @@ -674,9 +681,12 @@ def run_GNN(config, stats=True, train=True, log=False): #train_data_len = int(len(data) * 0.8) train_data_len = config['n_train_samples'] # Train/valid/test split - data_tr, data_te = data[:train_data_len], data[train_data_len:] - gnn = config['gnn'] + data_tr = data[seed*train_data_len: seed*train_data_len + train_data_len] + data_te = data.get_test_data(seed, train_data_len) + #data_tr, data_te = data[:train_data_len], data[train_data_len:] + + gnn = config['gnn'](**config['model_config']) if hasattr(gnn._loss,'__name__') and gnn._loss.__name__ == "MSE_moments": tr_output = [g.y for g in data_tr] @@ -708,9 +718,12 @@ def run_GNN(config, stats=True, train=True, log=False): # batch_size 500, ideally 500 epochs, patience 35 if train: + print("gnn ", gnn) # gnn.run_eagerly = True train_targets = gnn.fit(loader_tr, loader_va, loader_te) + learning_time = time.process_time() - learning_time_start + # states = gnn._states # if len(states) > 0: # min_key = np.min(list(states.keys())) @@ -727,7 +740,7 @@ def run_GNN(config, stats=True, train=True, log=False): targets, predictions = gnn.predict(loader_te) predictions = np.squeeze(predictions) - learning_time = time.process_time() - learning_time_start + #print("learning time ", learning_time) targets = np.array(targets) @@ -760,7 +773,8 @@ def run_GNN(config, stats=True, train=True, log=False): config['l_0_hdf_path'], config['mesh'], config['conv_layer'], batch_size, log, stats=stats, - corr_field_config=config['corr_field_config']) + corr_field_config=config['corr_field_config'], + seed=seed) #predict_l_0_time = time.process_time() - predict_l_0_start_time if stats: @@ -784,8 +798,8 @@ def run_GNN(config, stats=True, train=True, log=False): l_0_predictions) -def predict_level_zero(nn, output_dir, hdf_path, mesh, conv_layer, batch_size=1000, log=False, stats=False, corr_field_config=None): - graph_creator(output_dir, hdf_path, mesh, level=0) +def predict_level_zero(nn, output_dir, hdf_path, mesh, conv_layer, batch_size=1000, log=False, stats=False, corr_field_config=None, seed=1234): + #graph_creator(output_dir, hdf_path, mesh, level=0) # Load data sample_time = 0 @@ -794,7 +808,12 @@ def predict_level_zero(nn, output_dir, hdf_path, mesh, conv_layer, batch_size=10 data = FlowDataset(output_dir=output_dir, log=log)#, mesh=mesh, corr_field_config=corr_field_config) #data = data # [:10000] - + data.shuffle(seed=seed) + + print("output_dir ", output_dir) + print("len(data) ", len(data)) + print("data[0] ", data[0]) + predict_time_start = time.process_time() data.a = conv_layer.preprocess(data.a) data.a = sp_matrix_to_sp_tensor(data.a) diff --git a/src/mlmc/metamodel/create_graph.py b/src/mlmc/metamodel/create_graph.py index 2c2eecbb..0c77fdf4 100644 --- a/src/mlmc/metamodel/create_graph.py +++ b/src/mlmc/metamodel/create_graph.py @@ -2,7 +2,7 @@ import os.path import numpy as np import networkx as nx -import matplotlib.pyplot as plt +#import matplotlib.pyplot as plt from mlmc.tool import gmsh_io from mlmc.tool.hdf5 import HDF5 from spektral.data import Graph diff --git a/src/mlmc/metamodel/flow_task_CNN.py b/src/mlmc/metamodel/flow_task_CNN.py index e4d5805b..cc3d99fb 100644 --- a/src/mlmc/metamodel/flow_task_CNN.py +++ b/src/mlmc/metamodel/flow_task_CNN.py @@ -23,7 +23,7 @@ from spektral.data import MixedLoader from mlmc.metamodel.flow_dataset import FlowDataset from spektral.layers import GCNConv, GlobalSumPool, ChebConv, GraphSageConv, ARMAConv, GATConv, APPNPConv, GINConv -from spektral.layers.ops import sp_matrix_to_sp_tensor +from spektral.utils.sparse import sp_matrix_to_sp_tensor from tensorflow.keras.layers.experimental import preprocessing from mlmc.metamodel.custom_methods import abs_activation diff --git a/src/mlmc/metamodel/flow_task_GNN.py b/src/mlmc/metamodel/flow_task_GNN.py index 83bb63c3..365ae28d 100644 --- a/src/mlmc/metamodel/flow_task_GNN.py +++ b/src/mlmc/metamodel/flow_task_GNN.py @@ -13,7 +13,7 @@ from spektral.data import MixedLoader from mlmc.metamodel.flow_dataset import FlowDataset from spektral.layers import GCNConv, GlobalSumPool, ChebConv, GraphSageConv, ARMAConv, GATConv, APPNPConv, GINConv -from spektral.layers.ops import sp_matrix_to_sp_tensor +from spektral.utils.sparse import sp_matrix_to_sp_tensor diff --git a/src/mlmc/metamodel/flow_task_GNN_2.py b/src/mlmc/metamodel/flow_task_GNN_2.py index 34af162a..35f0dcb7 100644 --- a/src/mlmc/metamodel/flow_task_GNN_2.py +++ b/src/mlmc/metamodel/flow_task_GNN_2.py @@ -11,7 +11,7 @@ from spektral.data import MixedLoader from mlmc.metamodel.flow_dataset import FlowDataset from spektral.layers import GCNConv, GlobalSumPool, ChebConv, GraphSageConv, ARMAConv, GATConv, APPNPConv, GINConv -from spektral.layers.ops import sp_matrix_to_sp_tensor +from spektral.utils.sparse import sp_matrix_to_sp_tensor from tensorflow.keras.layers.experimental import preprocessing from mlmc.metamodel.custom_methods import abs_activation, var_loss_function @@ -32,7 +32,7 @@ def __init__(self, **kwargs): #self._n_hidden_layers = kwargs.get('n_hidden_layers', 1) #self._n_hidden_neurons = kwargs.get('n_hidden_neurons', [64]) # Number of hidden neurons for each hidden layer - self._loss = kwargs.get('loss', mean_squared_error) + self._loss = kwargs.get('loss', MeanSquaredError) self._accuracy_func = kwargs.get('accuracy_func', mean_squared_error) self._optimizer = kwargs.get('optimizer', tf.optimizers.Adam(learning_rate=0.001)) self._normalizer = kwargs.get('normalizer', preprocessing.Normalization()) @@ -47,7 +47,13 @@ def __init__(self, **kwargs): self._states = {} self._total_n_steps = 0 - model = kwargs.get('model') + if 'model_class' in kwargs: + model_class = kwargs.get('model_class') + net_model_config = kwargs.get('net_model_config') + model = model_class(**net_model_config) + print("model class model ", model) + else: + model = kwargs.get('model') if model is None: self._model = Net1(conv_layer=self._conv_layer, hidden_activation=self._hidden_activation, diff --git a/src/mlmc/metamodel/random_field_time.py b/src/mlmc/metamodel/random_field_time.py index da931af6..1eade569 100644 --- a/src/mlmc/metamodel/random_field_time.py +++ b/src/mlmc/metamodel/random_field_time.py @@ -5,8 +5,8 @@ def corr_field_sample_time(mesh_file=None, corr_length_config=None): - import matplotlib - from matplotlib import ticker, cm + # import matplotlib + # from matplotlib import ticker, cm #matplotlib.rcParams.update({'font.size': 22}) dim = 2 log = True From 3cf2b853409b4256033d39c8ff83c85741815151 Mon Sep 17 00:00:00 2001 From: Martin Spetlik Date: Thu, 21 Oct 2021 13:24:46 +0200 Subject: [PATCH 28/67] postprocessing --- src/mlmc/metamodel/postprocessing.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/mlmc/metamodel/postprocessing.py b/src/mlmc/metamodel/postprocessing.py index ec554741..00529876 100644 --- a/src/mlmc/metamodel/postprocessing.py +++ b/src/mlmc/metamodel/postprocessing.py @@ -1,11 +1,11 @@ import os import random import copy -import matplotlib.pyplot as plt +#import matplotlib.pyplot as plt from scipy.stats import ks_2samp -from mlmc.tool import plot -import mlmc.tool.simple_distribution -import mlmc.estimator +#from mlmc.tool import plot +#import mlmc.tool.simple_distribution +#import mlmc.estimator import mlmc.quantity_estimate as qe from mlmc.sample_storage import Memory from mlmc.quantity_spec import QuantitySpec, ChunkSpec @@ -13,7 +13,7 @@ from mlmc.sample_storage_hdf import SampleStorageHDF from mlmc.moments import Legendre, Monomial from mlmc.quantity import make_root_quantity -import mlmc.tool.simple_distribution +#import mlmc.tool.simple_distribution QUANTILE = 0.01 @@ -498,8 +498,9 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti ###### ### Get n ops ###### - n_ops, field_times, coarse_flow, fine_flow = get_sample_times_mlmc(mlmc_file) - #n_ops_2, _, _ = get_sample_times(sampling_info_path) + #n_ops, field_times, coarse_flow, fine_flow = get_sample_times_mlmc(mlmc_file) + n_ops, _, _ = get_sample_times(sampling_info_path) + n_ops = n_ops[n_levels-1:] if n_ops is None: n_ops = sample_storage.get_n_ops() From 50cab9b80778ec0f3716d5dc7fb655ba720dd787 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Fri, 22 Oct 2021 12:36:40 +0200 Subject: [PATCH 29/67] plot progress --- src/mlmc/metamodel/analyze_nn.py | 50 +++++++++++--- src/mlmc/metamodel/postprocessing.py | 48 ++++++++++++-- test/metamodels/metamodel_test.py | 98 ++++++++++++++++++++++------ 3 files changed, 162 insertions(+), 34 deletions(-) diff --git a/src/mlmc/metamodel/analyze_nn.py b/src/mlmc/metamodel/analyze_nn.py index a1e92c8e..b7b282d9 100644 --- a/src/mlmc/metamodel/analyze_nn.py +++ b/src/mlmc/metamodel/analyze_nn.py @@ -9,8 +9,8 @@ from mlmc.metamodel.create_graph import graph_creator from mlmc.moments import Legendre_tf, Monomial from mlmc.metamodel.random_field_time import corr_field_sample_time -#from mlmc.tool import plot -#import matplotlib.pyplot as plt +from mlmc.tool import plot +import matplotlib.pyplot as plt # Make numpy printouts easier to read. # np.set_printoptions(precision=9, suppress=True) @@ -19,7 +19,7 @@ from scipy.stats import ks_2samp import sklearn.model_selection from mlmc.metamodel.custom_methods import abs_activation, MSE_moments -from mlmc.metamodel.postprocessing import analyze_results, plot_loss, estimate_density, process_mlmc +from mlmc.metamodel.postprocessing import analyze_results, plot_loss, estimate_density, process_mlmc, plot_progress from mlmc.metamodel.flow_task_NN import DNN from mlmc.metamodel.flow_task_CNN import CNN @@ -309,7 +309,7 @@ def run_SVR(config, stats=True, train=True, log=False, seed=1234): # stats=stats) return svr_rbf, targets, predictions, learning_time, train_targets, train_predictions, \ - val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, total_steps + val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, total_steps, None, None, None save_times(config['save_path'], False, (preprocess_time, len(data)), learning_time, (predict_l_0_time, len(l_0_targets))) save_load_data(config['save_path'], False, targets, predictions, train_targets, train_predictions, val_targets, l_0_targets, @@ -370,7 +370,8 @@ def statistics(config): os.makedirs(iter_dir) model, targets, predictions, learning_time, train_targets, train_predictions, \ - val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, total_steps = \ + val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, total_steps,\ + conv_layers, flatten_output, dense_layers = \ mch_l_model(config, stats=True, train=True, log=log, seed=i) if config['save_model']: @@ -387,6 +388,19 @@ def statistics(config): model_data["l0_sample_time"] = l0_sample_time model_data["total_steps"] = total_steps model_data["learning_times"] = learning_time + + model_data["conv_layers"] = [] + model_data["dense_layers"] = [] + + if conv_layers is not None: + for index, c_layer in enumerate(conv_layers): + model_data["conv_layers"].append([c_layer._inputs, c_layer._weights, c_layer._outputs]) + + model_data["flatten_output"] = flatten_output + + for index, (d_layer, (inputs, outputs)) in enumerate(dense_layers.items()): + model_data["dense_layers"].append([inputs, d_layer.get_weights(), outputs]) + save_statistics(iter_dir, model_data) # save_times(save_path, False, (preprocess_time, len(data)), learning_time, (predict_l_0_time, len(l_0_targets))) @@ -434,6 +448,9 @@ def load_statistics(dir_path): models_data["total_steps"] = [] models_data["learning_times"] = [] models_data["log"] = [] + models_data["conv_layers"] = [] + models_data["flatten_output"] = [] + models_data["dense_layers"] = [] #dirs = (os.path.split(dir_path)[-1]).split("_") n_iters = 25 @@ -447,7 +464,13 @@ def load_statistics(dir_path): for file in glob.glob(os.path.join(data_dir_path, "*.npy")): file_name = os.path.split(file)[-1] file_name = file_name.split(".")[0] - models_data[file_name].append(np.load(file)) + # if file_name not in models_data: + # print("file name ", file_name) + # models_data[file_name] = [] + # print("np.load(file, allow_pickle=True) ", np.load(file, allow_pickle=True)) + # exit() + + models_data[file_name].append(np.load(file, allow_pickle=True)) return models_data @@ -498,8 +521,6 @@ def analyze_statistics(config): nn_means_mse = [] for i in range(len(data_dict["test_targets"])): - if i == 2: - break predictions = data_dict["test_predictions"][i] targets = data_dict["test_targets"][i] train_predictions = data_dict["train_predictions"][i] @@ -523,6 +544,9 @@ def analyze_statistics(config): replace_level=config['replace_level'], stats=True) + plot_progress(data_dict["conv_layers"][i], data_dict["flatten_output"][i], data_dict["dense_layers"][i]) + + mlmc_n_collected_all.append(mlmc_n_collected) nn_n_collected_all.append(nn_mlmc_n_collected) n_ops_all.append(n_ops) @@ -740,6 +764,8 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): #val_targets = gnn.val_targets total_steps = gnn._total_n_steps + gnn._model.clear_progress() + targets, predictions = gnn.predict(loader_te) predictions = np.squeeze(predictions) @@ -792,8 +818,14 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): # l_0_predictions, l1_sample_time, l0_sample_time, nn_level=level, replace_level=replace_level, # stats=stats) + conv_layers, flatten_output, dense_layers = gnn._model.plot_progress(stats) + #gnn._model.plot_progress(stats) + return gnn._model, targets, predictions, learning_time, train_targets, train_predictions,\ - val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, total_steps + val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, total_steps,\ + conv_layers, flatten_output, dense_layers + + gnn._model.plot_progress(stats) save_times(config['save_path'], False, (preprocess_time, len(data)), learning_time, (predict_l_0_time, len(l_0_targets))) save_load_data(config['save_path'], False, targets, predictions, train_targets, train_predictions, val_targets, l_0_targets, diff --git a/src/mlmc/metamodel/postprocessing.py b/src/mlmc/metamodel/postprocessing.py index d2acb76a..8cb9ff2e 100644 --- a/src/mlmc/metamodel/postprocessing.py +++ b/src/mlmc/metamodel/postprocessing.py @@ -1,11 +1,11 @@ import os import random import copy -#import matplotlib.pyplot as plt +import matplotlib.pyplot as plt from scipy.stats import ks_2samp #from mlmc.tool import plot #import mlmc.tool.simple_distribution -#import mlmc.estimator +import mlmc.estimator import mlmc.quantity_estimate as qe from mlmc.sample_storage import Memory from mlmc.quantity_spec import QuantitySpec, ChunkSpec @@ -445,6 +445,42 @@ def cut_samples(data, sample_storage, new_n_collected, new_l_0=0): return sample_storage +def plot_progress(conv_layers, flatten_output, dense_layers): + + for inputs, weights, outputs in conv_layers: + plt.matshow(weights[-1][0]) + plt.show() + + for index, input in enumerate(inputs[::10]): + plt.matshow(input[0]) + plt.show() + plt.matshow(outputs[index][0]) + plt.show() + + # print("shape ", c_layer._outputs[index][0].shape) + plt.matshow(np.sum(outputs[index][0], axis=0, keepdims=True)) + plt.show() + + # plt.matshow(self._inputs[-1][0]) + # plt.show() + # plt.matshow(self._outputs[-1][0]) + # plt.show() + + # print("output flatten ", self._output_flatten) + # print("final output ", final_output) + + plt.matshow([flatten_output[-1]]) + plt.show() + # plt.matshow(final_output[-1]) + # plt.show() + + for inputs, weights, outputs in dense_layers: + plt.matshow(inputs[-1]) + plt.show() + plt.matshow(outputs[-1]) + plt.show() + + def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predictions, train_targets, train_predictions, val_targets, l_0_targets=None, l_0_predictions=None, l1_sample_time=None, l0_sample_time=None, nn_level=0, replace_level=False, stats=False): @@ -507,9 +543,9 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti ###### ### Get n ops ###### - #n_ops, field_times, coarse_flow, fine_flow = get_sample_times_mlmc(mlmc_file) - n_ops, _, _ = get_sample_times(sampling_info_path) - n_ops = n_ops[n_levels-1:] + n_ops, field_times, coarse_flow, fine_flow = get_sample_times_mlmc(mlmc_file) + # n_ops, _, _ = get_sample_times(sampling_info_path) + # n_ops = n_ops[n_levels-1:] if n_ops is None: n_ops = sample_storage.get_n_ops() @@ -521,7 +557,7 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti level_samples = estimator.get_level_samples(level_id=l_id) data_mlmc.append(level_samples) - print("original level params" , sample_storage.get_level_parameters()) + print("original level params", sample_storage.get_level_parameters()) sample_storage = create_quantity_mlmc(data_mlmc, level_parameters=sample_storage.get_level_parameters()) print("Original storage") orig_storage_n_collected, orig_storage_max_vars = get_storage_info(sample_storage) diff --git a/test/metamodels/metamodel_test.py b/test/metamodels/metamodel_test.py index 94a0138c..67c89d7b 100644 --- a/test/metamodels/metamodel_test.py +++ b/test/metamodels/metamodel_test.py @@ -14,6 +14,7 @@ from tensorflow.keras.losses import MeanSquaredError, KLDivergence, MeanAbsoluteError from mlmc.metamodel.custom_methods import abs_activation, MSE_moments from tensorflow.keras import Model +from tensorflow.keras.models import Sequential from tensorflow.keras.layers import Dense from spektral.layers import GlobalSumPool, GlobalMaxPool, GlobalAvgPool import tensorflow as tf @@ -26,7 +27,7 @@ def get_gnn(): # Parameters # conv_layer = GCNConv conv_layer = ChebConv # Seems better than GCNConv, good distribution of predictions - # conv_layer = OwnChebConv + conv_layer = OwnChebConv # conv_layer = GraphSageConv # Seems better than ChebConv, good loss but very narrow distribution of predictions # # conv_layer = ARMAConv # Seems worse than GraphSageConv # conv_layer = GATConv # Slow and not better than GraphSageConv @@ -53,7 +54,7 @@ def get_gnn(): "normalizer": preprocessing.Normalization() } - model = Net(**net_model_config) + #model = Net(**net_model_config) model_config = {"loss": loss, "optimizer": optimizer, @@ -71,10 +72,11 @@ class Net(Model): def __init__(self, conv_layer, hidden_activation, output_activation, kernel_regularization, normalizer, **kwargs): super().__init__(**kwargs) + # self.normalizer = normalizer # self.norm_layer = tf.keras.layers.LayerNormalization(axis=1) - self.conv1 = conv_layer(8, K=1, activation=hidden_activation, kernel_regularizer=kernel_regularization) - # self.conv2 = conv_layer(64, K=1, activation=hidden_activation, kernel_regularizer=kernel_regularization) + self._conv_layers = [conv_layer(8, K=1, activation=hidden_activation, kernel_regularizer=kernel_regularization)]#, + #conv_layer(64, K=1, activation=hidden_activation, kernel_regularizer=kernel_regularization)] # self.conv3 = conv_layer(32, K=1, activation=hidden_activation, kernel_regularizer=kernel_regularization) # self.conv4 = conv_layer(32, K=1, activation=hidden_activation, kernel_regularizer=kernel_regularization) #self.conv2 = conv_layer(32, K=2, activation=hidden_activation, kernel_regularizer=kernel_regularization) @@ -83,27 +85,85 @@ def __init__(self, conv_layer, hidden_activation, output_activation, kernel_regu # self.conv4 = conv_layer(4, activation=hidden_activation, kernel_regularizer=kernel_regularization) # self.conv3 = conv_layer(64, activation=hidden_activation, kernel_regularizer=kernel_regularization) self.flatten = GlobalSumPool() + + #self._submodel = Sequential() + + self._dense_layers = {Dense(1): ([], [])} + + # for d_layer in self._dense_layers: + # self._submodel.add(d_layer) # self.fc1 = Dense(32, activation=hidden_activation) - self.fc2 = Dense(1)#, activation=output_activation) # linear activation for output neuron + #, activation=output_activation) # linear activation for output neuron def call(self, inputs): x, a = inputs - # print("x ", x) - # x = self.normalizer(x) - # x = self.norm_layer(x) - # print("normalized x ", x) - # print("x[0,0,:] ", x[0, 0, :]) - x = self.conv1([x, a]) - # print("x.shape ", x.shape) - # x = self.conv2([x, a]) - # # # print("conv2 x shape", x.shape) - # x = self.conv3([x, a]) - # x = self.conv4([x, a]) - output1 = self.flatten(x) - # output2 = self.fc1(output1) - output = self.fc2(output1) + + for c_layer in self._conv_layers: + x = c_layer([x, a]) + + output = self.flatten(x) + self._output_flatten = output + input = output + + for d_layer, (inputs, outputs) in self._dense_layers.items(): + inputs.append(input) + output = d_layer(input) + outputs.append(output) + return output + def clear_progress(self): + for c_layer in self._conv_layers: + c_layer.clear_progress() + + new_dense_layers = {} + for d_layer, _ in self._dense_layers.items(): + new_dense_layers = {d_layer: ([], [])} + + self._dense_layers = new_dense_layers + + def plot_progress(self, stats=False): + if not stats: + import matplotlib.pyplot as plt + + if not stats: + + for c_layer in self._conv_layers: + plt.matshow(c_layer._weights[-1][0]) + plt.show() + + for index, input in enumerate(c_layer._inputs[::10]): + plt.matshow(input[0]) + plt.show() + plt.matshow(c_layer._outputs[index][0]) + plt.show() + + #print("shape ", c_layer._outputs[index][0].shape) + plt.matshow(np.sum(c_layer._outputs[index][0], axis=0, keepdims=True)) + plt.show() + + # plt.matshow(self._inputs[-1][0]) + # plt.show() + # plt.matshow(self._outputs[-1][0]) + # plt.show() + + #print("output flatten ", self._output_flatten) + #print("final output ", final_output) + + plt.matshow([self._output_flatten[-1]]) + plt.show() + # plt.matshow(final_output[-1]) + # plt.show() + + for dense_layer, (input, output) in self._dense_layers.items(): + plt.matshow(input[-1]) + plt.show() + plt.matshow(output[-1]) + plt.show() + + else: + return self._conv_layers, self._output_flatten, self._dense_layers + def get_config(data_dir, case=0): if case == 0: From f43c3c22f96e390d60d920b7c429317218caa4fe Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Fri, 22 Oct 2021 13:26:57 +0200 Subject: [PATCH 30/67] requirements update --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 423e7e91..2c6421f8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,4 @@ attrs gstools memoization tensorflow +pandas From 5cc78dd483956b012c929d5537f64ba7fadf9eb5 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Fri, 22 Oct 2021 13:52:52 +0200 Subject: [PATCH 31/67] custom cheb conv layer --- src/mlmc/metamodel/own_cheb_conv.py | 416 ++++++++++++++++++++++++++++ 1 file changed, 416 insertions(+) create mode 100644 src/mlmc/metamodel/own_cheb_conv.py diff --git a/src/mlmc/metamodel/own_cheb_conv.py b/src/mlmc/metamodel/own_cheb_conv.py new file mode 100644 index 00000000..79528978 --- /dev/null +++ b/src/mlmc/metamodel/own_cheb_conv.py @@ -0,0 +1,416 @@ +import copy +import warnings + +import numpy as np +from scipy import linalg +from scipy import sparse as sp +from scipy.sparse.linalg import ArpackNoConvergence + +import tensorflow as tf +from tensorflow.keras import backend as K + +from spektral.layers import ops +from spektral.layers.convolutional.conv import Conv +import matplotlib.pyplot as plt + + +class OwnChebConv(Conv): + r""" + A Chebyshev convolutional layer from the paper + + > [Convolutional Neural Networks on Graphs with Fast Localized Spectral + Filtering](https://arxiv.org/abs/1606.09375)
+ > Michaël Defferrard et al. + + **Mode**: single, disjoint, mixed, batch. + + This layer computes: + $$ + \X' = \sum \limits_{k=0}^{K - 1} \T^{(k)} \W^{(k)} + \b^{(k)}, + $$ + where \( \T^{(0)}, ..., \T^{(K - 1)} \) are Chebyshev polynomials of \(\tilde \L\) + defined as + $$ + \T^{(0)} = \X \\ + \T^{(1)} = \tilde \L \X \\ + \T^{(k \ge 2)} = 2 \cdot \tilde \L \T^{(k - 1)} - \T^{(k - 2)}, + $$ + where + $$ + \tilde \L = \frac{2}{\lambda_{max}} \cdot (\I - \D^{-1/2} \A \D^{-1/2}) - \I. + $$ + + **Input** + + - Node features of shape `([batch], n_nodes, n_node_features)`; + - A list of K Chebyshev polynomials of shape + `[([batch], n_nodes, n_nodes), ..., ([batch], n_nodes, n_nodes)]`; can be computed with + `spektral.utils.convolution.chebyshev_filter`. + + **Output** + + - Node features with the same shape of the input, but with the last + dimension changed to `channels`. + + **Arguments** + + - `channels`: number of output channels; + - `K`: order of the Chebyshev polynomials; + - `activation`: activation function; + - `use_bias`: bool, add a bias vector to the output; + - `kernel_initializer`: initializer for the weights; + - `bias_initializer`: initializer for the bias vector; + - `kernel_regularizer`: regularization applied to the weights; + - `bias_regularizer`: regularization applied to the bias vector; + - `activity_regularizer`: regularization applied to the output; + - `kernel_constraint`: constraint applied to the weights; + - `bias_constraint`: constraint applied to the bias vector. + + """ + + def __init__( + self, + channels, + K=1, + activation=None, + use_bias=True, + kernel_initializer="glorot_uniform", + bias_initializer="zeros", + kernel_regularizer=None, + bias_regularizer=None, + activity_regularizer=None, + kernel_constraint=None, + bias_constraint=None, + **kwargs + ): + super().__init__( + activation=activation, + use_bias=use_bias, + kernel_initializer=kernel_initializer, + bias_initializer=bias_initializer, + kernel_regularizer=kernel_regularizer, + bias_regularizer=bias_regularizer, + activity_regularizer=activity_regularizer, + kernel_constraint=kernel_constraint, + bias_constraint=bias_constraint, + **kwargs + ) + self.channels = channels + self.K = K + + self._inputs = [] + self._weights = [] + self._outputs = [] + # self.use_bias = use_bias + # + # print("use bias ", use_bias) + # print("self use bias ", self.use_bias) + + def build(self, input_shape): + assert len(input_shape) >= 2 + input_dim = input_shape[0][-1] + + self.kernel = self.add_weight( + shape=(self.K, input_dim, self.channels), + initializer=self.kernel_initializer, + name="kernel", + regularizer=self.kernel_regularizer, + constraint=self.kernel_constraint, + ) + if self.use_bias: + self.bias = self.add_weight( + shape=(self.channels,), + initializer=self.bias_initializer, + name="bias", + regularizer=self.bias_regularizer, + constraint=self.bias_constraint, + ) + else: + self.bias = None + self.built = True + + def call(self, inputs): + x, a = inputs + + # print("self. kernel shape ", self.kernel.shape) + # print("x shape ", x.shape) + # print("type x ", type(x).__name__) + + if 'EagerTensor' in type(x).__name__: + self._inputs.append(x.numpy()) + + #print("a.shape ", a.shape) + + T_0 = x + # print("T_0 ", T_0[0, 0]) + # print("self.kernel ", self.kernel[0].shape) + output = K.dot(T_0, self.kernel[0]) + + # print("type(seklf.kernel) ", type(self.kernel)) + # print("kernel ", self.kernel) + + if 'UnliftedInitializerVariable' in type(self.kernel).__name__: + try: + #print("self.kernel shape ", self.kernel.shape) + self._weights.append(self.kernel.numpy()) + except: + pass + + if self.K > 1: + T_1 = ops.modal_dot(a, x) + output += K.dot(T_1, self.kernel[1]) + + # print("T_1 ", T_1) + # print("self.kernel[1] ", self.kernel[1]) + + for k in range(2, self.K): + T_2 = 2 * ops.modal_dot(a, T_1) - T_0 + output += K.dot(T_2, self.kernel[k]) + T_0, T_1 = T_1, T_2 + + #print("self use bias ", self.use_bias) + if self.use_bias: + #print("use bias") + output = K.bias_add(output, self.bias) + output = self.activation(output) + + # print("output shape ", output.shape) + # exit() + + if 'EagerTensor' in type(output).__name__: + self._outputs.append(output.numpy()) + + return output + + def clear_progress(self): + self._inputs = [] + self._outputs = [] + #self._weights = [] + + @property + def config(self): + return {"channels": self.channels, "K": self.K} + + @staticmethod + def preprocess(a): + a = normalized_laplacian(a) + a = rescale_laplacian(a) + return a + + +def degree_matrix(A): + """ + Computes the degree matrix of the given adjacency matrix. + :param A: rank 2 array or sparse matrix. + :return: if A is a dense array, a dense array; if A is sparse, a sparse + matrix in DIA format. + """ + degrees = np.array(A.sum(1)).flatten() + if sp.issparse(A): + D = sp.diags(degrees) + else: + D = np.diag(degrees) + return D + + +def degree_power(A, k): + r""" + Computes \(\D^{k}\) from the given adjacency matrix. Useful for computing + normalised Laplacian. + :param A: rank 2 array or sparse matrix. + :param k: exponent to which elevate the degree matrix. + :return: if A is a dense array, a dense array; if A is sparse, a sparse + matrix in DIA format. + """ + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + degrees = np.power(np.array(A.sum(1)), k).ravel() + degrees[np.isinf(degrees)] = 0.0 + if sp.issparse(A): + D = sp.diags(degrees) + else: + D = np.diag(degrees) + return D + + +def normalized_adjacency(A, symmetric=True): + r""" + Normalizes the given adjacency matrix using the degree matrix as either + \(\D^{-1}\A\) or \(\D^{-1/2}\A\D^{-1/2}\) (symmetric normalization). + :param A: rank 2 array or sparse matrix; + :param symmetric: boolean, compute symmetric normalization; + :return: the normalized adjacency matrix. + """ + if symmetric: + print("symmetric") + normalized_D = degree_power(A, -0.5) + print("normalized D") + return normalized_D.dot(A).dot(normalized_D) + else: + normalized_D = degree_power(A, -1.0) + return normalized_D.dot(A) + + +def laplacian(A): + r""" + Computes the Laplacian of the given adjacency matrix as \(\D - \A\). + :param A: rank 2 array or sparse matrix; + :return: the Laplacian. + """ + return degree_matrix(A) - A + + +def normalized_laplacian(A, symmetric=True): + r""" + Computes a normalized Laplacian of the given adjacency matrix as + \(\I - \D^{-1}\A\) or \(\I - \D^{-1/2}\A\D^{-1/2}\) (symmetric normalization). + :param A: rank 2 array or sparse matrix; + :param symmetric: boolean, compute symmetric normalization; + :return: the normalized Laplacian. + """ + if sp.issparse(A): + I = sp.eye(A.shape[-1], dtype=A.dtype) + else: + I = np.eye(A.shape[-1], dtype=A.dtype) + normalized_adj = normalized_adjacency(A, symmetric=symmetric) + return I - normalized_adj + + +def rescale_laplacian(L, lmax=None): + """ + Rescales the Laplacian eigenvalues in [-1,1], using lmax as largest eigenvalue. + :param L: rank 2 array or sparse matrix; + :param lmax: if None, compute largest eigenvalue with scipy.linalg.eisgh. + If the eigendecomposition fails, lmax is set to 2 automatically. + If scalar, use this value as largest eigenvalue when rescaling. + :return: + """ + if lmax is None: + try: + if sp.issparse(L): + lmax = sp.linalg.eigsh(L, 1, which="LM", return_eigenvectors=False)[0] + else: + n = L.shape[-1] + lmax = linalg.eigh(L, eigvals_only=True, eigvals=[n - 2, n - 1])[-1] + except ArpackNoConvergence: + lmax = 2 + if sp.issparse(L): + I = sp.eye(L.shape[-1], dtype=L.dtype) + else: + I = np.eye(L.shape[-1], dtype=L.dtype) + L_scaled = (2.0 / lmax) * L - I + return L_scaled + + +def gcn_filter(A, symmetric=True): + r""" + Computes the graph filter described in + [Kipf & Welling (2017)](https://arxiv.org/abs/1609.02907). + :param A: array or sparse matrix with rank 2 or 3; + :param symmetric: boolean, whether to normalize the matrix as + \(\D^{-\frac{1}{2}}\A\D^{-\frac{1}{2}}\) or as \(\D^{-1}\A\); + :return: array or sparse matrix with rank 2 or 3, same as A; + """ + out = copy.deepcopy(A) + if isinstance(A, list) or (isinstance(A, np.ndarray) and A.ndim == 3): + for i in range(len(A)): + out[i] = A[i] + out[i][np.diag_indices_from(out[i])] += 1 + out[i] = normalized_adjacency(out[i], symmetric=symmetric) + else: + if hasattr(out, "tocsr"): + out = out.tocsr() + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + out[np.diag_indices_from(out)] += 1 + out = normalized_adjacency(out, symmetric=symmetric) + + if sp.issparse(out): + out.sort_indices() + return out + + +def chebyshev_polynomial(X, k): + """ + Calculates Chebyshev polynomials of X, up to order k. + :param X: rank 2 array or sparse matrix; + :param k: the order up to which compute the polynomials, + :return: a list of k + 1 arrays or sparse matrices with one element for each + degree of the polynomial. + """ + T_k = list() + if sp.issparse(X): + T_k.append(sp.eye(X.shape[0], dtype=X.dtype).tocsr()) + else: + T_k.append(np.eye(X.shape[0], dtype=X.dtype)) + T_k.append(X) + + def chebyshev_recurrence(T_k_minus_one, T_k_minus_two, X): + if sp.issparse(X): + X_ = sp.csr_matrix(X, copy=True) + else: + X_ = np.copy(X) + return 2 * X_.dot(T_k_minus_one) - T_k_minus_two + + for _ in range(2, k + 1): + T_k.append(chebyshev_recurrence(T_k[-1], T_k[-2], X)) + + return T_k + + +def chebyshev_filter(A, k, symmetric=True): + r""" + Computes the Chebyshev filter from the given adjacency matrix, as described + in [Defferrard et at. (2016)](https://arxiv.org/abs/1606.09375). + :param A: rank 2 array or sparse matrix; + :param k: integer, the order of the Chebyshev polynomial; + :param symmetric: boolean, whether to normalize the matrix as + \(\D^{-\frac{1}{2}}\A\D^{-\frac{1}{2}}\) or as \(\D^{-1}\A\); + :return: a list of k + 1 arrays or sparse matrices with one element for each + degree of the polynomial. + """ + normalized_adj = normalized_adjacency(A, symmetric) + if sp.issparse(A): + I = sp.eye(A.shape[0], dtype=A.dtype) + else: + I = np.eye(A.shape[0], dtype=A.dtype) + L = I - normalized_adj # Compute Laplacian + + # Rescale Laplacian + L_scaled = rescale_laplacian(L) + + # Compute Chebyshev polynomial approximation + T_k = chebyshev_polynomial(L_scaled, k) + + # Sort indices + if sp.issparse(T_k[0]): + for i in range(len(T_k)): + T_k[i].sort_indices() + + return T_k + + +def add_self_loops(a, value=1): + """ + Sets the inner diagonals of `a` to `value`. + :param a: a np.array or scipy.sparse matrix, the innermost two dimensions + must be equal. + :param value: value to set the diagonals to. + :return: a np.array or scipy.sparse matrix with the same shape as `a`. + """ + a = a.copy() + if len(a.shape) < 2: + raise ValueError("a must have at least rank 2") + n = a.shape[-1] + if n != a.shape[-2]: + raise ValueError( + "Innermost two dimensions must be equal. Got {}".format(a.shape) + ) + if sp.issparse(a): + a = a.tolil() + a.setdiag(value) + return a.tocsr() + else: + idx = np.arange(n) + a[..., idx, idx] = value + return a From 9b8f834bbf8e00cb6c7540165bab28e40e0a1baf Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Mon, 25 Oct 2021 12:24:57 +0200 Subject: [PATCH 32/67] display weights --- src/mlmc/metamodel/analyze_nn.py | 3 +-- src/mlmc/metamodel/create_graph.py | 16 ++++++++++- src/mlmc/metamodel/postprocessing.py | 40 ++++++++++++++++++++++++---- 3 files changed, 51 insertions(+), 8 deletions(-) diff --git a/src/mlmc/metamodel/analyze_nn.py b/src/mlmc/metamodel/analyze_nn.py index b7b282d9..125a8091 100644 --- a/src/mlmc/metamodel/analyze_nn.py +++ b/src/mlmc/metamodel/analyze_nn.py @@ -544,8 +544,7 @@ def analyze_statistics(config): replace_level=config['replace_level'], stats=True) - plot_progress(data_dict["conv_layers"][i], data_dict["flatten_output"][i], data_dict["dense_layers"][i]) - + plot_progress(data_dict["conv_layers"][i], data_dict["flatten_output"][i], data_dict["dense_layers"][i], mesh_file=config["mesh"]) mlmc_n_collected_all.append(mlmc_n_collected) nn_n_collected_all.append(nn_mlmc_n_collected) diff --git a/src/mlmc/metamodel/create_graph.py b/src/mlmc/metamodel/create_graph.py index 0c77fdf4..d07b1f03 100644 --- a/src/mlmc/metamodel/create_graph.py +++ b/src/mlmc/metamodel/create_graph.py @@ -25,7 +25,7 @@ # HDF_PATH = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L5/mlmc_5.hdf5" -def extract_mesh_gmsh_io(mesh_file): +def extract_mesh_gmsh_io(mesh_file, get_points=False): """ Extract mesh from file :param mesh_file: Mesh file path @@ -61,6 +61,20 @@ def extract_mesh_gmsh_io(mesh_file): ele_ids[i] = id_bulk ele_nodes[id_bulk] = i_nodes + if get_points: + + min_pt = np.min(centers, axis=0) + max_pt = np.max(centers, axis=0) + diff = max_pt - min_pt + min_axis = np.argmin(diff) + non_zero_axes = [0, 1, 2] + # TODO: be able to use this mesh_dimension in fields + if diff[min_axis] < 1e-10: + non_zero_axes.pop(min_axis) + points = centers[:, non_zero_axes] + + return {'points': points, 'point_region_ids': point_region_ids, 'ele_ids': ele_ids, 'region_map': region_map} + return ele_nodes diff --git a/src/mlmc/metamodel/postprocessing.py b/src/mlmc/metamodel/postprocessing.py index 8cb9ff2e..ca8903d5 100644 --- a/src/mlmc/metamodel/postprocessing.py +++ b/src/mlmc/metamodel/postprocessing.py @@ -6,6 +6,7 @@ #from mlmc.tool import plot #import mlmc.tool.simple_distribution import mlmc.estimator +from mlmc.tool import gmsh_io import mlmc.quantity_estimate as qe from mlmc.sample_storage import Memory from mlmc.quantity_spec import QuantitySpec, ChunkSpec @@ -13,6 +14,7 @@ from mlmc.sample_storage_hdf import SampleStorageHDF from mlmc.moments import Legendre, Monomial from mlmc.quantity import make_root_quantity +from mlmc.metamodel.create_graph import extract_mesh_gmsh_io #import mlmc.tool.simple_distribution QUANTILE = 0.01 @@ -445,17 +447,45 @@ def cut_samples(data, sample_storage, new_n_collected, new_l_0=0): return sample_storage -def plot_progress(conv_layers, flatten_output, dense_layers): +def plot_progress(conv_layers, flatten_output, dense_layers, mesh_file=None): + + if mesh_file is not None: + #mesh = gmsh_io.GmshIO(fields_mesh) + mesh_data = extract_mesh_gmsh_io(mesh_file, get_points=True) + points = mesh_data['points'] + X = points[:, 0] + Y = points[:, 1] for inputs, weights, outputs in conv_layers: + plt.matshow(weights[-1][0]) plt.show() + # Note: weights have different shape than the mesh for index, input in enumerate(inputs[::10]): - plt.matshow(input[0]) - plt.show() - plt.matshow(outputs[index][0]) - plt.show() + if mesh_file: + fig, ax = plt.subplots(1, 1, figsize=(15, 10)) + + cont = ax.tricontourf(X, Y, input[0].ravel(), levels=16) + fig.colorbar(cont) + plt.title("input") + plt.show() + + print("range(outputs[index][0].shape[1]) ", range(outputs[index][0].shape[1])) + for i in range(outputs[index][0].shape[1]): + channel_output = outputs[index][0][:, i] + fig, ax = plt.subplots(1, 1, figsize=(15, 10)) + cont = ax.tricontourf(X, Y, channel_output, levels=16) + fig.colorbar(cont) + plt.title("channel {}".format(i)) + + plt.show() + + else: + plt.matshow(input[0]) + plt.show() + plt.matshow(outputs[index][0]) + plt.show() # print("shape ", c_layer._outputs[index][0].shape) plt.matshow(np.sum(outputs[index][0], axis=0, keepdims=True)) From 3dbc59993111ef6ebc3ca813ded61793d26bb711 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Wed, 27 Oct 2021 15:32:32 +0200 Subject: [PATCH 33/67] save model --- src/mlmc/metamodel/analyze_nn.py | 91 +++++++++++++------- src/mlmc/metamodel/flow_task_GNN_2.py | 1 - src/mlmc/metamodel/own_cheb_conv.py | 38 --------- src/mlmc/metamodel/postprocessing.py | 28 +++---- test/metamodels/metamodel_test.py | 114 +++++++++++++------------- 5 files changed, 129 insertions(+), 143 deletions(-) diff --git a/src/mlmc/metamodel/analyze_nn.py b/src/mlmc/metamodel/analyze_nn.py index 125a8091..95a5abc7 100644 --- a/src/mlmc/metamodel/analyze_nn.py +++ b/src/mlmc/metamodel/analyze_nn.py @@ -348,7 +348,7 @@ def predict_level_zero_SVR(nn, output_dir, hdf_path, mesh, batch_size=1000, log= def statistics(config): - n_subsamples = 25 + n_subsamples = 2 model_title, mch_l_model, log = config['machine_learning_model'] model_data = {} @@ -370,8 +370,7 @@ def statistics(config): os.makedirs(iter_dir) model, targets, predictions, learning_time, train_targets, train_predictions, \ - val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, total_steps,\ - conv_layers, flatten_output, dense_layers = \ + val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, total_steps = \ mch_l_model(config, stats=True, train=True, log=log, seed=i) if config['save_model']: @@ -389,18 +388,6 @@ def statistics(config): model_data["total_steps"] = total_steps model_data["learning_times"] = learning_time - model_data["conv_layers"] = [] - model_data["dense_layers"] = [] - - if conv_layers is not None: - for index, c_layer in enumerate(conv_layers): - model_data["conv_layers"].append([c_layer._inputs, c_layer._weights, c_layer._outputs]) - - model_data["flatten_output"] = flatten_output - - for index, (d_layer, (inputs, outputs)) in enumerate(dense_layers.items()): - model_data["dense_layers"].append([inputs, d_layer.get_weights(), outputs]) - save_statistics(iter_dir, model_data) # save_times(save_path, False, (preprocess_time, len(data)), learning_time, (predict_l_0_time, len(l_0_targets))) @@ -448,9 +435,6 @@ def load_statistics(dir_path): models_data["total_steps"] = [] models_data["learning_times"] = [] models_data["log"] = [] - models_data["conv_layers"] = [] - models_data["flatten_output"] = [] - models_data["dense_layers"] = [] #dirs = (os.path.split(dir_path)[-1]).split("_") n_iters = 25 @@ -492,6 +476,60 @@ def plot_sse(data_nn, data_mlmc, x_label="ith moment", y_label="MSE", title=""): fig.show() +def predict_data(config, model, mesh_file, log=True): + batch_size = config['batch_size'] + data = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log) + data = data # [:10000] + + data.a = config['conv_layer'].preprocess(data.a) + data.a = sp_matrix_to_sp_tensor(data.a) + + # idx = 0 + # data_te = data.get_test_data(idx, train_data_len) + data_te = data[:1] + + print("len(datate) ", len(data_te)) + print("batch size ", batch_size) + + # We use a MixedLoader since the dataset is in mixed mode + loader = MixedLoader(data_te, batch_size=batch_size) + + conv_layers = {} + dense_layers = {} + flatten_input = [] + flatten_output = [] + + step = 0 + for batch in loader: + if step == loader.steps_per_epoch: + break + inputs, target = batch + x, a = inputs + + for conv_index, conv_layer in enumerate(model._conv_layers): + if conv_index not in conv_layers: + conv_layers[conv_index] = [[], [], []] + conv_layers[conv_index][0].extend(x) # inputs + conv_layers[conv_index][1].extend(conv_layer.kernel.numpy()) # weights (kernel) + conv_out = conv_layer([x, a]) + conv_layers[conv_index][2].extend(conv_out) # outputs + + flatten_input = conv_layers[conv_index][2][-1] + flatten_output = model.flatten(conv_out) + + for index, dense_layer in enumerate(model._dense_layers): + if index not in dense_layers: + dense_layers[index] = [[], [], []] + + dense_layers[index][0].extend(flatten_output) # inputs + dense_layers[index][1].extend(dense_layer.weights) # weights (kernel) + dense_layers[index][2].extend(dense_layer(model.flatten(conv_out))) # outputs + + step += 1 + + plot_progress(conv_layers, dense_layers, flatten_output, mesh_file=mesh_file) + + def analyze_statistics(config): if not os.path.isdir(config['save_path']): @@ -532,6 +570,10 @@ def analyze_statistics(config): l1_sample_time = data_dict["l1_sample_time"][i] l0_sample_time = data_dict["l0_sample_time"][i] + model = data_dict["model"][i] + + predict_data(config, model, mesh_file=config["mesh"]) + mlmc_n_collected, nn_mlmc_n_collected, n_ops, n_ops_predict, orig_moments_mean, predict_moments_mean,\ ref_moments_mean, orig_level_params, nn_level_params = process_mlmc(config['hdf_path'], config['sampling_info_path'], @@ -544,8 +586,6 @@ def analyze_statistics(config): replace_level=config['replace_level'], stats=True) - plot_progress(data_dict["conv_layers"][i], data_dict["flatten_output"][i], data_dict["dense_layers"][i], mesh_file=config["mesh"]) - mlmc_n_collected_all.append(mlmc_n_collected) nn_n_collected_all.append(nn_mlmc_n_collected) n_ops_all.append(n_ops) @@ -710,7 +750,6 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): data_te = data.get_test_data(seed, train_data_len) #data_tr, data_te = data[:train_data_len], data[train_data_len:] - gnn = config['gnn'](**config['model_config']) # if hasattr(gnn._loss,'__name__') and gnn._loss.__name__ == "MSE_moments": @@ -763,8 +802,6 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): #val_targets = gnn.val_targets total_steps = gnn._total_n_steps - gnn._model.clear_progress() - targets, predictions = gnn.predict(loader_te) predictions = np.squeeze(predictions) @@ -817,14 +854,8 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): # l_0_predictions, l1_sample_time, l0_sample_time, nn_level=level, replace_level=replace_level, # stats=stats) - conv_layers, flatten_output, dense_layers = gnn._model.plot_progress(stats) - #gnn._model.plot_progress(stats) - return gnn._model, targets, predictions, learning_time, train_targets, train_predictions,\ - val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, total_steps,\ - conv_layers, flatten_output, dense_layers - - gnn._model.plot_progress(stats) + val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, total_steps save_times(config['save_path'], False, (preprocess_time, len(data)), learning_time, (predict_l_0_time, len(l_0_targets))) save_load_data(config['save_path'], False, targets, predictions, train_targets, train_predictions, val_targets, l_0_targets, diff --git a/src/mlmc/metamodel/flow_task_GNN_2.py b/src/mlmc/metamodel/flow_task_GNN_2.py index 35f0dcb7..ca07a93d 100644 --- a/src/mlmc/metamodel/flow_task_GNN_2.py +++ b/src/mlmc/metamodel/flow_task_GNN_2.py @@ -51,7 +51,6 @@ def __init__(self, **kwargs): model_class = kwargs.get('model_class') net_model_config = kwargs.get('net_model_config') model = model_class(**net_model_config) - print("model class model ", model) else: model = kwargs.get('model') diff --git a/src/mlmc/metamodel/own_cheb_conv.py b/src/mlmc/metamodel/own_cheb_conv.py index 79528978..1d830e1a 100644 --- a/src/mlmc/metamodel/own_cheb_conv.py +++ b/src/mlmc/metamodel/own_cheb_conv.py @@ -98,13 +98,7 @@ def __init__( self.channels = channels self.K = K - self._inputs = [] - self._weights = [] - self._outputs = [] # self.use_bias = use_bias - # - # print("use bias ", use_bias) - # print("self use bias ", self.use_bias) def build(self, input_shape): assert len(input_shape) >= 2 @@ -132,30 +126,9 @@ def build(self, input_shape): def call(self, inputs): x, a = inputs - # print("self. kernel shape ", self.kernel.shape) - # print("x shape ", x.shape) - # print("type x ", type(x).__name__) - - if 'EagerTensor' in type(x).__name__: - self._inputs.append(x.numpy()) - - #print("a.shape ", a.shape) - T_0 = x - # print("T_0 ", T_0[0, 0]) - # print("self.kernel ", self.kernel[0].shape) output = K.dot(T_0, self.kernel[0]) - # print("type(seklf.kernel) ", type(self.kernel)) - # print("kernel ", self.kernel) - - if 'UnliftedInitializerVariable' in type(self.kernel).__name__: - try: - #print("self.kernel shape ", self.kernel.shape) - self._weights.append(self.kernel.numpy()) - except: - pass - if self.K > 1: T_1 = ops.modal_dot(a, x) output += K.dot(T_1, self.kernel[1]) @@ -174,19 +147,8 @@ def call(self, inputs): output = K.bias_add(output, self.bias) output = self.activation(output) - # print("output shape ", output.shape) - # exit() - - if 'EagerTensor' in type(output).__name__: - self._outputs.append(output.numpy()) - return output - def clear_progress(self): - self._inputs = [] - self._outputs = [] - #self._weights = [] - @property def config(self): return {"channels": self.channels, "K": self.K} diff --git a/src/mlmc/metamodel/postprocessing.py b/src/mlmc/metamodel/postprocessing.py index ca8903d5..c438de7f 100644 --- a/src/mlmc/metamodel/postprocessing.py +++ b/src/mlmc/metamodel/postprocessing.py @@ -447,7 +447,7 @@ def cut_samples(data, sample_storage, new_n_collected, new_l_0=0): return sample_storage -def plot_progress(conv_layers, flatten_output, dense_layers, mesh_file=None): +def plot_progress(conv_layers, dense_layers, output_flatten, mesh_file=None): if mesh_file is not None: #mesh = gmsh_io.GmshIO(fields_mesh) @@ -456,9 +456,9 @@ def plot_progress(conv_layers, flatten_output, dense_layers, mesh_file=None): X = points[:, 0] Y = points[:, 1] - for inputs, weights, outputs in conv_layers: - - plt.matshow(weights[-1][0]) + for idx, conv_layer in conv_layers.items(): + inputs, weights, outputs = conv_layer[0], conv_layer[1], conv_layer[2] + plt.matshow(weights[-1]) plt.show() # Note: weights have different shape than the mesh @@ -466,14 +466,13 @@ def plot_progress(conv_layers, flatten_output, dense_layers, mesh_file=None): if mesh_file: fig, ax = plt.subplots(1, 1, figsize=(15, 10)) - cont = ax.tricontourf(X, Y, input[0].ravel(), levels=16) + cont = ax.tricontourf(X, Y, input.ravel(), levels=16) fig.colorbar(cont) plt.title("input") plt.show() - print("range(outputs[index][0].shape[1]) ", range(outputs[index][0].shape[1])) - for i in range(outputs[index][0].shape[1]): - channel_output = outputs[index][0][:, i] + for i in range(outputs[index].shape[1]): + channel_output = outputs[index][:, i] fig, ax = plt.subplots(1, 1, figsize=(15, 10)) cont = ax.tricontourf(X, Y, channel_output, levels=16) fig.colorbar(cont) @@ -488,7 +487,7 @@ def plot_progress(conv_layers, flatten_output, dense_layers, mesh_file=None): plt.show() # print("shape ", c_layer._outputs[index][0].shape) - plt.matshow(np.sum(outputs[index][0], axis=0, keepdims=True)) + plt.matshow(np.sum(outputs[index], axis=0, keepdims=True)) plt.show() # plt.matshow(self._inputs[-1][0]) @@ -499,15 +498,14 @@ def plot_progress(conv_layers, flatten_output, dense_layers, mesh_file=None): # print("output flatten ", self._output_flatten) # print("final output ", final_output) - plt.matshow([flatten_output[-1]]) + plt.matshow([output_flatten[-1]]) plt.show() - # plt.matshow(final_output[-1]) - # plt.show() - for inputs, weights, outputs in dense_layers: - plt.matshow(inputs[-1]) + for idx, dense_layer in dense_layers.items(): + inputs, weights, outputs = dense_layer[0], dense_layer[1], dense_layer[2] + plt.matshow([inputs[-1]]) plt.show() - plt.matshow(outputs[-1]) + plt.matshow([outputs[-1]]) plt.show() diff --git a/test/metamodels/metamodel_test.py b/test/metamodels/metamodel_test.py index 67c89d7b..08362d47 100644 --- a/test/metamodels/metamodel_test.py +++ b/test/metamodels/metamodel_test.py @@ -88,7 +88,7 @@ def __init__(self, conv_layer, hidden_activation, output_activation, kernel_regu #self._submodel = Sequential() - self._dense_layers = {Dense(1): ([], [])} + self._dense_layers = [Dense(1)] # for d_layer in self._dense_layers: # self._submodel.add(d_layer) @@ -102,67 +102,63 @@ def call(self, inputs): x = c_layer([x, a]) output = self.flatten(x) - self._output_flatten = output - input = output - for d_layer, (inputs, outputs) in self._dense_layers.items(): - inputs.append(input) - output = d_layer(input) - outputs.append(output) + for d_layer in self._dense_layers: + output = d_layer(output) return output - def clear_progress(self): - for c_layer in self._conv_layers: - c_layer.clear_progress() - - new_dense_layers = {} - for d_layer, _ in self._dense_layers.items(): - new_dense_layers = {d_layer: ([], [])} - - self._dense_layers = new_dense_layers - - def plot_progress(self, stats=False): - if not stats: - import matplotlib.pyplot as plt - - if not stats: - - for c_layer in self._conv_layers: - plt.matshow(c_layer._weights[-1][0]) - plt.show() - - for index, input in enumerate(c_layer._inputs[::10]): - plt.matshow(input[0]) - plt.show() - plt.matshow(c_layer._outputs[index][0]) - plt.show() - - #print("shape ", c_layer._outputs[index][0].shape) - plt.matshow(np.sum(c_layer._outputs[index][0], axis=0, keepdims=True)) - plt.show() - - # plt.matshow(self._inputs[-1][0]) - # plt.show() - # plt.matshow(self._outputs[-1][0]) - # plt.show() - - #print("output flatten ", self._output_flatten) - #print("final output ", final_output) - - plt.matshow([self._output_flatten[-1]]) - plt.show() - # plt.matshow(final_output[-1]) - # plt.show() - - for dense_layer, (input, output) in self._dense_layers.items(): - plt.matshow(input[-1]) - plt.show() - plt.matshow(output[-1]) - plt.show() + # def clear_progress(self): + # for c_layer in self._conv_layers: + # c_layer.clear_progress() + # + # new_dense_layers = [] + # for d_layer in self._dense_layers: + # new_dense_layers.append(d_layer) + # + # self._dense_layers = new_dense_layers - else: - return self._conv_layers, self._output_flatten, self._dense_layers + # def plot_progress(self, conv_layers, dense_layers, output_flatten, stats=False): + # if not stats: + # import matplotlib.pyplot as plt + # + # if not stats: + # + # for inputs, weights, outputs in conv_layers: + # plt.matshow(weights[-1][0]) + # plt.show() + # + # for index, input in enumerate(inputs[::10]): + # plt.matshow(input[0]) + # plt.show() + # plt.matshow(outputs[index][0]) + # plt.show() + # + # #print("shape ", c_layer._outputs[index][0].shape) + # plt.matshow(np.sum(outputs[index][0], axis=0, keepdims=True)) + # plt.show() + # + # # plt.matshow(self._inputs[-1][0]) + # # plt.show() + # # plt.matshow(self._outputs[-1][0]) + # # plt.show() + # + # #print("output flatten ", self._output_flatten) + # #print("final output ", final_output) + # + # plt.matshow([output_flatten[-1]]) + # plt.show() + # # plt.matshow(final_output[-1]) + # # plt.show() + # + # for inputs, weights, outputs in dense_layers: + # plt.matshow(inputs[-1]) + # plt.show() + # plt.matshow(outputs[-1]) + # plt.show() + # + # else: + # return self._conv_layers, self._output_flatten, self._dense_layers def get_config(data_dir, case=0): @@ -626,7 +622,7 @@ def get_arguments(arguments): # #machine_learning_model = ("DNN_mesh_L3_6", run_DNN, True) machine_learning_model = ("GCN_mesh_L3_log_16", run_GNN, True) - machine_learning_model = ("mesh_L3_log_test", run_GNN, True) + machine_learning_model = ("mesh_L3_log_test_saved_model", run_GNN, True) #machine_learning_model = ("mesh_L3_seed", run_GNN, False) @@ -661,7 +657,7 @@ def get_arguments(arguments): 'epochs': 5, 'learning_rate': 0.001, 'graph_creation_time': graph_creation_time, - 'save_model': False + 'save_model': True } #statistics(config) From 4a3fd9230de8db7f39eec7b28c7fdc1c253d149f Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Mon, 1 Nov 2021 11:42:28 +0100 Subject: [PATCH 34/67] process mlmc in progress --- src/mlmc/metamodel/analyze_nn.py | 392 ++++++++++++++++++++++++--- src/mlmc/metamodel/postprocessing.py | 130 ++++++--- test/metamodels/metamodel_test.py | 64 +---- 3 files changed, 447 insertions(+), 139 deletions(-) diff --git a/src/mlmc/metamodel/analyze_nn.py b/src/mlmc/metamodel/analyze_nn.py index 95a5abc7..c49ebd33 100644 --- a/src/mlmc/metamodel/analyze_nn.py +++ b/src/mlmc/metamodel/analyze_nn.py @@ -2,6 +2,7 @@ import numpy as np import time import glob +import copy import pickle os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only @@ -12,7 +13,7 @@ from mlmc.tool import plot import matplotlib.pyplot as plt # Make numpy printouts easier to read. - +from scipy import stats # np.set_printoptions(precision=9, suppress=True) import tensorflow as tf from tensorflow import keras @@ -486,7 +487,7 @@ def predict_data(config, model, mesh_file, log=True): # idx = 0 # data_te = data.get_test_data(idx, train_data_len) - data_te = data[:1] + data_te = data[-10:] print("len(datate) ", len(data_te)) print("batch size ", batch_size) @@ -530,6 +531,56 @@ def predict_data(config, model, mesh_file, log=True): plot_progress(conv_layers, dense_layers, flatten_output, mesh_file=mesh_file) +def remove_empty(data_1, data_2): + new_data_1 = [] + new_data_2 = [] + + # print("data 1 ", data_1) + # print("data 2 ", data_2) + + for d1, d2 in zip(data_1, data_2): + if len(d1) > 0 and len(d2) > 0: + new_data_1.append(d1) + new_data_2.append(d2) + + # print("new data ", new_data_1) + # print("new data ", new_data_2) + return np.array(new_data_1), np.array(new_data_2) + + +def remove_outliers(data, limit): + new_data = [] + for d in data: + if d < limit: + new_data.append(d) + return new_data + + +def process_data(data_dict): + new_dict = data_dict + for tag, d in data_dict.items(): + print("tag ", tag) + print("d ", d) + + if tag in ["train_predictions", "train_targets", "test_predictions", "test_targets"]: + dt = [] + + min_length = 10**9 + for item in data_dict[tag]: + if len(item) < min_length and len(item) > 0: + min_length = len(item) + + for item in data_dict[tag]: + dt.append(item[:min_length]) + # print("dt " ,dt) + # print("array dt shape ", np.array(dt).shape) + # print("tag ", tag) + + new_dict[tag] = np.array(dt) + + return new_dict + + def analyze_statistics(config): if not os.path.isdir(config['save_path']): @@ -538,6 +589,14 @@ def analyze_statistics(config): data_dict = load_statistics(config['save_path']) + data_dict = process_data(data_dict) + + print("train predictions type ", type(data_dict["train_predictions"])) + print("train predictions type ", type(data_dict["train_predictions"][0])) + print("train predictions shape ", np.array(data_dict["train_predictions"]).shape) + #print("train predictions ", data_dict["train_predictions"]) + # print("train predictions as matrix shape", np.asmatrix(np.array(data_dict["train_predictions"])).shape) + #print("data dict ", data_dict) # for key, data_dict in models_data.items(): @@ -548,17 +607,53 @@ def analyze_statistics(config): n_ops_all = [] n_ops_predict_all = [] + all_mlmc_moments_mean = [] + all_nn_moments_mean = [] + + all_mlmc_moments_var = [] + all_nn_moments_var = [] + mlmc_times = [] nn_times = [] + mlmc_times_levels = [] + nn_times_levels = [] mlmc_l_vars = [] + mlmc_vars = [] + nn_vars = [] nn_l_vars = [] + mlmc_l_means = [] + nn_l_means = [] mlmc_vars_mse = [] nn_vars_mse = [] mlmc_means_mse = [] nn_means_mse = [] + kl_mlmc_all = [] + kl_nn_all = [] + + orth_mlmc_means_mse = [] + orth_nn_means_mse = [] + + limit = 100#0.008#0.01#0.0009 for i in range(len(data_dict["test_targets"])): + # if i == 3: + # break + + print("index ", i) + + # if i not in [2, 5, 7]: + # continue + + # if i in [2, 11, 12]: + # continue + + # if i in [7, 13, 14]: + # continue + + # if i not in [13]: + # continue + predictions = data_dict["test_predictions"][i] targets = data_dict["test_targets"][i] train_predictions = data_dict["train_predictions"][i] @@ -569,13 +664,28 @@ def analyze_statistics(config): l_0_targets = data_dict["l_0_targets"][i] l1_sample_time = data_dict["l1_sample_time"][i] l0_sample_time = data_dict["l0_sample_time"][i] + total_steps = data_dict["total_steps"][i] + learning_time = data_dict["learning_times"][i] + print("learning time ", learning_time) + + iter_test_MSE = np.mean((predictions - targets) ** 2) + iter_train_MSE = np.mean((train_predictions - train_targets) ** 2) + + if iter_test_MSE > limit: + continue - model = data_dict["model"][i] + print("iter test MSE ", iter_test_MSE) + print("iter train MSE ", iter_train_MSE) - predict_data(config, model, mesh_file=config["mesh"]) + # if "current_patience" in data_dict: + # current_patience = data_dict["current_patience"][i] + # print("current patience ", current_patience) + print("total steps ", total_steps) + #try: mlmc_n_collected, nn_mlmc_n_collected, n_ops, n_ops_predict, orig_moments_mean, predict_moments_mean,\ - ref_moments_mean, orig_level_params, nn_level_params = process_mlmc(config['hdf_path'], + ref_moments_mean, orig_level_params, nn_level_params, kl_mlmc, kl_nn, target_variance, \ + orig_orth_moments, predict_orth_moments, ref_orth_moments = process_mlmc(config['hdf_path'], config['sampling_info_path'], config['ref_mlmc_file'], targets, predictions, train_targets, @@ -583,75 +693,169 @@ def analyze_statistics(config): val_targets, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, nn_level=config['level'], + mlmc_hdf_file=config['mlmc_hdf_path'], replace_level=config['replace_level'], stats=True) + # except: + # continue mlmc_n_collected_all.append(mlmc_n_collected) nn_n_collected_all.append(nn_mlmc_n_collected) n_ops_all.append(n_ops) n_ops_predict_all.append(n_ops_predict) - + mlmc_times_levels.append(np.array(mlmc_n_collected) * np.array(n_ops)) mlmc_times.append(np.sum(np.array(mlmc_n_collected) * np.array(n_ops))) nn_times.append(np.sum(np.array(nn_mlmc_n_collected) * np.array(n_ops_predict))) + nn_times_levels.append(np.array(nn_mlmc_n_collected) * np.array(n_ops_predict)) mlmc_l_vars.append(orig_moments_mean.l_vars) nn_l_vars.append(predict_moments_mean.l_vars) + mlmc_vars.append(orig_moments_mean.var) + nn_vars.append(predict_moments_mean.var) + + mlmc_l_means.append(orig_moments_mean.l_means) + nn_l_means.append(predict_moments_mean.l_means) + mlmc_vars_mse.append((ref_moments_mean.var - orig_moments_mean.var) ** 2) nn_vars_mse.append((ref_moments_mean.var - predict_moments_mean.var) ** 2) mlmc_means_mse.append((ref_moments_mean.mean - orig_moments_mean.mean) ** 2) - nn_means_mse.append((ref_moments_mean.mean - predict_moments_mean.mean) ** 2) + nn_means_mse.append((ref_moments_mean.mean) ** 2 - predict_moments_mean.mean) + + #print("np.min(len(ref_orth_moments.mean), len(orig_orth_moments.mean)) ", np.min(len(ref_orth_moments.mean), len(orig_orth_moments.mean))) + #print("ref_orth_moments.mean[:np.min(len(ref_orth_moments.mean), len(orig_orth_moments.mean))] ", ref_orth_moments.mean[:np.min(len(ref_orth_moments.mean), len(orig_orth_moments.mean))]) + if ref_orth_moments is not None: + orth_mlmc_means_mse.append((ref_orth_moments.mean[:np.min((len(ref_orth_moments.mean), len(orig_orth_moments.mean)))] - + orig_orth_moments.mean[:np.min((len(ref_orth_moments.mean), len(orig_orth_moments.mean)))]) ** 2) + orth_nn_means_mse.append((ref_orth_moments.mean[:np.min((len(ref_orth_moments.mean), len(predict_orth_moments.mean)))] + - predict_orth_moments.mean[:np.min((len(ref_orth_moments.mean), len(predict_orth_moments.mean)))]) ** 2) + + print("orig moments mean ", orig_moments_mean.mean) + all_mlmc_moments_mean.append(orig_moments_mean.mean) + all_nn_moments_mean.append(predict_moments_mean.mean) + + all_mlmc_moments_var.append(orig_moments_mean.var) + all_nn_moments_var.append(predict_moments_mean.var) + + kl_mlmc_all.append(kl_mlmc) + kl_nn_all.append(kl_nn) + + moments_plot = plot.MomentsPlots(log_var_y=True) + + print("all mlmc moments mean ", all_mlmc_moments_mean) + print("np.mean(all_mlmc_moments_var, axis=0) ", np.mean(all_mlmc_moments_var, axis=0)) + moments_plot.add_moments((np.mean(all_mlmc_moments_mean, axis=0), + np.mean(all_mlmc_moments_var, axis=0)), label="mlmc moments") + moments_plot.add_moments((np.mean(all_nn_moments_mean, axis=0), + np.mean(all_nn_moments_var, axis=0)), label="nn moments") + moments_plot.add_moments((orig_moments_mean.mean, + orig_moments_mean.var), label="orig moments") + moments_plot.show(None) + + display_vars(mlmc_vars, nn_vars, target_variance=target_variance) + + print("mlmc l vars ", np.mean(mlmc_l_vars, axis=0)) + print("nn l vars ", np.mean(nn_l_vars, axis=0)) + + print("var mlmc l vars ", np.var(mlmc_l_vars, axis=0)) + print("var nn l vars ", np.var(nn_l_vars, axis=0)) + + print("MAX mlmc l vars ", np.max(np.mean(mlmc_l_vars, axis=0), axis=1)) + print("MAX nn l vars ", np.max(np.mean(nn_l_vars, axis=0), axis=1)) + # + print("mlmc means MSE ", np.mean(mlmc_means_mse, axis=0)) + print("nn means MSE ", np.mean(nn_means_mse, axis=0)) + + print("mlmc times ", mlmc_times) + print("nn times ", nn_times) + + print("mlmc times levels ", mlmc_times_levels) + print("nn times levels ", nn_times_levels) + + print("n ops all ", n_ops_all) + print("n ops predict all ", n_ops_predict_all) + print("len(nn times) ", len(nn_times)) mlmc_total_time = np.mean(mlmc_times) nn_total_time = np.mean(nn_times) + + print("#############################") print("mlmc total time ", mlmc_total_time) print("nn total time ", nn_total_time) + print("#############################") + print("KL mlmc ", np.mean(kl_mlmc_all)) + print("KL nn ", np.mean(kl_nn_all)) + + print("MC: to 10: {}, above: {}".format(np.sum(np.mean(mlmc_means_mse, axis=0)[:10]), np.sum(np.mean(mlmc_means_mse, axis=0)[10:]))) + print("NN: to 10: {}, above: {}".format(np.sum(np.mean(nn_means_mse, axis=0)[:10]), np.sum(np.mean(nn_means_mse, axis=0)[10:]))) n_ops_mlmc_mean = np.mean(n_ops_all, axis=0) n_ops_nn_mean = np.mean(n_ops_predict_all, axis=0) - print("n ops mlmc mean ", n_ops_mlmc_mean) - print("n ops nn mean ", n_ops_nn_mean) + # print("n ops all ", n_ops_all) + # print("n ops predict all ", n_ops_predict_all) + # + # print("n ops mlmc mean ", n_ops_mlmc_mean) + # print("n ops nn mean ", n_ops_nn_mean) mlmc_n_collected = np.mean(mlmc_n_collected_all, axis=0) nn_n_collected = np.mean(nn_n_collected_all, axis=0) - print("mlmc n collected ", mlmc_n_collected_all) - print("nn n collected all ", nn_n_collected_all) - print("mlmc n collected ", mlmc_n_collected) - print("nn n collected ", nn_n_collected) + # print("mlmc n collected ", mlmc_n_collected_all) + # print("nn n collected all ", nn_n_collected_all) + # print("mlmc n collected ", mlmc_n_collected) + # print("nn n collected ", nn_n_collected) plt_var = plot.Variance() + plt_var.set_n_ops(np.mean(n_ops_predict_all, axis=0)) l_vars = np.mean(mlmc_l_vars, axis=0) - orig_level_params = orig_level_params # np.squeeze(orig_level_params) + # print("np.squeeze(orig_level_params) ", orig_level_params) + # print("l vars ", l_vars) + # print("np.squeeze(orig_level_params) shape", orig_level_params.shape) + # print("l vars shape", l_vars.shape) + print("orig level params ", orig_level_params) + #plt_var.add_level_variances(np.squeeze(orig_level_params), l_vars) plt_var.add_level_variances(orig_level_params, l_vars) - plt_var.show("mlmc_vars") - plt_var = plot.Variance() + # plt_var.show(None) + # plt_var.show("mlmc_vars") + # + # plt_var = plot.Variance() l_vars = np.mean(nn_l_vars, axis=0) - print("l vars shape ", l_vars.shape) - print("nn level parsm ", nn_level_params) + # print("nn l vars ", l_vars) + # print("nn level parsm ", nn_level_params) level_params = np.squeeze(nn_level_params) - level_params[0] /= 2 - plt_var.add_level_variances(level_params, l_vars) + level_params[0] *= 2 + plt_var.add_level_variances_nn(level_params, l_vars) plt_var.show("nn_vars") + plt_var.show(None) plot_sse(nn_vars_mse, mlmc_vars_mse, title="moments_var") plot_sse(nn_means_mse, mlmc_means_mse, title="moments_mean") + plot_sse(mlmc_means_mse, mlmc_means_mse, title="mlmc moments_mean") + + # if ref_orth_moments is not None: + # print("orth nn means mse ", orth_nn_means_mse) + # print("orth mlmc means mse ", orth_mlmc_means_mse) + # plot_sse(orth_nn_means_mse, orth_mlmc_means_mse, title="orthogonal moments_mean") data_dict["test_targets"] = np.array(data_dict["test_targets"]) data_dict["test_predictions"] = np.array(data_dict["test_predictions"]) data_dict["train_targets"] = np.array(data_dict["train_targets"]) data_dict["train_predictions"] = np.array(data_dict["train_predictions"]) - if data_dict["log"][0]: - data_dict["test_targets"] = np.exp(data_dict["test_targets"]) - data_dict["test_predictions"] = np.exp(data_dict["test_predictions"]) - data_dict["train_targets"] = np.exp(data_dict["train_targets"]) - data_dict["train_predictions"] = np.exp(data_dict["train_predictions"]) + print("data dict train predictions ", data_dict["train_predictions"]) - # print("test targets ", data_dict["test_targets"]) + # if data_dict["log"][0]: + # # print("np.exp(10)", np.exp(10)) + # print("test targets ", data_dict["test_targets"]) + # print("type test targets ", type(data_dict["test_targets"])) + # + # data_dict["test_targets"], data_dict["test_predictions"] = exp_values(data_dict["test_targets"], data_dict["test_predictions"]) + # data_dict["train_targets"], data_dict["train_predictions"] = exp_values(data_dict["train_targets"], data_dict["train_predictions"]) + # + # # print("test targets ", data_dict["test_targets"]) # print("test predictions ", data_dict["test_predictions"]) # # print("orig max vars ", data_dict["orig_max_vars"]) @@ -665,44 +869,150 @@ def analyze_statistics(config): # print("mean predict vars ", mean_predict_vars) print("total steps ", total_steps) - print("test targets ", data_dict["test_targets"]) - print("test predictions ", data_dict["test_predictions"]) - print("test diff ", data_dict["test_predictions"] - data_dict["test_targets"]) - print("test diff squared ", (data_dict["test_predictions"] - data_dict["test_targets"])**2) + # print("test targets ", data_dict["test_targets"]) + # print("test predictions ", data_dict["test_predictions"]) + # print("test diff ", data_dict["test_predictions"] - data_dict["test_targets"]) + # print("test diff squared ", (data_dict["test_predictions"] - data_dict["test_targets"])**2) + + #print("(test_predictions - test_targets)**2 ", (data_dict["test_predictions"] - data_dict["test_targets"])**2) + + #print("test targets shape ", data_dict["test_targets"].shape) + test_MSE = np.mean((data_dict["test_predictions"] - data_dict["test_targets"])**2, axis=1) test_RMSE = np.sqrt(test_MSE) + test_MAE = np.mean(np.abs(data_dict["test_predictions"] - data_dict["test_targets"]), axis=1) + print("test MSE ", test_MSE) + + all_test_RSE = [] + for index, t_targets in enumerate(data_dict["test_targets"]): + if test_MSE[index] > limit: + continue + mean_t = np.mean(t_targets) + RSE = np.sum((data_dict["test_predictions"][index] - t_targets) ** 2) / np.sum((t_targets - mean_t) ** 2) + all_test_RSE.append(RSE) + + + all_train_RSE = [] + try: + for index, t_targets in enumerate(data_dict["train_targets"]): + if test_MSE[index] > limit: + continue + mean_t = np.mean(t_targets) + #print("train predictions index ", data_dict["train_predictions"][index]) + RSE = np.sum((data_dict["train_predictions"][index] - t_targets) ** 2) / np.sum((t_targets - mean_t) ** 2) + all_train_RSE.append(RSE) + except: + pass + + #print("all test RSE ", np.mean(all_test_RSE)) + + # Relative squared error + test_RSE = np.sum((data_dict["test_predictions"] - data_dict["test_targets"])**2) /\ + np.sum((data_dict["test_targets"] - np.mean(data_dict["test_targets"]))**2) + + print("test RSE ", test_RSE) + + test_RAE = np.sqrt(np.sum((data_dict["test_predictions"] - data_dict["test_targets"]) ** 2)) / \ + np.sqrt(np.sum((data_dict["test_targets"]) ** 2)) + + + print("test MSE / mean targets" , np.mean(test_MSE)/np.mean(data_dict["test_targets"])) + + print("test RSE ", test_RSE) + print("test RAE ", test_RAE) + print("test_MSE ", test_MSE) + + t_mse_sum = [] + for t_mse in test_MSE: + # Note: je mozne odstranit vetsi hodnoty MSE pro L4, protoze by slo dosahnout mensich hodnot pokud by se navysil pocet iteraci nebo by se vysledek pro nejlepsi train + val MSE a ne posledni vysledek + if t_mse > limit:# 0.009: + continue + t_mse_sum.append(t_mse) + + print("t mse ", t_mse_sum) + print("LEN t mse ", len(t_mse_sum)) + print("T MSE sum ", np.mean(t_mse_sum)) + + print("train_predictions ", np.array(data_dict["train_predictions"]).shape) + print("train_targets ", data_dict["train_targets"]) + + data_dict["train_predictions"], data_dict["train_targets"] = remove_empty(data_dict["train_predictions"], data_dict["train_targets"]) + + print("remove empty train targets ", data_dict["train_targets"]) + + #data_dict["train_predictions"] = np.squeeze(data_dict["train_predictions"]) + + print("train_predictions - train_targets ", data_dict["train_predictions"] - data_dict["train_targets"]) + train_MSE = np.mean((data_dict["train_predictions"] - data_dict["train_targets"]) ** 2, axis=1) train_RMSE = np.sqrt(train_MSE) - train_MAE = np.mean(np.abs( data_dict["train_predictions"] - data_dict["train_targets"]), axis=1) + train_MAE = np.mean(np.abs(data_dict["train_predictions"] - data_dict["train_targets"]), axis=1) learning_times = data_dict["learning_times"] + # Relative squared error + train_RSE = np.sum((data_dict["train_predictions"] - data_dict["train_targets"]) ** 2) / \ + np.sum((data_dict["train_targets"] - np.mean(data_dict["train_targets"]))**2) + + # Relative absolute error + train_RAE = np.sqrt(np.sum((data_dict["train_predictions"] - data_dict["train_targets"]) ** 2)) / \ + np.sqrt(np.sum((data_dict["train_targets"]) ** 2)) + + print("train RSE ", train_RSE) + print("train REA ", train_RAE) + # plot_data(test_MSE, label="test MSE") # plot_data(test_MAE, label="test MAE") - print("test_MSE ", test_MSE) - print("NN moments MSE sum ", np.sum(np.mean(nn_means_mse, axis=0))) print("mean test MSE ", np.mean(test_MSE)) - print("mean test RMSE ", np.mean(test_RMSE)) - print("mean test MAE ", np.mean(test_MAE)) + #print("mean test RSE ", np.mean(test_RSE)) + #print("mean test RMSE ", np.mean(test_RMSE)) + #print("mean test MAE ", np.mean(test_MAE)) print("max test MSE ", np.max(test_MSE)) - print("max test RMSE ", np.max(test_RMSE)) - print("max test MAE ", np.max(test_MAE)) + #print("max test RMSE ", np.max(test_RMSE)) + #print("max test MAE ", np.max(test_MAE)) + + print("train_MSE ", train_MSE) print("mean train MSE ", np.mean(train_MSE)) - print("mean train RMSE ", np.mean(train_RMSE)) - print("mean train MAE ", np.mean(train_MAE)) + + print("test RSE ", np.mean(all_test_RSE)) + print("test RSE ", np.mean(all_train_RSE)) + #print("mean train RSE ", np.mean(train_RSE)) + #print("mean train RMSE ", np.mean(train_RMSE)) + #print("mean train MAE ", np.mean(train_MAE)) print("max train MSE ", np.max(train_MSE)) - print("max train RMSE ", np.max(train_RMSE)) - print("max train MAE ", np.max(train_MAE)) + #print("max train RMSE ", np.max(train_RMSE)) + #print("max train MAE ", np.max(train_MAE)) print("mean learning time ", np.mean(learning_times)) print("max learning time ", np.max(learning_times)) + + test_MSE = remove_outliers(test_MSE, limit) + train_MSE = remove_outliers(train_MSE, limit) + print("############# OUTPUT ################") + print("len(train MSE) ", len(train_MSE)) + print("train MSE ", np.mean(train_MSE)) + #print("train MSE sqrt var", np.sqrt(np.var(train_MSE))) + #print("train MSE std", np.std(train_MSE)) + print("train MSE ", train_MSE) + print("stats.sem(train_MSE) ", stats.sem(train_MSE)) + print("test MSE ", np.mean(test_MSE)) + print("test MSE ", test_MSE) + print("stats.sem(test_MSE) ", stats.sem(test_MSE)) + #print("test MSE std", np.sqrt(np.var(test_MSE))) + print("train RSE ", np.mean(all_train_RSE)) + print("test RSE ", np.mean(all_test_RSE)) + + print("nn total time ", nn_total_time) + print("mlmc total time ", mlmc_total_time) + print("######################################") + return train_MSE, test_MSE, np.mean(all_train_RSE), np.mean(all_test_RSE) def run_GNN(config, stats=True, train=True, log=False, seed=0): diff --git a/src/mlmc/metamodel/postprocessing.py b/src/mlmc/metamodel/postprocessing.py index c438de7f..b7cfcc25 100644 --- a/src/mlmc/metamodel/postprocessing.py +++ b/src/mlmc/metamodel/postprocessing.py @@ -17,7 +17,9 @@ from mlmc.metamodel.create_graph import extract_mesh_gmsh_io #import mlmc.tool.simple_distribution -QUANTILE = 0.01 +QUANTILE = 1e-6 +N_MOMENTS = 25 +TARGET_VAR = 1e-5 def plot_loss(train_loss, val_loss): @@ -102,7 +104,7 @@ def estimate_density(values, title="Density"): estimator = mlmc.estimator.Estimate(quantity=value_quantity, sample_storage=sample_storage, moments_fn=moments_fn) reg_param = 0 - target_var = 1e-4 + target_var = TARGET_VAR distr_obj, info, result, moments_fn = estimator.construct_density( tol=distr_accuracy, reg_param=reg_param, @@ -350,39 +352,51 @@ def compare_moments(original_q_estimator, predict_q_estimator, ref_estimator): def compare_densities(estimator_1, estimator_2, ref_estimator, label_1="", label_2=""): - distr_plot = plot.ArticleDistribution(title="densities", log_density=True) - tol = 1e-10 + distr_plot = plot.ArticleDistributionPDF(title="densities", log_density=True) + tol = 1e-7 reg_param = 0 print("orig estimator") - distr_obj_1, result, _, _ = estimator_1.construct_density(tol=tol, reg_param=reg_param) + distr_obj_1, result, _, _, orig_orth_moments = estimator_1.construct_density(tol=tol, reg_param=reg_param, + orth_moments_tol=TARGET_VAR) #distr_plot.add_distribution(distr_obj_1, label=label_1, color="blue") print("predict estimator") - distr_obj_2, result, _, _ = estimator_2.construct_density(tol=tol, reg_param=reg_param) + distr_obj_2, result, _, _, predict_orth_moments = estimator_2.construct_density(tol=tol, reg_param=reg_param, orth_moments_tol=TARGET_VAR) #distr_plot.add_distribution(distr_obj_2, label=label_2, color="red", line_style="--") print("Ref estimator") - ref_distr_obj, result, _, _ = ref_estimator.construct_density(tol=tol, reg_param=reg_param) + ref_distr_obj, result, _, _, ref_orth_moments = ref_estimator.construct_density(tol=tol, reg_param=reg_param, orth_moments_tol=TARGET_VAR) #distr_plot.add_distribution(ref_distr_obj, label="MC reference", color="black", line_style=":") - kl_div_ref_mlmc = mlmc.tool.simple_distribution.KL_divergence(ref_distr_obj.density, distr_obj_1.density, ref_distr_obj.domain[0], ref_distr_obj.domain[1]) + ref_estimator_pdf = get_quantity_estimator(ref_estimator._sample_storage, true_domain=None, n_moments=25) + ref_distr_obj, result, _, _, ref_orth_moments_pdf = ref_estimator_pdf.construct_density(tol=tol, + reg_param=reg_param, + orth_moments_tol=TARGET_VAR) + + domain = [np.max([ref_distr_obj.domain[0], distr_obj_1.domain[0], distr_obj_2.domain[0]]), + np.min([ref_distr_obj.domain[1], distr_obj_1.domain[1], distr_obj_2.domain[1]])] + kl_div_ref_mlmc = mlmc.tool.simple_distribution.KL_divergence(ref_distr_obj.density, distr_obj_1.density, domain[0], domain[1]) print("KL div ref|mlmc: {}".format(kl_div_ref_mlmc)) - # domain = [np.min([ref_distr_obj.domain[0], distr_obj_2.domain[0]]), - # np.max([ref_distr_obj.domain[1], distr_obj_2.domain[1]])] - kl_div_ref_gnn = mlmc.tool.simple_distribution.KL_divergence(ref_distr_obj.density, distr_obj_2.density, ref_distr_obj.domain[0], - ref_distr_obj.domain[1]) + # domain = [np.max([ref_distr_obj.domain[0], distr_obj_2.domain[0]]), + # np.min([ref_distr_obj.domain[1], distr_obj_2.domain[1]])] + kl_div_ref_gnn = mlmc.tool.simple_distribution.KL_divergence(ref_distr_obj.density, distr_obj_2.density, domain[0], + domain[1]) print("KL div ref|mlmc prediction: {}".format(kl_div_ref_gnn)) - distr_plot.add_distribution(distr_obj_1, label=label_1 + ", KL(ref|orig):{:0.4g}".format(kl_div_ref_mlmc), color="blue") - distr_plot.add_distribution(distr_obj_2, label=label_2 + ", KL(ref|gnn):{:0.4g}".format(kl_div_ref_gnn), color="red", line_style="--") - distr_plot.add_distribution(ref_distr_obj, label="MC reference", color="black", line_style=":") + #distr_plot.add_distribution(distr_obj_1, label=label_1 + ", KL(ref|orig):{:0.4g}".format(kl_div_ref_mlmc), color="blue") + #distr_plot.add_distribution(distr_obj_2, label=label_2 + ", KL(ref|gnn):{:0.4g}".format(kl_div_ref_gnn), color="red", line_style="--") + distr_plot.add_distribution(distr_obj_1, label=r"$D_{MC}:$" + "{:0.4g}".format(kl_div_ref_mlmc), color="blue") + distr_plot.add_distribution(distr_obj_2, label=r"$D_{meta}:$" + "{:0.4g}".format(kl_div_ref_gnn), color="red", line_style="--") + distr_plot.add_distribution(ref_distr_obj, label="MC ref", color="black", line_style=":") + distr_plot.show(file="densities.pdf") distr_plot.show(file=None) + return kl_div_ref_mlmc, kl_div_ref_gnn, orig_orth_moments, predict_orth_moments, ref_orth_moments def get_quantity_estimator(sample_storage, true_domain=None): n_moments = 25 @@ -403,7 +417,7 @@ def get_quantity_estimator(sample_storage, true_domain=None): def get_n_estimated(sample_storage, estimator, n_ops=None): - target_var = 5e-5 + target_var = TARGET_VAR #moments, estimator, _, quantity = estimate_moments(sample_storage, true_domain=true_domain) n_level_samples = sample_storage.get_n_collected() @@ -411,6 +425,8 @@ def get_n_estimated(sample_storage, estimator, n_ops=None): print("n level samples ", n_level_samples) variances, n_samples = estimator.estimate_diff_vars() + print("variances ", variances) + print("n samples ", n_samples) #variances, est_n_ops = estimator.estimate_diff_vars_regression(n_level_samples) if n_ops is None: @@ -430,18 +446,28 @@ def get_storage_info(sample_storage): return n_collected, max_vars -def cut_samples(data, sample_storage, new_n_collected, new_l_0=0): +def cut_samples(data, sample_storage, new_n_collected, new_l_0=0, bootstrap=False): new_data = [] for l_id, (d, n_est) in enumerate(zip(data, new_n_collected)): + # print("len d :", d.shape[1]) + # print("n est ", n_est) if n_est > 0: if l_id == new_l_0: - print(d.shape) - print("np.min(d.shape[1], n_est) ", np.min([d.shape[1], n_est])) - fine_samples = d[:, :np.min([d.shape[1], n_est]), 0].reshape(1, np.min([d.shape[1], n_est]), 1) + if bootstrap: + sample_idx = np.random.choice(list(range(0, d.shape[1]-1)), size=n_est, replace=True) + fine_samples = d[:, sample_idx, 0].reshape(1, np.min([d.shape[1], len(sample_idx)]), 1) + else: + fine_samples = d[:, :np.min([d.shape[1], n_est]), 0].reshape(1, np.min([d.shape[1], n_est]), 1) + coarse_samples = np.zeros(fine_samples.shape) new_data.append(np.concatenate((fine_samples, coarse_samples), axis=2)) else: - new_data.append(d[:, :np.min([d.shape[1], n_est]), :]) + if bootstrap: + sample_idx = np.random.choice(list(range(0, d.shape[1] - 1)), size=n_est, replace=True) + new_data.append(d[:, sample_idx, :]) + else: + new_data.append(d[:, :np.min([d.shape[1], n_est]), :]) + sample_storage = create_quantity_mlmc(new_data, level_parameters=sample_storage.get_level_parameters()) return sample_storage @@ -459,16 +485,20 @@ def plot_progress(conv_layers, dense_layers, output_flatten, mesh_file=None): for idx, conv_layer in conv_layers.items(): inputs, weights, outputs = conv_layer[0], conv_layer[1], conv_layer[2] plt.matshow(weights[-1]) + plt.title("weights") + plt.savefig("weights.pdf") plt.show() # Note: weights have different shape than the mesh - for index, input in enumerate(inputs[::10]): + used_inputs = inputs[::10] + for index, input in enumerate(used_inputs): if mesh_file: fig, ax = plt.subplots(1, 1, figsize=(15, 10)) cont = ax.tricontourf(X, Y, input.ravel(), levels=16) fig.colorbar(cont) plt.title("input") + plt.savefig("input.pdf") plt.show() for i in range(outputs[index].shape[1]): @@ -477,7 +507,7 @@ def plot_progress(conv_layers, dense_layers, output_flatten, mesh_file=None): cont = ax.tricontourf(X, Y, channel_output, levels=16) fig.colorbar(cont) plt.title("channel {}".format(i)) - + plt.savefig("channel_{}.pdf".format(i)) plt.show() else: @@ -501,18 +531,24 @@ def plot_progress(conv_layers, dense_layers, output_flatten, mesh_file=None): plt.matshow([output_flatten[-1]]) plt.show() - for idx, dense_layer in dense_layers.items(): - inputs, weights, outputs = dense_layer[0], dense_layer[1], dense_layer[2] - plt.matshow([inputs[-1]]) - plt.show() - plt.matshow([outputs[-1]]) - plt.show() + # for idx, dense_layer in dense_layers.items(): + # inputs, weights, outputs = dense_layer[0], dense_layer[1], dense_layer[2] + # plt.matshow([inputs[-1]]) + # plt.title("flatten") + # plt.savefig("flatten.pdf") + # plt.show() + # plt.matshow([outputs[-1]]) + # plt.show() -def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predictions, train_targets, train_predictions, - val_targets, l_0_targets=None, l_0_predictions=None, +def process_mlmc(nn_mlmc_file, sampling_info_path, ref_mlmc_file, targets, predictions, train_targets, train_predictions, + val_targets, l_0_targets=None, l_0_predictions=None, mlmc_hdf_file=None, l1_sample_time=None, l0_sample_time=None, nn_level=0, replace_level=False, stats=False): - #level_zero = False + # level_zero = False + cut_est = True + + if mlmc_hdf_file is None: + mlmc_hdf_file = nn_mlmc_file if not stats: print("nn_level ", nn_level) @@ -560,29 +596,39 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti ####### ### Create storage fromm original MLMC data ###### - sample_storage = SampleStorageHDF(file_path=mlmc_file) - print("get n collected ", sample_storage.get_n_collected()) + sample_storage = SampleStorageHDF(file_path=mlmc_hdf_file) + + print("mlmc sample storage get N collected ", sample_storage.get_n_collected()) + n_levels = len(sample_storage.get_level_ids()) original_moments, estimator, original_true_domain, _ = estimate_moments(sample_storage) + sample_storage_nn = SampleStorageHDF(file_path=nn_mlmc_file) + original_moments_nn, estimator_nn, original_true_domain_nn, _ = estimate_moments(sample_storage_nn) + orig_max_vars = np.max(original_moments.l_vars, axis=1) - print("orig max vars ", orig_max_vars) + # print("orig max vars ", orig_max_vars) ###### ### Get n ops ###### - n_ops, field_times, coarse_flow, fine_flow = get_sample_times_mlmc(mlmc_file) + n_ops, field_times, coarse_flow, fine_flow = get_sample_times_mlmc(mlmc_hdf_file) # n_ops, _, _ = get_sample_times(sampling_info_path) - # n_ops = n_ops[n_levels-1:] + print("n ops ", n_ops) if n_ops is None: n_ops = sample_storage.get_n_ops() field_times = np.zeros(len(n_ops)) flow_times = np.zeros(len(n_ops)) + # Test storage creation data_mlmc = [] - for l_id in range(n_levels): - level_samples = estimator.get_level_samples(level_id=l_id) + mlmc_n_collected = estimator._sample_storage.get_n_collected() + for l_id, l_n_collected in zip(range(n_levels), mlmc_n_collected): + # if l_id == 1: + # continue + level_samples = estimator.get_level_samples(level_id=l_id, n_samples=l_n_collected) + # print("len level samples ", len(np.squeeze(level_samples))) data_mlmc.append(level_samples) print("original level params", sample_storage.get_level_parameters()) @@ -789,8 +835,10 @@ def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predicti # label_2="gnn N: {}".format(sample_storage_predict.get_n_collected())) if stats: - return sample_storage.get_n_collected(), sample_storage_predict.get_n_collected(), n_ops, n_ops_predict, orig_moments_mean,\ - predict_moments_mean, ref_moments_mean, sample_storage.get_level_parameters(), sample_storage_predict.get_level_parameters() + return n_estimated_orig, n_estimated_nn, n_ops, n_ops_predict, orig_moments_mean, \ + predict_moments_mean, ref_moments_mean, sample_storage.get_level_parameters(), \ + sample_storage_predict.get_level_parameters(), kl_mlmc, kl_nn, TARGET_VAR, \ + orig_orth_moments, predict_orth_moments, ref_orth_moments plot_moments({"ref": ref_estimator, "orig": original_q_estimator, "nn": predict_q_estimator}) diff --git a/test/metamodels/metamodel_test.py b/test/metamodels/metamodel_test.py index 08362d47..de031b29 100644 --- a/test/metamodels/metamodel_test.py +++ b/test/metamodels/metamodel_test.py @@ -108,58 +108,6 @@ def call(self, inputs): return output - # def clear_progress(self): - # for c_layer in self._conv_layers: - # c_layer.clear_progress() - # - # new_dense_layers = [] - # for d_layer in self._dense_layers: - # new_dense_layers.append(d_layer) - # - # self._dense_layers = new_dense_layers - - # def plot_progress(self, conv_layers, dense_layers, output_flatten, stats=False): - # if not stats: - # import matplotlib.pyplot as plt - # - # if not stats: - # - # for inputs, weights, outputs in conv_layers: - # plt.matshow(weights[-1][0]) - # plt.show() - # - # for index, input in enumerate(inputs[::10]): - # plt.matshow(input[0]) - # plt.show() - # plt.matshow(outputs[index][0]) - # plt.show() - # - # #print("shape ", c_layer._outputs[index][0].shape) - # plt.matshow(np.sum(outputs[index][0], axis=0, keepdims=True)) - # plt.show() - # - # # plt.matshow(self._inputs[-1][0]) - # # plt.show() - # # plt.matshow(self._outputs[-1][0]) - # # plt.show() - # - # #print("output flatten ", self._output_flatten) - # #print("final output ", final_output) - # - # plt.matshow([output_flatten[-1]]) - # plt.show() - # # plt.matshow(final_output[-1]) - # # plt.show() - # - # for inputs, weights, outputs in dense_layers: - # plt.matshow(inputs[-1]) - # plt.show() - # plt.matshow(outputs[-1]) - # plt.show() - # - # else: - # return self._conv_layers, self._output_flatten, self._dense_layers - def get_config(data_dir, case=0): if case == 0: @@ -249,12 +197,12 @@ def get_config(data_dir, case=0): elif case == 7: # mesh size comparison data_dir = "/home/martin/Documents/metamodels/data/mesh_size/" cl = "cl_0_1_s_1" - level = 2 + level = 3 nn_level = 0 replace_level = False #mesh = os.path.join(data_dir, "l_step_1.0_common_files/mesh.msh".format(cl)) #L1, 7s mesh = os.path.join(data_dir, "l_step_0.27232698153315_common_files/mesh.msh".format(cl)) #L2 10.5 s - #mesh = os.path.join(data_dir, "l_step_0.07416198487095663_common_files/mesh.msh".format(cl)) #L3 12s + mesh = os.path.join(data_dir, "l_step_0.07416198487095663_common_files/mesh.msh".format(cl)) #L3 12s #mesh = os.path.join(data_dir, "l_step_0.020196309484414757_common_files/mesh.msh".format(cl)) #L4 22s #mesh = os.path.join(data_dir, "l_step_0.0055_common_files/mesh.msh".format(cl)) #L5 output_dir = os.path.join(data_dir, "{}/L1_{}/test/01_cond_field/output/".format(cl, level)) @@ -315,7 +263,7 @@ def get_config(data_dir, case=0): # sampling_info_path = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/" # ref_mlmc_file = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L1_benchmark/mlmc_1.hdf5" - return output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, replace_level, nn_level, + return output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, replace_level, nn_level, mlmc_hdf_path def plot_results_corr_length(): @@ -516,8 +464,8 @@ def get_arguments(arguments): work_dir = args.work_dir case = 7 #data_dir = "/home/martin/Documents/metamodels/data/1000_ele/" - output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, replace_level, nn_level = get_config( - data_dir, case) + output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file,\ + replace_level, nn_level, mlmc_hdf_path = get_config(data_dir, case) # plot_results_corr_length() # exit() @@ -623,6 +571,7 @@ def get_arguments(arguments): #machine_learning_model = ("DNN_mesh_L3_6", run_DNN, True) machine_learning_model = ("GCN_mesh_L3_log_16", run_GNN, True) machine_learning_model = ("mesh_L3_log_test_saved_model", run_GNN, True) + machine_learning_model = ("mesh_L3_log_50k_weights_2_K2", run_GNN, True) #machine_learning_model = ("mesh_L3_seed", run_GNN, False) @@ -640,6 +589,7 @@ def get_arguments(arguments): 'save_path': save_path, 'output_dir': output_dir, 'hdf_path': hdf_path, + 'mlmc_hdf_path': mlmc_hdf_path, 'mesh': mesh, 'l_0_output_dir': l_0_output_dir, 'l_0_hdf_path': l_0_hdf_path, From b10d3de446331be6cf52b655e6768903bfcdf90f Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Mon, 1 Nov 2021 12:09:33 +0100 Subject: [PATCH 35/67] metamodel files --- mlmc/metamodel/GCN.py | 102 +++ mlmc/metamodel/__init__.py | 0 mlmc/metamodel/analyze_nn.py | 993 ++++++++++++++++++++++++++++ mlmc/metamodel/create_graph.py | 186 ++++++ mlmc/metamodel/custom_methods.py | 42 ++ mlmc/metamodel/flow_dataset.py | 220 ++++++ mlmc/metamodel/flow_task_CNN.py | 65 ++ mlmc/metamodel/flow_task_GNN.py | 176 +++++ mlmc/metamodel/flow_task_GNN_2.py | 249 +++++++ mlmc/metamodel/flow_task_NN.py | 403 +++++++++++ mlmc/metamodel/graph_models.py | 128 ++++ mlmc/metamodel/main.py | 149 +++++ mlmc/metamodel/mnist.py | 167 +++++ mlmc/metamodel/own_cheb_conv.py | 378 +++++++++++ mlmc/metamodel/postprocessing.py | 965 +++++++++++++++++++++++++++ mlmc/metamodel/random_field_time.py | 93 +++ 16 files changed, 4316 insertions(+) create mode 100644 mlmc/metamodel/GCN.py create mode 100644 mlmc/metamodel/__init__.py create mode 100644 mlmc/metamodel/analyze_nn.py create mode 100644 mlmc/metamodel/create_graph.py create mode 100644 mlmc/metamodel/custom_methods.py create mode 100644 mlmc/metamodel/flow_dataset.py create mode 100644 mlmc/metamodel/flow_task_CNN.py create mode 100644 mlmc/metamodel/flow_task_GNN.py create mode 100644 mlmc/metamodel/flow_task_GNN_2.py create mode 100644 mlmc/metamodel/flow_task_NN.py create mode 100644 mlmc/metamodel/graph_models.py create mode 100644 mlmc/metamodel/main.py create mode 100644 mlmc/metamodel/mnist.py create mode 100644 mlmc/metamodel/own_cheb_conv.py create mode 100644 mlmc/metamodel/postprocessing.py create mode 100644 mlmc/metamodel/random_field_time.py diff --git a/mlmc/metamodel/GCN.py b/mlmc/metamodel/GCN.py new file mode 100644 index 00000000..d859c065 --- /dev/null +++ b/mlmc/metamodel/GCN.py @@ -0,0 +1,102 @@ +""" +This example shows how to perform regression of molecular properties with the +QM9 database, using a simple GNN in disjoint mode. +""" + +import numpy as np +import tensorflow as tf +from tensorflow.keras.layers import Dense, Input +from tensorflow.keras.losses import MeanSquaredError +from tensorflow.keras.models import Model +from tensorflow.keras.optimizers import Adam + +from spektral.data import DisjointLoader +from spektral.datasets import QM9 +from spektral.layers import ECCConv, GlobalSumPool + +################################################################################ +# PARAMETERS +################################################################################ +learning_rate = 1e-3 # Learning rate +epochs = 10 # Number of training epochs +batch_size = 32 # Batch size + +################################################################################ +# LOAD DATA +################################################################################ +dataset = QM9(amount=1000) # Set amount=None to train on whole dataset + +# Parameters +F = dataset.n_node_features # Dimension of node features +S = dataset.n_edge_features # Dimension of edge features +n_out = dataset.n_labels # Dimension of the target + +# Train/test split +idxs = np.random.permutation(len(dataset)) +split = int(0.9 * len(dataset)) +idx_tr, idx_te = np.split(idxs, [split]) +dataset_tr, dataset_te = dataset[idx_tr], dataset[idx_te] + +loader_tr = DisjointLoader(dataset_tr, batch_size=batch_size, epochs=epochs) +loader_te = DisjointLoader(dataset_te, batch_size=batch_size, epochs=1) + +################################################################################ +# BUILD MODEL +################################################################################ +X_in = Input(shape=(F,), name="X_in") +A_in = Input(shape=(None,), sparse=True, name="A_in") +E_in = Input(shape=(S,), name="E_in") +I_in = Input(shape=(), name="segment_ids_in", dtype=tf.int32) + +X_1 = ECCConv(32, activation="relu")([X_in, A_in, E_in]) +X_2 = ECCConv(32, activation="relu")([X_1, A_in, E_in]) +X_3 = GlobalSumPool()([X_2, I_in]) +output = Dense(n_out)(X_3) + +# Build model +model = Model(inputs=[X_in, A_in, E_in, I_in], outputs=output) +opt = Adam(lr=learning_rate) +loss_fn = MeanSquaredError() + + +################################################################################ +# FIT MODEL +################################################################################ +@tf.function(input_signature=loader_tr.tf_signature(), experimental_relax_shapes=True) +def train_step(inputs, target): + with tf.GradientTape() as tape: + predictions = model(inputs, training=True) + loss = loss_fn(target, predictions) + loss += sum(model.losses) + gradients = tape.gradient(loss, model.trainable_variables) + opt.apply_gradients(zip(gradients, model.trainable_variables)) + return loss + + +print("Fitting model") +current_batch = 0 +model_loss = 0 +for batch in loader_tr: + outs = train_step(*batch) + + model_loss += outs + current_batch += 1 + if current_batch == loader_tr.steps_per_epoch: + print("Loss: {}".format(model_loss / loader_tr.steps_per_epoch)) + model_loss = 0 + current_batch = 0 + +################################################################################ +# EVALUATE MODEL +################################################################################ +print("Testing model") +model_loss = 0 +for batch in loader_te: + inputs, target = batch + predictions = model(inputs, training=False) + + print("target ", target) + print("prediction ", predictions) + model_loss += loss_fn(target, predictions) +model_loss /= loader_te.steps_per_epoch +print("Done. Test loss: {}".format(model_loss)) diff --git a/mlmc/metamodel/__init__.py b/mlmc/metamodel/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/mlmc/metamodel/analyze_nn.py b/mlmc/metamodel/analyze_nn.py new file mode 100644 index 00000000..95a5abc7 --- /dev/null +++ b/mlmc/metamodel/analyze_nn.py @@ -0,0 +1,993 @@ +import os +import numpy as np +import time +import glob +import pickle + +os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only +from mlmc.metamodel.flow_dataset import FlowDataset +from mlmc.metamodel.create_graph import graph_creator +from mlmc.moments import Legendre_tf, Monomial +from mlmc.metamodel.random_field_time import corr_field_sample_time +from mlmc.tool import plot +import matplotlib.pyplot as plt +# Make numpy printouts easier to read. + +# np.set_printoptions(precision=9, suppress=True) +import tensorflow as tf +from tensorflow import keras +from scipy.stats import ks_2samp +import sklearn.model_selection +from mlmc.metamodel.custom_methods import abs_activation, MSE_moments +from mlmc.metamodel.postprocessing import analyze_results, plot_loss, estimate_density, process_mlmc, plot_progress +from mlmc.metamodel.flow_task_NN import DNN +from mlmc.metamodel.flow_task_CNN import CNN + +from mlmc.metamodel.flow_task_GNN_2 import GNN +from tensorflow.keras.losses import MeanSquaredError +from spektral.data import MixedLoader +from spektral.utils.sparse import sp_matrix_to_sp_tensor + +print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU'))) +epochs = 100 + +def prepare_data(data): + data = np.squeeze(np.stack(data.to_numpy(), axis=0)) + return np.asarray(data).astype('float64') + + +def split_dataset(dataset): + # Load data + dataset = dataset.dropna() + train_x, test_x, train_y, test_y = sklearn.model_selection.train_test_split(dataset.x, dataset.y, + test_size=0.2, random_state=123) + + train_x = prepare_data(train_x) + train_y = prepare_data(train_y) + + test_x = prepare_data(test_x) + test_y = prepare_data(test_y) + + return train_x, train_y, test_x, test_y + + +def run(): + # Parameters + loss = "mean_squared_error" + optimizer = tf.optimizers.Adam(learning_rate=0.001) + + data = FlowDataset() + # dataset = data.dataset[:10000] + # test_dataset = data.dataset[10000:50000] + + dataset = data.dataset[:50000] + test_dataset = data.dataset[50000:] + + train_input = prepare_data(dataset.x) + train_output = prepare_data(dataset.y) + + # train_input, train_output, test__input, test_output = split_dataset(dataset) + # print("len test(output) ", len(test_output)) + + dnn = DNN(loss=loss, optimizer=optimizer, output_activation=abs_activation, hidden_activation='relu', epochs=150) + dnn.fit(train_input, train_output) + + test_input = prepare_data(test_dataset.x) + test_output = prepare_data(test_dataset.y) + + predictions = dnn.predict(test_input) + predictions = np.squeeze(predictions) + + print("len(predictions) ", len(predictions)) + + plot_loss(dnn.history.history['loss'], dnn.history.history['val_loss']) + analyze_results(test_output, predictions) + + estimate_density(test_output) + estimate_density(predictions) + + +def run_CNN(output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level, log): + # Parameters + loss = "mean_squared_error" + optimizer = tf.optimizers.Adam(learning_rate=0.01) + + data = FlowDataset(output_dir=output_dir, level=level, log=log) + dataset = data.dataset[:] + + train_input, train_output, test_input, test_output = split_dataset(dataset) + + train_input = train_input[:2000] + train_output = train_output[:2000] + + print("len test(output) ", len(test_output)) + + train_input = np.expand_dims(train_input, axis=-1) + test_input = np.expand_dims(test_input, axis=-1) + print("train input shape ", train_input.shape) + + dnn = CNN(loss=loss, optimizer=optimizer, output_activation=abs_activation, hidden_activation='relu') + + dnn.fit(train_input, train_output) + + test_dataset = data.dataset[2000:] + test_input = prepare_data(test_dataset.x) + test_input = np.expand_dims(test_input, axis=-1) + print("test input shape ", test_input.shape) + test_output = prepare_data(test_dataset.y) + + predictions = dnn.predict(test_input) + predictions = np.squeeze(predictions) + + plot_loss(dnn.history.history['loss'], dnn.history.history['val_loss']) + + analyze_results(test_output, predictions) + + # estimate_density(test_output) + # estimate_density(predictions) + + +def bootstrap(): + loss = "mean_absolute_error" + optimizer = tf.optimizers.Adam(learning_rate=0.001) + n_subsamples = 10 + size = 10000 + + train_losses = [] + val_losses = [] + all_test_outputs = [] + all_predictions = [] + ks_statistics = [] + ks_p_values = [] + + data = FlowDataset() + dataset = data.dataset.dropna() + + for i in range(n_subsamples): + dset = dataset.sample(size, replace=True) + train_input, train_output, test_input, test_output = split_dataset(dset) + + print("Size TRAIN in: {}, out: {}, TEST in: {}, out: {}".format(len(train_input), len(train_output), + len(test_input), len(test_output))) + + dnn = DNN(loss=loss, optimizer=optimizer, output_activation=abs_activation, hidden_activation='relu') + dnn.fit(train_input, train_output) + + predictions = dnn.predict(test_input) + predictions = np.squeeze(predictions) + + train_losses.append(dnn.history.history['loss']) + val_losses.append(dnn.history.history['val_loss']) + + all_test_outputs.append(test_output) + all_predictions.append(predictions) + + statistics, pvalue = ks_2samp(test_output, predictions) + ks_statistics.append(statistics) + ks_p_values.append(pvalue) + + analyze_results(np.mean(all_test_outputs, axis=0), np.mean(all_predictions, axis=0)) + analyze_results(np.var(all_test_outputs, axis=0), np.var(all_predictions, axis=0)) + # + # estimate_density(np.mean(all_test_outputs, axis=0), title="Test outputs") + # estimate_density(np.mean(all_predictions, axis=0), title="Predictions") + + +def run_SVR(config, stats=True, train=True, log=False, seed=1234): + from sklearn.svm import SVR + print("seed ", seed) + + batch_size = 200 + epochs = 1000 + hidden_regularization = None # l2(2e-10) + graph_creation_time = config['graph_creation_time'] + if graph_creation_time == 0: + graph_creator_preproces_time = time.process_time() + graph_creator(config['output_dir'], config['hdf_path'], config['mesh'], level=config['level']) + graph_creation_time = time.process_time() - graph_creator_preproces_time + print("graph creation time ", graph_creation_time) + exit() + + preprocess_start_time = time.process_time() + # Load data + data = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log) + data.shuffle(seed=seed) + + dataset = data.dataset + dataset = dataset.sample(frac=1) + + train = dataset[:config['n_train_samples']] + test = dataset[config['n_train_samples']:] + + train_input, train_output = train.x, train.y + test_input, test_output = test.x, test.y + + train_input = prepare_data(train_input) + train_output = prepare_data(train_output) + + test_input = prepare_data(test_input) + test_output = prepare_data(test_output) + + # sc_X = StandardScaler() + # sc_y = StandardScaler() + # train_input = sc_X.fit_transform(train_input) + # train_output = sc_y.fit_transform(train_output.reshape(-1,1)) + # test_input = sc_X.fit_transform(test_input) + # test_output = sc_y.fit_transform(test_output.reshape(-1,1)) + #train_input, train_output, test_input, test_output = split_dataset(dataset) + + preprocess_time = time.process_time() - preprocess_start_time + preprocess_time = preprocess_time + graph_creation_time + learning_time_start = time.process_time() + + print("train input ", train_input.shape) + print("train output ", train_output.shape) + + svr_rbf = SVR(kernel='rbf', verbose=True) # 'linear' kernel fitting is never-ending and 'poly' kernel gives very bad score (e.g. -2450), sigmoid gives also bad score (e.g. -125) + svr_rbf.fit(train_input, train_output) + train_error = svr_rbf.score(train_input, train_output) + + #print("svr_rbf.get_params() ", svr_rbf.get_params()) + total_steps = 0 + + #test_input = sc_X.fit_transform(test_input) + test_error = svr_rbf.score(test_input, test_output) + + targets = test_output + train_targets = train_output + + # test_y = sc_y.fit_transform(test.y.to_numpy().reshape(-1,1)) + + predictions = svr_rbf.predict(test_input) + + print("train error ", train_error) + print("test error ", test_error) + + train_predictions = svr_rbf.predict(train_input) + #train_predictions = np.squeeze(train_predictions) + + learning_time = time.process_time() - learning_time_start + print("learning time ", learning_time) + + val_targets = [] + + orig_targets = targets + orig_predictions = predictions + print("MSE ", np.mean((predictions - targets) ** 2)) + + if log: + targets = np.exp(targets) + predictions = np.exp(predictions) + + print("np.var(target-predictions) ", np.var(targets - predictions)) + + if not stats: + #plot_loss(gnn._train_loss, gnn._val_loss) + analyze_results(targets, predictions) + + import matplotlib.pyplot as plt + + # plt.hist(train_output, bins=50, alpha=0.5, label='train target', density=True) + # plt.hist(train_predictions, bins=50, alpha=0.5, label='train predictions', density=True) + # + # # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) + # plt.legend(loc='upper right') + # # plt.xlim(-0.5, 1000) + # plt.yscale('log') + # plt.show() + + plt.hist(targets, bins=50, alpha=0.5, label='target', density=True) + plt.hist(predictions, bins=50, alpha=0.5, label='predictions', density=True) + + # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) + plt.legend(loc='upper right') + # plt.xlim(-0.5, 1000) + plt.yscale('log') + plt.show() + + #predict_l_0_start_time = time.process_time() + l_0_targets, l_0_predictions, predict_l_0_time = predict_level_zero_SVR(svr_rbf, config['l_0_output_dir'], + config['l_0_hdf_path'], + config['mesh'], batch_size, log, + stats=stats, + corr_field_config=config['corr_field_config'], + seed=seed) + + val_predictions = [] + + if stats: + l1_sample_time = preprocess_time / len(data) + learning_time / len(data) + l0_sample_time = predict_l_0_time / len(l_0_targets) + + # print("targets ", targets) + # print("predictions ", predictions) + + # orig_max_vars, predict_max_vars = process_mlmc(hdf_path, sampling_info_path, ref_mlmc_file, targets, predictions, train_targets, + # train_predictions, + # val_targets, l_0_targets, + # l_0_predictions, l1_sample_time, l0_sample_time, nn_level=level, replace_level=replace_level, + # stats=stats) + + return svr_rbf, targets, predictions, learning_time, train_targets, train_predictions, \ + val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, total_steps, None, None, None + + save_times(config['save_path'], False, (preprocess_time, len(data)), learning_time, (predict_l_0_time, len(l_0_targets))) + save_load_data(config['save_path'], False, targets, predictions, train_targets, train_predictions, val_targets, l_0_targets, + l_0_predictions) + + +def predict_level_zero_SVR(nn, output_dir, hdf_path, mesh, batch_size=1000, log=False, stats=False, corr_field_config=None, seed=1234): + #graph_creator(output_dir, hdf_path, mesh, level=0) + sample_time = 0 + if corr_field_config: + sample_time = corr_field_sample_time(mesh, corr_field_config) + + # Load data + data = FlowDataset(output_dir=output_dir, log=log) + data.shuffle(seed=seed) + dataset = data.dataset[:] + + predict_time_start = time.process_time() + test_input = prepare_data(dataset.x) + targets = prepare_data(dataset.y) + #print("data prepared") + + predictions = [] + for i in range(0, len(test_input), batch_size): + predictions.extend(nn.predict(test_input[i:i + batch_size])) + predictions = np.array(predictions) + #print("predictison shape ", predictions.shape) + predictions = np.squeeze(predictions) + + if log: + targets = np.exp(targets) + predictions = np.exp(predictions) + # analyze_results(targets, predictions) + predict_time = time.process_time() - predict_time_start + return targets, predictions, predict_time + sample_time * len(data) + + +def statistics(config): + n_subsamples = 2 + + model_title, mch_l_model, log = config['machine_learning_model'] + model_data = {} + model_data["log"] = log + + # seeds = [] + # for i in range(n_subsamples): + # seeds.append(i * 125) + + if not os.path.isdir(config['save_path']): + os.makedirs(config['save_path']) + else: + print("dir exists {}".format(config['save_path'])) + exit() + + for i in range(n_subsamples): + iter_dir = os.path.join(config['save_path'], "{}".format(i)) + if not os.path.isdir(iter_dir): + os.makedirs(iter_dir) + + model, targets, predictions, learning_time, train_targets, train_predictions, \ + val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, total_steps = \ + mch_l_model(config, stats=True, train=True, log=log, seed=i) + + if config['save_model']: + model_data["model"] = model + model_data["test_targets"] = targets + model_data["test_predictions"] = predictions + model_data["train_targets"] = train_targets + model_data["train_predictions"] = train_predictions + model_data["val_targets"] = val_targets + model_data["val_predictions"] = val_predictions + model_data["l_0_targets"] = l_0_targets + model_data["l_0_predictions"] = l_0_predictions + model_data["l1_sample_time"] = l1_sample_time + model_data["l0_sample_time"] = l0_sample_time + model_data["total_steps"] = total_steps + model_data["learning_times"] = learning_time + + save_statistics(iter_dir, model_data) + + # save_times(save_path, False, (preprocess_time, len(data)), learning_time, (predict_l_0_time, len(l_0_targets))) + # save_load_data(save_path, False, targets, predictions, train_targets, val_targets, l_0_targets, l_0_predictions) + + # for i in range(len(train_losses)): + # print("train loss ", train_losses[i]) + # print("test loss ", test_losses[i]) + # analyze_results(all_targets[i], all_predictions[i]) + # print("learning time ", learning_times[i]) + # print("##################################################") + + + analyze_statistics(config) + + # plot_loss(train_losses, val_losses) + # analyze_results(np.mean(all_test_outputs, axis=0), np.mean(all_predictions, axis=0)) + # analyze_results(np.var(all_test_outputs, axis=0), np.var(all_predictions, axis=0)) + # + # estimate_density(np.mean(all_test_outputs, axis=0), title="Test outputs") + # estimate_density(np.mean(all_predictions, axis=0), title="Predictions") + + +def save_statistics(save_dir_path, model_data): + for file_name, data in model_data.items(): + if file_name == "model" and data is not None: + data.save(os.path.join(save_dir_path, file_name)) + else: + np.save(os.path.join(save_dir_path, file_name), data) + + +def load_statistics(dir_path): + models_data = {} + models_data["model"] = [] + models_data["test_targets"] = [] + models_data["test_predictions"] = [] + models_data["train_targets"] = [] + models_data["train_predictions"] = [] + models_data["val_targets"] = [] + models_data["val_predictions"] = [] + models_data["l_0_targets"] = [] + models_data["l_0_predictions"] = [] + models_data["l1_sample_time"] = [] + models_data["l0_sample_time"] = [] + models_data["total_steps"] = [] + models_data["learning_times"] = [] + models_data["log"] = [] + + #dirs = (os.path.split(dir_path)[-1]).split("_") + n_iters = 25 + for i in range(n_iters): + data_dir_path = os.path.join(dir_path, str(i)) + if not os.path.isdir(data_dir_path): + print("data dir not exists {}".format(data_dir_path)) + break + if os.path.exists(os.path.join(data_dir_path,'model')): + models_data['model'].append(keras.models.load_model(os.path.join(data_dir_path, 'model'))) + for file in glob.glob(os.path.join(data_dir_path, "*.npy")): + file_name = os.path.split(file)[-1] + file_name = file_name.split(".")[0] + # if file_name not in models_data: + # print("file name ", file_name) + # models_data[file_name] = [] + # print("np.load(file, allow_pickle=True) ", np.load(file, allow_pickle=True)) + # exit() + + models_data[file_name].append(np.load(file, allow_pickle=True)) + + return models_data + + +def plot_sse(data_nn, data_mlmc, x_label="ith moment", y_label="MSE", title=""): + import matplotlib + #matplotlib.rcParams.update({'font.size': 38}) + matplotlib.rcParams.update({'lines.markersize': 14}) + fig, axes = plt.subplots(1, 1, figsize=(22, 10)) + data = np.array(data_nn) + x = range(data.shape[1]) + axes.set_title(title) + axes.set_xlabel(x_label) + axes.set_ylabel(y_label) + axes.errorbar(x, np.mean(data_nn, axis=0), yerr=np.sqrt(np.var(data_nn, axis=0)), fmt='o', label="NN MLMC", color="red") + axes.errorbar(x, np.mean(data_mlmc, axis=0), yerr=np.sqrt(np.var(data_mlmc, axis=0)), fmt='o', label="MLMC", color="blue") + fig.legend() + fig.savefig("{}.pdf".format(title)) + fig.show() + + +def predict_data(config, model, mesh_file, log=True): + batch_size = config['batch_size'] + data = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log) + data = data # [:10000] + + data.a = config['conv_layer'].preprocess(data.a) + data.a = sp_matrix_to_sp_tensor(data.a) + + # idx = 0 + # data_te = data.get_test_data(idx, train_data_len) + data_te = data[:1] + + print("len(datate) ", len(data_te)) + print("batch size ", batch_size) + + # We use a MixedLoader since the dataset is in mixed mode + loader = MixedLoader(data_te, batch_size=batch_size) + + conv_layers = {} + dense_layers = {} + flatten_input = [] + flatten_output = [] + + step = 0 + for batch in loader: + if step == loader.steps_per_epoch: + break + inputs, target = batch + x, a = inputs + + for conv_index, conv_layer in enumerate(model._conv_layers): + if conv_index not in conv_layers: + conv_layers[conv_index] = [[], [], []] + conv_layers[conv_index][0].extend(x) # inputs + conv_layers[conv_index][1].extend(conv_layer.kernel.numpy()) # weights (kernel) + conv_out = conv_layer([x, a]) + conv_layers[conv_index][2].extend(conv_out) # outputs + + flatten_input = conv_layers[conv_index][2][-1] + flatten_output = model.flatten(conv_out) + + for index, dense_layer in enumerate(model._dense_layers): + if index not in dense_layers: + dense_layers[index] = [[], [], []] + + dense_layers[index][0].extend(flatten_output) # inputs + dense_layers[index][1].extend(dense_layer.weights) # weights (kernel) + dense_layers[index][2].extend(dense_layer(model.flatten(conv_out))) # outputs + + step += 1 + + plot_progress(conv_layers, dense_layers, flatten_output, mesh_file=mesh_file) + + +def analyze_statistics(config): + + if not os.path.isdir(config['save_path']): + print("dir not exists") + exit() + + data_dict = load_statistics(config['save_path']) + + #print("data dict ", data_dict) + + # for key, data_dict in models_data.items(): + # print("model: {}".format(key)) + + mlmc_n_collected_all = [] + nn_n_collected_all = [] + n_ops_all = [] + n_ops_predict_all = [] + + mlmc_times = [] + nn_times = [] + + mlmc_l_vars = [] + nn_l_vars = [] + mlmc_vars_mse = [] + nn_vars_mse = [] + mlmc_means_mse = [] + nn_means_mse = [] + + for i in range(len(data_dict["test_targets"])): + predictions = data_dict["test_predictions"][i] + targets = data_dict["test_targets"][i] + train_predictions = data_dict["train_predictions"][i] + train_targets = data_dict["train_targets"][i] + val_predictions = data_dict["val_predictions"][i] + val_targets = data_dict["val_targets"][i] + l_0_predictions = data_dict["l_0_predictions"][i] + l_0_targets = data_dict["l_0_targets"][i] + l1_sample_time = data_dict["l1_sample_time"][i] + l0_sample_time = data_dict["l0_sample_time"][i] + + model = data_dict["model"][i] + + predict_data(config, model, mesh_file=config["mesh"]) + + mlmc_n_collected, nn_mlmc_n_collected, n_ops, n_ops_predict, orig_moments_mean, predict_moments_mean,\ + ref_moments_mean, orig_level_params, nn_level_params = process_mlmc(config['hdf_path'], + config['sampling_info_path'], + config['ref_mlmc_file'], targets, + predictions, train_targets, + train_predictions, + val_targets, l_0_targets, + l_0_predictions, l1_sample_time, l0_sample_time, + nn_level=config['level'], + replace_level=config['replace_level'], + stats=True) + + mlmc_n_collected_all.append(mlmc_n_collected) + nn_n_collected_all.append(nn_mlmc_n_collected) + n_ops_all.append(n_ops) + n_ops_predict_all.append(n_ops_predict) + + mlmc_times.append(np.sum(np.array(mlmc_n_collected) * np.array(n_ops))) + nn_times.append(np.sum(np.array(nn_mlmc_n_collected) * np.array(n_ops_predict))) + + mlmc_l_vars.append(orig_moments_mean.l_vars) + nn_l_vars.append(predict_moments_mean.l_vars) + + mlmc_vars_mse.append((ref_moments_mean.var - orig_moments_mean.var) ** 2) + nn_vars_mse.append((ref_moments_mean.var - predict_moments_mean.var) ** 2) + + mlmc_means_mse.append((ref_moments_mean.mean - orig_moments_mean.mean) ** 2) + nn_means_mse.append((ref_moments_mean.mean - predict_moments_mean.mean) ** 2) + + mlmc_total_time = np.mean(mlmc_times) + nn_total_time = np.mean(nn_times) + print("mlmc total time ", mlmc_total_time) + print("nn total time ", nn_total_time) + + n_ops_mlmc_mean = np.mean(n_ops_all, axis=0) + n_ops_nn_mean = np.mean(n_ops_predict_all, axis=0) + + print("n ops mlmc mean ", n_ops_mlmc_mean) + print("n ops nn mean ", n_ops_nn_mean) + + mlmc_n_collected = np.mean(mlmc_n_collected_all, axis=0) + nn_n_collected = np.mean(nn_n_collected_all, axis=0) + + print("mlmc n collected ", mlmc_n_collected_all) + print("nn n collected all ", nn_n_collected_all) + print("mlmc n collected ", mlmc_n_collected) + print("nn n collected ", nn_n_collected) + + plt_var = plot.Variance() + l_vars = np.mean(mlmc_l_vars, axis=0) + orig_level_params = orig_level_params # np.squeeze(orig_level_params) + plt_var.add_level_variances(orig_level_params, l_vars) + plt_var.show("mlmc_vars") + + plt_var = plot.Variance() + l_vars = np.mean(nn_l_vars, axis=0) + print("l vars shape ", l_vars.shape) + print("nn level parsm ", nn_level_params) + level_params = np.squeeze(nn_level_params) + level_params[0] /= 2 + plt_var.add_level_variances(level_params, l_vars) + plt_var.show("nn_vars") + + plot_sse(nn_vars_mse, mlmc_vars_mse, title="moments_var") + plot_sse(nn_means_mse, mlmc_means_mse, title="moments_mean") + + data_dict["test_targets"] = np.array(data_dict["test_targets"]) + data_dict["test_predictions"] = np.array(data_dict["test_predictions"]) + data_dict["train_targets"] = np.array(data_dict["train_targets"]) + data_dict["train_predictions"] = np.array(data_dict["train_predictions"]) + + if data_dict["log"][0]: + data_dict["test_targets"] = np.exp(data_dict["test_targets"]) + data_dict["test_predictions"] = np.exp(data_dict["test_predictions"]) + data_dict["train_targets"] = np.exp(data_dict["train_targets"]) + data_dict["train_predictions"] = np.exp(data_dict["train_predictions"]) + + # print("test targets ", data_dict["test_targets"]) + # print("test predictions ", data_dict["test_predictions"]) + # + # print("orig max vars ", data_dict["orig_max_vars"]) + # print("predict max vars ", data_dict["predict_max_vars"]) + + # mean_orig_vars = np.mean(data_dict["orig_max_vars"], axis=0) + # mean_predict_vars = np.mean(data_dict["predict_max_vars"], axis=0) + total_steps = np.mean(data_dict["total_steps"]) + + # print("mean orig vars ", mean_orig_vars) + # print("mean predict vars ", mean_predict_vars) + print("total steps ", total_steps) + + print("test targets ", data_dict["test_targets"]) + print("test predictions ", data_dict["test_predictions"]) + print("test diff ", data_dict["test_predictions"] - data_dict["test_targets"]) + print("test diff squared ", (data_dict["test_predictions"] - data_dict["test_targets"])**2) + + test_MSE = np.mean((data_dict["test_predictions"] - data_dict["test_targets"])**2, axis=1) + test_RMSE = np.sqrt(test_MSE) + test_MAE = np.mean(np.abs(data_dict["test_predictions"] - data_dict["test_targets"]), axis=1) + + train_MSE = np.mean((data_dict["train_predictions"] - data_dict["train_targets"]) ** 2, axis=1) + train_RMSE = np.sqrt(train_MSE) + train_MAE = np.mean(np.abs( data_dict["train_predictions"] - data_dict["train_targets"]), axis=1) + learning_times = data_dict["learning_times"] + + # plot_data(test_MSE, label="test MSE") + # plot_data(test_MAE, label="test MAE") + + print("test_MSE ", test_MSE) + + print("NN moments MSE sum ", np.sum(np.mean(nn_means_mse, axis=0))) + + print("mean test MSE ", np.mean(test_MSE)) + print("mean test RMSE ", np.mean(test_RMSE)) + print("mean test MAE ", np.mean(test_MAE)) + print("max test MSE ", np.max(test_MSE)) + print("max test RMSE ", np.max(test_RMSE)) + print("max test MAE ", np.max(test_MAE)) + + print("mean train MSE ", np.mean(train_MSE)) + print("mean train RMSE ", np.mean(train_RMSE)) + print("mean train MAE ", np.mean(train_MAE)) + print("max train MSE ", np.max(train_MSE)) + print("max train RMSE ", np.max(train_RMSE)) + print("max train MAE ", np.max(train_MAE)) + + print("mean learning time ", np.mean(learning_times)) + print("max learning time ", np.max(learning_times)) + print("######################################") + + +def run_GNN(config, stats=True, train=True, log=False, seed=0): + print("seed ", seed) + + loss = MeanSquaredError() # var_loss_function# + accuracy_func = MSE_moments + # loss = MeanAbsoluteError() + # loss = MeanSquaredLogarithmicError() + #loss = KLDivergence() + # loss = total_loss_function + optimizer = tf.optimizers.Adam(learning_rate=config['learning_rate']) + batch_size = config['batch_size']#2000 + epochs = config['epochs']#1000 + hidden_regularization = None # l2(2e-10) + + graph_creation_time = config['graph_creation_time'] + if graph_creation_time == 0: + graph_creator_preproces_time = time.process_time() + graph_creator(config['output_dir'], config['hdf_path'], config['mesh'], level=config['level']) + graph_creation_time = time.process_time() - graph_creator_preproces_time + print("graph creation time ", graph_creation_time) + exit() + + preprocess_start_time = time.process_time() + # Load data + data = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log) + data = data#[:10000] + + #print("len data ", len(data)) + #data.shuffle(seed=seed) + preprocess_time = time.process_time() - preprocess_start_time + #print("preproces time ", preprocess_time) + preprocess_time = preprocess_time + graph_creation_time + #print("total preprocess time ", preprocess_time) + + learning_time_start = time.process_time() + data.a = config['conv_layer'].preprocess(data.a) + data.a = sp_matrix_to_sp_tensor(data.a) + #train_data_len = int(len(data) * 0.8) + train_data_len = config['n_train_samples'] + # Train/valid/test split + + data_tr = data[seed*train_data_len: seed*train_data_len + train_data_len] + data_te = data.get_test_data(seed, train_data_len) + #data_tr, data_te = data[:train_data_len], data[train_data_len:] + + gnn = config['gnn'](**config['model_config']) + + # if hasattr(gnn._loss,'__name__') and gnn._loss.__name__ == "MSE_moments": + # tr_output = [g.y for g in data_tr] + # n_moments = 3 + # quantile = 0.001 + # domain = np.percentile(tr_output, [100 * quantile, 100 * (1 - quantile)]) + # moments_fn = Legendre_tf(n_moments, domain) + # #accuracy_func = MSE_moments(moments_fn=moments_fn) + # gnn._loss = MSE_moments(moments_fn=moments_fn) + + np.random.shuffle(data_tr) + val_data_len = int(len(data_tr) * config['val_samples_ratio']) + data_tr, data_va = data_tr[:-val_data_len], data_tr[-val_data_len:] + + # print("data_tr len ", len(data_tr)) + # print("data_va len ", len(data_va)) + # print("data_te len ", len(data_te)) + + # We use a MixedLoader since the dataset is in mixed mode + loader_tr = MixedLoader(data_tr, batch_size=batch_size, epochs=epochs) + loader_va = MixedLoader(data_va, batch_size=batch_size) + loader_te = MixedLoader(data_te, batch_size=batch_size) + # + if gnn is None: + gnn = GNN(loss=loss, optimizer=optimizer, conv_layer=config['conv_layer'], output_activation=abs_activation, + hidden_activation='relu', patience=150, hidden_reqularizer=hidden_regularization, + model=config['model'], accuracy_func=accuracy_func) # tanh takes to much time + # ideally patience = 150 + # batch_size 500, ideally 500 epochs, patience 35 + + if train: + print("gnn ", gnn) + # gnn.run_eagerly = True + train_targets = gnn.fit(loader_tr, loader_va, loader_te) + + learning_time = time.process_time() - learning_time_start + + # states = gnn._states + # if len(states) > 0: + # min_key = np.min(list(states.keys())) + # gnn = states[min_key] + + train_targets, train_predictions = gnn.predict(loader_tr) + train_predictions = np.squeeze(train_predictions) + + val_targets, val_predictions = gnn.predict(loader_va) + val_predictions = np.squeeze(val_predictions) + + #val_targets = gnn.val_targets + total_steps = gnn._total_n_steps + + targets, predictions = gnn.predict(loader_te) + predictions = np.squeeze(predictions) + + #print("learning time ", learning_time) + + targets = np.array(targets) + predictions = np.array(predictions) + + #print("MSE ", np.mean((predictions-targets)**2)) + + if log: + targets = np.exp(targets) + predictions = np.exp(predictions) + + if not stats: + analyze_results(targets, predictions) + plot_loss(gnn._train_loss, gnn._val_loss) + analyze_results(targets, predictions) + + import matplotlib.pyplot as plt + + plt.hist(targets, bins=50, alpha=0.5, label='target', density=True) + plt.hist(predictions, bins=50, alpha=0.5, label='predictions', density=True) + + # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) + plt.legend(loc='upper right') + # plt.xlim(-0.5, 1000) + plt.yscale('log') + plt.show() + + #predict_l_0_start_time = time.process_time() + l_0_targets, l_0_predictions, predict_l_0_time = predict_level_zero(gnn, config['l_0_output_dir'], + config['l_0_hdf_path'], config['mesh'], + config['conv_layer'], batch_size, log, + stats=stats, + corr_field_config=config['corr_field_config'], + seed=seed) + #predict_l_0_time = time.process_time() - predict_l_0_start_time + + if stats: + l1_sample_time = preprocess_time / len(data) + learning_time / len(data) + l0_sample_time = predict_l_0_time / len(l_0_targets) + + # print("targets ", targets) + # print("predictions ", predictions) + + # orig_max_vars, predict_max_vars = process_mlmc(hdf_path, sampling_info_path, ref_mlmc_file, targets, predictions, train_targets, + # train_predictions, + # val_targets, l_0_targets, + # l_0_predictions, l1_sample_time, l0_sample_time, nn_level=level, replace_level=replace_level, + # stats=stats) + + return gnn._model, targets, predictions, learning_time, train_targets, train_predictions,\ + val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, total_steps + + save_times(config['save_path'], False, (preprocess_time, len(data)), learning_time, (predict_l_0_time, len(l_0_targets))) + save_load_data(config['save_path'], False, targets, predictions, train_targets, train_predictions, val_targets, l_0_targets, + l_0_predictions) + + +def predict_level_zero(nn, output_dir, hdf_path, mesh, conv_layer, batch_size=1000, log=False, stats=False, + corr_field_config=None, seed=1234): + # graph_creator(output_dir, hdf_path, mesh, level=0) + # Load data + sample_time = 0 + if corr_field_config: + sample_time = corr_field_sample_time(mesh, corr_field_config) + + data = FlowDataset(output_dir=output_dir, log=log)#, mesh=mesh, corr_field_config=corr_field_config) + #data = data # [:10000] + data.shuffle(seed=seed) + + print("output_dir ", output_dir) + print("len(data) ", len(data)) + print("data[0] ", data[0]) + + predict_time_start = time.process_time() + data.a = conv_layer.preprocess(data.a) + data.a = sp_matrix_to_sp_tensor(data.a) + + loader_te = MixedLoader(data, batch_size=batch_size) + + targets, predictions = nn.predict(loader_te) + predictions = np.squeeze(predictions) + + if not stats: + analyze_results(targets, predictions) + + if log: + targets = np.exp(targets) + predictions = np.exp(predictions) + if not stats: + analyze_results(targets, predictions) + + predict_time = time.process_time() - predict_time_start + + return targets, predictions, predict_time + sample_time * len(data) + + +def save_times(path, load=False, preprocess=None, learning_time=None, predict_l_0=None): + if load: + preprocess_time = None + preprocess_n = None + predict_time = None + predict_n = None + if os.path.exists(os.path.join(path, "preprocess_time.npy")): + preprocess_time = np.load(os.path.join(path, "preprocess_time.npy")) + if os.path.exists(os.path.join(path, "preprocess_n.npy")): + preprocess_n = np.load(os.path.join(path, "preprocess_n.npy")) + if os.path.exists(os.path.join(path, "learning_time.npy")): + learning_time = np.load(os.path.join(path, "learning_time.npy")) + if os.path.exists(os.path.join(path, "predict_l_0_time.npy")): + predict_time = np.load(os.path.join(path, "predict_l_0_time.npy")) + if os.path.exists(os.path.join(path, "predict_l_0_n.npy")): + predict_n = np.load(os.path.join(path, "predict_l_0_n.npy")) + return preprocess_time, preprocess_n, learning_time, predict_time, predict_n + else: + if preprocess is not None: + np.save(os.path.join(path, "preprocess_time"), preprocess[0]) + np.save(os.path.join(path, "preprocess_n"), preprocess[1]) + if learning_time is not None: + np.save(os.path.join(path, "learning_time"), learning_time) + if preprocess is not None: + np.save(os.path.join(path, "predict_l_0_time"), predict_l_0[0]) + np.save(os.path.join(path, "predict_l_0_n"), predict_l_0[1]) + + +def save_load_data(path, load=False, targets=None, predictions=None, train_targets=None, train_predictions=None, + val_targets=None, l_0_targets=None, + l_0_predictions=None): + if load: + if os.path.exists(os.path.join(path, "targets.npy")): + targets = np.load(os.path.join(path, "targets.npy")) + if os.path.exists(os.path.join(path, "predictions.npy")): + predictions = np.load(os.path.join(path, "predictions.npy")) + if os.path.exists(os.path.join(path, "train_targets.npy")): + train_targets = np.load(os.path.join(path, "train_targets.npy")) + if os.path.exists(os.path.join(path, "train_predictions.npy")): + train_predictions = np.load(os.path.join(path, "train_predictions.npy")) + if os.path.exists(os.path.join(path, "val_targets.npy")): + val_targets = np.load(os.path.join(path, "val_targets.npy")) + if os.path.exists(os.path.join(path, "l_0_targets.npy")): + l_0_targets = np.load(os.path.join(path, "l_0_targets.npy")) + if os.path.exists(os.path.join(path, "l_0_predictions.npy")): + l_0_predictions = np.load(os.path.join(path, "l_0_predictions.npy")) + return targets, predictions, train_targets, train_predictions, val_targets, l_0_targets, l_0_predictions + else: + if targets is not None: + np.save(os.path.join(path, "targets"), targets) + if predictions is not None: + np.save(os.path.join(path, "predictions"), predictions) + if train_targets is not None: + np.save(os.path.join(path, "train_targets"), train_targets) + if train_predictions is not None: + np.save(os.path.join(path, "train_predictions"), train_predictions) + if val_targets is not None: + np.save(os.path.join(path, "val_targets"), val_targets) + if l_0_targets is not None: + np.save(os.path.join(path, "l_0_targets"), l_0_targets) + if l_0_predictions is not None: + np.save(os.path.join(path, "l_0_predictions"), l_0_predictions) + + +def process_results(hdf_path, sampling_info_path, ref_mlmc_file, save_path, nn_level, replace_level): + targets, predictions, train_targets, train_predictions, val_targets, l_0_targets, l_0_predictions = save_load_data( + save_path, load=True) + preprocess_time, preprocess_n, learning_time, predict_l_0_time, predict_l_0_n = save_times(save_path, load=True) + + l1_sample_time = preprocess_time / preprocess_n + learning_time / preprocess_n + l0_sample_time = predict_l_0_time / predict_l_0_n + + print("preprocess_time ", preprocess_time) + print("preprocess_n ", preprocess_n) + print("learning_time ", learning_time) + print("predict_l_0_time ", predict_l_0_time) + print("predict_l_0_n ", predict_l_0_n) + + print("l1 sample time ", l1_sample_time) + print("l0 sample time ", l0_sample_time) + + print("len targets ", len(targets)) + print("len predictions ", len(predictions)) + + print("len train targets ", len(train_targets)) + print("len val targets ", len(val_targets)) + + process_mlmc(hdf_path, sampling_info_path, ref_mlmc_file, targets, predictions, train_targets, train_predictions, + val_targets, l_0_targets, + l_0_predictions, l1_sample_time, l0_sample_time, nn_level=nn_level, replace_level=replace_level) diff --git a/mlmc/metamodel/create_graph.py b/mlmc/metamodel/create_graph.py new file mode 100644 index 00000000..d07b1f03 --- /dev/null +++ b/mlmc/metamodel/create_graph.py @@ -0,0 +1,186 @@ +import os +import os.path +import numpy as np +import networkx as nx +#import matplotlib.pyplot as plt +from mlmc.tool import gmsh_io +from mlmc.tool.hdf5 import HDF5 +from spektral.data import Graph +from mlmc.metamodel.flow_dataset import FlowDataset + + +MESH = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/l_step_0.055_common_files/mesh.msh" +#FIELDS_SAMPLE_MESH = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/output/L00_S0000000/fine_fields_sample.msh" +FIELDS_SAMPLE = "fine_fields_sample.msh" +# OUTPUT_DIR = "/home/martin/Documents/metamodels/data/1000_ele/test/01_cond_field/output/" +# HDF_PATH = "/home/martin/Documents/metamodels/data/1000_ele/test/01_cond_field/mlmc_1.hdf5" + +# OUTPUT_DIR = "/home/martin/Documents/metamodels/data/cl_0_3_s_4/L5/test/01_cond_field/output/" +# HDF_PATH = "/home/martin/Documents/metamodels/data/cl_0_3_s_4/L5/mlmc_5.hdf5" + +# OUTPUT_DIR = "/home/martin/Documents/metamodels/data/cl_0_1_s_1/L5/test/01_cond_field/output/" +# HDF_PATH = "/home/martin/Documents/metamodels/data/cl_0_1_s_1/L5/mlmc_5.hdf5" + +# OUTPUT_DIR = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L5/test/01_cond_field/output/" +# HDF_PATH = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L5/mlmc_5.hdf5" + + +def extract_mesh_gmsh_io(mesh_file, get_points=False): + """ + Extract mesh from file + :param mesh_file: Mesh file path + :return: Dict + """ + mesh = gmsh_io.GmshIO(mesh_file) + is_bc_region = {} + region_map = {} + for name, (id, _) in mesh.physical.items(): + unquoted_name = name.strip("\"'") + is_bc_region[id] = (unquoted_name[0] == '.') + region_map[unquoted_name] = id + + bulk_elements = [] + + for id, el in mesh.elements.items(): + _, tags, i_nodes = el + region_id = tags[0] + if not is_bc_region[region_id]: + bulk_elements.append(id) + + n_bulk = len(bulk_elements) + centers = np.empty((n_bulk, 3)) + ele_ids = np.zeros(n_bulk, dtype=int) + ele_nodes = {} + point_region_ids = np.zeros(n_bulk, dtype=int) + + for i, id_bulk in enumerate(bulk_elements): + _, tags, i_nodes = mesh.elements[id_bulk] + region_id = tags[0] + centers[i] = np.average(np.array([mesh.nodes[i_node] for i_node in i_nodes]), axis=0) + point_region_ids[i] = region_id + ele_ids[i] = id_bulk + ele_nodes[id_bulk] = i_nodes + + if get_points: + + min_pt = np.min(centers, axis=0) + max_pt = np.max(centers, axis=0) + diff = max_pt - min_pt + min_axis = np.argmin(diff) + non_zero_axes = [0, 1, 2] + # TODO: be able to use this mesh_dimension in fields + if diff[min_axis] < 1e-10: + non_zero_axes.pop(min_axis) + points = centers[:, non_zero_axes] + + return {'points': points, 'point_region_ids': point_region_ids, 'ele_ids': ele_ids, 'region_map': region_map} + + return ele_nodes + + +def get_node_features(fields_mesh): + mesh = gmsh_io.GmshIO(fields_mesh) + element_data = mesh.current_elem_data + features = list(element_data.values()) + return features + + +def create_adjacency_matrix(ele_nodes): + + adjacency_matrix = np.zeros((len(ele_nodes), len(ele_nodes))) + #adjacency_matrix = sparse.csr_matrix((len(ele_nodes), len(ele_nodes))) # + + nodes = list(ele_nodes.values()) + for i in range(adjacency_matrix.shape[0]): + ele_nodes = nodes[i] + + for j in range(i+1, len(nodes)): + if i == j: + continue + ele_n = nodes[j] + + if len(list(set(ele_nodes).intersection(ele_n))) == 2: + adjacency_matrix[j][i] = adjacency_matrix[i][j] = 1 + + print(np.count_nonzero(adjacency_matrix)) + assert np.allclose(adjacency_matrix, adjacency_matrix.T) # symmetry + return adjacency_matrix + + +def plot_graph(adjacency_matrix): + #G = nx.from_scipy_sparse_matrix(adjacency_matrix) + G = nx.from_numpy_matrix(adjacency_matrix) + nx.draw_kamada_kawai(G, with_labels=True, node_size=1, font_size=6) + plt.axis('equal') + plt.show() + + +def reject_outliers(data, m=2): + #print("abs(data - np.mean(data)) < m * np.std(data) ", abs(data - np.mean(data)) < m * np.std(data)) + #return data[abs(data - np.mean(data)) < m * np.std(data)] + return abs(data - np.mean(data)) < m * np.std(data) + + +def graph_creator(output_dir, hdf_path, mesh, level=0): + adjacency_matrix = create_adjacency_matrix(extract_mesh_gmsh_io(mesh)) + np.save(os.path.join(output_dir, "adjacency_matrix"), adjacency_matrix, allow_pickle=True) + loaded_adjacency_matrix = np.load(os.path.join(output_dir, "adjacency_matrix.npy"), allow_pickle=True) + + #plot_graph(loaded_adjacency_matrix) + + hdf = HDF5(file_path=hdf_path, load_from_file=True) + level_group = hdf.add_level_group(level_id=str(level)) + + # print("collected ", level_group.collected()[:, 0, :]) + # indices = reject_outliers(np.squeeze(level_group.collected()[:, 0, :]), m=6) + # print("removed outliers ", np.count_nonzero(~indices)) + indices = np.ones(len(level_group.collected())) + + collected = zip(level_group.get_collected_ids(), level_group.collected()) + + graphs = [] + data = [] + i = 0 + for keep, (sample_id, col_values) in zip(indices, collected): + if not keep: + continue + output_value = col_values[0, 0] + + sample_dir = os.path.join(output_dir, sample_id) + field_mesh = os.path.join(sample_dir, FIELDS_SAMPLE) + if os.path.exists(field_mesh): + # i += 1 + # if i > 150: + # break + + features = get_node_features(field_mesh) + np.save(os.path.join(sample_dir, "nodes_features"), features) + np.save(os.path.join(sample_dir, "output"), output_value) + + #graphs.append(Graph(x=features, y=output_value)) # , a=self.adjacency_matrix)) + # Save data for pandas dataframe creation, not used with Graph neural network + #data.append({'x': features, 'y': output_value}) + + #loaded_features = np.load(os.path.join(sample_dir, "nodes_features.npy")) + #print("loaded features ", loaded_features) + + #FlowDataset.pickle_data(graphs, FlowDataset.GRAPHS_FILE) + #FlowDataset.pickle_data(data, FlowDataset.DATA_FILE) + + +if __name__ == "__main__": + + # mesh = "/home/martin/Documents/metamodels/data/5_ele/cl_0_1_s_1/L5/l_step_0.020196309484414757_common_files/mesh.msh" + # output_dir = "/home/martin/Documents/metamodels/data/5_ele/cl_0_3_s_4/L1_3/test/01_cond_field/output/" + # hdf_path = "/home/martin/Documents/metamodels/data/5_ele/cl_0_1_s_1/L1_3/mlmc_1.hdf5" + + import cProfile + import pstats + pr = cProfile.Profile() + pr.enable() + + my_result = graph_creator(output_dir, hdf_path, mesh) + + pr.disable() + ps = pstats.Stats(pr).sort_stats('cumtime') + ps.print_stats() diff --git a/mlmc/metamodel/custom_methods.py b/mlmc/metamodel/custom_methods.py new file mode 100644 index 00000000..3a66ce0c --- /dev/null +++ b/mlmc/metamodel/custom_methods.py @@ -0,0 +1,42 @@ +import numpy as np +import tensorflow as tf +from tensorflow.keras import backend as K +# import tensorflow.experimental.numpy as tnp +# tnp.experimental_enable_numpy_behavior() +from mlmc.moments import Monomial, Legendre + + +def abs_activation(x): + return K.abs(x) + + +def var_loss_function(y_true, y_predict): + if tf.is_tensor(y_true): + y_true = float(y_true) + + # else: + # print("diff shape ", (y_true - K.squeeze(y_predict, axis=1)).shape) + + return K.var(y_true - K.squeeze(y_predict, axis=1)) + #return K.sum(K.abs(y_true - K.squeeze(y_predict, axis=1))) + + +def total_loss_function(y_true, y_predict): + if tf.is_tensor(y_true): + y_true = float(y_true) + + # else: + # print("diff shape ", (y_true - K.squeeze(y_predict, axis=1)).shape) + + #return K.var(K.abs(y_true - K.squeeze(y_predict, axis=1))) + + return K.mean((y_true - K.squeeze(y_predict, axis=1))**2) + K.var(K.abs(y_true - K.squeeze(y_predict, axis=1))) + + +def MSE_moments(moments_fn=None): + if moments_fn is None: + raise ValueError + + def calc_err(y_true, y_predict): + return K.mean(K.sum((moments_fn.eval_all(y_true) - moments_fn.eval_all(y_predict))**2, axis=1)) + return calc_err \ No newline at end of file diff --git a/mlmc/metamodel/flow_dataset.py b/mlmc/metamodel/flow_dataset.py new file mode 100644 index 00000000..ed025700 --- /dev/null +++ b/mlmc/metamodel/flow_dataset.py @@ -0,0 +1,220 @@ +import os +import re +import copy +import random +import numpy as np +import pandas as pd +from mlmc.tool import gmsh_io +from mlmc.tool.flow_mc import FlowSim, create_corr_field +from spektral.data import Dataset, Graph +import pickle + +#MESH = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/l_step_0.055_common_files/mesh.msh" +FIELDS_SAMPLE = "fine_fields_sample.msh" +#OUTPUT_DIR = "/home/martin/Documents/metamodels/data/1000_ele/test/01_cond_field/output/" +#OUTPUT_DIR = "/home/martin/Documents/metamodels/data/cl_0_3_s_4/L5/test/01_cond_field/output/" +#OUTPUT_DIR = "/home/martin/Documents/metamodels/data/cl_0_1_s_1/L5/test/01_cond_field/output/" +#OUTPUT_DIR = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L5/test/01_cond_field/output/" + + +class FlowDataset(Dataset): + GRAPHS_FILE = "graphs" + DATA_FILE = "data" + + def __init__(self, output_dir=None, level=0, log=False, mesh=None, corr_field_config=None, **kwargs): + self._output_dir = output_dir + # if self._output_dir is None: + # self._output_dir = OUTPUT_DIR + self._log = log + self.level = level + self._mesh = mesh + self._corr_field_config = corr_field_config + self.adjacency_matrix = np.load(os.path.join(self._output_dir, "adjacency_matrix.npy"), allow_pickle=True) # adjacency matrix + self.data = [] + super().__init__(**kwargs) + #self.a = self.adjacency_matrix + self.dataset = pd.DataFrame(self.data) + + def get_test_data(self, index, length): + new_dataset = self.dataset[0:index * length] + self.dataset[index * length + length:] + new_obj = copy.deepcopy(self) + new_obj.dataset = new_dataset + + return new_obj + + def shuffle(self, seed=None): + if seed is not None: + random.seed(seed) + + random.shuffle(self.data) + self.dataset = pd.DataFrame(self.data) + + # def generate_data(self): + # n_samples = 10**5 + # graphs = [] + # mesh_data = FlowSim.extract_mesh(self._mesh) + # fields = create_corr_field(model="exp", dim=2, + # sigma=self._corr_field_config['sigma'], + # corr_length=self._corr_field_config['corr_length'], + # log=self._corr_field_config['log']) + # + # # # Create fields both fine and coarse + # fields = FlowSim.make_fields(fields, mesh_data, None) + # + # for i in range(n_samples): + # fine_input_sample, coarse_input_sample = FlowSim.generate_random_sample(fields, coarse_step=0, + # n_fine_elements=len( + # mesh_data['points'])) + # # print("len fine input sample ", len(fine_input_sample["conductivity"])) + # # print("fine input sample ", fine_input_sample["conductivity"]) + # + # features = fine_input_sample["conductivity"] + # output = 1 + # + # # gmsh_io.GmshIO().write_fields('fields_sample.msh', mesh_data['ele_ids'], fine_input_sample) + # # + # # mesh = gmsh_io.GmshIO('fields_sample.msh') + # # element_data = mesh.current_elem_data + # # features = list(element_data.values()) + # + # if self._log: + # features = np.log(features) + # output = np.log(output) + # # features = (features - minimum) / (maximum - minimum) + # graphs.append(Graph(x=features, y=output)) # , a=self.adjacency_matrix)) + # # Save data for pandas dataframe creation, not used with Graph neural network + # self.data.append({'x': features, 'y': output}) + # + # self.a = self.adjacency_matrix + # return graphs + + def read(self): + # if self._mesh is not None: + # return self.generate_data() + + # with open(os.path.join(OUTPUT_DIR, FlowDataset.GRAPHS_FILE), 'rb') as reader: + # graphs = pickle.loads(reader) + # + # if os.path.exists(os.path.join(OUTPUT_DIR, FlowDataset.DATA_FILE)): + # with open(os.path.join(OUTPUT_DIR, FlowDataset.DATA_FILE), 'rb') as reader: + # self.data = pickle.loads(reader) + # + # return graphs + + # i = 0 + all_outputs = [] + all_features = [] + # + # for s_dir in os.listdir(self._output_dir): + # try: + # l = re.findall(r'L(\d+)_S', s_dir)[0] + # if int(l) != self.level: + # continue + # except IndexError: + # continue + # if os.path.isdir(os.path.join(self._output_dir, s_dir)): + # sample_dir = os.path.join(self._output_dir, s_dir) + # if os.path.exists(os.path.join(sample_dir, "nodes_features.npy")): + # features = np.load(os.path.join(sample_dir, "nodes_features.npy")) + # output = np.load(os.path.join(sample_dir, "output.npy")) + # all_outputs.append(output) + # all_features.append(features) + # + # #print("all outputs ", np.array(all_outputs).shape) + # min_output = np.min(all_outputs) + # max_output = np.max(all_outputs) + # + # maximum = np.max(all_features) + # minimum = np.min(all_features) + # + # if self._log: + # minimum = np.log(minimum) + # maximum = np.log(maximum) + # + # self.min_output = min_output + # self.max_output = max_output + # self.min_feature = minimum + # self.max_feature = maximum + + graphs = [] + for s_dir in os.listdir(self._output_dir): + try: + l = re.findall(r'L(\d+)_S', s_dir)[0] + if int(l) != self.level: + continue + except IndexError: + continue + + if os.path.isdir(os.path.join(self._output_dir, s_dir)): + sample_dir = os.path.join(self._output_dir, s_dir) + if os.path.exists(os.path.join(sample_dir, "nodes_features.npy")): + features = np.load(os.path.join(sample_dir, "nodes_features.npy")) + output = np.load(os.path.join(sample_dir, "output.npy")) + + #features = (features - minimum) / (maximum - minimum) + # + # output = (output - min_output) / (max_output - min_output) + # print("max ", maximum) + # print("max ", minimum) + # + # print("new featuers max ", np.max(new_features)) + # print("new featuers min ", np.min(new_features)) + # exit() + + if self._log: + features = np.log(features) + output = np.log(output) + + #features = (features - minimum) / (maximum - minimum) + + graphs.append(Graph(x=features, y=output))#, a=self.adjacency_matrix)) + + # Save data for pandas dataframe creation, not used with Graph neural network + self.data.append({'x': features, 'y': output}) + + self.a = self.adjacency_matrix + return graphs + + @staticmethod + def pickle_data(data, output_dir, file_path): + with open(os.path.join(output_dir, file_path), 'wb') as writer: + pickle.dump(data, writer) + + +def extract_mesh_gmsh_io(mesh_file): + """ + Extract mesh from file + :param mesh_file: Mesh file path + :return: Dict + """ + mesh = gmsh_io.GmshIO(mesh_file) + is_bc_region = {} + region_map = {} + for name, (id, _) in mesh.physical.items(): + unquoted_name = name.strip("\"'") + is_bc_region[id] = (unquoted_name[0] == '.') + region_map[unquoted_name] = id + + bulk_elements = [] + + for id, el in mesh.elements.items(): + _, tags, i_nodes = el + region_id = tags[0] + if not is_bc_region[region_id]: + bulk_elements.append(id) + + n_bulk = len(bulk_elements) + centers = np.empty((n_bulk, 3)) + ele_ids = np.zeros(n_bulk, dtype=int) + ele_nodes = {} + point_region_ids = np.zeros(n_bulk, dtype=int) + + for i, id_bulk in enumerate(bulk_elements): + _, tags, i_nodes = mesh.elements[id_bulk] + region_id = tags[0] + centers[i] = np.average(np.array([mesh.nodes[i_node] for i_node in i_nodes]), axis=0) + point_region_ids[i] = region_id + ele_ids[i] = id_bulk + ele_nodes[id_bulk] = i_nodes + + return ele_nodes diff --git a/mlmc/metamodel/flow_task_CNN.py b/mlmc/metamodel/flow_task_CNN.py new file mode 100644 index 00000000..3d9cf41f --- /dev/null +++ b/mlmc/metamodel/flow_task_CNN.py @@ -0,0 +1,65 @@ +import tensorflow as tf +from mlmc.metamodel.graph_models import cnn_model +from tensorflow.keras.layers.experimental import preprocessing + +# Following 3 lines prevent "Failed to get convolution algorithm. This is probably because cuDNN failed to initialize" +config = tf.compat.v1.ConfigProto() +config.gpu_options.allow_growth = True +session = tf.compat.v1.InteractiveSession(config=config) + +import os +import numpy as np +#os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only +import tensorflow as tf +from tensorflow.keras.losses import MeanSquaredError, SparseCategoricalCrossentropy, KLDivergence +from tensorflow.keras.metrics import mean_squared_error, kl_divergence +from tensorflow.keras.callbacks import History +from tensorflow.keras.optimizers import Adam +from tensorflow.keras.regularizers import l2 +from mlmc.metamodel.postprocessing import analyze_results, plot_loss +from spektral.data import MixedLoader +from mlmc.metamodel.flow_dataset import FlowDataset +from spektral.layers import GCNConv, GlobalSumPool, ChebConv, GraphSageConv, ARMAConv, GATConv, APPNPConv, GINConv +from spektral.utils.sparse import sp_matrix_to_sp_tensor +from tensorflow.keras.layers.experimental import preprocessing +from mlmc.metamodel.custom_methods import abs_activation + +from mlmc.metamodel.graph_models import Net1 + + + +################################## +# Convolutional neural network # +################################## + +class CNN: + def __init__(self, **kwargs): + self._epochs = kwargs.get('epochs', 100) + self._val_split = kwargs.get('var_split', 0.2) + self._verbose = kwargs.get('verbose', False) + + self._loss = kwargs.get('loss', 'mean_squared_error') + self._optimizer = kwargs.get('optimizer', tf.optimizers.Adam(learning_rate=0.001)) + self._normalizer = kwargs.get('normalizer', preprocessing.Normalization()) + + self.history = None # Set in fit method + self._model = cnn_model() + self._model.compile(loss=self._loss, optimizer=self._optimizer) + + def fit(self, train_input, train_output): + self.history = self._model.fit(train_input, train_output, validation_split=self._val_split, + verbose=self._verbose, epochs=self._epochs) + + def predict(self, test_input): + return self._model.predict(test_input) + + def summary(self): + """ + Should be called after fit method + """ + return self._model.summary() + + + + + diff --git a/mlmc/metamodel/flow_task_GNN.py b/mlmc/metamodel/flow_task_GNN.py new file mode 100644 index 00000000..30181119 --- /dev/null +++ b/mlmc/metamodel/flow_task_GNN.py @@ -0,0 +1,176 @@ +import os +import numpy as np +import matplotlib.pyplot as plt +#os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only +import tensorflow as tf +from tensorflow.keras.regularizers import l2 + +from tensorflow.keras.losses import MeanSquaredError, SparseCategoricalCrossentropy, KLDivergence +from tensorflow.keras.metrics import mean_squared_error, kl_divergence +from tensorflow.keras.optimizers import Adam + +from mlmc.metamodel.postprocessing import analyze_results, plot_loss +from mlmc.metamodel.custom_methods import abs_activation +from spektral.data import MixedLoader +from mlmc.metamodel.flow_dataset import FlowDataset +from spektral.layers import GCNConv, GlobalSumPool, ChebConv, GraphSageConv, ARMAConv, GATConv, APPNPConv, GINConv +from spektral.utils.sparse import sp_matrix_to_sp_tensor + + + + +#conv_layer = GCNConv +conv_layer = ChebConv # Seems better than GCNConv, good distribution of predictions +#conv_layer = GraphSageConv # Seems better than ChebConv, good loss but very narrow distribution of predictions +#conv_layer = ARMAConv # Seems worse than GraphSageConv +#conv_layer = GATConv # Slow and not better than GraphSageConv +# conv_layer = APPNPConv # Not bad but worse than GraphSageConv +# conv_layer = GINConv # it is comparable to APPNPConv +act_func = "relu"#"tanh"#"elu" # ReLU keep predictions above zero + +optimizer = Adam() +loss_fn = MeanSquaredError() +#loss_fn = KLDivergence() + +acc_fn = mean_squared_error +#acc_fn = kl_divergence + +print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU'))) + +# Parameters +batch_size = 10000 # Batch size +epochs = 1000 # Number of training epochs +patience = 10 # Patience for early stopping +l2_reg = 0#5e-4 # Regularization rate for l2 + + +kernel_regularization = l2(l2_reg) + +# Create model +model = Net1(conv_layer=conv_layer, hidden_activation='relu', output_activation=abs_activation, + kernel_regularization=kernel_regularization) +#model = GeneralGNN(output=1, activation=abs_activation) + + +# Load data +data = FlowDataset() + +#print("data.a ", data.a) + +data.a = conv_layer.preprocess(data.a) +data.a = sp_matrix_to_sp_tensor(data.a) + +# Train/valid/test split +data_tr, data_te = data[:10000], data[10000:], +np.random.shuffle(data_tr) +data_tr, data_va = data_tr[:8000], data_tr[8000:] + +# We use a MixedLoader since the dataset is in mixed mode +loader_tr = MixedLoader(data_tr, batch_size=batch_size, epochs=epochs) +loader_va = MixedLoader(data_va, batch_size=batch_size) +loader_te = MixedLoader(data_te, batch_size=batch_size) + + + + + + + +# Training function +@tf.function +def train_on_batch(inputs, target): + with tf.GradientTape() as tape: + print("inputs data shape ", inputs[0].shape) # (number of train samples, number of vertices, number of properties for each vertex) + predictions = model(inputs, training=True) + #@TODO: zkusit pridat k loss function KLDivergence + # print(KLDivergence(target, predictions)) + # exit() + loss = loss_fn(target, predictions) + sum(model.losses)# + KLDivergence(target, predictions)#+ sum(model.losses) + acc = tf.reduce_mean(acc_fn(target, predictions)) + + gradients = tape.gradient(loss, model.trainable_variables) + optimizer.apply_gradients(zip(gradients, model.trainable_variables)) + return loss, acc + + +# Evaluation function +def evaluate(loader): + step = 0 + results = [] + for batch in loader: + step += 1 + inputs, target = batch + print("loader ", loader) + print("inputs data shape ", inputs[0].shape) # (number of validation or test samples, number of vertices, number of properties for each vertex) + predictions = model(inputs, training=False) + + loss = loss_fn(target, predictions) + acc = tf.reduce_mean(acc_fn(target, predictions)) + results.append((loss, acc, len(target))) # Keep track of batch size + if step == loader.steps_per_epoch: + results = np.array(results) + print("np.average(results[:, :-1], axis=0, weights=results[:, -1]) ", + np.average(results[:, :-1], axis=0, weights=results[:, -1])) + exit() + return np.average(results[:, :-1], axis=0, weights=results[:, -1]), target, predictions + + +# Setup training +best_val_loss = np.inf +current_patience = patience +step = 0 + +# Training loop +results_tr = [] +for batch in loader_tr: + step += 1 + + # Training step + inputs, target = batch + loss, acc = train_on_batch(inputs, target) + results_tr.append((loss, acc, len(target))) + + all_targets = [] + all_predictions = [] + + if step == loader_tr.steps_per_epoch: + results_va, target, predictions = evaluate(loader_va) + + print("results_va[0] ", results_va[0]) + exit() + if results_va[0] < best_val_loss: + best_val_loss = results_va[0] + current_patience = patience + results_te, target, predictions = evaluate(loader_te) + + print("target ", target) + print("predictions ", np.squeeze(predictions.numpy())) + print("len(target) ", len(target)) + print("len(predictions) ", len(np.squeeze(predictions.numpy()))) + + analyze_results(target, np.squeeze(predictions.numpy())) + + else: + current_patience -= 1 + if current_patience == 0: + print("Early stopping") + break + + # Print results + results_tr = np.array(results_tr) + results_tr = np.average(results_tr[:, :-1], 0, weights=results_tr[:, -1]) + print( + "Train loss: {:.4f}, acc: {:.4f} | " + "Valid loss: {:.4f}, acc: {:.4f} | " + "Test loss: {:.4f}, acc: {:.4f}".format( + *results_tr, *results_va, *results_te + ) + ) + + # Reset epoch + results_tr = [] + step = 0 + + + + diff --git a/mlmc/metamodel/flow_task_GNN_2.py b/mlmc/metamodel/flow_task_GNN_2.py new file mode 100644 index 00000000..ca07a93d --- /dev/null +++ b/mlmc/metamodel/flow_task_GNN_2.py @@ -0,0 +1,249 @@ +import os +import numpy as np +#os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only +import tensorflow as tf +from tensorflow.keras.losses import MeanSquaredError, SparseCategoricalCrossentropy, KLDivergence +from tensorflow.keras.metrics import mean_squared_error, kl_divergence +from tensorflow.keras.callbacks import History +from tensorflow.keras.optimizers import Adam +from tensorflow.keras.regularizers import l2 +from mlmc.metamodel.postprocessing import analyze_results, plot_loss, estimate_density +from spektral.data import MixedLoader +from mlmc.metamodel.flow_dataset import FlowDataset +from spektral.layers import GCNConv, GlobalSumPool, ChebConv, GraphSageConv, ARMAConv, GATConv, APPNPConv, GINConv +from spektral.utils.sparse import sp_matrix_to_sp_tensor +from tensorflow.keras.layers.experimental import preprocessing +from mlmc.metamodel.custom_methods import abs_activation, var_loss_function + +from mlmc.metamodel.graph_models import Net1 + + +class GNN: + def __init__(self, **kwargs): + + self._epochs = kwargs.get('epochs', 100) + #self._val_split = kwargs.get('var_split', 0.2) + #self._verbose = kwargs.get('verbose', False) + + self._hidden_activation = kwargs.get('hidden_activation', 'relu') + self._hidden_regularizer = kwargs.get('hidden_reqularizer', None) + self._output_activation = kwargs.get('output_activation', 'linear') + self._conv_layer = kwargs.get('conv_layer', None) + #self._n_hidden_layers = kwargs.get('n_hidden_layers', 1) + #self._n_hidden_neurons = kwargs.get('n_hidden_neurons', [64]) # Number of hidden neurons for each hidden layer + + self._loss = kwargs.get('loss', MeanSquaredError) + self._accuracy_func = kwargs.get('accuracy_func', mean_squared_error) + self._optimizer = kwargs.get('optimizer', tf.optimizers.Adam(learning_rate=0.001)) + self._normalizer = kwargs.get('normalizer', preprocessing.Normalization()) + self._patience = kwargs.get('patience', 20) + self._verbose = kwargs.get('verbose', True) + + self._train_loss = [] + self._val_loss = [] + self._test_loss = [] + + self.val_targets = [] + self._states = {} + self._total_n_steps = 0 + + if 'model_class' in kwargs: + model_class = kwargs.get('model_class') + net_model_config = kwargs.get('net_model_config') + model = model_class(**net_model_config) + else: + model = kwargs.get('model') + + if model is None: + self._model = Net1(conv_layer=self._conv_layer, hidden_activation=self._hidden_activation, + output_activation=self._output_activation, + kernel_regularization=self._hidden_regularizer, + normalizer=self._normalizer) + else: + self._model = model + # self._model = model(conv_layer=self._conv_layer, hidden_activation=self._hidden_activation, + # output_activation=self._output_activation, + # kernel_regularization=self._hidden_regularizer, + # normalizer=self._normalizer) + #self._model = model(n_labels=1, output_activation="relu") + + def fit(self, loader_tr, loader_va, loader_te): + """ + Training procedure + """ + # Setup training + best_val_loss = np.inf + current_patience = self._patience + step = 0 + self._total_n_steps = 0 + + train_targets = True + train_targets_list = [] + + # Training loop + results_tr = [] + for batch in loader_tr: + step += 1 + self._total_n_steps += 1 + + # Training step + inputs, target = batch + + if train_targets: + train_targets_list.extend(target) + + loss, acc = self.train_on_batch(inputs, target) + self._train_loss.append(loss) + results_tr.append((loss, acc, len(target))) + + results_va = self.evaluate(loader_va) + self._val_loss.append(results_va[0]) + + if step == loader_tr.steps_per_epoch: # step_per_epoch = int(np.ceil(len(self.dataset) / self.batch_size)) + train_targets = False + # results_va = self.evaluate(loader_va) + # self._val_loss.append(results_va[0]) + #print("results_va[0] ", results_va[0]) + + if results_va[0] < best_val_loss: + best_val_loss = results_va[0] + current_patience = self._patience + self._states = {} + results_te = self.evaluate(loader_te) + self._test_loss.append(results_te[0]) + else: + current_patience -= 1 + #results_tr_0 = np.array(results_tr) + loss_tr = results_va[0] + self._states[loss_tr] = self + if current_patience == 0: + print("Early stopping") + break + + # Print results + results_tr = np.array(results_tr) + results_tr = np.average(results_tr[:, :-1], 0, weights=results_tr[:, -1]) + if self._verbose: + print( + "Train loss: {:.4f}, acc: {:.4f} | " + "Valid loss: {:.4f}, acc: {:.4f} | " + "Test loss: {:.4f}, acc: {:.4f}".format( + *results_tr, *results_va, *results_te + ) + ) + # Reset epoch + results_tr = [] + step = 0 + + return train_targets_list + + # Training function + @tf.function + def train_on_batch(self, inputs, target): + with tf.GradientTape() as tape: + predictions = self._model(inputs, training=True) + loss = self._loss(target, predictions) + sum(self._model.losses) #+ 5 * var_loss_function(target, predictions) + #loss = 100 * var_loss_function(target, predictions) + acc = tf.reduce_mean(self._accuracy_func(target, predictions)) + + gradients = tape.gradient(loss, self._model.trainable_variables) + self._optimizer.apply_gradients(zip(gradients, self._model.trainable_variables)) + + return loss, acc + + def evaluate(self, loader): + step = 0 + results = [] + + if len(self.val_targets) > 0: + val_targets = False + else: + val_targets = True + + for batch in loader: + step += 1 + inputs, target = batch + + if val_targets: + self.val_targets.extend(target) + + predictions = self._model(inputs, training=False) + + loss = self._loss(target, predictions) + #print("target ", target) + #print("loss ", np.mean((target - predictions)**2)) + acc = tf.reduce_mean(self._accuracy_func(target, predictions)) + results.append((loss, acc, len(target))) # Keep track of batch size + if step == loader.steps_per_epoch: + results = np.array(results) + return np.average(results[:, :-1], axis=0, weights=results[:, -1]) + + def predict(self, loader): + targets = [] + predictions = [] + step = 0 + for batch in loader: + + step += 1 + inputs, target = batch + targets.extend(target) + predictions.extend(self._model(inputs, training=False)) + + if step == loader.steps_per_epoch: + return targets, predictions + + return targets, predictions + + +# if __name__ == "__main__": +# # Parameters +# #conv_layer = GCNConv +# conv_layer = ChebConv # Seems better than GCNConv, good distribution of predictions +# # # conv_layer = GraphSageConv # Seems better than ChebConv, good loss but very narrow distribution of predictions +# # # conv_layer = ARMAConv # Seems worse than GraphSageConv +# # # conv_layer = GATConv # Slow and not better than GraphSageConv +# # # conv_layer = APPNPConv # Not bad but worse than GraphSageConv +# # # conv_layer = GINConv # it is comparable to APPNPConv +# # act_func = "relu" # "tanh"#"elu" # ReLU keep predictions above zero +# loss = MeanSquaredError() +# optimizer = tf.optimizers.Adam(learning_rate=0.001) +# batch_size = 500 +# epochs = 100 +# +# # Load data +# data = FlowDataset() +# data = data#[:10000] +# #data.a = conv_layer.preprocess(data.a) +# data.a = sp_matrix_to_sp_tensor(data.a) +# +# train_data_len = int(len(data) * 0.8) +# train_data_len = 10000 +# +# # Train/valid/test split +# data_tr, data_te = data[:train_data_len], data[train_data_len:], +# np.random.shuffle(data_tr) +# +# val_data_len = int(len(data_tr) * 0.2) +# data_tr, data_va = data_tr[:-val_data_len], data_tr[-val_data_len:] +# +# print("data_tr len ", len(data_tr)) +# print("data_va len ", len(data_va)) +# print("data_te len ", len(data_te)) +# +# # We use a MixedLoader since the dataset is in mixed mode +# loader_tr = MixedLoader(data_tr, batch_size=batch_size, epochs=epochs) +# loader_va = MixedLoader(data_va, batch_size=batch_size) +# loader_te = MixedLoader(data_te, batch_size=batch_size) +# +# gnn = GNN(loss=loss, optimizer=optimizer, conv_layer=conv_layer, output_activation=abs_activation, +# hidden_activation='relu', patience=20) +# gnn.fit(loader_tr, loader_va, loader_te) +# +# targets, predictions = gnn.predict(loader_te) +# predictions = np.squeeze(predictions) +# +# plot_loss(gnn._train_loss, gnn._val_loss) +# analyze_results(targets, predictions) +# +# estimate_density(targets) +# estimate_density(predictions) diff --git a/mlmc/metamodel/flow_task_NN.py b/mlmc/metamodel/flow_task_NN.py new file mode 100644 index 00000000..98c80686 --- /dev/null +++ b/mlmc/metamodel/flow_task_NN.py @@ -0,0 +1,403 @@ +import tensorflow as tf +import numpy as np +from tensorflow.keras.metrics import mean_squared_error, kl_divergence +from tensorflow.keras.layers.experimental import preprocessing +from mlmc.metamodel.graph_models import dnn_model +from tensorflow import keras +from tensorflow.keras import layers +from tensorflow.keras.layers.experimental import preprocessing + + +# epochs = 200 +# +# +# # Parameters +# loss_fn = "mean_squared_error" +# optimizer_nn = tf.optimizers.Adam(learning_rate=0.1) +# optimizer_dnn = tf.optimizers.Adam(learning_rate=0.001) +# +# +# # Load data +# data = FlowDataset() +# +# dataset = data.dataset[:5000] +# print("len(dataset) ", len(dataset)) +# dataset = dataset.dropna() +# +# train_dataset = dataset.sample(frac=0.8, random_state=0) +# test_dataset = dataset.drop(train_dataset.index) +# +# train_x = np.squeeze(np.stack(train_dataset.x.to_numpy(), axis=0)) +# train_x = np.asarray(train_x).astype('float64') +# train_y = train_dataset.y.to_numpy() +# train_y = np.asarray(train_y).astype('float64') +# +# test_x = np.squeeze(np.stack(test_dataset.x.to_numpy(), axis=0)) +# test_x = np.asarray(test_x).astype('float32') +# test_y = test_dataset.y.to_numpy() +# test_y = np.asarray(test_y).astype('float32') + + +# #################### +# ## Neural network - very bad for our purposes ## +# #################### +# +# normalizer = preprocessing.Normalization() +# linear_model = tf.keras.Sequential([ +# normalizer, +# layers.Dense(units=1) +# ]) +# +# linear_model.compile( +# optimizer=optimizer_nn, +# loss=loss_fn) +# # +# # history = linear_model.fit( +# # train_x, train_y, +# # epochs=100, +# # # suppress logging +# # verbose=0, +# # # Calculate validation results on 20% of the training data +# # validation_split=0.2) +# # +# # linear_model.summary() +# # +# # +# # plot_loss(history) +# # predictions = np.squeeze(linear_model.predict(test_x)) +# # analyze_results(target=test_y, predictions=predictions) + + + +######################### +# Deep neural network # +######################### + + +class DNN: + def __init__(self, **kwargs): + self._epochs = kwargs.get('epochs', 100) + self._val_split = kwargs.get('var_split', 0.2) + self._verbose = kwargs.get('verbose', False) + + self._hidden_activation = kwargs.get('hidden_activation', 'relu') + self._hidden_regularizer = kwargs.get('hidden_reqularizer', None) + self._output_activation = kwargs.get('output_activation', 'linear') + self._n_hidden_layers = kwargs.get('n_hidden_layers', 1) + self._n_hidden_neurons = kwargs.get('n_hidden_neurons', [64]) # Number of hidden neurons for each hidden layer + + self._loss = kwargs.get('loss', 'mean_squared_error') + self._optimizer = kwargs.get('optimizer', tf.optimizers.Adam(learning_rate=0.001)) + self._normalizer = kwargs.get('normalizer', preprocessing.Normalization()) + + self.history = None # Set in fit method + self._create_model() + + def _create_model(self): + hidden_layers = [] + for i in range(self._n_hidden_layers): + if self._hidden_regularizer is not None: + hidden_layers.append( + layers.Dense(self._n_hidden_neurons[i], + kernel_regularizer=self._hidden_regularizer, + activation=self._hidden_activation)) + else: + hidden_layers.append( + layers.Dense(self._n_hidden_neurons[i],activation=self._hidden_activation)) + + self._model = keras.Sequential([ + self._normalizer, + *hidden_layers, + layers.Dense(1, activation=self._output_activation) + ]) + + self._model.compile(loss=self._loss, optimizer=self._optimizer) + + def fit(self, train_input, train_output): + self.history = self._model.fit(train_input, train_output, validation_split=self._val_split, + verbose=self._verbose, epochs=self._epochs) + + def predict(self, test_input): + return self._model.predict(test_input) + + def summary(self): + """ + Should be called after fit method + """ + return self._model.summary() + + +class DNN_2: + def __init__(self, **kwargs): + print("######## Create GNN #########") + + self._epochs = kwargs.get('epochs', 100) + #self._val_split = kwargs.get('var_split', 0.2) + #self._verbose = kwargs.get('verbose', False) + + self._hidden_activation = kwargs.get('hidden_activation', 'relu') + self._hidden_regularizer = kwargs.get('hidden_reqularizer', None) + self._output_activation = kwargs.get('output_activation', 'linear') + self._conv_layer = kwargs.get('conv_layer', None) + #self._n_hidden_layers = kwargs.get('n_hidden_layers', 1) + #self._n_hidden_neurons = kwargs.get('n_hidden_neurons', [64]) # Number of hidden neurons for each hidden layer + + self._loss = kwargs.get('loss', mean_squared_error) + self._final_loss = kwargs.get('final_loss', mean_squared_error) + self._accuracy_func = kwargs.get('accuracy_func', mean_squared_error) + self._optimizer = kwargs.get('optimizer', tf.optimizers.Adam(learning_rate=0.001)) + self._normalizer = kwargs.get('normalizer', preprocessing.Normalization()) + self._patience = kwargs.get('patience', 20) + self._verbose = kwargs.get('verbose', True) + #self._loss_changed = False + + self._train_loss = [] + self._val_loss = [] + self._test_loss = [] + + self._loss_params = {} + self._n_moments = 3 + + self.val_targets = [] + self._states = {} + self._total_n_steps = 0 + + if 'model_class' in kwargs: + model_class = kwargs.get('model_class') + net_model_config = kwargs.get('net_model_config') + model = model_class(**net_model_config) + print("model class model ", model) + else: + model = kwargs.get('model') + + if model is None: + self._model = DNNNet(conv_layer=self._conv_layer, hidden_activation=self._hidden_activation, + output_activation=self._output_activation, + kernel_regularization=self._hidden_regularizer, + normalizer=self._normalizer) + else: + self._model = model + # self._model = model(conv_layer=self._conv_layer, hidden_activation=self._hidden_activation, + # output_activation=self._output_activation, + # kernel_regularization=self._hidden_regularizer, + # normalizer=self._normalizer) + #self._model = model(n_labels=1, output_activation="relu") + + def fit(self, train, validation, test): + """ + Training procedure + """ + print("fit init loss ", self._loss) + # Setup training + self._best_val_loss = np.inf + self._current_patience = self._patience + step = 0 + self._total_n_steps = 0 + + train_targets = True + train_targets_list = [] + + # Training loop + results_tr = [] + + step += 1 + self._total_n_steps += 1 + + # Training step + inputs, target = train.x, train.y + + if train_targets: + train_targets_list.extend(target) + + loss, acc = self.train_on_batch(inputs, target) + results_tr.append((loss, acc, len(target))) + + results_va = self.evaluate(validation) + self._val_loss.append(results_va[0]) + + + train_targets = False + # results_va = self.evaluate(loader_va) + # self._val_loss.append(results_va[0]) + #print("results_va[0] ", results_va[0]) + + #print("self best val loss ", self._best_val_loss) + + if (results_va[0] + results_tr[-1][0]) < self._best_val_loss:# or (self._val_loss[-1] < self._val_loss[-2] and self._val_loss[-2] < self._val_loss[-3]): # Continue to learn if validation loss is decreasing + self._best_val_loss = (results_va[0] + results_tr[-1][0])#results_va[0] + self._current_patience = self._patience + self._states = {} + results_te = self.evaluate(test) + self._test_loss.append(results_te[0]) + else: + self._current_patience -= 1 + #results_tr_0 = np.array(results_tr) + loss_tr = results_va[0] + self._states[loss_tr] = self + + if self._current_patience == 0: + #if self._update_loss(patience=True): + print("Early stopping") + return + + # Print results + results_tr = np.array(results_tr) + results_tr = np.average(results_tr[:, :-1], 0, weights=results_tr[:, -1]) + + self._train_loss.append(results_tr[0]) + if self._verbose: + print( + "Train loss: {:.4f}, acc: {:.4f} | " + "Valid loss: {:.4f}, acc: {:.4f} | " + "Test loss: {:.4f}, acc: {:.4f}".format( + *results_tr, *results_va, *results_te + ) + ) + #self._update_loss() + # Reset epoch + results_tr = [] + step = 0 + + return train_targets_list + + def _update_loss(self, patience=False): + condition_max_loss = self._loss_params["loss_max"] #/ self._n_moments + #condition_max_loss = self._loss_params["loss_max"] + # print("self.train_loss ", self._train_loss) + m_increment = 1 + + if patience and self._n_moments <= self._loss_params["max_moments"]: + self._n_moments += m_increment + moments_fn = self._loss_params['moments_class'](self._n_moments, self._loss_params["domain"]) + self._loss = self._final_loss(moments_fn=moments_fn) + # self._loss = MSE_moments_2(moments_fn=moments_fn) + + self._best_val_loss = np.inf + print("self._loss ", self._loss) + elif patience: + return True + + # if self._train_loss[-1] > 1e10: + # moments_fn = self._loss_params['moments_class'](self._n_moments, self._loss_params["domain"]) + # #self._loss = self._final_loss(moments_fn=moments_fn) + # self._loss = MSE_moments_2(moments_fn=moments_fn) + # else: + # moments_fn = self._loss_params['moments_class'](self._n_moments, self._loss_params["domain"]) + # self._loss = self._final_loss(moments_fn=moments_fn) + + if self._n_moments <= self._loss_params["max_moments"] and len(self._train_loss) > 0\ + and self._train_loss[-1] < condition_max_loss and self._val_loss[-1] < condition_max_loss: + # print("self._train_loss ", self._train_loss) + # print("change loss, n_moments {}, last train loss: {}".format(self._n_moments, self._train_loss[-1])) + #self._n_moments = self._loss_params["max_moments"] + + print("self._n_moments ", self._n_moments) + self._n_moments += m_increment + moments_fn = self._loss_params['moments_class'](self._n_moments, self._loss_params["domain"]) + self._loss = self._final_loss(moments_fn=moments_fn) + #self._loss = MSE_moments_2(moments_fn=moments_fn) + + self._best_val_loss = np.inf + print("self._loss ", self._loss) + + # Training function + #@tf.function + def train_on_batch(self, inputs, target): + with tf.GradientTape() as tape: + predictions = self._model(inputs, training=True) + loss = self._loss(target, predictions) + sum(self._model.losses) #+ 5 * var_loss_function(target, predictions) + #loss = 100 * var_loss_function(target, predictions) + acc = tf.reduce_mean(self._accuracy_func(target, predictions)) + + gradients = tape.gradient(loss, self._model.trainable_variables) + self._optimizer.apply_gradients(zip(gradients, self._model.trainable_variables)) + + return loss, acc + + def evaluate(self, data): + step = 0 + results = [] + + if len(self.val_targets) > 0: + val_targets = False + else: + val_targets = True + + step += 1 + inputs, target = data.x, data.y + + if val_targets: + self.val_targets.extend(target) + + predictions = self._model(inputs, training=False) + + #print("evaluate loss function ", self._loss) + + loss = self._loss(target, predictions) + #print("target ", target) + #print("loss ", np.mean((target - predictions)**2)) + acc = tf.reduce_mean(self._accuracy_func(target, predictions)) + results.append((loss, acc, len(target))) # Keep track of batch size + results = np.array(results) + return np.average(results[:, :-1], axis=0, weights=results[:, -1]) + + def predict(self, data): + targets = [] + predictions = [] + + inputs, target = data.x, data.y + targets.extend(target) + predictions.extend(self._model(inputs, training=False)) + + return targets, predictions + + +# def build_and_compile_model(normalizer): +# model = keras.Sequential([ +# normalizer, +# layers.Dense(450, activation='relu'), #64 +# #layers.Dense(64, activation='relu'), +# layers.Dense(1, activation=abs_activation) +# ]) +# +# model.compile(loss=loss_fn, +# optimizer=optimizer_dnn) +# return model +# +# +# normalizer = preprocessing.Normalization() +# dnn_model = build_and_compile_model(normalizer) +# dnn_history = dnn_model.fit( +# train_x, train_y, +# validation_split=0.2, +# verbose=0, epochs=epochs) +# dnn_model.summary() +# +# plot_loss(dnn_history) +# +# predictions = np.squeeze(dnn_model.predict(test_x)) +# +# print("target ", test_y) +# print("predictions ", predictions) +# +# +# for index, (t, p) in enumerate(zip(test_y, predictions)): +# if index > 100: +# break +# print("t: {}, p: {}".format(t, p)) +# +# print("target mean ", np.mean(test_y)) +# print("predictions mean ", np.mean(predictions)) +# +# print("target var ", np.var(test_y)) +# print("predictions var ", np.var(predictions)) +# +# analyze_results(target=test_y, predictions=predictions) + + + + + + + + diff --git a/mlmc/metamodel/graph_models.py b/mlmc/metamodel/graph_models.py new file mode 100644 index 00000000..beb7aea7 --- /dev/null +++ b/mlmc/metamodel/graph_models.py @@ -0,0 +1,128 @@ +from tensorflow.keras import Model +from tensorflow.keras.layers import Dense +from spektral.layers import GlobalSumPool, GlobalMaxPool, GlobalAvgPool +import tensorflow as tf +from tensorflow import keras +from tensorflow.keras import layers +from mlmc.metamodel.custom_methods import abs_activation +from tensorflow.keras.layers.experimental import preprocessing + + +# Build model +class Net1(Model): + def __init__(self, conv_layer, hidden_activation, output_activation, kernel_regularization, normalizer, **kwargs): + super().__init__(**kwargs) + #self.normalizer = normalizer + #self.norm_layer = tf.keras.layers.LayerNormalization(axis=1) + self.conv1 = conv_layer(256, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv2 = conv_layer(128, K=2, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv3 = conv_layer(16, K=2, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv4 = conv_layer(8, K=2,activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv2 = conv_layer(32, K=2, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv3 = conv_layer(16, K=2, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv3 = conv_layer(8, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv4 = conv_layer(4, activation=hidden_activation, kernel_regularizer=kernel_regularization) + #self.conv3 = conv_layer(64, activation=hidden_activation, kernel_regularizer=kernel_regularization) + self.flatten = GlobalSumPool() + #self.fc1 = Dense(32, activation=hidden_activation) + self.fc2 = Dense(1)#, activation=output_activation) # linear activation for output neuron + + def call(self, inputs): + x, a = inputs + #print("x ", x) + #x = self.normalizer(x) + #x = self.norm_layer(x) + #print("normalized x ", x) + + #print("x[0,0,:] ", x[0, 0, :]) + x = self.conv1([x, a]) + + #print("x[0,0,:] ", x[0,0,:]) + # print("x[0, 0, :] ", tf.make_ndarray(x[0,0,:].op.get_attr('net1/strided_slice_1:0'))) + #print("x.shape ", x.shape) + # x = self.conv2([x, a]) + # # print("conv2 x shape", x.shape) + # x = self.conv3([x, a]) + # x = self.conv4([x, a]) + output1 = self.flatten(x) + #output2 = self.fc1(output1) + output = self.fc2(output1) + + # print("x1 " ,x1) + # print("output1 ", output1) + # print("output2 ", output2) + # print("output ", output) + #print("output ", output.shape) + + return output + + +# Build model +class NetGCN(Model): + # Setup from https://arxiv.org/pdf/1901.06181.pdf + def __init__(self, conv_layer, hidden_activation, output_activation, kernel_regularization, normalizer, **kwargs): + super().__init__(**kwargs) + + #self.normalizer = normalizer + #self.norm_layer = tf.keras.layers.LayerNormalization(axis=1) + self.conv1 = conv_layer(256, activation=hidden_activation, kernel_regularizer=kernel_regularization) + #.conv2 = conv_layer(32, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv3 = conv_layer(16, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv4 = conv_layer(16, activation=hidden_activation, kernel_regularizer=kernel_regularization) + # self.conv5 = conv_layer(32, activation=hidden_activation, kernel_regularizer=kernel_regularization) + self.flatten = GlobalSumPool() + #self.fc1 = Dense(16, activation=hidden_activation) + self.fc2 = Dense(1)#, activation=output_activation) # linear activation for output neuron + + def call(self, inputs): + x, a = inputs + #print("x ", x) + #x = self.normalizer(x) + #x = self.norm_layer(x) + #print("normalized x ", x) + + #print("x[0,0,:] ", x[0, 0, :]) + x = self.conv1([x, a]) + #x = self.conv2([x, a]) + # x = self.conv3([x, a]) + # x = self.conv4([x, a]) + # x = self.conv5([x, a]) + # + # #print("x[0,0,:] ", x[0,0,:]) + # print("x[0, 0, :] ", tf.make_ndarray(x[0,0,:].op.get_attr('net1/strided_slice_1:0'))) + # print("x.shape ", x.shape) + # x = self.conv2([x, a]) + # x = self.conv3([x, a]) + output = self.flatten(x) + #output = self.fc1(output) + output = self.fc2(output) + + #print("output ", output.shape) + + return output + + +def cnn_model(): + return keras.Sequential([ + #@TODO: Try normalization + #self._normalizer, # Seems worse results with normalization + layers.Conv1D(filters=256, kernel_size=3, activation='relu', input_shape=(958, 1)),#input_shape=(958, 1)), + #layers.BatchNormalization(), + layers.AveragePooling1D(pool_size=2), + layers.Conv1D(filters=128, kernel_size=3, activation='relu'), + #layers.BatchNormalization(), + layers.AveragePooling1D(pool_size=2), + + layers.Flatten(), + layers.Dense(64, activation='relu'), + layers.Dense(1, activation=abs_activation) + ]) + + +def dnn_model(): + return keras.Sequential([ + preprocessing.Normalization(), + layers.Dense(32, activation="relu"), + #layers.Dense(32, activation="relu"), + layers.Dense(1, activation=abs_activation) + ]) \ No newline at end of file diff --git a/mlmc/metamodel/main.py b/mlmc/metamodel/main.py new file mode 100644 index 00000000..464fd9ae --- /dev/null +++ b/mlmc/metamodel/main.py @@ -0,0 +1,149 @@ +import os +import numpy as np +import pandas as pd +from mlmc.tool.hdf5 import HDF5 +import matplotlib.pyplot as plt + +from sklearn.model_selection import train_test_split + +DATA_PATH = "/home/martin/Documents/metamodels/data" + +def get_inputs(dir): + fields_file = os.path.join(dir, "fine_fields_sample.msh") + input = [] + with open(fields_file, "r") as r: + fields = r.readlines() + for f in fields[12:]: + line = f.split(" ") + if len(line) > 1: + input.append(float(line[1])) + else: + break + return input + + +def preprocess_data(): + dir_path = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/output" + hdf = HDF5(file_path="/home/martin/Documents/metamodels/data/L1/test/01_cond_field/mlmc_1.hdf5", + load_from_file=True) + level_group = hdf.add_level_group(level_id=str(0)) + collected = zip(level_group.get_collected_ids(), level_group.collected()) + + df_values = [] + for sample_id, col_values in collected: + output_value = col_values[0, 0] + sample_dir = os.path.join(dir_path, sample_id) + if os.path.isdir(sample_dir): + input = get_inputs(sample_dir) + d = {'x': np.array(input), 'y': output_value} + df_values.append(d) + + df = pd.DataFrame(df_values) + df.to_pickle(os.path.join(DATA_PATH, "data.pkl")) + + +def load_data(): + df = pd.read_pickle(os.path.join(DATA_PATH, "data.pkl")) + return df + + +def data_analysis(df): + print(df.info()) + print(df.y.describe()) + + # df.y.plot.hist(bins=50, logx=True) + # plt.show() + df.y.plot.kde(bw_method=0.3) + plt.xlim([-5, df.y.max()]) + plt.show() + + +def support_vector_regression(df): + from sklearn.svm import SVR + from sklearn.preprocessing import StandardScaler + print("df. info ", df.info) + train, test = train_test_split(df[:8000], test_size=0.2) + print("train describe", train.describe()) + print("test describe ", test.describe()) + + x = np.stack(train.x.to_numpy(), axis=0) + y = train.y.to_numpy() + + sc_X = StandardScaler() + sc_y = StandardScaler() + x = sc_X.fit_transform(x) + # print("y.shape ", y.shape) + # print("y.reshape(-1, 1) ", y.reshape(-1, 1).shape) + y = sc_y.fit_transform(y.reshape(-1, 1)) + + test_x = sc_X.fit_transform(np.stack(test.x.to_numpy(), axis=0)) + test_y = test.y.to_numpy().reshape(-1, 1) + test_y = sc_y.fit_transform(test.y.to_numpy().reshape(-1, 1)) + + # plt.hist(y, bins=50, alpha=0.5, label='train', density=True) + # plt.hist(test_y, bins=50, alpha=0.5, label='test', density=True) + # plt.legend(loc='upper right') + # # plt.xlim(-0.5, 1000) + # plt.yscale('log') + # plt.show() + # exit() + + # print("x.shape ", x.shape) + # print("y.shape ", y.shape) + # exit() + + #svr_rbf = SVR(kernel='rbf', verbose=True) # 'linear' kernel fitting is never-ending and 'poly' kernel gives very bad score (e.g. -2450), sigmoid gives also bad score (e.g. -125) + svr_rbf = SVR(kernel='poly', degree=50, verbose=True) + svr_rbf.fit(x, y) + + train_error = svr_rbf.score(x, y) + + test_error = svr_rbf.score(test_x, test_y) + + predictions = svr_rbf.predict(test_x) + + plt.hist(test_y, bins=50, alpha=0.5, label='target', density=True) + plt.hist(predictions, bins=50, alpha=0.5, label='predictions', density=True) + + # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) + plt.legend(loc='upper right') + # plt.xlim(-0.5, 1000) + plt.yscale('log') + plt.show() + + plt.hist(test_y, bins=50, alpha=0.5, label='target', density=True) + plt.hist(predictions, bins=50, alpha=0.5, label='predictions', density=True) + + # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) + plt.legend(loc='upper right') + # plt.xlim(-0.5, 1000) + #plt.yscale('log') + plt.show() + + exit() + + print("train error ", train_error) + print("test error ", test_error) + + +def svr_run(): + # preprocess_data() + df = load_data() + # data_analysis(df) + support_vector_regression(df) + + +if __name__ == "__main__": + # import cProfile + # import pstats + # pr = cProfile.Profile() + # pr.enable() + svr_run() + # my_result = svr_run() + # + # pr.disable() + # ps = pstats.Stats(pr).sort_stats('cumtime') + # ps.print_stats() + + + diff --git a/mlmc/metamodel/mnist.py b/mlmc/metamodel/mnist.py new file mode 100644 index 00000000..59a2158c --- /dev/null +++ b/mlmc/metamodel/mnist.py @@ -0,0 +1,167 @@ +import numpy as np +import tensorflow as tf +from tensorflow.keras import Model +from tensorflow.keras.layers import Dense +from tensorflow.keras.losses import SparseCategoricalCrossentropy +from tensorflow.keras.metrics import sparse_categorical_accuracy +from tensorflow.keras.optimizers import Adam +from tensorflow.keras.regularizers import l2 + +from spektral.data import MixedLoader +from spektral.datasets.mnist import MNIST +from spektral.layers import GCNConv, GlobalSumPool +from spektral.layers.ops import sp_matrix_to_sp_tensor + + + +print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU'))) + + +# Parameters +batch_size = 32 # Batch size +epochs = 1000 # Number of training epochs +patience = 10 # Patience for early stopping +l2_reg = 5e-4 # Regularization rate for l2 + +# Load data +data = MNIST() + +print("data.a ", data.a) + + +# The adjacency matrix is stored as an attribute of the dataset. +# Create filter for GCN and convert to sparse tensor. +data.a = GCNConv.preprocess(data.a) +data.a = sp_matrix_to_sp_tensor(data.a) + + +# Train/valid/test split +data_tr, data_te = data[:-10000], data[-10000:] +np.random.shuffle(data_tr) +data_tr, data_va = data_tr[:-10000], data_tr[-10000:] + + + +# for tr in data_tr[:10]: +# #print(tr.x) +# print(tr.y) +# exit() + +print("data_tr[0] ", data_tr[0].n_node_features) + + +# We use a MixedLoader since the dataset is in mixed mode +loader_tr = MixedLoader(data_tr, batch_size=batch_size, epochs=epochs) +loader_va = MixedLoader(data_va, batch_size=batch_size) +loader_te = MixedLoader(data_te, batch_size=batch_size) + + +# Build model +class Net(Model): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.conv1 = GCNConv(32, activation="elu", kernel_regularizer=l2(l2_reg)) + self.conv2 = GCNConv(32, activation="elu", kernel_regularizer=l2(l2_reg)) + self.flatten = GlobalSumPool() + self.fc1 = Dense(512, activation="relu") + self.fc2 = Dense(10, activation="softmax") # MNIST has 10 classes + + def call(self, inputs): + x, a = inputs + x = self.conv1([x, a]) + x = self.conv2([x, a]) + output = self.flatten(x) + output = self.fc1(output) + output = self.fc2(output) + + return output + + +# Create model +model = Net() +optimizer = Adam() +loss_fn = SparseCategoricalCrossentropy() + + +# Training function +@tf.function +def train_on_batch(inputs, target): + with tf.GradientTape() as tape: + predictions = model(inputs, training=True) + + print("predictions ", predictions) + print("target" , target) + loss = loss_fn(target, predictions) + sum(model.losses) + acc = tf.reduce_mean(sparse_categorical_accuracy(target, predictions)) + + gradients = tape.gradient(loss, model.trainable_variables) + optimizer.apply_gradients(zip(gradients, model.trainable_variables)) + return loss, acc + + +# Evaluation function +def evaluate(loader): + step = 0 + results = [] + for batch in loader: + step += 1 + inputs, target = batch + predictions = model(inputs, training=False) + loss = loss_fn(target, predictions) + acc = tf.reduce_mean(sparse_categorical_accuracy(target, predictions)) + results.append((loss, acc, len(target))) # Keep track of batch size + if step == loader.steps_per_epoch: + results = np.array(results) + return np.average(results[:, :-1], 0, weights=results[:, -1]) + + +# Setup training +best_val_loss = 99999 +current_patience = patience +step = 0 + +# Training loop +results_tr = [] +for batch in loader_tr: + step += 1 + + # Training step + inputs, target = batch + + # print("inputs ", inputs) + # print("target ", target) + # + # print("len(inputs) ", len(inputs)) + # print("len(target) ", len(target)) + # print("inputs shape ", np.array(inputs).shape) + # exit() + + loss, acc = train_on_batch(inputs, target) + results_tr.append((loss, acc, len(target))) + + if step == loader_tr.steps_per_epoch: + results_va = evaluate(loader_va) + if results_va[0] < best_val_loss: + best_val_loss = results_va[0] + current_patience = patience + results_te = evaluate(loader_te) + else: + current_patience -= 1 + if current_patience == 0: + print("Early stopping") + break + + # Print results + results_tr = np.array(results_tr) + results_tr = np.average(results_tr[:, :-1], 0, weights=results_tr[:, -1]) + print( + "Train loss: {:.4f}, acc: {:.4f} | " + "Valid loss: {:.4f}, acc: {:.4f} | " + "Test loss: {:.4f}, acc: {:.4f}".format( + *results_tr, *results_va, *results_te + ) + ) + + # Reset epoch + results_tr = [] + step = 0 diff --git a/mlmc/metamodel/own_cheb_conv.py b/mlmc/metamodel/own_cheb_conv.py new file mode 100644 index 00000000..1d830e1a --- /dev/null +++ b/mlmc/metamodel/own_cheb_conv.py @@ -0,0 +1,378 @@ +import copy +import warnings + +import numpy as np +from scipy import linalg +from scipy import sparse as sp +from scipy.sparse.linalg import ArpackNoConvergence + +import tensorflow as tf +from tensorflow.keras import backend as K + +from spektral.layers import ops +from spektral.layers.convolutional.conv import Conv +import matplotlib.pyplot as plt + + +class OwnChebConv(Conv): + r""" + A Chebyshev convolutional layer from the paper + + > [Convolutional Neural Networks on Graphs with Fast Localized Spectral + Filtering](https://arxiv.org/abs/1606.09375)
+ > Michaël Defferrard et al. + + **Mode**: single, disjoint, mixed, batch. + + This layer computes: + $$ + \X' = \sum \limits_{k=0}^{K - 1} \T^{(k)} \W^{(k)} + \b^{(k)}, + $$ + where \( \T^{(0)}, ..., \T^{(K - 1)} \) are Chebyshev polynomials of \(\tilde \L\) + defined as + $$ + \T^{(0)} = \X \\ + \T^{(1)} = \tilde \L \X \\ + \T^{(k \ge 2)} = 2 \cdot \tilde \L \T^{(k - 1)} - \T^{(k - 2)}, + $$ + where + $$ + \tilde \L = \frac{2}{\lambda_{max}} \cdot (\I - \D^{-1/2} \A \D^{-1/2}) - \I. + $$ + + **Input** + + - Node features of shape `([batch], n_nodes, n_node_features)`; + - A list of K Chebyshev polynomials of shape + `[([batch], n_nodes, n_nodes), ..., ([batch], n_nodes, n_nodes)]`; can be computed with + `spektral.utils.convolution.chebyshev_filter`. + + **Output** + + - Node features with the same shape of the input, but with the last + dimension changed to `channels`. + + **Arguments** + + - `channels`: number of output channels; + - `K`: order of the Chebyshev polynomials; + - `activation`: activation function; + - `use_bias`: bool, add a bias vector to the output; + - `kernel_initializer`: initializer for the weights; + - `bias_initializer`: initializer for the bias vector; + - `kernel_regularizer`: regularization applied to the weights; + - `bias_regularizer`: regularization applied to the bias vector; + - `activity_regularizer`: regularization applied to the output; + - `kernel_constraint`: constraint applied to the weights; + - `bias_constraint`: constraint applied to the bias vector. + + """ + + def __init__( + self, + channels, + K=1, + activation=None, + use_bias=True, + kernel_initializer="glorot_uniform", + bias_initializer="zeros", + kernel_regularizer=None, + bias_regularizer=None, + activity_regularizer=None, + kernel_constraint=None, + bias_constraint=None, + **kwargs + ): + super().__init__( + activation=activation, + use_bias=use_bias, + kernel_initializer=kernel_initializer, + bias_initializer=bias_initializer, + kernel_regularizer=kernel_regularizer, + bias_regularizer=bias_regularizer, + activity_regularizer=activity_regularizer, + kernel_constraint=kernel_constraint, + bias_constraint=bias_constraint, + **kwargs + ) + self.channels = channels + self.K = K + + # self.use_bias = use_bias + + def build(self, input_shape): + assert len(input_shape) >= 2 + input_dim = input_shape[0][-1] + + self.kernel = self.add_weight( + shape=(self.K, input_dim, self.channels), + initializer=self.kernel_initializer, + name="kernel", + regularizer=self.kernel_regularizer, + constraint=self.kernel_constraint, + ) + if self.use_bias: + self.bias = self.add_weight( + shape=(self.channels,), + initializer=self.bias_initializer, + name="bias", + regularizer=self.bias_regularizer, + constraint=self.bias_constraint, + ) + else: + self.bias = None + self.built = True + + def call(self, inputs): + x, a = inputs + + T_0 = x + output = K.dot(T_0, self.kernel[0]) + + if self.K > 1: + T_1 = ops.modal_dot(a, x) + output += K.dot(T_1, self.kernel[1]) + + # print("T_1 ", T_1) + # print("self.kernel[1] ", self.kernel[1]) + + for k in range(2, self.K): + T_2 = 2 * ops.modal_dot(a, T_1) - T_0 + output += K.dot(T_2, self.kernel[k]) + T_0, T_1 = T_1, T_2 + + #print("self use bias ", self.use_bias) + if self.use_bias: + #print("use bias") + output = K.bias_add(output, self.bias) + output = self.activation(output) + + return output + + @property + def config(self): + return {"channels": self.channels, "K": self.K} + + @staticmethod + def preprocess(a): + a = normalized_laplacian(a) + a = rescale_laplacian(a) + return a + + +def degree_matrix(A): + """ + Computes the degree matrix of the given adjacency matrix. + :param A: rank 2 array or sparse matrix. + :return: if A is a dense array, a dense array; if A is sparse, a sparse + matrix in DIA format. + """ + degrees = np.array(A.sum(1)).flatten() + if sp.issparse(A): + D = sp.diags(degrees) + else: + D = np.diag(degrees) + return D + + +def degree_power(A, k): + r""" + Computes \(\D^{k}\) from the given adjacency matrix. Useful for computing + normalised Laplacian. + :param A: rank 2 array or sparse matrix. + :param k: exponent to which elevate the degree matrix. + :return: if A is a dense array, a dense array; if A is sparse, a sparse + matrix in DIA format. + """ + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + degrees = np.power(np.array(A.sum(1)), k).ravel() + degrees[np.isinf(degrees)] = 0.0 + if sp.issparse(A): + D = sp.diags(degrees) + else: + D = np.diag(degrees) + return D + + +def normalized_adjacency(A, symmetric=True): + r""" + Normalizes the given adjacency matrix using the degree matrix as either + \(\D^{-1}\A\) or \(\D^{-1/2}\A\D^{-1/2}\) (symmetric normalization). + :param A: rank 2 array or sparse matrix; + :param symmetric: boolean, compute symmetric normalization; + :return: the normalized adjacency matrix. + """ + if symmetric: + print("symmetric") + normalized_D = degree_power(A, -0.5) + print("normalized D") + return normalized_D.dot(A).dot(normalized_D) + else: + normalized_D = degree_power(A, -1.0) + return normalized_D.dot(A) + + +def laplacian(A): + r""" + Computes the Laplacian of the given adjacency matrix as \(\D - \A\). + :param A: rank 2 array or sparse matrix; + :return: the Laplacian. + """ + return degree_matrix(A) - A + + +def normalized_laplacian(A, symmetric=True): + r""" + Computes a normalized Laplacian of the given adjacency matrix as + \(\I - \D^{-1}\A\) or \(\I - \D^{-1/2}\A\D^{-1/2}\) (symmetric normalization). + :param A: rank 2 array or sparse matrix; + :param symmetric: boolean, compute symmetric normalization; + :return: the normalized Laplacian. + """ + if sp.issparse(A): + I = sp.eye(A.shape[-1], dtype=A.dtype) + else: + I = np.eye(A.shape[-1], dtype=A.dtype) + normalized_adj = normalized_adjacency(A, symmetric=symmetric) + return I - normalized_adj + + +def rescale_laplacian(L, lmax=None): + """ + Rescales the Laplacian eigenvalues in [-1,1], using lmax as largest eigenvalue. + :param L: rank 2 array or sparse matrix; + :param lmax: if None, compute largest eigenvalue with scipy.linalg.eisgh. + If the eigendecomposition fails, lmax is set to 2 automatically. + If scalar, use this value as largest eigenvalue when rescaling. + :return: + """ + if lmax is None: + try: + if sp.issparse(L): + lmax = sp.linalg.eigsh(L, 1, which="LM", return_eigenvectors=False)[0] + else: + n = L.shape[-1] + lmax = linalg.eigh(L, eigvals_only=True, eigvals=[n - 2, n - 1])[-1] + except ArpackNoConvergence: + lmax = 2 + if sp.issparse(L): + I = sp.eye(L.shape[-1], dtype=L.dtype) + else: + I = np.eye(L.shape[-1], dtype=L.dtype) + L_scaled = (2.0 / lmax) * L - I + return L_scaled + + +def gcn_filter(A, symmetric=True): + r""" + Computes the graph filter described in + [Kipf & Welling (2017)](https://arxiv.org/abs/1609.02907). + :param A: array or sparse matrix with rank 2 or 3; + :param symmetric: boolean, whether to normalize the matrix as + \(\D^{-\frac{1}{2}}\A\D^{-\frac{1}{2}}\) or as \(\D^{-1}\A\); + :return: array or sparse matrix with rank 2 or 3, same as A; + """ + out = copy.deepcopy(A) + if isinstance(A, list) or (isinstance(A, np.ndarray) and A.ndim == 3): + for i in range(len(A)): + out[i] = A[i] + out[i][np.diag_indices_from(out[i])] += 1 + out[i] = normalized_adjacency(out[i], symmetric=symmetric) + else: + if hasattr(out, "tocsr"): + out = out.tocsr() + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + out[np.diag_indices_from(out)] += 1 + out = normalized_adjacency(out, symmetric=symmetric) + + if sp.issparse(out): + out.sort_indices() + return out + + +def chebyshev_polynomial(X, k): + """ + Calculates Chebyshev polynomials of X, up to order k. + :param X: rank 2 array or sparse matrix; + :param k: the order up to which compute the polynomials, + :return: a list of k + 1 arrays or sparse matrices with one element for each + degree of the polynomial. + """ + T_k = list() + if sp.issparse(X): + T_k.append(sp.eye(X.shape[0], dtype=X.dtype).tocsr()) + else: + T_k.append(np.eye(X.shape[0], dtype=X.dtype)) + T_k.append(X) + + def chebyshev_recurrence(T_k_minus_one, T_k_minus_two, X): + if sp.issparse(X): + X_ = sp.csr_matrix(X, copy=True) + else: + X_ = np.copy(X) + return 2 * X_.dot(T_k_minus_one) - T_k_minus_two + + for _ in range(2, k + 1): + T_k.append(chebyshev_recurrence(T_k[-1], T_k[-2], X)) + + return T_k + + +def chebyshev_filter(A, k, symmetric=True): + r""" + Computes the Chebyshev filter from the given adjacency matrix, as described + in [Defferrard et at. (2016)](https://arxiv.org/abs/1606.09375). + :param A: rank 2 array or sparse matrix; + :param k: integer, the order of the Chebyshev polynomial; + :param symmetric: boolean, whether to normalize the matrix as + \(\D^{-\frac{1}{2}}\A\D^{-\frac{1}{2}}\) or as \(\D^{-1}\A\); + :return: a list of k + 1 arrays or sparse matrices with one element for each + degree of the polynomial. + """ + normalized_adj = normalized_adjacency(A, symmetric) + if sp.issparse(A): + I = sp.eye(A.shape[0], dtype=A.dtype) + else: + I = np.eye(A.shape[0], dtype=A.dtype) + L = I - normalized_adj # Compute Laplacian + + # Rescale Laplacian + L_scaled = rescale_laplacian(L) + + # Compute Chebyshev polynomial approximation + T_k = chebyshev_polynomial(L_scaled, k) + + # Sort indices + if sp.issparse(T_k[0]): + for i in range(len(T_k)): + T_k[i].sort_indices() + + return T_k + + +def add_self_loops(a, value=1): + """ + Sets the inner diagonals of `a` to `value`. + :param a: a np.array or scipy.sparse matrix, the innermost two dimensions + must be equal. + :param value: value to set the diagonals to. + :return: a np.array or scipy.sparse matrix with the same shape as `a`. + """ + a = a.copy() + if len(a.shape) < 2: + raise ValueError("a must have at least rank 2") + n = a.shape[-1] + if n != a.shape[-2]: + raise ValueError( + "Innermost two dimensions must be equal. Got {}".format(a.shape) + ) + if sp.issparse(a): + a = a.tolil() + a.setdiag(value) + return a.tocsr() + else: + idx = np.arange(n) + a[..., idx, idx] = value + return a diff --git a/mlmc/metamodel/postprocessing.py b/mlmc/metamodel/postprocessing.py new file mode 100644 index 00000000..c438de7f --- /dev/null +++ b/mlmc/metamodel/postprocessing.py @@ -0,0 +1,965 @@ +import os +import random +import copy +import matplotlib.pyplot as plt +from scipy.stats import ks_2samp +#from mlmc.tool import plot +#import mlmc.tool.simple_distribution +import mlmc.estimator +from mlmc.tool import gmsh_io +import mlmc.quantity_estimate as qe +from mlmc.sample_storage import Memory +from mlmc.quantity_spec import QuantitySpec, ChunkSpec +import numpy as np +from mlmc.sample_storage_hdf import SampleStorageHDF +from mlmc.moments import Legendre, Monomial +from mlmc.quantity import make_root_quantity +from mlmc.metamodel.create_graph import extract_mesh_gmsh_io +#import mlmc.tool.simple_distribution + +QUANTILE = 0.01 + + +def plot_loss(train_loss, val_loss): + plt.plot(train_loss, label='loss') + plt.plot(val_loss, label='val_loss') + #plt.ylim([0, 10]) + plt.yscale("log") + plt.xlabel('Epoch') + plt.ylabel('Error') + plt.legend() + plt.grid(True) + plt.show() + + +def analyze_results(target, predictions): + #statistics, pvalue = ks_2samp(target, predictions) + + print("Target mean: {}, var: {}, Q25: {}, Q50: {}, Q75: {}".format(np.mean(target), + np.var(target), + np.quantile(target, 0.25), + np.quantile(target, 0.5), + np.quantile(target, 0.75))) + print("Predic mean: {}, var: {}, Q25: {}, Q50: {}, Q75: {}".format(np.mean(predictions), + np.var(predictions), + np.quantile(predictions, 0.25), + np.quantile(predictions, 0.5), + np.quantile(predictions, 0.75))) + + #print("KS statistics: {}, pvalue: {}".format(statistics, pvalue)) + # The closer KS statistic is to 0 the more likely it is that the two samples were drawn from the same distribution + + plt.hist(target, alpha=0.5, label='target', density=True) + plt.hist(predictions, alpha=0.5, label='predictions', density=True) + plt.legend(loc='upper right') + plt.show() + + +def estimate_density(values, title="Density"): + sample_storage = Memory() + n_levels = 1 + n_moments = 25 + distr_accuracy = 1e-7 + + distr_plot = plot.Distribution(title=title, + log_density=True) + + result_format = [QuantitySpec(name="flow", unit="m", shape=(1,), times=[0], locations=['0'])] + + sample_storage.save_global_data(result_format=result_format, level_parameters=np.ones(n_levels)) + + successful_samples = {} + failed_samples = {} + n_ops = {} + n_successful = len(values) + for l_id in range(n_levels): + sizes = [] + for quantity_spec in result_format: + sizes.append(np.prod(quantity_spec.shape) * len(quantity_spec.times) * len(quantity_spec.locations)) + + # Dict[level_id, List[Tuple[sample_id:str, Tuple[fine_result: ndarray, coarse_result: ndarray]]]] + successful_samples[l_id] = [] + for sample_id in range(len(values)): + successful_samples[l_id].append((str(sample_id), (values[sample_id], 0))) + + n_ops[l_id] = [random.random(), n_successful] + + sample_storage.save_scheduled_samples(l_id, samples=["S{:07d}".format(i) for i in range(n_successful)]) + + sample_storage.save_samples(successful_samples, failed_samples) + sample_storage.save_n_ops(list(n_ops.items())) + + quantity = make_root_quantity(storage=sample_storage, q_specs=result_format) + length = quantity['flow'] + time = length[0] + location = time['0'] + value_quantity = location[0] + + quantile = QUANTILE + true_domain = mlmc.estimator.Estimate.estimate_domain(value_quantity, sample_storage, quantile=quantile) + moments_fn = Legendre(n_moments, true_domain) + + estimator = mlmc.estimator.Estimate(quantity=value_quantity, sample_storage=sample_storage, moments_fn=moments_fn) + + reg_param = 0 + target_var = 1e-4 + distr_obj, info, result, moments_fn = estimator.construct_density( + tol=distr_accuracy, + reg_param=reg_param, + orth_moments_tol=target_var) + + samples = value_quantity.samples(ChunkSpec(level_id=0, n_samples=sample_storage.get_n_collected()[0]))[..., 0] + + distr_plot.add_raw_samples(np.squeeze(samples)) + + distr_plot.add_distribution(distr_obj, label="") + + # kl = mlmc.tool.simple_distribution.KL_divergence(self.cut_distr.pdf, distr_obj.density, + # self.cut_distr.domain[0], self.cut_distr.domain[1]) + #kl_divergences.append(kl) + + distr_plot.show(file=None) + + + return estimator.estimate_moments() + + +def create_quantity(target, predictions): + sample_storage = Memory() + n_levels = 2 + + result_format = [QuantitySpec(name="conductivity", unit="m", shape=(1, 1), times=[1], locations=['0'])] + + sample_storage.save_global_data(result_format=result_format, level_parameters=np.ones(n_levels)) + + successful_samples = {} + failed_samples = {} + n_ops = {} + n_successful = len(target) + for l_id in range(n_levels): + sizes = [] + for quantity_spec in result_format: + sizes.append(np.prod(quantity_spec.shape) * len(quantity_spec.times) * len(quantity_spec.locations)) + + successful_samples[l_id] = [] + for sample_id in range(n_successful): + if l_id == 0: + fine_result = predictions[sample_id] + coarse_result = (np.zeros((np.sum(sizes),))) + else: + fine_result = target[sample_id] + coarse_result = predictions[sample_id] + + successful_samples[l_id].append((str(sample_id), (fine_result, coarse_result))) + + n_ops[l_id] = [random.random(), n_successful] + sample_storage.save_scheduled_samples(l_id, samples=["S{:07d}".format(i) for i in range(n_successful)]) + + sample_storage.save_samples(successful_samples, failed_samples) + sample_storage.save_n_ops(list(n_ops.items())) + + quantity = make_root_quantity(storage=sample_storage, q_specs=result_format) + length = quantity['flow'] + time = length[0] + location = time['0'] + value_quantity = location[0] + + return value_quantity, sample_storage + + +def diff_moments(target, predictions): + n_moments = 25 + quantity, target_sample_storage = create_quantity(target, predictions) + + quantile = QUANTILE + true_domain = mlmc.estimator.Estimate.estimate_domain(quantity, target_sample_storage, quantile=quantile) + + moments_fn = Legendre(n_moments, true_domain) + + quantity_moments = qe.moments(quantity, moments_fn) + + + moments_mean = qe.estimate_mean(quantity_moments) + + print("moments l means ", moments_mean.l_means) + print("moments l vars ", moments_mean.l_vars) + + print("np.max values mean l vars ", np.max(moments_mean.l_vars, axis=1)) + + print("moments mean ", moments_mean.mean) + print("moments var ", moments_mean.var) + + +def create_quantity_mlmc(data, level_parameters, num_ops=None): + sample_storage = Memory() + n_levels = len(data) + + result_format = [QuantitySpec(name="conductivity", unit="m", shape=(1, 1), times=[1], locations=['0'])] + sample_storage.save_global_data(result_format=result_format, level_parameters=level_parameters) + + successful_samples = {} + failed_samples = {} + n_ops = {} + for l_id in range(n_levels): + n_successful = data[l_id].shape[1] + sizes = [] + for quantity_spec in result_format: + sizes.append(np.prod(quantity_spec.shape) * len(quantity_spec.times) * len(quantity_spec.locations)) + + # Dict[level_id, List[Tuple[sample_id:str, Tuple[fine_result: ndarray, coarse_result: ndarray]]]] + successful_samples[l_id] = [] + for sample_id in range(n_successful): + + fine_result = data[l_id][:, sample_id, 0] + if l_id == 0: + coarse_result = (np.zeros((np.sum(sizes),))) + else: + coarse_result = data[l_id][:, sample_id, 1] + + successful_samples[l_id].append((str(sample_id), (fine_result, coarse_result))) + + # if num_ops is not None: + # n_ops[l_id] = [num_ops[l_id], n_successful] + # else: + n_ops[l_id] = [random.random(), n_successful] + + sample_storage.save_scheduled_samples(l_id, samples=["S{:07d}".format(i) for i in range(n_successful)]) + + # + # print("successful samples ") + # print("l 0", successful_samples[0][:10]) + # print("l 1", successful_samples[1][:10]) + # print("l 2", successful_samples[2][:10]) + # print("l 3", successful_samples[3][:10]) + + sample_storage.save_samples(successful_samples, failed_samples) + sample_storage.save_n_ops(list(n_ops.items())) + + return sample_storage + + +def estimate_moments(sample_storage, true_domain=None): + n_moments = 15 + result_format = sample_storage.load_result_format() + root_quantity = make_root_quantity(sample_storage, result_format) + + conductivity = root_quantity['conductivity'] + time = conductivity[1] # times: [1] + location = time['0'] # locations: ['0'] + q_value = location[0, 0] + + if true_domain is None: + quantile = QUANTILE + true_domain = mlmc.estimator.Estimate.estimate_domain(q_value, sample_storage, quantile=quantile) + moments_fn = Legendre(n_moments, true_domain) + + estimator = mlmc.estimator.Estimate(quantity=q_value, sample_storage=sample_storage, moments_fn=moments_fn) + #means, vars = estimator.estimate_moments(moments_fn) + + moments_mean = qe.estimate_mean(qe.moments(q_value, moments_fn)) + return moments_mean, estimator, true_domain, q_value + + +def ref_storage(mlmc_file): + sample_storage = SampleStorageHDF(file_path=mlmc_file) + return sample_storage + + +def get_largest_domain(storages): + true_domains = [] + for storage in storages: + result_format = storage.load_result_format() + root_quantity = make_root_quantity(storage, result_format) + + conductivity = root_quantity['conductivity'] + time = conductivity[1] # times: [1] + location = time['0'] # locations: ['0'] + q_value = location[0, 0] + + # @TODO: How to estimate true_domain? + quantile = QUANTILE + domain = mlmc.estimator.Estimate.estimate_domain(q_value, storage, quantile=quantile) + + true_domains.append([domain[0], domain[1]]) + + true_domains = np.array(true_domains) + + #print("true domains ", true_domains) + true_domain = [np.min(true_domains[:, 0]), np.max(true_domains[:, 1])] + #true_domain = [np.max(true_domains[:, 0]), np.min(true_domains[:, 1])] + #true_domain = [np.mean(true_domains[:, 0]), np.mean(true_domains[:, 1])] + + + #true_domain = true_domain[-1] + + return true_domain + + +def compare_moments(original_q_estimator, predict_q_estimator, ref_estimator): + original_q_estimator.estimate_moments() + orig_moments_mean = original_q_estimator.moments_mean + + predict_q_estimator.estimate_moments() + predict_moments_mean = predict_q_estimator.moments_mean + + ref_estimator.estimate_moments() + ref_moments_mean = ref_estimator.moments_mean + + #print("ref moments mean ", ref_moments_mean.mean) + print("orig moments mean ", orig_moments_mean.mean) + print("predict moments mean ", predict_moments_mean.mean) + + print("ref orig mean SSE ", np.sum((ref_moments_mean.mean - orig_moments_mean.mean)**2)) + print("ref predict mean SSE ", np.sum((ref_moments_mean.mean - predict_moments_mean.mean) ** 2)) + # + print("ref orig mean SE ", np.sum(np.abs((ref_moments_mean.mean - orig_moments_mean.mean)))) + print("ref predict mean SE ", np.sum(np.abs((ref_moments_mean.mean - predict_moments_mean.mean)))) + + #print("ref moments var ", ref_moments_mean.var) + print("orig moments var ", orig_moments_mean.var) + print("predict moments var ", predict_moments_mean.var) + + # print("MAX orig moments var ", np.max(orig_moments_mean.l_vars, axis=1)) + # print("MAX predict moments var ", np.max(predict_moments_mean.l_vars, axis=1)) + + print("ref orig var SSE ", np.sum((ref_moments_mean.var - orig_moments_mean.var) ** 2)) + print("ref predict var SSE ", np.sum((ref_moments_mean.var - predict_moments_mean.var) ** 2)) + # + print("ref orig var SE ", np.sum(np.abs((ref_moments_mean.var - orig_moments_mean.var)))) + print("ref predict var SE ", np.sum(np.abs((ref_moments_mean.var - predict_moments_mean.var)))) + + return orig_moments_mean, predict_moments_mean, ref_moments_mean + + # l_0_samples = predict_q_estimator.get_level_samples(level_id=0) + # l_1_samples = predict_q_estimator.get_level_samples(level_id=1) + # l_2_samples = predict_q_estimator.get_level_samples(level_id=2) + # + # print("l 0 samples shape ", np.squeeze(l_0_samples).shape) + # print("l 1 samples shape ", np.squeeze(l_1_samples[..., 0]).shape) + # + # print("l_0_samples.var ", np.var(np.squeeze(l_0_samples))) + # print("l_1_samples ", l_1_samples) + # + # diff = l_1_samples[..., 0] - l_1_samples[..., 1] + # + # print("l1 diff ", diff) + # print("var l1 diff ", np.var(diff)) + # print("fine l_1_samples.var ", np.var(np.squeeze(l_1_samples[..., 0]))) + # print("fine l_2_samples.var ", np.var(np.squeeze(l_2_samples[..., 0]))) + + +def compare_densities(estimator_1, estimator_2, ref_estimator, label_1="", label_2=""): + + distr_plot = plot.ArticleDistribution(title="densities", log_density=True) + tol = 1e-10 + reg_param = 0 + + print("orig estimator") + distr_obj_1, result, _, _ = estimator_1.construct_density(tol=tol, reg_param=reg_param) + #distr_plot.add_distribution(distr_obj_1, label=label_1, color="blue") + + print("predict estimator") + distr_obj_2, result, _, _ = estimator_2.construct_density(tol=tol, reg_param=reg_param) + #distr_plot.add_distribution(distr_obj_2, label=label_2, color="red", line_style="--") + + print("Ref estimator") + ref_distr_obj, result, _, _ = ref_estimator.construct_density(tol=tol, reg_param=reg_param) + #distr_plot.add_distribution(ref_distr_obj, label="MC reference", color="black", line_style=":") + + kl_div_ref_mlmc = mlmc.tool.simple_distribution.KL_divergence(ref_distr_obj.density, distr_obj_1.density, ref_distr_obj.domain[0], ref_distr_obj.domain[1]) + + print("KL div ref|mlmc: {}".format(kl_div_ref_mlmc)) + + # domain = [np.min([ref_distr_obj.domain[0], distr_obj_2.domain[0]]), + # np.max([ref_distr_obj.domain[1], distr_obj_2.domain[1]])] + kl_div_ref_gnn = mlmc.tool.simple_distribution.KL_divergence(ref_distr_obj.density, distr_obj_2.density, ref_distr_obj.domain[0], + ref_distr_obj.domain[1]) + + print("KL div ref|mlmc prediction: {}".format(kl_div_ref_gnn)) + + distr_plot.add_distribution(distr_obj_1, label=label_1 + ", KL(ref|orig):{:0.4g}".format(kl_div_ref_mlmc), color="blue") + distr_plot.add_distribution(distr_obj_2, label=label_2 + ", KL(ref|gnn):{:0.4g}".format(kl_div_ref_gnn), color="red", line_style="--") + distr_plot.add_distribution(ref_distr_obj, label="MC reference", color="black", line_style=":") + + distr_plot.show(file=None) + + +def get_quantity_estimator(sample_storage, true_domain=None): + n_moments = 25 + result_format = sample_storage.load_result_format() + root_quantity = make_root_quantity(sample_storage, result_format) + conductivity = root_quantity['conductivity'] + time = conductivity[1] # times: [1] + location = time['0'] # locations: ['0'] + quantity = location[0, 0] + + if true_domain is None: + quantile = QUANTILE + true_domain = mlmc.estimator.Estimate.estimate_domain(quantity, sample_storage, quantile=quantile) + + moments_fn = Legendre(n_moments, true_domain) + + return mlmc.estimator.Estimate(quantity=quantity, sample_storage=sample_storage, moments_fn=moments_fn) + + +def get_n_estimated(sample_storage, estimator, n_ops=None): + target_var = 5e-5 + #moments, estimator, _, quantity = estimate_moments(sample_storage, true_domain=true_domain) + + n_level_samples = sample_storage.get_n_collected() + # New estimation according to already finished samples + + print("n level samples ", n_level_samples) + variances, n_samples = estimator.estimate_diff_vars() + #variances, est_n_ops = estimator.estimate_diff_vars_regression(n_level_samples) + + if n_ops is None: + n_ops = n_samples + print("get n estimated n ops ", n_ops) + n_estimated = mlmc.estimator.estimate_n_samples_for_target_variance(target_var, variances, n_ops, + n_levels=len(n_level_samples)) + return n_estimated, variances, n_samples + + +def get_storage_info(sample_storage): + moments, estimator, _, _ = estimate_moments(sample_storage) + n_collected = sample_storage.get_n_collected() + max_vars = np.max(np.array(moments.l_vars) / np.array(sample_storage.get_n_collected())[:, np.newaxis], axis=1) + print("n collected ", n_collected) + print("moments.l_vars max ", max_vars) + return n_collected, max_vars + + +def cut_samples(data, sample_storage, new_n_collected, new_l_0=0): + new_data = [] + for l_id, (d, n_est) in enumerate(zip(data, new_n_collected)): + if n_est > 0: + if l_id == new_l_0: + print(d.shape) + print("np.min(d.shape[1], n_est) ", np.min([d.shape[1], n_est])) + fine_samples = d[:, :np.min([d.shape[1], n_est]), 0].reshape(1, np.min([d.shape[1], n_est]), 1) + coarse_samples = np.zeros(fine_samples.shape) + new_data.append(np.concatenate((fine_samples, coarse_samples), axis=2)) + else: + new_data.append(d[:, :np.min([d.shape[1], n_est]), :]) + sample_storage = create_quantity_mlmc(new_data, level_parameters=sample_storage.get_level_parameters()) + + return sample_storage + + +def plot_progress(conv_layers, dense_layers, output_flatten, mesh_file=None): + + if mesh_file is not None: + #mesh = gmsh_io.GmshIO(fields_mesh) + mesh_data = extract_mesh_gmsh_io(mesh_file, get_points=True) + points = mesh_data['points'] + X = points[:, 0] + Y = points[:, 1] + + for idx, conv_layer in conv_layers.items(): + inputs, weights, outputs = conv_layer[0], conv_layer[1], conv_layer[2] + plt.matshow(weights[-1]) + plt.show() + # Note: weights have different shape than the mesh + + for index, input in enumerate(inputs[::10]): + if mesh_file: + fig, ax = plt.subplots(1, 1, figsize=(15, 10)) + + cont = ax.tricontourf(X, Y, input.ravel(), levels=16) + fig.colorbar(cont) + plt.title("input") + plt.show() + + for i in range(outputs[index].shape[1]): + channel_output = outputs[index][:, i] + fig, ax = plt.subplots(1, 1, figsize=(15, 10)) + cont = ax.tricontourf(X, Y, channel_output, levels=16) + fig.colorbar(cont) + plt.title("channel {}".format(i)) + + plt.show() + + else: + plt.matshow(input[0]) + plt.show() + plt.matshow(outputs[index][0]) + plt.show() + + # print("shape ", c_layer._outputs[index][0].shape) + plt.matshow(np.sum(outputs[index], axis=0, keepdims=True)) + plt.show() + + # plt.matshow(self._inputs[-1][0]) + # plt.show() + # plt.matshow(self._outputs[-1][0]) + # plt.show() + + # print("output flatten ", self._output_flatten) + # print("final output ", final_output) + + plt.matshow([output_flatten[-1]]) + plt.show() + + for idx, dense_layer in dense_layers.items(): + inputs, weights, outputs = dense_layer[0], dense_layer[1], dense_layer[2] + plt.matshow([inputs[-1]]) + plt.show() + plt.matshow([outputs[-1]]) + plt.show() + + +def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predictions, train_targets, train_predictions, + val_targets, l_0_targets=None, l_0_predictions=None, + l1_sample_time=None, l0_sample_time=None, nn_level=0, replace_level=False, stats=False): + #level_zero = False + + if not stats: + print("nn_level ", nn_level) + print("replace level ", replace_level) + + if not stats: + # print("nn_level ", nn_level) + # print("replace level ", replace_level) + + # targets = np.exp(targets) + # predictions = np.exp(predictions) + # l_0_predictions = np.exp(l_0_predictions) + # l_0_targets = np.exp(l_0_targets) + + # print("targets ", targets) + # print("predictions ", predictions) + plt.hist(targets, bins=50, alpha=0.5, label='target', density=True) + plt.hist(predictions, bins=50, alpha=0.5, label='predictions', density=True) + + #plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) + plt.legend(loc='upper right') + #plt.xlim(-0.5, 1000) + plt.yscale('log') + plt.show() + + plt.hist(l_0_targets, bins=50, alpha=0.5, label='l_0_target', density=True) + plt.hist(l_0_predictions, bins=50, alpha=0.5, label='l_0_predictions', density=True) + + # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) + plt.legend(loc='upper right') + # plt.xlim(-0.5, 1000) + plt.yscale('log') + plt.show() + + # targets = np.exp(targets) + # predictions = np.exp(predictions) + + # # Creating plot + # plt.boxplot(targets) + # plt.boxplot(predictions) + # # show plot + # plt.show() + #exit() + + ####### + ### Create storage fromm original MLMC data + ###### + sample_storage = SampleStorageHDF(file_path=mlmc_file) + print("get n collected ", sample_storage.get_n_collected()) + n_levels = len(sample_storage.get_level_ids()) + original_moments, estimator, original_true_domain, _ = estimate_moments(sample_storage) + + orig_max_vars = np.max(original_moments.l_vars, axis=1) + print("orig max vars ", orig_max_vars) + + ###### + ### Get n ops + ###### + n_ops, field_times, coarse_flow, fine_flow = get_sample_times_mlmc(mlmc_file) + # n_ops, _, _ = get_sample_times(sampling_info_path) + # n_ops = n_ops[n_levels-1:] + + if n_ops is None: + n_ops = sample_storage.get_n_ops() + field_times = np.zeros(len(n_ops)) + flow_times = np.zeros(len(n_ops)) + # Test storage creation + data_mlmc = [] + for l_id in range(n_levels): + level_samples = estimator.get_level_samples(level_id=l_id) + data_mlmc.append(level_samples) + + print("original level params", sample_storage.get_level_parameters()) + sample_storage = create_quantity_mlmc(data_mlmc, level_parameters=sample_storage.get_level_parameters()) + print("Original storage") + orig_storage_n_collected, orig_storage_max_vars = get_storage_info(sample_storage) + + if replace_level: + n_lev = n_levels + level_params = [*sample_storage.get_level_parameters()] + else: + n_lev = n_levels - nn_level + 1 + level_params = [sample_storage.get_level_parameters()[nn_level], *sample_storage.get_level_parameters()[nn_level:]] + + # Use predicted data as zero level results and level one coarse results + data_nn = [] + n_ops_predict = [] + + nn_lev_sim_time = 0 + for l_id in range(nn_level): + nn_lev_sim_time = n_ops[l_id] - nn_lev_sim_time + + for l_id in range(n_lev): + if l_id == 0: + level_samples = l_0_predictions.reshape(1, len(l_0_predictions), 1) + #level_samples = np.ones((1, len(l_0_predictions), 1)) + ft_index = nn_level + if nn_level > 0: + ft_index = nn_level - 1 + n_ops_predict.append(l0_sample_time)# + field_times[ft_index] / 2) + #level_samples = level_samples[:, :50000, :] + else: + if replace_level: + level_id = l_id + level_samples = estimator.get_level_samples(level_id=level_id) + n_ops_predict.append(n_ops[level_id]) + else: + if l_id == 1: + # print("l id replaced level ", l_id) + # print("len(predictions ) ", len(predictions)) + + coarse_level_samples = predictions.reshape(1, len(predictions), 1) + #coarse_level_samples = np.ones((1, len(predictions), 1)) + fine_level_samples = targets.reshape(1, len(targets), 1) + + # coarse_level_samples = np.concatenate((coarse_level_samples, + # train_predictions.reshape(1, len(train_predictions), 1)), axis=1) + # + # fine_level_samples = np.concatenate((fine_level_samples, + # train_targets.reshape(1, len(train_targets), 1)), axis=1) + + # coarse_level_samples = coarse_level_samples[:, :len(predictions)//2, :] + # fine_level_samples = fine_level_samples[:, :len(predictions)//2, :] + level_samples = np.concatenate((fine_level_samples, coarse_level_samples), axis=2) + + n_ops_predict.append(n_ops[nn_level] - nn_lev_sim_time + + l1_sample_time) + else: + if replace_level: + level_id = l_id #- 1 + else: + level_id = l_id + nn_level - 1 + #print("level id ", level_id) + level_samples = estimator.get_level_samples(level_id=level_id) + n_ops_predict.append(n_ops[level_id]) + + data_nn.append(level_samples) + + print("level params ", level_params) + sample_storage_predict = create_quantity_mlmc(data_nn, level_parameters=level_params) + print("n ops predict ", n_ops_predict) + print("Storage predict info") + predict_storage_n_collected, predict_storage_max_vars = get_storage_info(sample_storage_predict) + + # if stats: + # return orig_storage_max_vars, predict_storage_max_vars + + ###### + ### Create estimators + ###### + ref_sample_storage = ref_storage(ref_mlmc_file) + domain = get_largest_domain([sample_storage, sample_storage_predict, ref_sample_storage]) + original_q_estimator = get_quantity_estimator(sample_storage, true_domain=domain) + predict_q_estimator = get_quantity_estimator(sample_storage_predict, true_domain=domain) + #ref_estimator = get_quantity_estimator(ref_sample_storage, true_domain=domain) + + ####### + ### Calculate N estimated samples + ####### + print("n ops ", n_ops) + print("n ops predict ", n_ops_predict) + + # # # remove levels + # # # #@TODO: remove asap + # new_n_samples = [0, 0, sample_storage.get_n_collected()[-3], sample_storage.get_n_collected()[-2], sample_storage.get_n_collected()[-1]] + # sample_storage = cut_samples(data_mlmc, sample_storage, new_n_samples, new_l_0=2) + # print("Cut storage info") + # get_storage_info(sample_storage) + # original_q_estimator = get_quantity_estimator(sample_storage, true_domain=domain) + # n_ops = n_ops[2:] + # # # ##### + + #### Original data + n_ops_est = copy.deepcopy(n_ops) + #n_ops_est[0] = n_ops_est[0] / 1000 + n_estimated_orig, l_vars_orig, n_samples_orig = get_n_estimated(sample_storage, original_q_estimator, n_ops=n_ops_est) + print("n estimated orig ", n_estimated_orig) + #print("l vars orig ", np.array(l_vars_orig) / np.array(sample_storage.get_n_collected())[:, np.newaxis]) + + sample_storage = cut_samples(data_mlmc, sample_storage, n_estimated_orig) + # + # sample_storage_predict = cut_samples(data_nn, sample_storage_predict, [sample_storage_predict.get_n_collected()[0], + # *n_estimated_orig]) + # + # predict_q_estimator = get_quantity_estimator(sample_storage_predict, true_domain=domain) + # + # print("sample storage predict n collected ", sample_storage_predict.get_n_collected()) + # print("sample storage n collected ", sample_storage.get_n_collected()) + + n_ops_predict_orig = copy.deepcopy(n_ops_predict) + #n_ops_predict_orig[0] = n_ops_predict_orig[0] /5 + #n_ops_predict = np.array(n_ops_predict)**2 + #n_ops_predict[0] = n_ops_predict[0] / 1000 + print("n ops predict for estimate ", n_ops_predict) + # n_samples = [10000, 2000, 500, 150, 40, 11] + + n_estimated_nn, l_var_nn, n_samples_nn = get_n_estimated(sample_storage_predict, predict_q_estimator, + n_ops=n_ops_predict) + #n_estimated_nn = [50000, 10000, 850] + sample_storage_predict = cut_samples(data_nn, sample_storage_predict, n_estimated_nn) + + ####### + ## Estimate total time + ####### + print("NN estimated ", n_estimated_nn) + print("MLMC estimated ", n_estimated_orig) + NN_time_levels = n_ops_predict_orig * np.array(n_estimated_nn) + n_collected_times = n_ops * np.array(n_estimated_orig) + print("NN time levels ", NN_time_levels) + print("MLMC time levels", n_collected_times) + nn_total_time = np.sum(NN_time_levels) + print("NN total time ", nn_total_time) + mlmc_total_time = np.sum(n_collected_times) + print("MLMC total time ", mlmc_total_time) + + #original_moments, estimator, original_true_domain, _ = estimate_moments(sample_storage) + + # # Use train and test data without validation data + # data = [] + # for l_id in range(n_levels): + # level_samples = estimator.get_level_samples(level_id=l_id) + # if l_id == 0: + # level_samples = np.concatenate((train_targets.reshape(1, len(train_targets), 1), + # targets.reshape(1, len(targets), 1)), axis=1) + # data.append(level_samples) + # sample_storage_nn = create_quantity_mlmc(data) + # moments_nn, estimator_nn, _, _ = estimate_moments(sample_storage_nn, true_domain=original_true_domain) + + + + # n0 = 100 + # nL = 10 + # num_levels = n_levels + 1 + # initial_n_samples = np.round(np.exp2(np.linspace(np.log2(n0), np.log2(nL), num_levels))).astype(int) + # if len(initial_n_samples) == len(data): + # for i in range(len(data)): + # print(data[i].shape) + # data[i] = data[i][:, :initial_n_samples[i], :] + # print("data[i].shape ", data[i].shape) + + #level_params = [sample_storage.get_level_parameters()[0], *sample_storage.get_level_parameters()] + + + # print("means nn ", moments_nn.mean) + # print("means_predict ", moments_predict.mean) + # + # print("means nn - means predict ", moments_nn.mean - moments_predict.mean) + # print("abs means nn - means predict ", np.abs(moments_nn.mean - moments_predict.mean)) + # + # print("vars nn ", moments_nn.var) + # print("vars predict ", moments_predict.var) + # + # print("moments_nn.l_means ", moments_nn.l_means[0]) + # print("moments_predict.l_means ", moments_predict.l_means[0]) + # + # print("moments nn n samples ", moments_nn.n_samples) + # print("moments nn n removed samples ", moments_predict.n_rm_samples) + # print("moments predict n samples ", moments_predict.n_samples) + # print("moments predict n removed samples ", moments_predict.n_rm_samples) + # + # for l_id, (l_mom, l_mom_pred) in enumerate(zip(moments_nn.l_means, moments_predict.l_means)): + # print("L id: {}, mom diff: {}".format(l_id, l_mom - l_mom_pred)) + + domain = get_largest_domain([sample_storage, sample_storage_predict, ref_sample_storage]) + original_q_estimator = get_quantity_estimator(sample_storage, true_domain=domain) + predict_q_estimator = get_quantity_estimator(sample_storage_predict, true_domain=domain) + ref_estimator = get_quantity_estimator(ref_sample_storage, true_domain=domain) + + orig_moments_mean, predict_moments_mean, ref_moments_mean = compare_moments(original_q_estimator, predict_q_estimator, ref_estimator) + + kl_mlmc, kl_nn = -1, -1 + # kl_mlmc, kl_nn = compare_densities(original_q_estimator, predict_q_estimator, ref_estimator, + # label_1="orig N: {}".format(sample_storage.get_n_collected()), + # label_2="gnn N: {}".format(sample_storage_predict.get_n_collected())) + + if stats: + return sample_storage.get_n_collected(), sample_storage_predict.get_n_collected(), n_ops, n_ops_predict, orig_moments_mean,\ + predict_moments_mean, ref_moments_mean, sample_storage.get_level_parameters(), sample_storage_predict.get_level_parameters() + + plot_moments({"ref": ref_estimator, "orig": original_q_estimator, "nn": predict_q_estimator}) + + + +def plot_moments(mlmc_estimators): + n_moments = 25 + moments_plot = mlmc.tool.plot.MomentsPlots( + title="Legendre {} moments".format(n_moments)) + + # moments_plot = mlmc.tool.plot.PlotMoments( + # title="Monomial {} moments".format(self.n_moments), log_mean_y=False) + + for nl, estimator in mlmc_estimators.items(): + + moments_mean = qe.estimate_mean(qe.moments(estimator._quantity, estimator._moments_fn)) + est_moments = moments_mean.mean + est_vars = moments_mean.var + + n_collected = [str(n_c) for n_c in estimator._sample_storage.get_n_collected()] + moments_plot.add_moments((moments_mean.mean, moments_mean.var), label="#{} N:".format(nl) + ", ".join(n_collected)) + + # print("moments level means ", moments_mean.l_means) + # print("moments level vars ", moments_mean.l_vars) + # print("moments level max vars ", np.max(moments_mean.l_vars, axis=1)) + # print("est moments ", est_moments) + # print("est_vars ", est_vars) + # print("np.max(est_vars) ", np.max(est_vars)) + + moments_plot.show(None) + #moments_plot.show(file=os.path.join(self.work_dir, "{}_moments".format(self.n_moments))) + moments_plot.reset() + + +def analyze_mlmc_data(): + n_levels = 5 + # mlmc_file = "/home/martin/Documents/metamodels/data/cl_0_3_s_4/L5/mlmc_5.hdf5" + mlmc_file = "/home/martin/Documents/metamodels/data/cl_0_1_s_1/L5/mlmc_5.hdf5" + + sample_storage = SampleStorageHDF(file_path=mlmc_file) + original_moments, estimator, original_true_domain = estimate_moments(sample_storage) + + # Test storage creation + data = [] + for l_id in range(n_levels): + level_samples = estimator.get_level_samples(level_id=l_id) + l_fine = np.squeeze(level_samples[..., 0]) + + print("mean l_fine ", np.mean(l_fine)) + plt.hist(l_fine, alpha=0.5, label='{}'.format(l_id), density=True) + data.append(level_samples) + + plt.legend(loc='upper right') + plt.show() + sample_storage_2 = create_quantity_mlmc(data) + moments_2, estimator_2, _ = estimate_moments(sample_storage_2) + assert np.allclose(original_moments.mean, moments_2.mean) + assert np.allclose(original_moments.var, moments_2.var) + + +def get_sample_times_mlmc(mlmc_file): + sample_storage = SampleStorageHDF(file_path=mlmc_file) + + n_ops = sample_storage.get_n_ops() + generate_rnd = sample_storage.get_generate_rnd_times() + extract_mesh = sample_storage.get_extract_mesh_times() + make_fields = sample_storage.get_make_field_times() + coarse_flow = sample_storage.get_coarse_flow_times() + fine_flow = sample_storage.get_fine_flow_times() + + def time_for_sample_func(data): + new_n_ops = [] + for nop in data: + nop = np.squeeze(nop) + if len(nop) > 0: + new_n_ops.append(nop[0] / nop[1]) + return new_n_ops + + print("generated rnd ", generate_rnd) + + generate_rnd = time_for_sample_func(generate_rnd) + extract_mesh = time_for_sample_func(extract_mesh) + make_fields = time_for_sample_func(make_fields) + coarse_flow = time_for_sample_func(coarse_flow) + fine_flow = time_for_sample_func(fine_flow) + + field_times = generate_rnd + extract_mesh + make_fields + + print("n ops ", n_ops) + print("field times ", field_times) + print("coarse flow ", coarse_flow) + print("fine flow ", fine_flow) + + return n_ops, field_times, coarse_flow, fine_flow + + +def get_sample_times(sampling_info_path): + n_levels = [5] + for nl in n_levels: + variances = [] + n_ops = [] + times = [] + + times_scheduled_samples = [] + running_times = [] + flow_running_times = [] + + for i in range(0, 100): + sampling_info_path_iter = os.path.join(sampling_info_path, str(i)) + if os.path.isdir(sampling_info_path_iter): + variances.append(np.load(os.path.join(sampling_info_path_iter, "variances.npy"))) + n_ops.append(np.load(os.path.join(sampling_info_path_iter, "n_ops.npy"))) + times.append(np.load(os.path.join(sampling_info_path_iter, "time.npy"))) + + running_times.append(np.load(os.path.join(sampling_info_path_iter, "running_times.npy"))) + flow_running_times.append(np.load(os.path.join(sampling_info_path_iter, "flow_running_times.npy"))) + if os.path.exists(os.path.join(sampling_info_path_iter, "scheduled_samples_time.npy")): + times_scheduled_samples.append( + np.load(os.path.join(sampling_info_path_iter, "scheduled_samples_time.npy"))) + else: + break + + def time_for_sample_func(data): + new_n_ops = [] + for nop in data: + nop = np.squeeze(nop) + if len(nop) > 0: + new_n_ops.append(nop[:, 0]/nop[:, 1]) + return new_n_ops + + n_ops = time_for_sample_func(n_ops) + running_times = time_for_sample_func(running_times) + flow_running_times = time_for_sample_func(flow_running_times) + + + field_times = np.mean(np.array(running_times) - np.array(flow_running_times) - np.array(flow_running_times), + axis=0) + + flow_times = np.mean(np.array(flow_running_times), axis=0) + n_ops = np.mean(n_ops, axis=0) + + # print("n ops ", n_ops) + # print("running times ", np.mean(running_times, axis=0)) + # print("flow running times ", flow_times) + # exit() + + #n_ops = np.mean(running_times, axis=0) # CPU time of simulation (fields + flow for both coarse and fine sample) + + print("field times ", field_times) + + print("n ops ", n_ops) + print("type n ops ", type(n_ops)) + + if np.isnan(np.all(n_ops)): + n_ops = None + + return n_ops, field_times, flow_times + + +def plot_data(data, label): + plt.hist(data, alpha=0.5, label=label, density=True) + + #plt.hist(predictions, bins=50, alpha=0.5, label='predictions', density=True) + # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) + plt.legend(loc='upper right') + # plt.xlim(-0.5, 1000) + #plt.yscale('log') + plt.show() + + +if __name__ == "__main__": + analyze_mlmc_data() diff --git a/mlmc/metamodel/random_field_time.py b/mlmc/metamodel/random_field_time.py new file mode 100644 index 00000000..1eade569 --- /dev/null +++ b/mlmc/metamodel/random_field_time.py @@ -0,0 +1,93 @@ +import time +import numpy as np +from mlmc.tool import gmsh_io +from mlmc.tool.flow_mc import FlowSim, create_corr_field + + +def corr_field_sample_time(mesh_file=None, corr_length_config=None): + # import matplotlib + # from matplotlib import ticker, cm + #matplotlib.rcParams.update({'font.size': 22}) + dim = 2 + log = True + cl = 0.1 + s = 1 + + if mesh_file is None: + #mesh_file = "/home/martin/Sync/Documents/flow123d_results/flow_experiments/Exponential/corr_length_0_01/l_step_0.0055_common_files/mesh.msh" + #mesh_file = "/home/martin/Documents/metamodels/data/5_ele/cl_0_1_s_1/L5/l_step_0.020196309484414757_common_files/mesh.msh" + mesh_file = "/home/martin/Documents/metamodels/data/1000_ele/l_step_0.055_common_files/mesh.msh" + + + start_time = time.process_time() + mesh_data = FlowSim.extract_mesh(mesh_file) + if corr_length_config is not None: + fields = create_corr_field(model="exp", dim=dim, + sigma=corr_length_config['sigma'], + corr_length=corr_length_config['corr_length'], + log=corr_length_config['log']) + else: + fields = create_corr_field(model="exp", dim=dim, + sigma=s, + corr_length=cl, + log=log) + # # Create fields both fine and coarse + fields = FlowSim.make_fields(fields, mesh_data, None) + + n_samples = 200 + for i in range(n_samples): + + fine_input_sample, coarse_input_sample = FlowSim.generate_random_sample(fields, coarse_step=0, + n_fine_elements=len( + mesh_data['points'])) + + len(fine_input_sample["conductivity"]) + features_log = np.log(fine_input_sample["conductivity"]) + output = 1 + # + # print("fine input sample ", fine_input_sample["conductivity"].shape) + # + # gmsh_io.GmshIO().write_fields('fields_sample.msh', mesh_data['ele_ids'], fine_input_sample) + # + # mesh = gmsh_io.GmshIO('fields_sample.msh') + # element_data = mesh.current_elem_data + # features = list(element_data.values()) + # print("features ", np.array(features).shape) + + rnd_time = time.process_time() - start_time + print("rnd_time / n_samples ", rnd_time / n_samples) + return rnd_time / n_samples + + #Xfinal, Yfinal = fields.fields[0].correlated_field.points[:, 0], fields.fields[0].correlated_field.points[:, 1] + + # cont = ax.tricontourf(Xfinal, + # Yfinal, + # fine_input_sample['conductivity'].ravel())#, locator=ticker.LogLocator()) + + # fig.colorbar(cont) + # fig.savefig("cl_{}_var_{}.pdf".format(cl, s ** 2)) + # plt.show() + + # print("fields ", fields) + # model = gs.Exponential(dim=2, len_scale=cl) + # srf = gs.SRF(model, mesh_type="unstructed", seed=20170519, mode_no=1000, generator='RandMeth') + # print("model.var ", model.var) + # field = srf( + # (fields.fields[0].correlated_field.points[:, 0], fields.fields[0].correlated_field.points[:, 1])) + # srf.vtk_export("field") + # ax = srf.plot() + # ax.set_aspect("equal") + + +if __name__ == "__main__": + import cProfile + import pstats + pr = cProfile.Profile() + pr.enable() + + my_result = corr_field_sample_time() + + pr.disable() + ps = pstats.Stats(pr).sort_stats('cumtime') + ps.print_stats() + From 0b0310f312419d6adc3f4a92f6b817caa1672e6b Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Mon, 1 Nov 2021 16:04:58 +0100 Subject: [PATCH 36/67] postprocessing --- mlmc/metamodel/analyze_nn.py | 511 +++++++++++++++--- mlmc/metamodel/postprocessing.py | 824 ++++++++++++++++++------------ mlmc/plot/plots.py | 56 ++ requirements.txt | 1 + test/metamodels/metamodel_test.py | 14 +- 5 files changed, 1026 insertions(+), 380 deletions(-) diff --git a/mlmc/metamodel/analyze_nn.py b/mlmc/metamodel/analyze_nn.py index 95a5abc7..a6a1c53a 100644 --- a/mlmc/metamodel/analyze_nn.py +++ b/mlmc/metamodel/analyze_nn.py @@ -2,6 +2,7 @@ import numpy as np import time import glob +import copy import pickle os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only @@ -9,10 +10,10 @@ from mlmc.metamodel.create_graph import graph_creator from mlmc.moments import Legendre_tf, Monomial from mlmc.metamodel.random_field_time import corr_field_sample_time -from mlmc.tool import plot +from mlmc.plot import plots import matplotlib.pyplot as plt # Make numpy printouts easier to read. - +from scipy import stats # np.set_printoptions(precision=9, suppress=True) import tensorflow as tf from tensorflow import keras @@ -477,6 +478,8 @@ def plot_sse(data_nn, data_mlmc, x_label="ith moment", y_label="MSE", title=""): def predict_data(config, model, mesh_file, log=True): + if model is None: + return batch_size = config['batch_size'] data = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log) data = data # [:10000] @@ -530,15 +533,72 @@ def predict_data(config, model, mesh_file, log=True): plot_progress(conv_layers, dense_layers, flatten_output, mesh_file=mesh_file) -def analyze_statistics(config): +def remove_empty(data_1, data_2): + new_data_1 = [] + new_data_2 = [] + + # print("data 1 ", data_1) + # print("data 2 ", data_2) + + for d1, d2 in zip(data_1, data_2): + if len(d1) > 0 and len(d2) > 0: + new_data_1.append(d1) + new_data_2.append(d2) + + # print("new data ", new_data_1) + # print("new data ", new_data_2) + return np.array(new_data_1), np.array(new_data_2) + + +def remove_outliers(data, limit): + new_data = [] + for d in data: + if d < limit: + new_data.append(d) + return new_data + + +def process_data(data_dict): + new_dict = data_dict + for tag, d in data_dict.items(): + print("tag ", tag) + print("d ", d) + + if tag in ["train_predictions", "train_targets", "test_predictions", "test_targets"]: + dt = [] + + min_length = 10**9 + for item in data_dict[tag]: + if len(item) < min_length and len(item) > 0: + min_length = len(item) + + for item in data_dict[tag]: + dt.append(item[:min_length]) + # print("dt " ,dt) + # print("array dt shape ", np.array(dt).shape) + # print("tag ", tag) + + new_dict[tag] = np.array(dt) + + return new_dict + +def analyze_statistics(config): if not os.path.isdir(config['save_path']): print("dir not exists") exit() data_dict = load_statistics(config['save_path']) - #print("data dict ", data_dict) + data_dict = process_data(data_dict) + + print("train predictions type ", type(data_dict["train_predictions"])) + print("train predictions type ", type(data_dict["train_predictions"][0])) + print("train predictions shape ", np.array(data_dict["train_predictions"]).shape) + # print("train predictions ", data_dict["train_predictions"]) + # print("train predictions as matrix shape", np.asmatrix(np.array(data_dict["train_predictions"])).shape) + + # print("data dict ", data_dict) # for key, data_dict in models_data.items(): # print("model: {}".format(key)) @@ -548,17 +608,53 @@ def analyze_statistics(config): n_ops_all = [] n_ops_predict_all = [] + all_mlmc_moments_mean = [] + all_nn_moments_mean = [] + + all_mlmc_moments_var = [] + all_nn_moments_var = [] + mlmc_times = [] nn_times = [] + mlmc_times_levels = [] + nn_times_levels = [] mlmc_l_vars = [] + mlmc_vars = [] + nn_vars = [] nn_l_vars = [] + mlmc_l_means = [] + nn_l_means = [] mlmc_vars_mse = [] nn_vars_mse = [] mlmc_means_mse = [] nn_means_mse = [] + kl_mlmc_all = [] + kl_nn_all = [] + + orth_mlmc_means_mse = [] + orth_nn_means_mse = [] + + limit = 100 # 0.008#0.01#0.0009 for i in range(len(data_dict["test_targets"])): + # if i == 3: + # break + + print("index ", i) + + # if i not in [2, 5, 7]: + # continue + + # if i in [2, 11, 12]: + # continue + + # if i in [7, 13, 14]: + # continue + + # if i not in [13]: + # continue + predictions = data_dict["test_predictions"][i] targets = data_dict["test_targets"][i] train_predictions = data_dict["train_predictions"][i] @@ -569,93 +665,216 @@ def analyze_statistics(config): l_0_targets = data_dict["l_0_targets"][i] l1_sample_time = data_dict["l1_sample_time"][i] l0_sample_time = data_dict["l0_sample_time"][i] + total_steps = data_dict["total_steps"][i] + learning_time = data_dict["learning_times"][i] + print("learning time ", learning_time) - model = data_dict["model"][i] + try: + model = data_dict["model"][i] + except: + model = None predict_data(config, model, mesh_file=config["mesh"]) - mlmc_n_collected, nn_mlmc_n_collected, n_ops, n_ops_predict, orig_moments_mean, predict_moments_mean,\ - ref_moments_mean, orig_level_params, nn_level_params = process_mlmc(config['hdf_path'], - config['sampling_info_path'], - config['ref_mlmc_file'], targets, - predictions, train_targets, - train_predictions, - val_targets, l_0_targets, - l_0_predictions, l1_sample_time, l0_sample_time, - nn_level=config['level'], - replace_level=config['replace_level'], - stats=True) + iter_test_MSE = np.mean((predictions - targets) ** 2) + iter_train_MSE = np.mean((train_predictions - train_targets) ** 2) + + if iter_test_MSE > limit: + continue + + print("iter test MSE ", iter_test_MSE) + print("iter train MSE ", iter_train_MSE) + + # if "current_patience" in data_dict: + # current_patience = data_dict["current_patience"][i] + # print("current patience ", current_patience) + + print("total steps ", total_steps) + # try: + mlmc_n_collected, nn_mlmc_n_collected, n_ops, n_ops_predict, orig_moments_mean, predict_moments_mean, \ + ref_moments_mean, orig_level_params, nn_level_params, kl_mlmc, kl_nn, target_variance, \ + orig_orth_moments, predict_orth_moments, ref_orth_moments = process_mlmc(config['hdf_path'], + config['sampling_info_path'], + config['ref_mlmc_file'], targets, + predictions, train_targets, + train_predictions, + val_targets, l_0_targets, + l_0_predictions, l1_sample_time, + l0_sample_time, + nn_level=config['level'], + replace_level=config['replace_level'], + mlmc_hdf_file=config['mlmc_hdf_path'], + stats=True, + learning_time=learning_time) + # except: + # continue mlmc_n_collected_all.append(mlmc_n_collected) nn_n_collected_all.append(nn_mlmc_n_collected) n_ops_all.append(n_ops) n_ops_predict_all.append(n_ops_predict) - + mlmc_times_levels.append(np.array(mlmc_n_collected) * np.array(n_ops)) mlmc_times.append(np.sum(np.array(mlmc_n_collected) * np.array(n_ops))) nn_times.append(np.sum(np.array(nn_mlmc_n_collected) * np.array(n_ops_predict))) + nn_times_levels.append(np.array(nn_mlmc_n_collected) * np.array(n_ops_predict)) mlmc_l_vars.append(orig_moments_mean.l_vars) nn_l_vars.append(predict_moments_mean.l_vars) + mlmc_vars.append(orig_moments_mean.var) + nn_vars.append(predict_moments_mean.var) + + mlmc_l_means.append(orig_moments_mean.l_means) + nn_l_means.append(predict_moments_mean.l_means) + mlmc_vars_mse.append((ref_moments_mean.var - orig_moments_mean.var) ** 2) nn_vars_mse.append((ref_moments_mean.var - predict_moments_mean.var) ** 2) mlmc_means_mse.append((ref_moments_mean.mean - orig_moments_mean.mean) ** 2) - nn_means_mse.append((ref_moments_mean.mean - predict_moments_mean.mean) ** 2) + nn_means_mse.append((ref_moments_mean.mean) ** 2 - predict_moments_mean.mean) + + # print("np.min(len(ref_orth_moments.mean), len(orig_orth_moments.mean)) ", np.min(len(ref_orth_moments.mean), len(orig_orth_moments.mean))) + # print("ref_orth_moments.mean[:np.min(len(ref_orth_moments.mean), len(orig_orth_moments.mean))] ", ref_orth_moments.mean[:np.min(len(ref_orth_moments.mean), len(orig_orth_moments.mean))]) + if ref_orth_moments is not None: + orth_mlmc_means_mse.append( + (ref_orth_moments.mean[:np.min((len(ref_orth_moments.mean), len(orig_orth_moments.mean)))] - + orig_orth_moments.mean[:np.min((len(ref_orth_moments.mean), len(orig_orth_moments.mean)))]) ** 2) + orth_nn_means_mse.append( + (ref_orth_moments.mean[:np.min((len(ref_orth_moments.mean), len(predict_orth_moments.mean)))] + - predict_orth_moments.mean[ + :np.min((len(ref_orth_moments.mean), len(predict_orth_moments.mean)))]) ** 2) + + print("orig moments mean ", orig_moments_mean.mean) + all_mlmc_moments_mean.append(orig_moments_mean.mean) + all_nn_moments_mean.append(predict_moments_mean.mean) + + all_mlmc_moments_var.append(orig_moments_mean.var) + all_nn_moments_var.append(predict_moments_mean.var) + + kl_mlmc_all.append(kl_mlmc) + kl_nn_all.append(kl_nn) + + moments_plot = plots.MomentsPlots(log_var_y=True) + + print("all mlmc moments mean ", all_mlmc_moments_mean) + print("np.mean(all_mlmc_moments_var, axis=0) ", np.mean(all_mlmc_moments_var, axis=0)) + moments_plot.add_moments((np.mean(all_mlmc_moments_mean, axis=0), + np.mean(all_mlmc_moments_var, axis=0)), label="mlmc moments") + moments_plot.add_moments((np.mean(all_nn_moments_mean, axis=0), + np.mean(all_nn_moments_var, axis=0)), label="nn moments") + moments_plot.add_moments((orig_moments_mean.mean, + orig_moments_mean.var), label="orig moments") + moments_plot.show(None) + + display_vars(mlmc_vars, nn_vars, target_variance=target_variance) + + print("mlmc l vars ", np.mean(mlmc_l_vars, axis=0)) + print("nn l vars ", np.mean(nn_l_vars, axis=0)) + + print("var mlmc l vars ", np.var(mlmc_l_vars, axis=0)) + print("var nn l vars ", np.var(nn_l_vars, axis=0)) + + print("MAX mlmc l vars ", np.max(np.mean(mlmc_l_vars, axis=0), axis=1)) + print("MAX nn l vars ", np.max(np.mean(nn_l_vars, axis=0), axis=1)) + # + print("mlmc means MSE ", np.mean(mlmc_means_mse, axis=0)) + print("nn means MSE ", np.mean(nn_means_mse, axis=0)) + + print("mlmc times ", mlmc_times) + print("nn times ", nn_times) + + print("mlmc times levels ", mlmc_times_levels) + print("nn times levels ", nn_times_levels) + + print("n ops all ", n_ops_all) + print("n ops predict all ", n_ops_predict_all) + print("len(nn times) ", len(nn_times)) mlmc_total_time = np.mean(mlmc_times) nn_total_time = np.mean(nn_times) + + print("#############################") print("mlmc total time ", mlmc_total_time) print("nn total time ", nn_total_time) + print("#############################") + print("KL mlmc ", np.mean(kl_mlmc_all)) + print("KL nn ", np.mean(kl_nn_all)) + + print("MC: to 10: {}, above: {}".format(np.sum(np.mean(mlmc_means_mse, axis=0)[:10]), + np.sum(np.mean(mlmc_means_mse, axis=0)[10:]))) + print("NN: to 10: {}, above: {}".format(np.sum(np.mean(nn_means_mse, axis=0)[:10]), + np.sum(np.mean(nn_means_mse, axis=0)[10:]))) n_ops_mlmc_mean = np.mean(n_ops_all, axis=0) n_ops_nn_mean = np.mean(n_ops_predict_all, axis=0) - print("n ops mlmc mean ", n_ops_mlmc_mean) - print("n ops nn mean ", n_ops_nn_mean) + # print("n ops all ", n_ops_all) + # print("n ops predict all ", n_ops_predict_all) + # + # print("n ops mlmc mean ", n_ops_mlmc_mean) + # print("n ops nn mean ", n_ops_nn_mean) mlmc_n_collected = np.mean(mlmc_n_collected_all, axis=0) nn_n_collected = np.mean(nn_n_collected_all, axis=0) - print("mlmc n collected ", mlmc_n_collected_all) - print("nn n collected all ", nn_n_collected_all) - print("mlmc n collected ", mlmc_n_collected) - print("nn n collected ", nn_n_collected) + # print("mlmc n collected ", mlmc_n_collected_all) + # print("nn n collected all ", nn_n_collected_all) + # print("mlmc n collected ", mlmc_n_collected) + # print("nn n collected ", nn_n_collected) - plt_var = plot.Variance() + plt_var = plots.VarianceNN() + plt_var.set_n_ops(np.mean(n_ops_predict_all, axis=0)) l_vars = np.mean(mlmc_l_vars, axis=0) - orig_level_params = orig_level_params # np.squeeze(orig_level_params) + # print("np.squeeze(orig_level_params) ", orig_level_params) + # print("l vars ", l_vars) + # print("np.squeeze(orig_level_params) shape", orig_level_params.shape) + # print("l vars shape", l_vars.shape) + print("orig level params ", orig_level_params) + # plt_var.add_level_variances(np.squeeze(orig_level_params), l_vars) plt_var.add_level_variances(orig_level_params, l_vars) - plt_var.show("mlmc_vars") - plt_var = plot.Variance() + # plt_var.show(None) + # plt_var.show("mlmc_vars") + # + # plt_var = plot.Variance() l_vars = np.mean(nn_l_vars, axis=0) - print("l vars shape ", l_vars.shape) - print("nn level parsm ", nn_level_params) + # print("nn l vars ", l_vars) + # print("nn level parsm ", nn_level_params) level_params = np.squeeze(nn_level_params) - level_params[0] /= 2 - plt_var.add_level_variances(level_params, l_vars) + level_params[0] *= 2 + plt_var.add_level_variances_nn(level_params, l_vars) plt_var.show("nn_vars") + plt_var.show(None) plot_sse(nn_vars_mse, mlmc_vars_mse, title="moments_var") plot_sse(nn_means_mse, mlmc_means_mse, title="moments_mean") + plot_sse(mlmc_means_mse, mlmc_means_mse, title="mlmc moments_mean") + + # if ref_orth_moments is not None: + # print("orth nn means mse ", orth_nn_means_mse) + # print("orth mlmc means mse ", orth_mlmc_means_mse) + # plot_sse(orth_nn_means_mse, orth_mlmc_means_mse, title="orthogonal moments_mean") data_dict["test_targets"] = np.array(data_dict["test_targets"]) data_dict["test_predictions"] = np.array(data_dict["test_predictions"]) data_dict["train_targets"] = np.array(data_dict["train_targets"]) data_dict["train_predictions"] = np.array(data_dict["train_predictions"]) - if data_dict["log"][0]: - data_dict["test_targets"] = np.exp(data_dict["test_targets"]) - data_dict["test_predictions"] = np.exp(data_dict["test_predictions"]) - data_dict["train_targets"] = np.exp(data_dict["train_targets"]) - data_dict["train_predictions"] = np.exp(data_dict["train_predictions"]) + print("data dict train predictions ", data_dict["train_predictions"]) - # print("test targets ", data_dict["test_targets"]) - # print("test predictions ", data_dict["test_predictions"]) - # - # print("orig max vars ", data_dict["orig_max_vars"]) - # print("predict max vars ", data_dict["predict_max_vars"]) + # if data_dict["log"][0]: + # # print("np.exp(10)", np.exp(10)) + # print("test targets ", data_dict["test_targets"]) + # print("type test targets ", type(data_dict["test_targets"])) + # + # data_dict["test_targets"], data_dict["test_predictions"] = exp_values(data_dict["test_targets"], data_dict["test_predictions"]) + # data_dict["train_targets"], data_dict["train_predictions"] = exp_values(data_dict["train_targets"], data_dict["train_predictions"]) + # + # # print("test targets ", data_dict["test_targets"]) + # print("test predictions ", data_dict["test_predictions"]) + # + # print("orig max vars ", data_dict["orig_max_vars"]) + # print("predict max vars ", data_dict["predict_max_vars"]) # mean_orig_vars = np.mean(data_dict["orig_max_vars"], axis=0) # mean_predict_vars = np.mean(data_dict["predict_max_vars"], axis=0) @@ -665,44 +884,218 @@ def analyze_statistics(config): # print("mean predict vars ", mean_predict_vars) print("total steps ", total_steps) - print("test targets ", data_dict["test_targets"]) - print("test predictions ", data_dict["test_predictions"]) - print("test diff ", data_dict["test_predictions"] - data_dict["test_targets"]) - print("test diff squared ", (data_dict["test_predictions"] - data_dict["test_targets"])**2) + # print("test targets ", data_dict["test_targets"]) + # print("test predictions ", data_dict["test_predictions"]) + # print("test diff ", data_dict["test_predictions"] - data_dict["test_targets"]) + # print("test diff squared ", (data_dict["test_predictions"] - data_dict["test_targets"])**2) - test_MSE = np.mean((data_dict["test_predictions"] - data_dict["test_targets"])**2, axis=1) + # print("(test_predictions - test_targets)**2 ", (data_dict["test_predictions"] - data_dict["test_targets"])**2) + + # print("test targets shape ", data_dict["test_targets"].shape) + + test_MSE = np.mean((data_dict["test_predictions"] - data_dict["test_targets"]) ** 2, axis=1) test_RMSE = np.sqrt(test_MSE) + test_MAE = np.mean(np.abs(data_dict["test_predictions"] - data_dict["test_targets"]), axis=1) + print("test MSE ", test_MSE) + + all_test_RSE = [] + for index, t_targets in enumerate(data_dict["test_targets"]): + if test_MSE[index] > limit: + continue + mean_t = np.mean(t_targets) + RSE = np.sum((data_dict["test_predictions"][index] - t_targets) ** 2) / np.sum((t_targets - mean_t) ** 2) + all_test_RSE.append(RSE) + + all_train_RSE = [] + try: + for index, t_targets in enumerate(data_dict["train_targets"]): + if test_MSE[index] > limit: + continue + mean_t = np.mean(t_targets) + # print("train predictions index ", data_dict["train_predictions"][index]) + RSE = np.sum((data_dict["train_predictions"][index] - t_targets) ** 2) / np.sum((t_targets - mean_t) ** 2) + all_train_RSE.append(RSE) + except: + pass + + # print("all test RSE ", np.mean(all_test_RSE)) + + # Relative squared error + test_RSE = np.sum((data_dict["test_predictions"] - data_dict["test_targets"]) ** 2) / \ + np.sum((data_dict["test_targets"] - np.mean(data_dict["test_targets"])) ** 2) + + print("test RSE ", test_RSE) + + test_RAE = np.sqrt(np.sum((data_dict["test_predictions"] - data_dict["test_targets"]) ** 2)) / \ + np.sqrt(np.sum((data_dict["test_targets"]) ** 2)) + + print("test MSE / mean targets", np.mean(test_MSE) / np.mean(data_dict["test_targets"])) + + print("test RSE ", test_RSE) + print("test RAE ", test_RAE) + print("test_MSE ", test_MSE) + + t_mse_sum = [] + for t_mse in test_MSE: + # Note: je mozne odstranit vetsi hodnoty MSE pro L4, protoze by slo dosahnout mensich hodnot pokud by se navysil pocet iteraci nebo by se vysledek pro nejlepsi train + val MSE a ne posledni vysledek + if t_mse > limit: # 0.009: + continue + t_mse_sum.append(t_mse) + + print("t mse ", t_mse_sum) + print("LEN t mse ", len(t_mse_sum)) + print("T MSE sum ", np.mean(t_mse_sum)) + + print("train_predictions ", np.array(data_dict["train_predictions"]).shape) + print("train_targets ", data_dict["train_targets"]) + + data_dict["train_predictions"], data_dict["train_targets"] = remove_empty(data_dict["train_predictions"], + data_dict["train_targets"]) + + print("remove empty train targets ", data_dict["train_targets"]) + + # data_dict["train_predictions"] = np.squeeze(data_dict["train_predictions"]) + + print("train_predictions - train_targets ", data_dict["train_predictions"] - data_dict["train_targets"]) + train_MSE = np.mean((data_dict["train_predictions"] - data_dict["train_targets"]) ** 2, axis=1) train_RMSE = np.sqrt(train_MSE) - train_MAE = np.mean(np.abs( data_dict["train_predictions"] - data_dict["train_targets"]), axis=1) + train_MAE = np.mean(np.abs(data_dict["train_predictions"] - data_dict["train_targets"]), axis=1) learning_times = data_dict["learning_times"] + # Relative squared error + train_RSE = np.sum((data_dict["train_predictions"] - data_dict["train_targets"]) ** 2) / \ + np.sum((data_dict["train_targets"] - np.mean(data_dict["train_targets"])) ** 2) + + # Relative absolute error + train_RAE = np.sqrt(np.sum((data_dict["train_predictions"] - data_dict["train_targets"]) ** 2)) / \ + np.sqrt(np.sum((data_dict["train_targets"]) ** 2)) + + print("train RSE ", train_RSE) + print("train REA ", train_RAE) + # plot_data(test_MSE, label="test MSE") # plot_data(test_MAE, label="test MAE") - print("test_MSE ", test_MSE) - - print("NN moments MSE sum ", np.sum(np.mean(nn_means_mse, axis=0))) + print("NN moments MSE sum ", np.sum(np.mean(nn_means_mse, axis=0))) print("mean test MSE ", np.mean(test_MSE)) - print("mean test RMSE ", np.mean(test_RMSE)) - print("mean test MAE ", np.mean(test_MAE)) + # print("mean test RSE ", np.mean(test_RSE)) + # print("mean test RMSE ", np.mean(test_RMSE)) + # print("mean test MAE ", np.mean(test_MAE)) print("max test MSE ", np.max(test_MSE)) - print("max test RMSE ", np.max(test_RMSE)) - print("max test MAE ", np.max(test_MAE)) + # print("max test RMSE ", np.max(test_RMSE)) + # print("max test MAE ", np.max(test_MAE)) + + print("train_MSE ", train_MSE) print("mean train MSE ", np.mean(train_MSE)) - print("mean train RMSE ", np.mean(train_RMSE)) - print("mean train MAE ", np.mean(train_MAE)) + + print("test RSE ", np.mean(all_test_RSE)) + print("test RSE ", np.mean(all_train_RSE)) + # print("mean train RSE ", np.mean(train_RSE)) + # print("mean train RMSE ", np.mean(train_RMSE)) + # print("mean train MAE ", np.mean(train_MAE)) print("max train MSE ", np.max(train_MSE)) - print("max train RMSE ", np.max(train_RMSE)) - print("max train MAE ", np.max(train_MAE)) + # print("max train RMSE ", np.max(train_RMSE)) + # print("max train MAE ", np.max(train_MAE)) print("mean learning time ", np.mean(learning_times)) print("max learning time ", np.max(learning_times)) + + test_MSE = remove_outliers(test_MSE, limit) + train_MSE = remove_outliers(train_MSE, limit) + print("############# OUTPUT ################") + print("len(train MSE) ", len(train_MSE)) + print("train MSE ", np.mean(train_MSE)) + # print("train MSE sqrt var", np.sqrt(np.var(train_MSE))) + # print("train MSE std", np.std(train_MSE)) + print("train MSE ", train_MSE) + print("stats.sem(train_MSE) ", stats.sem(train_MSE)) + print("test MSE ", np.mean(test_MSE)) + print("test MSE ", test_MSE) + print("stats.sem(test_MSE) ", stats.sem(test_MSE)) + # print("test MSE std", np.sqrt(np.var(test_MSE))) + print("train RSE ", np.mean(all_train_RSE)) + print("test RSE ", np.mean(all_test_RSE)) + + print("nn total time ", nn_total_time) + print("mlmc total time ", mlmc_total_time) + print("######################################") + return train_MSE, test_MSE, np.mean(all_train_RSE), np.mean(all_test_RSE) + +def plot_sse(data_nn, data_mlmc, x_label="ith moment", y_label="MSE", title=""): + import matplotlib + matplotlib.rcParams.update({'font.size': 16}) + matplotlib.rcParams.update({'lines.markersize': 8}) + #fig, axes = plt.subplots(1, 1, figsize=(22, 10)) + fig, axes = plt.subplots(1, 1, figsize=(8, 5)) + data = np.array(data_nn) + x = range(data.shape[1]) + axes.set_title(title) + axes.set_xlabel(x_label) + axes.set_ylabel(y_label) + axes.errorbar(x, np.mean(data_mlmc, axis=0), yerr=np.sqrt(np.var(data_mlmc, axis=0)), fmt='o', label="MC", + color="blue") + axes.errorbar(x, np.mean(data_nn, axis=0), yerr=np.sqrt(np.var(data_nn, axis=0)), fmt='o', + label="MLMC with meta-model", color="red") + fig.legend() + fig.savefig("{}.pdf".format(title)) + fig.show() + + +def plot_sse_scatter(data_nn, data_mlmc, x_label="ith moment", y_label="MSE", title=""): + import matplotlib + #matplotlib.rcParams.update({'font.size': 38}) + matplotlib.rcParams.update({'lines.markersize': 14}) + fig, axes = plt.subplots(1, 1, figsize=(22, 10)) + data = np.array(data_nn) + + print("data.shape ", data.shape) + x = range(len(data[0])) + axes.set_title(title) + axes.set_xlabel(x_label) + axes.set_ylabel(y_label) + axes.scatter(x, np.mean(data_nn, axis=0), label="NN MLMC", color="red") + axes.scatter(x, np.mean(data_mlmc, axis=0), label="MLMC", color="blue") + fig.legend() + fig.savefig("{}.pdf".format(title)) + fig.show() + + +def display_vars(mlmc_vars, nn_vars, target_variance, title=""): + mlmc_mean_vars = np.mean(mlmc_vars, axis=0) + nn_mean_vars = np.mean(nn_vars, axis=0) + + import matplotlib + # matplotlib.rcParams.update({'font.size': 38}) + matplotlib.rcParams.update({'lines.markersize': 14}) + fig, axes = plt.subplots(1, 1, figsize=(22, 10)) + + x = range(0, len(mlmc_vars[0])) + axes.set_xlabel("moments") + axes.set_ylabel("var") + axes.set_yscale("log") + + axes.axhline(y=target_variance, label="target var: {}".format(target_variance)) + + print("mlmc error bar ", np.sqrt(np.var(mlmc_vars, axis=0))) + axes.errorbar(x, np.mean(mlmc_vars, axis=0), yerr=np.sqrt(np.var(mlmc_vars, axis=0)), fmt='o', label="MLMC vars", + color="blue") + + print("mlmc vars ", mlmc_vars) + print("nn vars ", nn_vars) + print("nn error bar ", np.sqrt(np.var(nn_vars, axis=0))) + axes.errorbar(x, np.mean(nn_vars, axis=0), yerr=np.sqrt(np.var(nn_vars, axis=0)), fmt='o', label="NN vars", + color="red") + + fig.legend() + fig.savefig("{}.pdf".format(title)) + fig.show() + def run_GNN(config, stats=True, train=True, log=False, seed=0): diff --git a/mlmc/metamodel/postprocessing.py b/mlmc/metamodel/postprocessing.py index c438de7f..a5825d4f 100644 --- a/mlmc/metamodel/postprocessing.py +++ b/mlmc/metamodel/postprocessing.py @@ -2,22 +2,479 @@ import random import copy import matplotlib.pyplot as plt -from scipy.stats import ks_2samp -#from mlmc.tool import plot -#import mlmc.tool.simple_distribution import mlmc.estimator from mlmc.tool import gmsh_io -import mlmc.quantity_estimate as qe +import mlmc.quantity.quantity_estimate as qe from mlmc.sample_storage import Memory -from mlmc.quantity_spec import QuantitySpec, ChunkSpec +from mlmc.quantity.quantity_spec import QuantitySpec, ChunkSpec import numpy as np from mlmc.sample_storage_hdf import SampleStorageHDF from mlmc.moments import Legendre, Monomial -from mlmc.quantity import make_root_quantity +from mlmc.quantity.quantity import make_root_quantity from mlmc.metamodel.create_graph import extract_mesh_gmsh_io -#import mlmc.tool.simple_distribution -QUANTILE = 0.01 + +QUANTILE = 1e-6 +N_MOMENTS = 25 +TARGET_VAR = 1e-5 + +def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predictions, train_targets, train_predictions, + val_targets, l_0_targets=None, l_0_predictions=None, + l1_sample_time=None, l0_sample_time=None, nn_level=0, replace_level=False, stats=False, mlmc_hdf_file=None, + learning_time=0): + # level_zero = False + cut_est = True + + if mlmc_hdf_file is None: + mlmc_hdf_file = nn_hdf_file + + if not stats: + print("nn_level ", nn_level) + print("replace level ", replace_level) + + if not stats: + # print("nn_level ", nn_level) + # print("replace level ", replace_level) + + targets = np.exp(targets) + predictions = np.exp(predictions) + l_0_predictions = np.exp(l_0_predictions) + l_0_targets = np.exp(l_0_targets) + + # print("targets ", targets) + # print("predictions ", predictions) + plt.hist(targets, bins=50, alpha=0.5, label='target', density=True) + plt.hist(predictions, bins=50, alpha=0.5, label='predictions', density=True) + + # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) + plt.legend(loc='upper right') + # plt.xlim(-0.5, 1000) + plt.yscale('log') + plt.show() + + plt.hist(l_0_targets, bins=50, alpha=0.5, label='l_0_target', density=True) + plt.hist(l_0_predictions, bins=50, alpha=0.5, label='l_0_predictions', density=True) + + # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) + plt.legend(loc='upper right') + # plt.xlim(-0.5, 1000) + plt.yscale('log') + plt.show() + + # targets = np.exp(targets) + # predictions = np.exp(predictions) + + # # Creating plot + # plt.boxplot(targets) + # plt.boxplot(predictions) + # # show plot + # plt.show() + #exit() + + ####### + ### Create storage fromm original MLMC data + ###### + sample_storage = SampleStorageHDF(file_path=mlmc_hdf_file) + + print("mlmc sample storage get N collected ", sample_storage.get_n_collected()) + + n_levels = len(sample_storage.get_level_ids()) + original_moments, estimator, original_true_domain, _ = estimate_moments(sample_storage) + + sample_storage_nn = SampleStorageHDF(file_path=nn_hdf_file) + original_moments_nn, estimator_nn, original_true_domain_nn, _ = estimate_moments(sample_storage_nn) + + orig_max_vars = np.max(original_moments.l_vars, axis=1) + # print("orig max vars ", orig_max_vars) + + ###### + ### Get n ops + ###### + n_ops, field_times, coarse_flow, fine_flow = get_sample_times_mlmc(mlmc_hdf_file) + # n_ops, _, _ = get_sample_times(sampling_info_path) + print("n ops ", n_ops) + + if n_ops is None: + n_ops = sample_storage.get_n_ops() + field_times = np.zeros(len(n_ops)) + flow_times = np.zeros(len(n_ops)) + # Test storage creation + data_mlmc = [] + mlmc_n_collected = estimator._sample_storage.get_n_collected() + for l_id, l_n_collected in zip(range(n_levels), mlmc_n_collected): + # if l_id == 1: + # continue + level_samples = estimator.get_level_samples(level_id=l_id, n_samples=l_n_collected) + # print("len level samples ", len(np.squeeze(level_samples))) + data_mlmc.append(level_samples) + + print("original level params", sample_storage.get_level_parameters()) + sample_storage = create_quantity_mlmc(data_mlmc, level_parameters=sample_storage.get_level_parameters()) + if cut_est: + n0 = 2000 + nL = 100 + n_levels = sample_storage.get_n_levels() + n_samples = np.round(np.exp2(np.linspace(np.log2(n0), np.log2(nL), n_levels))).astype(int) + print('n samples ', n_samples) + sample_storage_for_estimated = cut_samples(data_mlmc, sample_storage, n_samples) # [2300]) + + print("Original storage") + orig_storage_n_collected, orig_storage_max_vars = get_storage_info(sample_storage) + print("orig storage max vars ", orig_storage_max_vars) + + mlmc_nn_diff_level = 1 + if replace_level: + n_lev = n_levels + level_params = [*sample_storage.get_level_parameters()] + else: + n_lev = n_levels - nn_level + 1 + level_params = [sample_storage.get_level_parameters()[nn_level], + *sample_storage.get_level_parameters()[mlmc_nn_diff_level - 1:]] + + # Use predicted data as zero level results and level one coarse results + data_nn = [] + n_ops_predict = [] + + print("l0_sample_time ", l0_sample_time) + print("l1_sample_time ", l1_sample_time) + + ############################ + ### calculate level 1 n ops + ############################ + len_data = 20000 # 50000#80000 + len_train_data = 2000 + l0_predict_time = 1e-3 + preprocess_time = l1_sample_time * len_data - learning_time + preprocess_time_per_sample = preprocess_time / len_data + n_ops_train = preprocess_time_per_sample + (learning_time / len_train_data) + l0_predict_time + # n_ops_test = (preprocess_time / len_test_data) + + print("L1 sample time ", l1_sample_time) + l1_sample_time = n_ops_train + + print("learning time ", learning_time) + print("preprocess time ", preprocess_time) + print("preprocess time per sample ", preprocess_time_per_sample) + + print("new l1 sample time ", l1_sample_time) + + # print("n ops train ", n_ops_train) + # print("n ops test ", n_ops_test) + + # n_ops = n_ops_0 + (n_ops_train * (len_train_data / nn_estimated) + n_ops_test * (len_test_data / nn_estimated)) / ( + # nn_estimated) + # n_ops += predict_l0_time + + # l0_sample_time, l1_sample_time = l1_sample_time, l0_sample_time + nn_n_collected = estimator_nn._sample_storage.get_n_collected() + print("nn n collected ", nn_n_collected) + + nn_lev_sim_time = 0 + for l_id in range(nn_level): + nn_lev_sim_time = n_ops[l_id] - nn_lev_sim_time + + for l_id in range(n_lev): + if l_id == 0: + level_samples = l_0_predictions.reshape(1, len(l_0_predictions), 1) + # level_samples = np.ones((1, len(l_0_predictions), 1)) + ft_index = nn_level + if nn_level > 0: + ft_index = nn_level - 1 + n_ops_predict.append(l0_sample_time) # + field_times[ft_index] / 2) + else: + if replace_level: + level_id = l_id + level_samples = estimator_nn.get_level_samples(level_id=level_id, n_samples=nn_n_collected[level_id]) + n_ops_predict.append(n_ops[level_id]) + else: + if l_id < mlmc_nn_diff_level: + continue + if l_id == mlmc_nn_diff_level: + coarse_level_samples = predictions.reshape(1, len(predictions), 1) + fine_level_samples = targets.reshape(1, len(targets), 1) + + # coarse_level_samples = np.concatenate((coarse_level_samples, + # train_predictions.reshape(1, len(train_predictions), 1)), axis=1) + # + # fine_level_samples = np.concatenate((fine_level_samples, + # train_targets.reshape(1, len(train_targets), 1)), axis=1) + + # coarse_level_samples = coarse_level_samples[:, :len(predictions)//2, :] + # fine_level_samples = fine_level_samples[:, :len(predictions)//2, :] + level_samples = np.concatenate((fine_level_samples, coarse_level_samples), axis=2) + print("nn level ", nn_level) + + n_ops_predict.append(n_ops[mlmc_nn_diff_level - 1] - nn_lev_sim_time + l1_sample_time) + else: + if replace_level: + level_id = l_id # - 1 + else: + level_id = l_id + nn_level - 1 + level_samples = estimator_nn.get_level_samples(level_id=level_id, + n_samples=nn_n_collected[level_id]) + print('n ops ', n_ops) + print("level id ", level_id) + + n_ops_predict.append(n_ops[level_id]) + # print("n ops predict append", n_ops_predict) + + data_nn.append(level_samples) + + print("n ops predict ", n_ops_predict) + # n_ops_predict[1] += n_ops_predict[1]*0.2 + + print("level params ", level_params) + sample_storage_predict = create_quantity_mlmc(data_nn, level_parameters=level_params) + if cut_est: + n0 = 2000 + nL = 100 + n_levels = sample_storage_predict.get_n_levels() + n_samples = np.round(np.exp2(np.linspace(np.log2(n0), np.log2(nL), n_levels))).astype(int) + print('n samples predict', n_samples) + sample_storage_predict_for_estimate = cut_samples(data_nn, sample_storage_predict, n_samples) # [4500, 1500]) + # print("n ops predict ", n_ops_predict) + print("Storage predict info") + predict_storage_n_collected, predict_storage_max_vars = get_storage_info(sample_storage_predict) + + # if stats: + # return orig_storage_max_vars, predict_storage_max_vars + + ###### + ### Create estimators + ###### + ref_sample_storage = ref_storage(ref_mlmc_file) + domain = get_largest_domain([sample_storage, sample_storage_predict, ref_sample_storage]) + domain = None + original_q_estimator = get_quantity_estimator(sample_storage, true_domain=domain) + predict_q_estimator = get_quantity_estimator(sample_storage_predict, true_domain=domain) + + if cut_est: + original_q_estimator_est = get_quantity_estimator(sample_storage_for_estimated, true_domain=domain) + predict_q_estimator_est = get_quantity_estimator(sample_storage_predict_for_estimate, true_domain=domain) + # ref_estimator = get_quantity_estimator(ref_sample_storage, true_domain=domain) + + ####### + ### Calculate N estimated samples + ####### + print("n ops ", n_ops) + print("n ops predict ", n_ops_predict) + + # # # remove levels + # # # #@TODO: remove asap + # new_n_samples = [0, 0, sample_storage.get_n_collected()[-3], sample_storage.get_n_collected()[-2], sample_storage.get_n_collected()[-1]] + # sample_storage = cut_samples(data_mlmc, sample_storage, new_n_samples, new_l_0=2) + # print("Cut storage info") + # get_storage_info(sample_storage) + # original_q_estimator = get_quantity_estimator(sample_storage, true_domain=domain) + # n_ops = n_ops[2:] + # # # ##### + + #### Original data + n_ops_est = copy.deepcopy(n_ops) + # n_ops_est[0] = n_ops_est[0] / 1000 + n_estimated_orig, l_vars_orig, n_samples_orig = get_n_estimated(sample_storage, original_q_estimator, + n_ops=n_ops_est) + + print("n estimted orig ", n_estimated_orig) + print("n ops est ", n_ops_est) + + ###### + ## initial N geuss + if cut_est: + n_estimated_orig, l_vars_orig, n_samples_orig = get_n_estimated(sample_storage_for_estimated, + original_q_estimator_est, n_ops=n_ops_est) + sample_storage_for_estimated = cut_samples(data_mlmc, sample_storage_for_estimated, n_estimated_orig, + bootstrap=True) + + # print("new n estimated orig ", n_estimated_orig) + # print("l vars orig ", np.array(l_vars_orig) / np.array(sample_storage.get_n_collected())[:, np.newaxis]) + + print("new n estimated orig ", n_estimated_orig) + + sample_storage = cut_samples(data_mlmc, sample_storage, n_estimated_orig, bootstrap=True) + + # original_q_estimator.quantity = original_q_estimator.quantity.subsample(sample_vec=n_estimated_orig) + + # original_q_estimator = get_quantity_estimator(sample_storage, true_domain=None, quantity=original_q_estimator.quantity) + + n_ops_predict_orig = copy.deepcopy(n_ops_predict) + # n_ops_predict_orig[0] = n_ops_predict_orig[0] /5 + # n_ops_predict = np.array(n_ops_predict)**2 + # n_ops_predict[0] = n_ops_predict[0] / 1000 + # print("n ops predict for estimate ", n_ops_predict) + # n_samples = [10000, 2000, 500, 150, 40, 11] + + n_estimated_nn, l_var_nn, n_samples_nn = get_n_estimated(sample_storage_predict, predict_q_estimator, + n_ops=n_ops_predict) + ###### + ## initial N geuss + if cut_est: + n_estimated_nn, l_vars_nn, n_samples_nn = get_n_estimated(sample_storage_predict_for_estimate, + predict_q_estimator_est, n_ops=n_ops_predict) + sample_storage_predict_for_estimate = cut_samples(data_nn, sample_storage_predict, n_estimated_nn) + + # exit() + + print("l var nn ", l_var_nn) + print("n ops predict ", n_ops_predict) + print("n estimated nn ", n_estimated_nn) + + print("n estimated orig ", n_estimated_orig) + + # new_n_estimated_nn = [] + # for n_est in n_estimated_nn: + # new_n_estimated_nn.append(int(n_est + n_est * 0.2)) + # n_estimated_nn = new_n_estimated_nn + + # ("new n estimated nn ", n_estimated_nn) + # exit() + # n_estimated_nn = [50000, 10000, 850] + # sample_storage_predict_2 = cut_samples(data_nn, sample_storage_predict, n_estimated_nn) + # predict_q_estimator_2 = get_quantity_estimator(sample_storage_predict_2, true_domain=domain) + # + # + # new_n_estimated_nn, new_l_var_nn, new_n_samples_nn = get_n_estimated(sample_storage_predict_2, predict_q_estimator_2, + # n_ops=n_ops_predict) + + # print("new n estimated nn", new_n_estimated_nn) + + # n_estimated_nn = new_n_estimated_nn + sample_storage_predict_0 = copy.deepcopy(sample_storage_predict) + + # predict_q_estimator.quantity = predict_q_estimator.quantity.subsample(sample_vec=n_estimated_nn) + + n_ops_test = (preprocess_time_per_sample + l0_predict_time) # * (n_estimated_nn[1] - len_train_data) + if n_estimated_nn[1] > len_train_data: + orig_n_ops = n_ops_predict[1] - l1_sample_time + cost_tr = l1_sample_time * (len_train_data) # / n_estimated_nn[1]) + cost_te = n_ops_test * (n_estimated_nn[1] - len_train_data) # / n_estimated_nn[1]) + + # note L1 sample time is n_ops_train + + n_ops_predict[1] = orig_n_ops + ((cost_tr + cost_te) / n_estimated_nn[1]) + + print("preprocess time per sample", preprocess_time_per_sample) + print("orig n ops ", orig_n_ops) + print("cost_tr ", cost_tr) + print("cost te ", cost_te) + + n_estimated_nn, l_var_nn, n_samples_nn = get_n_estimated(sample_storage_predict, predict_q_estimator, + n_ops=n_ops_predict) + print("n ops ", n_ops) + print("new n ops predict ", n_ops_predict) + print("new n estimated nn ", n_estimated_nn) + + # exit() + + sample_storage_predict = cut_samples(data_nn, sample_storage_predict, n_estimated_nn) + + # predict_q_estimator = get_quantity_estimator(sample_storage_predict_0, true_domain=domain) + # predict_q_estimator.quantity = predict_q_estimator.quantity.subsample(sample_vec=n_estimated_nn) + + print("new n estimated nn ", n_estimated_nn) + print("new l var nn ", l_var_nn) + + # predict_moments = compute_moments(sample_storage_predict) + # print("predict moments var ", predict_moments.var) + + ####### + ## Estimate total time + ####### + print("NN estimated ", n_estimated_nn) + print("MLMC estimated ", n_estimated_orig) + print("n ops predict_orig ", n_ops_predict_orig) + print("n estimated nn[1] ", n_estimated_nn[1]) + # n_ops_predict_orig = n_ops_predict + # n_ops_test = preprocess_time / (n_estimated_nn[1] - len_train_data) + # if n_estimated_nn[1] > len_train_data: + # n_ops_predict_orig[1] = n_ops_predict_orig[1] - l1_sample_time + ((l1_sample_time * (len_train_data/n_estimated_nn[1]) + \ + # n_ops_test * ((n_estimated_nn[1] - len_train_data)/n_estimated_nn[1])) / n_estimated_nn[1]) + # n_ops_predict = n_ops_predict_orig + # # print("n ops predict ", n_ops_predict) + + NN_time_levels = n_ops_predict_orig * np.array(n_estimated_nn) + n_collected_times = n_ops * np.array(n_estimated_orig) + print("NN time levels ", NN_time_levels) + print("MLMC time levels", n_collected_times) + nn_total_time = np.sum(NN_time_levels) + print("NN total time ", nn_total_time) + mlmc_total_time = np.sum(n_collected_times) + print("MLMC total time ", mlmc_total_time) + + # original_moments, estimator, original_true_domain, _ = estimate_moments(sample_storage) + + # # Use train and test data without validation data + # data = [] + # for l_id in range(n_levels): + # level_samples = estimator.get_level_samples(level_id=l_id) + # if l_id == 0: + # level_samples = np.concatenate((train_targets.reshape(1, len(train_targets), 1), + # targets.reshape(1, len(targets), 1)), axis=1) + # data.append(level_samples) + # sample_storage_nn = create_quantity_mlmc(data) + # moments_nn, estimator_nn, _, _ = estimate_moments(sample_storage_nn, true_domain=original_true_domain) + + # n0 = 100 + # nL = 10 + # num_levels = n_levels + 1 + # initial_n_samples = np.round(np.exp2(np.linspace(np.log2(n0), np.log2(nL), num_levels))).astype(int) + # if len(initial_n_samples) == len(data): + # for i in range(len(data)): + # print(data[i].shape) + # data[i] = data[i][:, :initial_n_samples[i], :] + # print("data[i].shape ", data[i].shape) + + # level_params = [sample_storage.get_level_parameters()[0], *sample_storage.get_level_parameters()] + + # print("means nn ", moments_nn.mean) + # print("means_predict ", moments_predict.mean) + # + # print("means nn - means predict ", moments_nn.mean - moments_predict.mean) + # print("abs means nn - means predict ", np.abs(moments_nn.mean - moments_predict.mean)) + # + # print("vars nn ", moments_nn.var) + # print("vars predict ", moments_predict.var) + # + # print("moments_nn.l_means ", moments_nn.l_means[0]) + # print("moments_predict.l_means ", moments_predict.l_means[0]) + # + # print("moments nn n samples ", moments_nn.n_samples) + # print("moments nn n removed samples ", moments_predict.n_rm_samples) + # print("moments predict n samples ", moments_predict.n_samples) + # print("moments predict n removed samples ", moments_predict.n_rm_samples) + # + # for l_id, (l_mom, l_mom_pred) in enumerate(zip(moments_nn.l_means, moments_predict.l_means)): + # print("L id: {}, mom diff: {}".format(l_id, l_mom - l_mom_pred)) + + domain = get_largest_domain([sample_storage, sample_storage_predict, ref_sample_storage]) + common_domain = get_largest_domain([sample_storage, sample_storage_predict, ref_sample_storage]) + domain = None + original_q_estimator = get_quantity_estimator(sample_storage, true_domain=domain) + predict_q_estimator = get_quantity_estimator(sample_storage_predict, true_domain=domain) + + if cut_est: + original_q_estimator = get_quantity_estimator(sample_storage_for_estimated, true_domain=domain) + predict_q_estimator = get_quantity_estimator(sample_storage_predict_for_estimate, true_domain=domain) + + ref_estimator = get_quantity_estimator(ref_sample_storage, true_domain=domain) + + orig_moments_mean, predict_moments_mean, ref_moments_mean = compare_moments(original_q_estimator, + predict_q_estimator, ref_estimator) + + kl_mlmc, kl_nn = -1, -1 + orig_orth_moments, predict_orth_moments, ref_orth_moments = None, None, None + # kl_mlmc, kl_nn, orig_orth_moments, predict_orth_moments, ref_orth_moments = compare_densities(original_q_estimator, predict_q_estimator, ref_estimator, + # label_1="orig N: {}".format(n_estimated_orig), + # label_2="gnn N: {}".format(n_estimated_nn)) + + if stats: + return n_estimated_orig, n_estimated_nn, n_ops, n_ops_predict, orig_moments_mean, \ + predict_moments_mean, ref_moments_mean, sample_storage.get_level_parameters(), \ + sample_storage_predict.get_level_parameters(), kl_mlmc, kl_nn, TARGET_VAR, \ + orig_orth_moments, predict_orth_moments, ref_orth_moments + + plot_moments({"ref": ref_estimator, "orig": original_q_estimator, "nn": predict_q_estimator}) + def plot_loss(train_loss, val_loss): @@ -61,7 +518,7 @@ def estimate_density(values, title="Density"): n_moments = 25 distr_accuracy = 1e-7 - distr_plot = plot.Distribution(title=title, + distr_plot = plots.Distribution(title=title, log_density=True) result_format = [QuantitySpec(name="flow", unit="m", shape=(1,), times=[0], locations=['0'])] @@ -102,7 +559,7 @@ def estimate_density(values, title="Density"): estimator = mlmc.estimator.Estimate(quantity=value_quantity, sample_storage=sample_storage, moments_fn=moments_fn) reg_param = 0 - target_var = 1e-4 + target_var = TARGET_VAR distr_obj, info, result, moments_fn = estimator.construct_density( tol=distr_accuracy, reg_param=reg_param, @@ -350,39 +807,52 @@ def compare_moments(original_q_estimator, predict_q_estimator, ref_estimator): def compare_densities(estimator_1, estimator_2, ref_estimator, label_1="", label_2=""): - distr_plot = plot.ArticleDistribution(title="densities", log_density=True) - tol = 1e-10 + distr_plot = plot.ArticleDistributionPDF(title="densities", log_density=True) + tol = 1e-7 reg_param = 0 print("orig estimator") - distr_obj_1, result, _, _ = estimator_1.construct_density(tol=tol, reg_param=reg_param) + distr_obj_1, result, _, _, orig_orth_moments = estimator_1.construct_density(tol=tol, reg_param=reg_param, + orth_moments_tol=TARGET_VAR) #distr_plot.add_distribution(distr_obj_1, label=label_1, color="blue") print("predict estimator") - distr_obj_2, result, _, _ = estimator_2.construct_density(tol=tol, reg_param=reg_param) + distr_obj_2, result, _, _, predict_orth_moments = estimator_2.construct_density(tol=tol, reg_param=reg_param, orth_moments_tol=TARGET_VAR) #distr_plot.add_distribution(distr_obj_2, label=label_2, color="red", line_style="--") print("Ref estimator") - ref_distr_obj, result, _, _ = ref_estimator.construct_density(tol=tol, reg_param=reg_param) + ref_distr_obj, result, _, _, ref_orth_moments = ref_estimator.construct_density(tol=tol, reg_param=reg_param, orth_moments_tol=TARGET_VAR) #distr_plot.add_distribution(ref_distr_obj, label="MC reference", color="black", line_style=":") - kl_div_ref_mlmc = mlmc.tool.simple_distribution.KL_divergence(ref_distr_obj.density, distr_obj_1.density, ref_distr_obj.domain[0], ref_distr_obj.domain[1]) + ref_estimator_pdf = get_quantity_estimator(ref_estimator._sample_storage, true_domain=None, n_moments=25) + ref_distr_obj, result, _, _, ref_orth_moments_pdf = ref_estimator_pdf.construct_density(tol=tol, + reg_param=reg_param, + orth_moments_tol=TARGET_VAR) + + domain = [np.max([ref_distr_obj.domain[0], distr_obj_1.domain[0], distr_obj_2.domain[0]]), + np.min([ref_distr_obj.domain[1], distr_obj_1.domain[1], distr_obj_2.domain[1]])] + kl_div_ref_mlmc = mlmc.tool.simple_distribution.KL_divergence(ref_distr_obj.density, distr_obj_1.density, domain[0], domain[1]) print("KL div ref|mlmc: {}".format(kl_div_ref_mlmc)) - # domain = [np.min([ref_distr_obj.domain[0], distr_obj_2.domain[0]]), - # np.max([ref_distr_obj.domain[1], distr_obj_2.domain[1]])] - kl_div_ref_gnn = mlmc.tool.simple_distribution.KL_divergence(ref_distr_obj.density, distr_obj_2.density, ref_distr_obj.domain[0], - ref_distr_obj.domain[1]) + # domain = [np.max([ref_distr_obj.domain[0], distr_obj_2.domain[0]]), + # np.min([ref_distr_obj.domain[1], distr_obj_2.domain[1]])] + kl_div_ref_gnn = mlmc.tool.simple_distribution.KL_divergence(ref_distr_obj.density, distr_obj_2.density, domain[0], + domain[1]) print("KL div ref|mlmc prediction: {}".format(kl_div_ref_gnn)) - distr_plot.add_distribution(distr_obj_1, label=label_1 + ", KL(ref|orig):{:0.4g}".format(kl_div_ref_mlmc), color="blue") - distr_plot.add_distribution(distr_obj_2, label=label_2 + ", KL(ref|gnn):{:0.4g}".format(kl_div_ref_gnn), color="red", line_style="--") - distr_plot.add_distribution(ref_distr_obj, label="MC reference", color="black", line_style=":") + #distr_plot.add_distribution(distr_obj_1, label=label_1 + ", KL(ref|orig):{:0.4g}".format(kl_div_ref_mlmc), color="blue") + #distr_plot.add_distribution(distr_obj_2, label=label_2 + ", KL(ref|gnn):{:0.4g}".format(kl_div_ref_gnn), color="red", line_style="--") + distr_plot.add_distribution(distr_obj_1, label=r"$D_{MC}:$" + "{:0.4g}".format(kl_div_ref_mlmc), color="blue") + distr_plot.add_distribution(distr_obj_2, label=r"$D_{meta}:$" + "{:0.4g}".format(kl_div_ref_gnn), color="red", line_style="--") + distr_plot.add_distribution(ref_distr_obj, label="MC ref", color="black", line_style=":") + distr_plot.show(file="densities.pdf") distr_plot.show(file=None) + return kl_div_ref_mlmc, kl_div_ref_gnn, orig_orth_moments, predict_orth_moments, ref_orth_moments + def get_quantity_estimator(sample_storage, true_domain=None): n_moments = 25 @@ -403,7 +873,7 @@ def get_quantity_estimator(sample_storage, true_domain=None): def get_n_estimated(sample_storage, estimator, n_ops=None): - target_var = 5e-5 + target_var = TARGET_VAR #moments, estimator, _, quantity = estimate_moments(sample_storage, true_domain=true_domain) n_level_samples = sample_storage.get_n_collected() @@ -430,18 +900,28 @@ def get_storage_info(sample_storage): return n_collected, max_vars -def cut_samples(data, sample_storage, new_n_collected, new_l_0=0): +def cut_samples(data, sample_storage, new_n_collected, new_l_0=0, bootstrap=False): new_data = [] for l_id, (d, n_est) in enumerate(zip(data, new_n_collected)): + # print("len d :", d.shape[1]) + # print("n est ", n_est) if n_est > 0: if l_id == new_l_0: - print(d.shape) - print("np.min(d.shape[1], n_est) ", np.min([d.shape[1], n_est])) - fine_samples = d[:, :np.min([d.shape[1], n_est]), 0].reshape(1, np.min([d.shape[1], n_est]), 1) + if bootstrap: + sample_idx = np.random.choice(list(range(0, d.shape[1]-1)), size=n_est, replace=True) + fine_samples = d[:, sample_idx, 0].reshape(1, np.min([d.shape[1], len(sample_idx)]), 1) + else: + fine_samples = d[:, :np.min([d.shape[1], n_est]), 0].reshape(1, np.min([d.shape[1], n_est]), 1) + coarse_samples = np.zeros(fine_samples.shape) new_data.append(np.concatenate((fine_samples, coarse_samples), axis=2)) else: - new_data.append(d[:, :np.min([d.shape[1], n_est]), :]) + if bootstrap: + sample_idx = np.random.choice(list(range(0, d.shape[1] - 1)), size=n_est, replace=True) + new_data.append(d[:, sample_idx, :]) + else: + new_data.append(d[:, :np.min([d.shape[1], n_est]), :]) + sample_storage = create_quantity_mlmc(new_data, level_parameters=sample_storage.get_level_parameters()) return sample_storage @@ -509,292 +989,6 @@ def plot_progress(conv_layers, dense_layers, output_flatten, mesh_file=None): plt.show() -def process_mlmc(mlmc_file, sampling_info_path, ref_mlmc_file, targets, predictions, train_targets, train_predictions, - val_targets, l_0_targets=None, l_0_predictions=None, - l1_sample_time=None, l0_sample_time=None, nn_level=0, replace_level=False, stats=False): - #level_zero = False - - if not stats: - print("nn_level ", nn_level) - print("replace level ", replace_level) - - if not stats: - # print("nn_level ", nn_level) - # print("replace level ", replace_level) - - # targets = np.exp(targets) - # predictions = np.exp(predictions) - # l_0_predictions = np.exp(l_0_predictions) - # l_0_targets = np.exp(l_0_targets) - - # print("targets ", targets) - # print("predictions ", predictions) - plt.hist(targets, bins=50, alpha=0.5, label='target', density=True) - plt.hist(predictions, bins=50, alpha=0.5, label='predictions', density=True) - - #plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) - plt.legend(loc='upper right') - #plt.xlim(-0.5, 1000) - plt.yscale('log') - plt.show() - - plt.hist(l_0_targets, bins=50, alpha=0.5, label='l_0_target', density=True) - plt.hist(l_0_predictions, bins=50, alpha=0.5, label='l_0_predictions', density=True) - - # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) - plt.legend(loc='upper right') - # plt.xlim(-0.5, 1000) - plt.yscale('log') - plt.show() - - # targets = np.exp(targets) - # predictions = np.exp(predictions) - - # # Creating plot - # plt.boxplot(targets) - # plt.boxplot(predictions) - # # show plot - # plt.show() - #exit() - - ####### - ### Create storage fromm original MLMC data - ###### - sample_storage = SampleStorageHDF(file_path=mlmc_file) - print("get n collected ", sample_storage.get_n_collected()) - n_levels = len(sample_storage.get_level_ids()) - original_moments, estimator, original_true_domain, _ = estimate_moments(sample_storage) - - orig_max_vars = np.max(original_moments.l_vars, axis=1) - print("orig max vars ", orig_max_vars) - - ###### - ### Get n ops - ###### - n_ops, field_times, coarse_flow, fine_flow = get_sample_times_mlmc(mlmc_file) - # n_ops, _, _ = get_sample_times(sampling_info_path) - # n_ops = n_ops[n_levels-1:] - - if n_ops is None: - n_ops = sample_storage.get_n_ops() - field_times = np.zeros(len(n_ops)) - flow_times = np.zeros(len(n_ops)) - # Test storage creation - data_mlmc = [] - for l_id in range(n_levels): - level_samples = estimator.get_level_samples(level_id=l_id) - data_mlmc.append(level_samples) - - print("original level params", sample_storage.get_level_parameters()) - sample_storage = create_quantity_mlmc(data_mlmc, level_parameters=sample_storage.get_level_parameters()) - print("Original storage") - orig_storage_n_collected, orig_storage_max_vars = get_storage_info(sample_storage) - - if replace_level: - n_lev = n_levels - level_params = [*sample_storage.get_level_parameters()] - else: - n_lev = n_levels - nn_level + 1 - level_params = [sample_storage.get_level_parameters()[nn_level], *sample_storage.get_level_parameters()[nn_level:]] - - # Use predicted data as zero level results and level one coarse results - data_nn = [] - n_ops_predict = [] - - nn_lev_sim_time = 0 - for l_id in range(nn_level): - nn_lev_sim_time = n_ops[l_id] - nn_lev_sim_time - - for l_id in range(n_lev): - if l_id == 0: - level_samples = l_0_predictions.reshape(1, len(l_0_predictions), 1) - #level_samples = np.ones((1, len(l_0_predictions), 1)) - ft_index = nn_level - if nn_level > 0: - ft_index = nn_level - 1 - n_ops_predict.append(l0_sample_time)# + field_times[ft_index] / 2) - #level_samples = level_samples[:, :50000, :] - else: - if replace_level: - level_id = l_id - level_samples = estimator.get_level_samples(level_id=level_id) - n_ops_predict.append(n_ops[level_id]) - else: - if l_id == 1: - # print("l id replaced level ", l_id) - # print("len(predictions ) ", len(predictions)) - - coarse_level_samples = predictions.reshape(1, len(predictions), 1) - #coarse_level_samples = np.ones((1, len(predictions), 1)) - fine_level_samples = targets.reshape(1, len(targets), 1) - - # coarse_level_samples = np.concatenate((coarse_level_samples, - # train_predictions.reshape(1, len(train_predictions), 1)), axis=1) - # - # fine_level_samples = np.concatenate((fine_level_samples, - # train_targets.reshape(1, len(train_targets), 1)), axis=1) - - # coarse_level_samples = coarse_level_samples[:, :len(predictions)//2, :] - # fine_level_samples = fine_level_samples[:, :len(predictions)//2, :] - level_samples = np.concatenate((fine_level_samples, coarse_level_samples), axis=2) - - n_ops_predict.append(n_ops[nn_level] - nn_lev_sim_time - + l1_sample_time) - else: - if replace_level: - level_id = l_id #- 1 - else: - level_id = l_id + nn_level - 1 - #print("level id ", level_id) - level_samples = estimator.get_level_samples(level_id=level_id) - n_ops_predict.append(n_ops[level_id]) - - data_nn.append(level_samples) - - print("level params ", level_params) - sample_storage_predict = create_quantity_mlmc(data_nn, level_parameters=level_params) - print("n ops predict ", n_ops_predict) - print("Storage predict info") - predict_storage_n_collected, predict_storage_max_vars = get_storage_info(sample_storage_predict) - - # if stats: - # return orig_storage_max_vars, predict_storage_max_vars - - ###### - ### Create estimators - ###### - ref_sample_storage = ref_storage(ref_mlmc_file) - domain = get_largest_domain([sample_storage, sample_storage_predict, ref_sample_storage]) - original_q_estimator = get_quantity_estimator(sample_storage, true_domain=domain) - predict_q_estimator = get_quantity_estimator(sample_storage_predict, true_domain=domain) - #ref_estimator = get_quantity_estimator(ref_sample_storage, true_domain=domain) - - ####### - ### Calculate N estimated samples - ####### - print("n ops ", n_ops) - print("n ops predict ", n_ops_predict) - - # # # remove levels - # # # #@TODO: remove asap - # new_n_samples = [0, 0, sample_storage.get_n_collected()[-3], sample_storage.get_n_collected()[-2], sample_storage.get_n_collected()[-1]] - # sample_storage = cut_samples(data_mlmc, sample_storage, new_n_samples, new_l_0=2) - # print("Cut storage info") - # get_storage_info(sample_storage) - # original_q_estimator = get_quantity_estimator(sample_storage, true_domain=domain) - # n_ops = n_ops[2:] - # # # ##### - - #### Original data - n_ops_est = copy.deepcopy(n_ops) - #n_ops_est[0] = n_ops_est[0] / 1000 - n_estimated_orig, l_vars_orig, n_samples_orig = get_n_estimated(sample_storage, original_q_estimator, n_ops=n_ops_est) - print("n estimated orig ", n_estimated_orig) - #print("l vars orig ", np.array(l_vars_orig) / np.array(sample_storage.get_n_collected())[:, np.newaxis]) - - sample_storage = cut_samples(data_mlmc, sample_storage, n_estimated_orig) - # - # sample_storage_predict = cut_samples(data_nn, sample_storage_predict, [sample_storage_predict.get_n_collected()[0], - # *n_estimated_orig]) - # - # predict_q_estimator = get_quantity_estimator(sample_storage_predict, true_domain=domain) - # - # print("sample storage predict n collected ", sample_storage_predict.get_n_collected()) - # print("sample storage n collected ", sample_storage.get_n_collected()) - - n_ops_predict_orig = copy.deepcopy(n_ops_predict) - #n_ops_predict_orig[0] = n_ops_predict_orig[0] /5 - #n_ops_predict = np.array(n_ops_predict)**2 - #n_ops_predict[0] = n_ops_predict[0] / 1000 - print("n ops predict for estimate ", n_ops_predict) - # n_samples = [10000, 2000, 500, 150, 40, 11] - - n_estimated_nn, l_var_nn, n_samples_nn = get_n_estimated(sample_storage_predict, predict_q_estimator, - n_ops=n_ops_predict) - #n_estimated_nn = [50000, 10000, 850] - sample_storage_predict = cut_samples(data_nn, sample_storage_predict, n_estimated_nn) - - ####### - ## Estimate total time - ####### - print("NN estimated ", n_estimated_nn) - print("MLMC estimated ", n_estimated_orig) - NN_time_levels = n_ops_predict_orig * np.array(n_estimated_nn) - n_collected_times = n_ops * np.array(n_estimated_orig) - print("NN time levels ", NN_time_levels) - print("MLMC time levels", n_collected_times) - nn_total_time = np.sum(NN_time_levels) - print("NN total time ", nn_total_time) - mlmc_total_time = np.sum(n_collected_times) - print("MLMC total time ", mlmc_total_time) - - #original_moments, estimator, original_true_domain, _ = estimate_moments(sample_storage) - - # # Use train and test data without validation data - # data = [] - # for l_id in range(n_levels): - # level_samples = estimator.get_level_samples(level_id=l_id) - # if l_id == 0: - # level_samples = np.concatenate((train_targets.reshape(1, len(train_targets), 1), - # targets.reshape(1, len(targets), 1)), axis=1) - # data.append(level_samples) - # sample_storage_nn = create_quantity_mlmc(data) - # moments_nn, estimator_nn, _, _ = estimate_moments(sample_storage_nn, true_domain=original_true_domain) - - - - # n0 = 100 - # nL = 10 - # num_levels = n_levels + 1 - # initial_n_samples = np.round(np.exp2(np.linspace(np.log2(n0), np.log2(nL), num_levels))).astype(int) - # if len(initial_n_samples) == len(data): - # for i in range(len(data)): - # print(data[i].shape) - # data[i] = data[i][:, :initial_n_samples[i], :] - # print("data[i].shape ", data[i].shape) - - #level_params = [sample_storage.get_level_parameters()[0], *sample_storage.get_level_parameters()] - - - # print("means nn ", moments_nn.mean) - # print("means_predict ", moments_predict.mean) - # - # print("means nn - means predict ", moments_nn.mean - moments_predict.mean) - # print("abs means nn - means predict ", np.abs(moments_nn.mean - moments_predict.mean)) - # - # print("vars nn ", moments_nn.var) - # print("vars predict ", moments_predict.var) - # - # print("moments_nn.l_means ", moments_nn.l_means[0]) - # print("moments_predict.l_means ", moments_predict.l_means[0]) - # - # print("moments nn n samples ", moments_nn.n_samples) - # print("moments nn n removed samples ", moments_predict.n_rm_samples) - # print("moments predict n samples ", moments_predict.n_samples) - # print("moments predict n removed samples ", moments_predict.n_rm_samples) - # - # for l_id, (l_mom, l_mom_pred) in enumerate(zip(moments_nn.l_means, moments_predict.l_means)): - # print("L id: {}, mom diff: {}".format(l_id, l_mom - l_mom_pred)) - - domain = get_largest_domain([sample_storage, sample_storage_predict, ref_sample_storage]) - original_q_estimator = get_quantity_estimator(sample_storage, true_domain=domain) - predict_q_estimator = get_quantity_estimator(sample_storage_predict, true_domain=domain) - ref_estimator = get_quantity_estimator(ref_sample_storage, true_domain=domain) - - orig_moments_mean, predict_moments_mean, ref_moments_mean = compare_moments(original_q_estimator, predict_q_estimator, ref_estimator) - - kl_mlmc, kl_nn = -1, -1 - # kl_mlmc, kl_nn = compare_densities(original_q_estimator, predict_q_estimator, ref_estimator, - # label_1="orig N: {}".format(sample_storage.get_n_collected()), - # label_2="gnn N: {}".format(sample_storage_predict.get_n_collected())) - - if stats: - return sample_storage.get_n_collected(), sample_storage_predict.get_n_collected(), n_ops, n_ops_predict, orig_moments_mean,\ - predict_moments_mean, ref_moments_mean, sample_storage.get_level_parameters(), sample_storage_predict.get_level_parameters() - - plot_moments({"ref": ref_estimator, "orig": original_q_estimator, "nn": predict_q_estimator}) - - def plot_moments(mlmc_estimators): n_moments = 25 diff --git a/mlmc/plot/plots.py b/mlmc/plot/plots.py index d4f12dd0..b673805b 100644 --- a/mlmc/plot/plots.py +++ b/mlmc/plot/plots.py @@ -967,6 +967,62 @@ def show(self, file=""): _show_and_save(self.fig, file, self.title) +class MomentsPlots(Distribution): + def __init__(self, title="", quantity_name="i-th moment", legend_title="", log_mean_y=False, log_var_y=False): + """ + """ + self._domain = None + self._title = title + self._legend_title = legend_title + self.plot_matrix = [] + self.i_plot = 0 + + self.cmap = plt.cm.tab20 + self.ax_var = None + self.ax_log_density = None + self.x_lim = None + + # mean_colors = ["brown", "salmon", "orange", "goldenrod", "red"] + # var_colors = ["blue", "slateblue", "indigo", "darkseagreen", "green"] + # + # self.mean_color = iter(mean_colors) + # self.cdf_color = iter(var_colors) + + self.fig, axes = plt.subplots(1, 2, figsize=(22, 10)) + self.ax_mean = axes[0] + self.ax_var = axes[1] + + #self.fig.suptitle(title, y=0.99) + x_axis_label = quantity_name + + self.ax_mean.set_ylabel("Mean") + self.ax_mean.set_xlabel(x_axis_label) + self.ax_mean.tick_params(axis='y') + + self.ax_var.set_ylabel("Var") + #self.ax_var.tick_params(axis='y') + self.ax_var.set_xlabel(x_axis_label) + + if log_mean_y: + self.ax_mean.set_yscale('log') + + if log_var_y: + self.ax_var.set_yscale('log') + + def add_moments(self, moments, label=None): + means, vars = moments + X = range(0, len(means)) + print("vars ", vars) + self.ax_var.scatter(X, vars, color=self.cmap(self.i_plot), label=label) + self.ax_mean.scatter(X, means, color=self.cmap(self.i_plot), label=label) + #self._plot_borders(self.ax_cdf, self.cdf_color, domain) + self.i_plot += 1 + + def show(self, file=""): + self.ax_mean.legend() + self.ax_var.legend() + + _show_and_save(self.fig, file, self._title) class BSplots: diff --git a/requirements.txt b/requirements.txt index 54cf655c..c123f5c8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ gstools memoization tensorflow pandas +spektral diff --git a/test/metamodels/metamodel_test.py b/test/metamodels/metamodel_test.py index de031b29..bf145ee1 100644 --- a/test/metamodels/metamodel_test.py +++ b/test/metamodels/metamodel_test.py @@ -18,7 +18,7 @@ from tensorflow.keras.layers import Dense from spektral.layers import GlobalSumPool, GlobalMaxPool, GlobalAvgPool import tensorflow as tf -from mlmc.tool import plot +from mlmc.plot import plots from tensorflow.keras.layers.experimental import preprocessing warnings.filterwarnings("ignore", category=DeprecationWarning) @@ -340,14 +340,14 @@ def plot_results_corr_length(): te_RSE[cl] = np.mean(test_RSE) - plt_cl = plot.CorrLength() + plt_cl = plots.CorrLength() plt_cl.add_mse_test(te_MSE) plt_cl.add_mse_train(tr_MSE) plt_cl.show(None) plt_cl.show("corr_length_mse") - plt_cl = plot.CorrLength() + plt_cl = plots.CorrLength() plt_cl.add_mse_test(te_RSE) plt_cl.add_mse_train(tr_RSE) @@ -429,14 +429,14 @@ def plot_results_corr_length(): te_RSE[cl] = np.mean(test_RSE) - plt_cl = plot.CorrLength() + plt_cl = plots.CorrLength() plt_cl.add_mse_test(te_MSE) plt_cl.add_mse_train(tr_MSE) plt_cl.show(None) plt_cl.show("corr_length_mse") - plt_cl = plot.CorrLength() + plt_cl = plots.CorrLength() plt_cl.add_mse_test(te_RSE) plt_cl.add_mse_train(tr_RSE) @@ -571,7 +571,9 @@ def get_arguments(arguments): #machine_learning_model = ("DNN_mesh_L3_6", run_DNN, True) machine_learning_model = ("GCN_mesh_L3_log_16", run_GNN, True) machine_learning_model = ("mesh_L3_log_test_saved_model", run_GNN, True) - machine_learning_model = ("mesh_L3_log_50k_weights_2_K2", run_GNN, True) + machine_learning_model = ("mesh_L3_log_50k_weights_2", run_GNN, True) + + machine_learning_model = ("mesh_L3_log_50k", run_GNN, True) #machine_learning_model = ("mesh_L3_seed", run_GNN, False) From 8cc8e489e495d05cc1f16cb8a096775e046aff4a Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Tue, 2 Nov 2021 10:53:35 +0100 Subject: [PATCH 37/67] fix quantity concept test --- test/test_quantity_concept.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/test_quantity_concept.py b/test/test_quantity_concept.py index 9cf7030e..03f1f291 100644 --- a/test/test_quantity_concept.py +++ b/test/test_quantity_concept.py @@ -464,9 +464,6 @@ def fill_sample_storage(self, sample_storage): fine_result = np.random.randint(5 + 5*sample_id, high=5+5*(1+sample_id), size=(np.sum(sizes),)) - print("fine results shape ", fine_result.shape) - exit() - if l_id == 0: coarse_result = (np.zeros((np.sum(sizes),))) else: From 224135b2ed9638e1664f5058594706a779b65bfd Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Wed, 3 Nov 2021 12:37:49 +0100 Subject: [PATCH 38/67] postprocessing plots --- mlmc/estimator.py | 2 +- mlmc/metamodel/analyze_nn.py | 99 +++++++++++++++++-- mlmc/metamodel/create_graph.py | 2 +- mlmc/metamodel/postprocessing.py | 52 ++++++---- mlmc/plot/plots.py | 160 +++++++++++++++++++++++++++++++ 5 files changed, 286 insertions(+), 29 deletions(-) diff --git a/mlmc/estimator.py b/mlmc/estimator.py index 1c26338a..dcda7308 100644 --- a/mlmc/estimator.py +++ b/mlmc/estimator.py @@ -330,7 +330,7 @@ def construct_density(self, tol=1e-8, reg_param=0.0, orth_moments_tol=1e-4, exac domain=moments_obj.domain) result = distr_obj.estimate_density_minimize(tol, reg_param) # 0.95 two side quantile - return distr_obj, info, result, moments_obj + return distr_obj, info, result, moments_obj, moments_mean def get_level_samples(self, level_id, n_samples=None): """ diff --git a/mlmc/metamodel/analyze_nn.py b/mlmc/metamodel/analyze_nn.py index a6a1c53a..11bb9dbc 100644 --- a/mlmc/metamodel/analyze_nn.py +++ b/mlmc/metamodel/analyze_nn.py @@ -477,6 +477,62 @@ def plot_sse(data_nn, data_mlmc, x_label="ith moment", y_label="MSE", title=""): fig.show() +def check_loss(config, model, log=True): + if model is None: + return + batch_size = config['batch_size'] + data = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log) + data = data # [:10000] + + data.a = config['conv_layer'].preprocess(data.a) + data.a = sp_matrix_to_sp_tensor(data.a) + + train_data_len = config["n_train_samples"] + + idx = 1 + data_tr = data[idx * train_data_len: idx * train_data_len + train_data_len] + data_te = data.get_test_data(idx, train_data_len) + + print("len(datate) ", len(data_te)) + print("batch size ", batch_size) + + loader_tr = MixedLoader(data_tr, batch_size=batch_size) + loader_te = MixedLoader(data_te, batch_size=batch_size) + + + train_targets, train_predictions = model_predict(model, loader_tr) + train_predictions = np.squeeze(train_predictions) + + test_targets, test_predictions = model_predict(model, loader_te) + test_predictions = np.squeeze(test_predictions) + + train_MSE = np.mean((train_predictions - train_targets) ** 2) + test_MSE = np.mean((test_predictions - test_targets) ** 2) + + print("train MSE: {}, test MSE: {}".format(train_MSE, test_MSE)) + + conv_layers = {} + dense_layers = {} + flatten_input = [] + flatten_output = [] + + +def model_predict(model, loader): + targets = [] + predictions = [] + step = 0 + for batch in loader: + step += 1 + inputs, target = batch + targets.extend(target) + predictions.extend(model(inputs, training=False)) + + if step == loader.steps_per_epoch: + return targets, predictions + + return targets, predictions + + def predict_data(config, model, mesh_file, log=True): if model is None: return @@ -489,7 +545,7 @@ def predict_data(config, model, mesh_file, log=True): # idx = 0 # data_te = data.get_test_data(idx, train_data_len) - data_te = data[:1] + data_te = data[-1:] print("len(datate) ", len(data_te)) print("batch size ", batch_size) @@ -518,15 +574,46 @@ def predict_data(config, model, mesh_file, log=True): conv_layers[conv_index][2].extend(conv_out) # outputs flatten_input = conv_layers[conv_index][2][-1] - flatten_output = model.flatten(conv_out) + # flatten_output = model.flatten(conv_out) + # + # print("flatten output ", flatten_output) + + prev_layer_input = conv_out + prev_layer = model.flatten + + print("flatten output ", flatten_output) + print("model._dense_layers ", model._dense_layers) for index, dense_layer in enumerate(model._dense_layers): + # if index == 1: + # break + if index not in dense_layers: dense_layers[index] = [[], [], []] - dense_layers[index][0].extend(flatten_output) # inputs + if prev_layer is None: + prev_layer = model._dense_layers[index - 1] + + print("dense layer ", dense_layer) + print("dense layer ", dense_layer.weights) + print("prev layer ", prev_layer) + print("prev layer ", prev_layer.weights) + + # + # print("prev layer ", prev_layer.weights) + + #print("dense layer kernel", dense_layer.kernel) + #print("model.flatten(conv_out) ", model.flatten(conv_out)) + + print("prev layer input ", prev_layer_input) + + dense_layers[index][0].extend(prev_layer(prev_layer_input)) # inputs dense_layers[index][1].extend(dense_layer.weights) # weights (kernel) - dense_layers[index][2].extend(dense_layer(model.flatten(conv_out))) # outputs + dense_layers[index][2].extend(dense_layer(prev_layer(prev_layer_input))) # outputs + + prev_layer_input = prev_layer(prev_layer_input) + + prev_layer = None step += 1 @@ -674,7 +761,8 @@ def analyze_statistics(config): except: model = None - predict_data(config, model, mesh_file=config["mesh"]) + #check_loss(config, model) + #predict_data(config, model, mesh_file=config["mesh"]) iter_test_MSE = np.mean((predictions - targets) ** 2) iter_train_MSE = np.mean((train_predictions - train_targets) ** 2) @@ -1097,7 +1185,6 @@ def display_vars(mlmc_vars, nn_vars, target_variance, title=""): fig.show() - def run_GNN(config, stats=True, train=True, log=False, seed=0): print("seed ", seed) diff --git a/mlmc/metamodel/create_graph.py b/mlmc/metamodel/create_graph.py index d07b1f03..99369efa 100644 --- a/mlmc/metamodel/create_graph.py +++ b/mlmc/metamodel/create_graph.py @@ -2,7 +2,6 @@ import os.path import numpy as np import networkx as nx -#import matplotlib.pyplot as plt from mlmc.tool import gmsh_io from mlmc.tool.hdf5 import HDF5 from spektral.data import Graph @@ -108,6 +107,7 @@ def create_adjacency_matrix(ele_nodes): def plot_graph(adjacency_matrix): + import matplotlib.pyplot as plt #G = nx.from_scipy_sparse_matrix(adjacency_matrix) G = nx.from_numpy_matrix(adjacency_matrix) nx.draw_kamada_kawai(G, with_labels=True, node_size=1, font_size=6) diff --git a/mlmc/metamodel/postprocessing.py b/mlmc/metamodel/postprocessing.py index a5825d4f..da36d73f 100644 --- a/mlmc/metamodel/postprocessing.py +++ b/mlmc/metamodel/postprocessing.py @@ -12,12 +12,14 @@ from mlmc.moments import Legendre, Monomial from mlmc.quantity.quantity import make_root_quantity from mlmc.metamodel.create_graph import extract_mesh_gmsh_io +from mlmc.plot import plots QUANTILE = 1e-6 N_MOMENTS = 25 TARGET_VAR = 1e-5 + def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predictions, train_targets, train_predictions, val_targets, l_0_targets=None, l_0_predictions=None, l1_sample_time=None, l0_sample_time=None, nn_level=0, replace_level=False, stats=False, mlmc_hdf_file=None, @@ -463,9 +465,9 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic kl_mlmc, kl_nn = -1, -1 orig_orth_moments, predict_orth_moments, ref_orth_moments = None, None, None - # kl_mlmc, kl_nn, orig_orth_moments, predict_orth_moments, ref_orth_moments = compare_densities(original_q_estimator, predict_q_estimator, ref_estimator, - # label_1="orig N: {}".format(n_estimated_orig), - # label_2="gnn N: {}".format(n_estimated_nn)) + kl_mlmc, kl_nn, orig_orth_moments, predict_orth_moments, ref_orth_moments = compare_densities(original_q_estimator, predict_q_estimator, ref_estimator, + label_1="orig N: {}".format(n_estimated_orig), + label_2="gnn N: {}".format(n_estimated_nn)) if stats: return n_estimated_orig, n_estimated_nn, n_ops, n_ops_predict, orig_moments_mean, \ @@ -515,7 +517,7 @@ def analyze_results(target, predictions): def estimate_density(values, title="Density"): sample_storage = Memory() n_levels = 1 - n_moments = 25 + n_moments = N_MOMENTS distr_accuracy = 1e-7 distr_plot = plots.Distribution(title=title, @@ -696,7 +698,7 @@ def create_quantity_mlmc(data, level_parameters, num_ops=None): def estimate_moments(sample_storage, true_domain=None): - n_moments = 15 + n_moments = N_MOMENTS result_format = sample_storage.load_result_format() root_quantity = make_root_quantity(sample_storage, result_format) @@ -807,25 +809,26 @@ def compare_moments(original_q_estimator, predict_q_estimator, ref_estimator): def compare_densities(estimator_1, estimator_2, ref_estimator, label_1="", label_2=""): - distr_plot = plot.ArticleDistributionPDF(title="densities", log_density=True) + distr_plot = plots.ArticleDistributionPDF(title="densities", log_density=True) tol = 1e-7 reg_param = 0 print("orig estimator") - distr_obj_1, result, _, _, orig_orth_moments = estimator_1.construct_density(tol=tol, reg_param=reg_param, + distr_obj_1, _, result, _, orig_orth_moments = estimator_1.construct_density(tol=tol, reg_param=reg_param, orth_moments_tol=TARGET_VAR) #distr_plot.add_distribution(distr_obj_1, label=label_1, color="blue") print("predict estimator") - distr_obj_2, result, _, _, predict_orth_moments = estimator_2.construct_density(tol=tol, reg_param=reg_param, orth_moments_tol=TARGET_VAR) + + distr_obj_2, _, result, _, predict_orth_moments = estimator_2.construct_density(tol=tol, reg_param=reg_param, orth_moments_tol=TARGET_VAR) #distr_plot.add_distribution(distr_obj_2, label=label_2, color="red", line_style="--") print("Ref estimator") - ref_distr_obj, result, _, _, ref_orth_moments = ref_estimator.construct_density(tol=tol, reg_param=reg_param, orth_moments_tol=TARGET_VAR) + ref_distr_obj, _, result, _, ref_orth_moments = ref_estimator.construct_density(tol=tol, reg_param=reg_param, orth_moments_tol=TARGET_VAR) #distr_plot.add_distribution(ref_distr_obj, label="MC reference", color="black", line_style=":") - ref_estimator_pdf = get_quantity_estimator(ref_estimator._sample_storage, true_domain=None, n_moments=25) - ref_distr_obj, result, _, _, ref_orth_moments_pdf = ref_estimator_pdf.construct_density(tol=tol, + ref_estimator_pdf = get_quantity_estimator(ref_estimator._sample_storage, true_domain=None, n_moments=N_MOMENTS) + ref_distr_obj, _, result, _, ref_orth_moments_pdf = ref_estimator_pdf.construct_density(tol=tol, reg_param=reg_param, orth_moments_tol=TARGET_VAR) @@ -854,14 +857,16 @@ def compare_densities(estimator_1, estimator_2, ref_estimator, label_1="", label return kl_div_ref_mlmc, kl_div_ref_gnn, orig_orth_moments, predict_orth_moments, ref_orth_moments -def get_quantity_estimator(sample_storage, true_domain=None): - n_moments = 25 +def get_quantity_estimator(sample_storage, true_domain=None, quantity=None, n_moments=None): + if n_moments is None: + n_moments = N_MOMENTS result_format = sample_storage.load_result_format() - root_quantity = make_root_quantity(sample_storage, result_format) - conductivity = root_quantity['conductivity'] - time = conductivity[1] # times: [1] - location = time['0'] # locations: ['0'] - quantity = location[0, 0] + if quantity is None: + root_quantity = make_root_quantity(sample_storage, result_format) + conductivity = root_quantity['conductivity'] + time = conductivity[1] # times: [1] + location = time['0'] # locations: ['0'] + quantity = location[0, 0] if true_domain is None: quantile = QUANTILE @@ -872,6 +877,7 @@ def get_quantity_estimator(sample_storage, true_domain=None): return mlmc.estimator.Estimate(quantity=quantity, sample_storage=sample_storage, moments_fn=moments_fn) + def get_n_estimated(sample_storage, estimator, n_ops=None): target_var = TARGET_VAR #moments, estimator, _, quantity = estimate_moments(sample_storage, true_domain=true_domain) @@ -978,20 +984,24 @@ def plot_progress(conv_layers, dense_layers, output_flatten, mesh_file=None): # print("output flatten ", self._output_flatten) # print("final output ", final_output) - plt.matshow([output_flatten[-1]]) - plt.show() + if len(output_flatten) > 0: + plt.matshow([output_flatten[-1]]) + plt.show() for idx, dense_layer in dense_layers.items(): inputs, weights, outputs = dense_layer[0], dense_layer[1], dense_layer[2] + plt.matshow([inputs[-1]]) plt.show() plt.matshow([outputs[-1]]) plt.show() + #exit() + def plot_moments(mlmc_estimators): - n_moments = 25 + n_moments = N_MOMENTS moments_plot = mlmc.tool.plot.MomentsPlots( title="Legendre {} moments".format(n_moments)) diff --git a/mlmc/plot/plots.py b/mlmc/plot/plots.py index b673805b..08f766db 100644 --- a/mlmc/plot/plots.py +++ b/mlmc/plot/plots.py @@ -1781,3 +1781,163 @@ def plot_pbs_flow_job_time(): #ax.set_yscale('log') ax.plot(1/(level_params**2), n_ops) _show_and_save(fig, "flow_time", "flow_time") + + +class ArticleDistributionPDF(Distribution): + """ + mlmc.plot.Distribution + + Class for plotting distribution approximation: PDF and CDF (optional) + Provides methods to: add more plots, add exact PDF, add ECDF/histogram from single level MC + """ + def __init__(self, exact_distr=None, title="", quantity_name="Y", legend_title="", + log_density=False, cdf_plot=False, log_x=False, error_plot='l2', reg_plot=False, multipliers_plot=True): + """ + Plot configuration + :param exact_distr: Optional exact domain (for adding to plot and computing error) + :param title: Figure title. + :param quantity_name: Quantity for X axis label. + :param log_density: Plot logarithm of density value. + :param cdf_plot: Plot CDF as well (default) + :param log_x: Use logarithmic scale for X axis. + :param error_plot: None, 'diff', 'kl. Plot error of pdf using either difference or + integrand of KL divergence: exact_pdf * log(exact_pdf / approx_pdf). + Simple difference is used for CDF for both options. + """ + matplotlib.rcParams.update({'font.size': 16}) + #matplotlib.rcParams.update({'lines.markersize': 8}) + self._exact_distr = exact_distr + self._log_density = log_density + self._log_x = log_x + self._error_plot = error_plot + self._domain = None + self._title = title + self._legend_title = legend_title + self.plot_matrix = [] + self.i_plot = 0 + + self.ax_cdf = None + self.ax_log_density = None + self.x_lim = None + + self.pdf_color = "brown" + self.cdf_color = "blue" + + self.reg_plot = reg_plot + + #self.fig, self.ax_cdf = plt.subplots(1, 1, figsize=(22, 10)) + self.fig, self.ax_pdf = plt.subplots(1, 1, figsize=(8, 5)) + self.fig_cdf = None + #self.ax_pdf = self.ax_cdf.twinx() + + #self.fig.suptitle(title, y=0.99) + x_axis_label = quantity_name + + # PDF axes + self.ax_pdf.set_ylabel("PDF")#, color=self.pdf_color) + #self.ax_pdf.set_ylabel("probability density") + self.ax_pdf.set_xlabel(x_axis_label) + #self.ax_pdf.tick_params(axis='y', labelcolor=self.pdf_color) + if self._log_x: + self.ax_pdf.set_xscale('log') + x_axis_label = "log " + x_axis_label + # if self._log_density: + # self.ax_pdf.set_yscale('log') + + # if cdf_plot: + # # CDF axes + # #self.ax_cdf.set_title("CDF approximations") + # self.ax_cdf.set_ylabel("CDF", color=self.cdf_color) + # self.ax_cdf.tick_params(axis='y', labelcolor=self.cdf_color) + # self.ax_cdf.set_xlabel(x_axis_label) + # if self._log_x: + # self.ax_cdf.set_xscale('log') + + self.x_lim = [0, 2.6] + + #self.x_lim = [0, 5] + self.x_lim = [0, 2.5] + + self.ax_pdf.set_xlim(*self.x_lim) + #self.ax_cdf.set_xlim(*self.x_lim) + + # """adjust ax2 ylimit so that v2 in ax2 is aligned to v1 in ax1""" + # _, y1 = self.ax_pdf.transData.transform((0, 0)) + # _, y2 = self.ax_cdf.transData.transform((0, 0)) + # inv = self.ax_cdf.transData.inverted() + # _, dy = inv.transform((0, 0)) - inv.transform((0, y1 - y2)) + # miny, maxy = self.ax_cdf.get_ylim() + # self.ax_cdf.set_ylim(miny + dy, maxy + dy) + + def add_raw_samples(self, samples): + """ + Add histogram and ecdf for raw samples. + :param samples: + """ + # Histogram + domain = (np.min(samples), np.max(samples)) + self.adjust_domain(domain) + if self.x_lim is not None: + self._domain = self.x_lim + N = len(samples) + print("N samples ", N) + bins = self._grid(int(0.5 * np.sqrt(N))) + self.ax_pdf.hist(samples, density=True, color='red', bins=bins, alpha=0.3) + + # Ecdf + # X = np.sort(samples) + # Y = (np.arange(len(X)) + 0.5) / float(len(X)) + # X, Y = make_monotone(X, Y) + # if self.ax_cdf is not None: + # self.ax_cdf.plot(X, Y, ':', color='midnightblue', label="ecdf") + + # PDF approx as derivative of Bspline CDF approx + # size_8 = int(N / 8) + # w = np.ones_like(X) + # w[:size_8] = 1 / (Y[:size_8]) + # w[N - size_8:] = 1 / (1 - Y[N - size_8:]) + # spl = interpolate.UnivariateSpline(X, Y, w, k=3, s=1) + # sX = np.linspace(domain[0], domain[1], 1000) + # if self._reg_param == 0: + # self.ax_pdf.plot(sX, spl.derivative()(sX), color='red', alpha=0.4, label="derivative of Bspline CDF") + + def add_distribution(self, distr_object, label=None, size=0, mom_indices=None, reg_param=0, color=None, line_style=None): + """ + Add plot for distribution 'distr_object' with given label. + :param distr_object: Instance of Distribution, we use methods: density, cdf and attribute domain + :param label: string label for legend + :return: + """ + self._reg_param = reg_param + + # if label is None: + # label = "size {}".format(distr_object.moments_fn.size) + domain = distr_object.domain + self.adjust_domain(domain) + d_size = domain[1] - domain[0] + slack = 0 # 0.05 + extended_domain = (domain[0] - slack * d_size, domain[1] + slack * d_size) + X = self._grid(10000, domain=domain) + + line_styles = ['-', ':', '-.', '--'] + plots = [] + + Y_pdf = distr_object.density(X) + + if line_style is None: + line_style = "-" + + if color is None: + color = self.pdf_color + + self.ax_pdf.plot(X, Y_pdf, color=color, label=label, linestyle=line_style) + Y_cdf = distr_object.cdf(X) + + # if self.ax_cdf is not None: + # if line_style is None: + # line_style = "-" + # if color is None: + # color = self.cdf_color + # self.ax_cdf.plot(X, Y_cdf, color=color, linestyle=line_style) + # #self._plot_borders(self.ax_cdf, self.cdf_color, domain) + self.i_plot += 1 From a37e1357c322a6a2ff8e714fde5e98cc72f02cb5 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Wed, 3 Nov 2021 16:27:36 +0100 Subject: [PATCH 39/67] multiple features --- mlmc/metamodel/analyze_nn.py | 11 +- mlmc/metamodel/create_graph.py | 72 ++++- mlmc/metamodel/flow_task_GNN_2.py | 9 +- mlmc/sample_storage_hdf.py | 15 + mlmc/tool/gmsh_io.py | 8 +- test/metamodels/metamodel_test.py | 459 +++++++++++++++++------------- 6 files changed, 355 insertions(+), 219 deletions(-) diff --git a/mlmc/metamodel/analyze_nn.py b/mlmc/metamodel/analyze_nn.py index 11bb9dbc..443f5c47 100644 --- a/mlmc/metamodel/analyze_nn.py +++ b/mlmc/metamodel/analyze_nn.py @@ -1202,7 +1202,8 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): graph_creation_time = config['graph_creation_time'] if graph_creation_time == 0: graph_creator_preproces_time = time.process_time() - graph_creator(config['output_dir'], config['hdf_path'], config['mesh'], level=config['level']) + graph_creator(config['output_dir'], config['hdf_path'], config['mesh'], level=config['level'], + feature_names=config['feature_names']) graph_creation_time = time.process_time() - graph_creator_preproces_time print("graph creation time ", graph_creation_time) exit() @@ -1318,7 +1319,9 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): config['conv_layer'], batch_size, log, stats=stats, corr_field_config=config['corr_field_config'], - seed=seed) + seed=seed, + feature_names=config['feature_names'] + ) #predict_l_0_time = time.process_time() - predict_l_0_start_time if stats: @@ -1343,8 +1346,8 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): def predict_level_zero(nn, output_dir, hdf_path, mesh, conv_layer, batch_size=1000, log=False, stats=False, - corr_field_config=None, seed=1234): - # graph_creator(output_dir, hdf_path, mesh, level=0) + corr_field_config=None, seed=1234, feature_names=[]): + graph_creator(output_dir, hdf_path, mesh, level=0, feature_names=feature_names) # Load data sample_time = 0 if corr_field_config: diff --git a/mlmc/metamodel/create_graph.py b/mlmc/metamodel/create_graph.py index 99369efa..3c46a949 100644 --- a/mlmc/metamodel/create_graph.py +++ b/mlmc/metamodel/create_graph.py @@ -4,6 +4,8 @@ import networkx as nx from mlmc.tool import gmsh_io from mlmc.tool.hdf5 import HDF5 +from mlmc.sample_storage_hdf import SampleStorageHDF +from mlmc.quantity.quantity import make_root_quantity from spektral.data import Graph from mlmc.metamodel.flow_dataset import FlowDataset @@ -77,13 +79,46 @@ def extract_mesh_gmsh_io(mesh_file, get_points=False): return ele_nodes -def get_node_features(fields_mesh): +def get_node_features(fields_mesh, feature_names): + """ + Extract mesh from file + :param fields_mesh: Mesh file + :param feature_names: [[], []] - fields in each sublist are joint to one feature, each sublist corresponds to one vertex feature + :return: list + """ mesh = gmsh_io.GmshIO(fields_mesh) - element_data = mesh.current_elem_data - features = list(element_data.values()) + + features = [] + for f_names in feature_names: + joint_features = join_fields(mesh._fields, f_names) + + features.append(list(joint_features.values())) + return features +def join_fields(fields, f_names): + if len(f_names) > 0: + x_name = len(set([*fields[f_names[0]]])) + assert all(x_name == len(set([*fields[f_n]])) for f_n in f_names) + + # # Using defaultdict + # c = [collections.Counter(fields[f_n]) for f_n in f_names] + # Cdict = collections.defaultdict(int) + + joint_dict = {} + for f_n in f_names: + for key, item in fields[f_n].items(): + #print("key: {}, item: {}".format(key, np.squeeze(item))) + joint_dict.setdefault(key, 0) + + if joint_dict[key] != 0 and np.squeeze(item) != 0: + raise ValueError("Just one field value should be non zero for each element") + joint_dict[key] += np.squeeze(item) + + return joint_dict + + def create_adjacency_matrix(ele_nodes): adjacency_matrix = np.zeros((len(ele_nodes), len(ele_nodes))) @@ -101,7 +136,7 @@ def create_adjacency_matrix(ele_nodes): if len(list(set(ele_nodes).intersection(ele_n))) == 2: adjacency_matrix[j][i] = adjacency_matrix[i][j] = 1 - print(np.count_nonzero(adjacency_matrix)) + #print(np.count_nonzero(adjacency_matrix)) assert np.allclose(adjacency_matrix, adjacency_matrix.T) # symmetry return adjacency_matrix @@ -121,22 +156,34 @@ def reject_outliers(data, m=2): return abs(data - np.mean(data)) < m * np.std(data) -def graph_creator(output_dir, hdf_path, mesh, level=0): +def graph_creator(output_dir, hdf_path, mesh, level=0, feature_names=[['conductivity']], quantity_name="conductivity"): adjacency_matrix = create_adjacency_matrix(extract_mesh_gmsh_io(mesh)) np.save(os.path.join(output_dir, "adjacency_matrix"), adjacency_matrix, allow_pickle=True) loaded_adjacency_matrix = np.load(os.path.join(output_dir, "adjacency_matrix.npy"), allow_pickle=True) #plot_graph(loaded_adjacency_matrix) + sample_storage = SampleStorageHDF(file_path=hdf_path) + sample_storage.chunk_size = 1e8 + result_format = sample_storage.load_result_format() + root_quantity = make_root_quantity(sample_storage, result_format) + + #@TODO: + conductivity = root_quantity[quantity_name] + time = conductivity[1] # times: [1] + location = time['0'] # locations: ['0'] + q_value = location[0, 0] + hdf = HDF5(file_path=hdf_path, load_from_file=True) level_group = hdf.add_level_group(level_id=str(level)) - # print("collected ", level_group.collected()[:, 0, :]) - # indices = reject_outliers(np.squeeze(level_group.collected()[:, 0, :]), m=6) - # print("removed outliers ", np.count_nonzero(~indices)) - indices = np.ones(len(level_group.collected())) + chunk_spec = next(sample_storage.chunks(level_id=level, n_samples=sample_storage.get_n_collected()[int(level)])) + collected_values = q_value.samples(chunk_spec=chunk_spec)[0] - collected = zip(level_group.get_collected_ids(), level_group.collected()) + collected_ids = sample_storage.collected_ids(level_id=level) + + indices = np.ones(len(collected_values)) + collected = zip(collected_ids, collected_values) graphs = [] data = [] @@ -144,7 +191,8 @@ def graph_creator(output_dir, hdf_path, mesh, level=0): for keep, (sample_id, col_values) in zip(indices, collected): if not keep: continue - output_value = col_values[0, 0] + + output_value = col_values[0] sample_dir = os.path.join(output_dir, sample_id) field_mesh = os.path.join(sample_dir, FIELDS_SAMPLE) @@ -153,7 +201,7 @@ def graph_creator(output_dir, hdf_path, mesh, level=0): # if i > 150: # break - features = get_node_features(field_mesh) + features = get_node_features(field_mesh, feature_names) np.save(os.path.join(sample_dir, "nodes_features"), features) np.save(os.path.join(sample_dir, "output"), output_value) diff --git a/mlmc/metamodel/flow_task_GNN_2.py b/mlmc/metamodel/flow_task_GNN_2.py index ca07a93d..0db7da9f 100644 --- a/mlmc/metamodel/flow_task_GNN_2.py +++ b/mlmc/metamodel/flow_task_GNN_2.py @@ -99,7 +99,7 @@ def fit(self, loader_tr, loader_va, loader_te): results_va = self.evaluate(loader_va) self._val_loss.append(results_va[0]) - if step == loader_tr.steps_per_epoch: # step_per_epoch = int(np.ceil(len(self.dataset) / self.batch_size)) + if step == loader_tr.steps_per_epoch: # step_per_epoch = int(np.ceil(len(self.dataset) / self.batch_size)) train_targets = False # results_va = self.evaluate(loader_va) # self._val_loss.append(results_va[0]) @@ -125,12 +125,13 @@ def fit(self, loader_tr, loader_va, loader_te): results_tr = np.average(results_tr[:, :-1], 0, weights=results_tr[:, -1]) if self._verbose: print( - "Train loss: {:.4f}, acc: {:.4f} | " - "Valid loss: {:.4f}, acc: {:.4f} | " - "Test loss: {:.4f}, acc: {:.4f}".format( + "Train loss: {:.12f}, acc: {:.12f} | " + "Valid loss: {:.12f}, acc: {:.12f} | " + "Test loss: {:.12f}, acc: {:.12f}".format( *results_tr, *results_va, *results_te ) ) + # Reset epoch results_tr = [] step = 0 diff --git a/mlmc/sample_storage_hdf.py b/mlmc/sample_storage_hdf.py index 75f91e0d..c137cb09 100644 --- a/mlmc/sample_storage_hdf.py +++ b/mlmc/sample_storage_hdf.py @@ -206,6 +206,21 @@ def unfinished_ids(self): return unfinished + def collected_ids(self, level_id=None): + """ + List of colected ids + :param level_id: int + :return: list + """ + if level_id is not None: + return self._level_groups[level_id].get_collected_ids() + + unfinished = [] + for level in self._level_groups: + unfinished.extend(level.get_collected_ids()) + + return unfinished + def failed_samples(self): """ Dictionary of failed samples diff --git a/mlmc/tool/gmsh_io.py b/mlmc/tool/gmsh_io.py index c5a3ad36..9f12d93a 100644 --- a/mlmc/tool/gmsh_io.py +++ b/mlmc/tool/gmsh_io.py @@ -44,6 +44,8 @@ def reset(self): self.elements = {} self.physical = {} self.element_data = {} + self._fields = {} + self._field = None def read_element_data_head(self, mshfile): @@ -69,7 +71,6 @@ def read_element_data_head(self, mshfile): n_elem = float(columns[0]) return field, time, t_idx, n_comp, n_elem - def read(self, mshfile=None): """Read a Gmsh .msh file. @@ -100,6 +101,10 @@ def read(self, mshfile=None): readmode = 5 elif line == '$ElementData': field, time, t_idx, n_comp, n_ele = self.read_element_data_head(mshfile) + + self._fields.setdefault(field, {}) + self._field = field + field_times = self.element_data.setdefault(field, {}) assert t_idx not in field_times self.current_elem_data = {} @@ -115,6 +120,7 @@ def read(self, mshfile=None): comp_values = [float(col) for col in columns[1:]] assert len(comp_values) == self.current_n_components self.current_elem_data[ele_idx] = comp_values + self._fields[self._field] = self.current_elem_data if readmode == 5: if len(columns) == 3: diff --git a/test/metamodels/metamodel_test.py b/test/metamodels/metamodel_test.py index bf145ee1..44a74d94 100644 --- a/test/metamodels/metamodel_test.py +++ b/test/metamodels/metamodel_test.py @@ -84,11 +84,13 @@ def __init__(self, conv_layer, hidden_activation, output_activation, kernel_regu # self.conv3 = conv_layer(8, activation=hidden_activation, kernel_regularizer=kernel_regularization) # self.conv4 = conv_layer(4, activation=hidden_activation, kernel_regularizer=kernel_regularization) # self.conv3 = conv_layer(64, activation=hidden_activation, kernel_regularizer=kernel_regularization) - self.flatten = GlobalSumPool() + #self.flatten = GlobalSumPool() + + self.flatten = GlobalAvgPool() #self._submodel = Sequential() - self._dense_layers = [Dense(1)] + self._dense_layers = [Dense(32, activation=hidden_activation), Dense(1)] # for d_layer in self._dense_layers: # self._submodel.add(d_layer) @@ -110,6 +112,8 @@ def call(self, inputs): def get_config(data_dir, case=0): + feature_names = [['conductivity']] + if case == 0: cl = "cl_0_3_s_4" nn_level = 0 @@ -250,198 +254,251 @@ def get_config(data_dir, case=0): sampling_info_path = os.path.join(data_dir, "{}/sampling_info".format(cl)) ref_mlmc_file = os.path.join(data_dir, "{}/L1_benchmark/mlmc_1.hdf5".format(cl)) - # elif case == 6: - # nn_level = 0 - # replace_level = True - # mesh = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/l_step_0.055_common_files/mesh.msh" - # output_dir = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/output" - # hdf_path = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/mlmc_1.hdf5" - # - # save_path = "/home/martin/Documents/metamodels/data/L1/" - # l_0_output_dir = output_dir - # l_0_hdf_path = hdf_path - # sampling_info_path = "/home/martin/Documents/metamodels/data/L1/test/01_cond_field/" - # ref_mlmc_file = "/home/martin/Documents/metamodels/data/1000_ele/cl_0_1_s_1/L1_benchmark/mlmc_1.hdf5" - - return output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, replace_level, nn_level, mlmc_hdf_path - - -def plot_results_corr_length(): - cl_all = {"cl_0_001_s_1": 0.001, "cl_0_01_s_1": 0.01, "cl_0_1_s_1": 0.1, "cl_1_s_1": 1, "cl_10_s_1": 10} - - # cl_all = {"cl_0_001_s_1": 0.001, "cl_10_s_1": 1} - # - cl_all = {"cl_0_001_s_1": 0.001} - - tr_MSE = {} - te_MSE = {} - tr_RSE = {} - te_RSE = {} - - for cl_dir, cl in cl_all.items(): - data_dir = "/home/martin/Documents/metamodels/data/mesh_size/" - level = 3 + elif case == 12: # mesh size comparison + data_dir = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/" + # cl = "cl_0_1_s_1" + level = 1 nn_level = 0 replace_level = False - # mesh = os.path.join(data_dir, "l_step_1.0_common_files/mesh.msh".format(cl)) #L1, 7s - # mesh = os.path.join(data_dir, "l_step_0.27232698153315_common_files/mesh.msh".format(cl)) #L2 10.5 s - mesh = os.path.join(data_dir, "l_step_0.07416198487095663_common_files/mesh.msh".format(cl_dir)) # L3 12s + mesh = os.path.join(data_dir, "l_step_1.0_common_files/repo.msh") # L1, 7s + # mesh = os.path.join(data_dir, "l_step_0.27232698153315_common_files/repo.msh".format(cl)) #L2 10.5 s + # mesh = os.path.join(data_dir, "l_step_0.07416198487095663_common_files/mesh.msh".format(cl)) #L3 12s # mesh = os.path.join(data_dir, "l_step_0.020196309484414757_common_files/mesh.msh".format(cl)) #L4 22s # mesh = os.path.join(data_dir, "l_step_0.0055_common_files/mesh.msh".format(cl)) #L5 - output_dir = os.path.join(data_dir, "{}/L1_{}/test/01_cond_field/output/".format(cl_dir, level)) - hdf_path = os.path.join(data_dir, "{}/L1_{}/mlmc_1.hdf5".format(cl_dir, level)) - mlmc_hdf_path = os.path.join(data_dir, "{}/mlmc_hdf/L1_{}/mlmc_1.hdf5".format(cl_dir, level)) - save_path = os.path.join(data_dir, "{}".format(cl_dir)) - l_0_output_dir = os.path.join(data_dir, "{}/L0_MC/L1_{}/test/01_cond_field/output/".format(cl_dir, level)) - l_0_hdf_path = os.path.join(data_dir, "{}/L0_MC/L1_{}/mlmc_1.hdf5".format(cl_dir, level)) - sampling_info_path = os.path.join(data_dir, "{}/sampling_info".format(cl_dir)) - ref_mlmc_file = os.path.join(data_dir, "{}/L1_benchmark/mlmc_1.hdf5".format(cl_dir)) - - machine_learning_model = ("mesh_L3_log_15k", run_GNN, False) - - gnn, conv_layer, corr_field_config, model_config = get_gnn() - - save_path = os.path.join(save_path, machine_learning_model[0]) - - print("save path ", save_path) - graph_creation_time = 28 # 22#159#0#159#66 - - config = {'machine_learning_model': machine_learning_model, - 'save_path': save_path, - 'sampling_info_path': sampling_info_path, - 'output_dir': output_dir, - 'nn_hdf_path': hdf_path, - 'mlmc_hdf_path': mlmc_hdf_path, - 'mesh': mesh, - 'l_0_output_dir': l_0_output_dir, - 'l_0_hdf_path': l_0_hdf_path, - 'ref_mlmc_file': ref_mlmc_file, - 'level': nn_level, - 'conv_layer': conv_layer, - 'gnn': gnn, - 'model_config': model_config, - 'replace_level': replace_level, - 'corr_field_config': corr_field_config, - 'n_train_samples': 2000, - 'val_samples_ratio': 0.3, - 'batch_size': 200, - 'epochs': 2000, - 'learning_rate': 0.01, - 'graph_creation_time': graph_creation_time, - 'save_model': False, - 'loss_params': {'moments_class': Legendre_tf, "max_moments": 20, 'loss_max': 0.5, 'quantile': 1e-3} - } - - train_MSE, test_MSE, train_RSE, test_RSE = analyze_statistics(config) - - tr_MSE[cl] = np.mean(train_MSE) - te_MSE[cl] = np.mean(test_MSE) - tr_RSE[cl] = np.mean(train_RSE) - te_RSE[cl] = np.mean(test_RSE) - - - plt_cl = plots.CorrLength() - plt_cl.add_mse_test(te_MSE) - plt_cl.add_mse_train(tr_MSE) - - plt_cl.show(None) - plt_cl.show("corr_length_mse") - - plt_cl = plots.CorrLength() - plt_cl.add_mse_test(te_RSE) - plt_cl.add_mse_train(tr_RSE) - - plt_cl.show(None) - plt_cl.show("corr_length_mse") - - -def plot_results_corr_length(): - cl_all = {"cl_0_001_s_1": 0.001, "cl_0_01_s_1": 0.01, "cl_0_1_s_1": 0.1, "cl_1_s_1": 1, "cl_10_s_1": 10} - - # cl_all = {"cl_0_001_s_1": 0.001, "cl_10_s_1": 1} - # - cl_all = {"cl_0_001_s_1": 0.001} - - tr_MSE = {} - te_MSE = {} - tr_RSE = {} - te_RSE = {} - - for cl_dir, cl in cl_all.items(): - data_dir = "/home/martin/Documents/metamodels/data/mesh_size/" - level = 3 + output_dir = os.path.join(data_dir, "L1_{}/test/02_conc/output/".format(level)) + hdf_path = os.path.join(data_dir, "L1_{}/mlmc_1.hdf5".format(level)) + mlmc_hdf_path = os.path.join(data_dir, "/mlmc_hdf/L1_{}/mlmc_1.hdf5".format(level)) + save_path = data_dir + l_0_output_dir = os.path.join(data_dir, "L0_MC/L1_{}/test/02_conc/output/".format(level)) + l_0_hdf_path = os.path.join(data_dir, "L0_MC/L1_{}/mlmc_1.hdf5".format(level)) + sampling_info_path = os.path.join(data_dir, "sampling_info") + ref_mlmc_file = os.path.join(data_dir, "L1_benchmark/mlmc_1.hdf5") + feature_names = [['conductivity_top', 'conductivity_bot', 'conductivity_repo']] + + elif case == 13: # mesh size comparison + data_dir = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_por/" + # cl = "cl_0_1_s_1" + level = 1 nn_level = 0 replace_level = False - # mesh = os.path.join(data_dir, "l_step_1.0_common_files/mesh.msh".format(cl)) #L1, 7s - # mesh = os.path.join(data_dir, "l_step_0.27232698153315_common_files/mesh.msh".format(cl)) #L2 10.5 s - mesh = os.path.join(data_dir, "l_step_0.07416198487095663_common_files/mesh.msh".format(cl_dir)) # L3 12s + mesh = os.path.join(data_dir, "l_step_1.0_common_files/repo.msh") # L1, 7s + # mesh = os.path.join(data_dir, "l_step_0.27232698153315_common_files/repo.msh".format(cl)) #L2 10.5 s + # mesh = os.path.join(data_dir, "l_step_0.07416198487095663_common_files/mesh.msh".format(cl)) #L3 12s # mesh = os.path.join(data_dir, "l_step_0.020196309484414757_common_files/mesh.msh".format(cl)) #L4 22s # mesh = os.path.join(data_dir, "l_step_0.0055_common_files/mesh.msh".format(cl)) #L5 - output_dir = os.path.join(data_dir, "{}/L1_{}/test/01_cond_field/output/".format(cl_dir, level)) - hdf_path = os.path.join(data_dir, "{}/L1_{}/mlmc_1.hdf5".format(cl_dir, level)) - mlmc_hdf_path = os.path.join(data_dir, "{}/mlmc_hdf/L1_{}/mlmc_1.hdf5".format(cl_dir, level)) - save_path = os.path.join(data_dir, "{}".format(cl_dir)) - l_0_output_dir = os.path.join(data_dir, "{}/L0_MC/L1_{}/test/01_cond_field/output/".format(cl_dir, level)) - l_0_hdf_path = os.path.join(data_dir, "{}/L0_MC/L1_{}/mlmc_1.hdf5".format(cl_dir, level)) - sampling_info_path = os.path.join(data_dir, "{}/sampling_info".format(cl_dir)) - ref_mlmc_file = os.path.join(data_dir, "{}/L1_benchmark/mlmc_1.hdf5".format(cl_dir)) - - machine_learning_model = ("mesh_L3_log_15k", run_GNN, False) - - gnn, conv_layer, corr_field_config, model_config = get_gnn() - - save_path = os.path.join(save_path, machine_learning_model[0]) - - print("save path ", save_path) - graph_creation_time = 25 # 22#159#0#159#66 - - config = {'machine_learning_model': machine_learning_model, - 'save_path': save_path, - 'sampling_info_path': sampling_info_path, - 'output_dir': output_dir, - 'nn_hdf_path': hdf_path, - 'mlmc_hdf_path': mlmc_hdf_path, - 'mesh': mesh, - 'l_0_output_dir': l_0_output_dir, - 'l_0_hdf_path': l_0_hdf_path, - 'ref_mlmc_file': ref_mlmc_file, - 'level': nn_level, - 'conv_layer': conv_layer, - 'gnn': gnn, - 'model_config': model_config, - 'replace_level': replace_level, - 'corr_field_config': corr_field_config, - 'n_train_samples': 2000, - 'val_samples_ratio': 0.3, - 'batch_size': 200, - 'epochs': 2000, - 'learning_rate': 0.01, - 'graph_creation_time': graph_creation_time, - 'save_model': False, - 'loss_params': {'moments_class': Legendre_tf, "max_moments": 20, 'loss_max': 0.5, 'quantile': 1e-3} - } - - train_MSE, test_MSE, train_RSE, test_RSE = analyze_statistics(config) - - tr_MSE[cl] = np.mean(train_MSE) - te_MSE[cl] = np.mean(test_MSE) - tr_RSE[cl] = np.mean(train_RSE) - te_RSE[cl] = np.mean(test_RSE) - - - plt_cl = plots.CorrLength() - plt_cl.add_mse_test(te_MSE) - plt_cl.add_mse_train(tr_MSE) - - plt_cl.show(None) - plt_cl.show("corr_length_mse") - - plt_cl = plots.CorrLength() - plt_cl.add_mse_test(te_RSE) - plt_cl.add_mse_train(tr_RSE) - - plt_cl.show(None) - plt_cl.show("corr_length_mse") + output_dir = os.path.join(data_dir, "L1_{}/test/02_conc/output/".format(level)) + hdf_path = os.path.join(data_dir, "L1_{}/mlmc_1.hdf5".format(level)) + mlmc_hdf_path = os.path.join(data_dir, "/mlmc_hdf/L1_{}/mlmc_1.hdf5".format(level)) + save_path = data_dir + l_0_output_dir = os.path.join(data_dir, "L0_MC/L1_{}/test/02_conc/output/".format(level)) + l_0_hdf_path = os.path.join(data_dir, "L0_MC/L1_{}/mlmc_1.hdf5".format(level)) + sampling_info_path = os.path.join(data_dir, "sampling_info") + ref_mlmc_file = os.path.join(data_dir, "L1_benchmark/mlmc_1.hdf5") + feature_names = [['porosity_top', 'porosity_bot', 'porosity_repo']] + + elif case == 14: # mesh size comparison + data_dir = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_2_features/" + # cl = "cl_0_1_s_1" + level = 1 + nn_level = 0 + replace_level = False + mesh = os.path.join(data_dir, "l_step_1.0_common_files/repo.msh") # L1, 7s + # mesh = os.path.join(data_dir, "l_step_0.27232698153315_common_files/repo.msh".format(cl)) #L2 10.5 s + # mesh = os.path.join(data_dir, "l_step_0.07416198487095663_common_files/mesh.msh".format(cl)) #L3 12s + # mesh = os.path.join(data_dir, "l_step_0.020196309484414757_common_files/mesh.msh".format(cl)) #L4 22s + # mesh = os.path.join(data_dir, "l_step_0.0055_common_files/mesh.msh".format(cl)) #L5 + output_dir = os.path.join(data_dir, "L1_{}/test/02_conc/output/".format(level)) + hdf_path = os.path.join(data_dir, "L1_{}/mlmc_1.hdf5".format(level)) + mlmc_hdf_path = os.path.join(data_dir, "/mlmc_hdf/L1_{}/mlmc_1.hdf5".format(level)) + save_path = data_dir + l_0_output_dir = os.path.join(data_dir, "L0_MC/L1_{}/test/02_conc/output/".format(level)) + l_0_hdf_path = os.path.join(data_dir, "L0_MC/L1_{}/mlmc_1.hdf5".format(level)) + sampling_info_path = os.path.join(data_dir, "sampling_info") + ref_mlmc_file = os.path.join(data_dir, "L1_benchmark/mlmc_1.hdf5") + + + feature_names = [['conductivity_top', 'conductivity_bot', 'conductivity_repo'], + ['porosity_top', 'porosity_bot', 'porosity_repo']] + + return output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, replace_level, nn_level, mlmc_hdf_path, feature_names + + +# def plot_results_corr_length(): +# cl_all = {"cl_0_001_s_1": 0.001, "cl_0_01_s_1": 0.01, "cl_0_1_s_1": 0.1, "cl_1_s_1": 1, "cl_10_s_1": 10} +# +# # cl_all = {"cl_0_001_s_1": 0.001, "cl_10_s_1": 1} +# # +# cl_all = {"cl_0_001_s_1": 0.001} +# +# tr_MSE = {} +# te_MSE = {} +# tr_RSE = {} +# te_RSE = {} +# +# for cl_dir, cl in cl_all.items(): +# data_dir = "/home/martin/Documents/metamodels/data/mesh_size/" +# level = 3 +# nn_level = 0 +# replace_level = False +# # mesh = os.path.join(data_dir, "l_step_1.0_common_files/mesh.msh".format(cl)) #L1, 7s +# # mesh = os.path.join(data_dir, "l_step_0.27232698153315_common_files/mesh.msh".format(cl)) #L2 10.5 s +# mesh = os.path.join(data_dir, "l_step_0.07416198487095663_common_files/mesh.msh".format(cl_dir)) # L3 12s +# # mesh = os.path.join(data_dir, "l_step_0.020196309484414757_common_files/mesh.msh".format(cl)) #L4 22s +# # mesh = os.path.join(data_dir, "l_step_0.0055_common_files/mesh.msh".format(cl)) #L5 +# output_dir = os.path.join(data_dir, "{}/L1_{}/test/01_cond_field/output/".format(cl_dir, level)) +# hdf_path = os.path.join(data_dir, "{}/L1_{}/mlmc_1.hdf5".format(cl_dir, level)) +# mlmc_hdf_path = os.path.join(data_dir, "{}/mlmc_hdf/L1_{}/mlmc_1.hdf5".format(cl_dir, level)) +# save_path = os.path.join(data_dir, "{}".format(cl_dir)) +# l_0_output_dir = os.path.join(data_dir, "{}/L0_MC/L1_{}/test/01_cond_field/output/".format(cl_dir, level)) +# l_0_hdf_path = os.path.join(data_dir, "{}/L0_MC/L1_{}/mlmc_1.hdf5".format(cl_dir, level)) +# sampling_info_path = os.path.join(data_dir, "{}/sampling_info".format(cl_dir)) +# ref_mlmc_file = os.path.join(data_dir, "{}/L1_benchmark/mlmc_1.hdf5".format(cl_dir)) +# +# machine_learning_model = ("mesh_L3_log_15k", run_GNN, False) +# +# gnn, conv_layer, corr_field_config, model_config = get_gnn() +# +# save_path = os.path.join(save_path, machine_learning_model[0]) +# +# print("save path ", save_path) +# graph_creation_time = 28 # 22#159#0#159#66 +# +# config = {'machine_learning_model': machine_learning_model, +# 'save_path': save_path, +# 'sampling_info_path': sampling_info_path, +# 'output_dir': output_dir, +# 'nn_hdf_path': hdf_path, +# 'mlmc_hdf_path': mlmc_hdf_path, +# 'mesh': mesh, +# 'l_0_output_dir': l_0_output_dir, +# 'l_0_hdf_path': l_0_hdf_path, +# 'ref_mlmc_file': ref_mlmc_file, +# 'level': nn_level, +# 'conv_layer': conv_layer, +# 'gnn': gnn, +# 'model_config': model_config, +# 'replace_level': replace_level, +# 'corr_field_config': corr_field_config, +# 'n_train_samples': 2000, +# 'val_samples_ratio': 0.3, +# 'batch_size': 200, +# 'epochs': 2000, +# 'learning_rate': 0.01, +# 'graph_creation_time': graph_creation_time, +# 'save_model': False, +# 'loss_params': {'moments_class': Legendre_tf, "max_moments": 20, 'loss_max': 0.5, 'quantile': 1e-3} +# } +# +# train_MSE, test_MSE, train_RSE, test_RSE = analyze_statistics(config) +# +# tr_MSE[cl] = np.mean(train_MSE) +# te_MSE[cl] = np.mean(test_MSE) +# tr_RSE[cl] = np.mean(train_RSE) +# te_RSE[cl] = np.mean(test_RSE) +# +# +# plt_cl = plots.CorrLength() +# plt_cl.add_mse_test(te_MSE) +# plt_cl.add_mse_train(tr_MSE) +# +# plt_cl.show(None) +# plt_cl.show("corr_length_mse") +# +# plt_cl = plots.CorrLength() +# plt_cl.add_mse_test(te_RSE) +# plt_cl.add_mse_train(tr_RSE) +# +# plt_cl.show(None) +# plt_cl.show("corr_length_mse") + + +# def plot_results_corr_length(): +# cl_all = {"cl_0_001_s_1": 0.001, "cl_0_01_s_1": 0.01, "cl_0_1_s_1": 0.1, "cl_1_s_1": 1, "cl_10_s_1": 10} +# +# # cl_all = {"cl_0_001_s_1": 0.001, "cl_10_s_1": 1} +# # +# cl_all = {"cl_0_001_s_1": 0.001} +# +# tr_MSE = {} +# te_MSE = {} +# tr_RSE = {} +# te_RSE = {} +# +# for cl_dir, cl in cl_all.items(): +# data_dir = "/home/martin/Documents/metamodels/data/mesh_size/" +# level = 3 +# nn_level = 0 +# replace_level = False +# # mesh = os.path.join(data_dir, "l_step_1.0_common_files/mesh.msh".format(cl)) #L1, 7s +# # mesh = os.path.join(data_dir, "l_step_0.27232698153315_common_files/mesh.msh".format(cl)) #L2 10.5 s +# mesh = os.path.join(data_dir, "l_step_0.07416198487095663_common_files/mesh.msh".format(cl_dir)) # L3 12s +# # mesh = os.path.join(data_dir, "l_step_0.020196309484414757_common_files/mesh.msh".format(cl)) #L4 22s +# # mesh = os.path.join(data_dir, "l_step_0.0055_common_files/mesh.msh".format(cl)) #L5 +# output_dir = os.path.join(data_dir, "{}/L1_{}/test/01_cond_field/output/".format(cl_dir, level)) +# hdf_path = os.path.join(data_dir, "{}/L1_{}/mlmc_1.hdf5".format(cl_dir, level)) +# mlmc_hdf_path = os.path.join(data_dir, "{}/mlmc_hdf/L1_{}/mlmc_1.hdf5".format(cl_dir, level)) +# save_path = os.path.join(data_dir, "{}".format(cl_dir)) +# l_0_output_dir = os.path.join(data_dir, "{}/L0_MC/L1_{}/test/01_cond_field/output/".format(cl_dir, level)) +# l_0_hdf_path = os.path.join(data_dir, "{}/L0_MC/L1_{}/mlmc_1.hdf5".format(cl_dir, level)) +# sampling_info_path = os.path.join(data_dir, "{}/sampling_info".format(cl_dir)) +# ref_mlmc_file = os.path.join(data_dir, "{}/L1_benchmark/mlmc_1.hdf5".format(cl_dir)) +# +# machine_learning_model = ("mesh_L3_log_15k", run_GNN, False) +# +# gnn, conv_layer, corr_field_config, model_config = get_gnn() +# +# save_path = os.path.join(save_path, machine_learning_model[0]) +# +# print("save path ", save_path) +# graph_creation_time = 25 # 22#159#0#159#66 +# +# config = {'machine_learning_model': machine_learning_model, +# 'save_path': save_path, +# 'sampling_info_path': sampling_info_path, +# 'output_dir': output_dir, +# 'nn_hdf_path': hdf_path, +# 'mlmc_hdf_path': mlmc_hdf_path, +# 'mesh': mesh, +# 'l_0_output_dir': l_0_output_dir, +# 'l_0_hdf_path': l_0_hdf_path, +# 'ref_mlmc_file': ref_mlmc_file, +# 'level': nn_level, +# 'conv_layer': conv_layer, +# 'gnn': gnn, +# 'model_config': model_config, +# 'replace_level': replace_level, +# 'corr_field_config': corr_field_config, +# 'n_train_samples': 2000, +# 'val_samples_ratio': 0.3, +# 'batch_size': 200, +# 'epochs': 2000, +# 'learning_rate': 0.01, +# 'graph_creation_time': graph_creation_time, +# 'save_model': False, +# 'loss_params': {'moments_class': Legendre_tf, "max_moments": 20, 'loss_max': 0.5, 'quantile': 1e-3} +# } +# +# train_MSE, test_MSE, train_RSE, test_RSE = analyze_statistics(config) +# +# tr_MSE[cl] = np.mean(train_MSE) +# te_MSE[cl] = np.mean(test_MSE) +# tr_RSE[cl] = np.mean(train_RSE) +# te_RSE[cl] = np.mean(test_RSE) +# +# +# plt_cl = plots.CorrLength() +# plt_cl.add_mse_test(te_MSE) +# plt_cl.add_mse_train(tr_MSE) +# +# plt_cl.show(None) +# plt_cl.show("corr_length_mse") +# +# plt_cl = plots.CorrLength() +# plt_cl.add_mse_test(te_RSE) +# plt_cl.add_mse_train(tr_RSE) +# +# plt_cl.show(None) +# plt_cl.show("corr_length_mse") def get_arguments(arguments): @@ -462,10 +519,10 @@ def get_arguments(arguments): args = get_arguments(sys.argv[1:]) data_dir = args.data_dir work_dir = args.work_dir - case = 7 + case = 12 #data_dir = "/home/martin/Documents/metamodels/data/1000_ele/" output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file,\ - replace_level, nn_level, mlmc_hdf_path = get_config(data_dir, case) + replace_level, nn_level, mlmc_hdf_path, feature_names = get_config(data_dir, case) # plot_results_corr_length() # exit() @@ -571,21 +628,26 @@ def get_arguments(arguments): #machine_learning_model = ("DNN_mesh_L3_6", run_DNN, True) machine_learning_model = ("GCN_mesh_L3_log_16", run_GNN, True) machine_learning_model = ("mesh_L3_log_test_saved_model", run_GNN, True) - machine_learning_model = ("mesh_L3_log_50k_weights_2", run_GNN, True) + machine_learning_model = ("mesh_L3_log_50k_weights_5", run_GNN, True) - machine_learning_model = ("mesh_L3_log_50k", run_GNN, True) + machine_learning_model = ("test_02_conc", run_GNN, False) - #machine_learning_model = ("mesh_L3_seed", run_GNN, False) + #machine_learning_model = ("mesh_L3_log_50k_weights_test_dense", run_GNN, False) #machine_learning_model = ("mesh_L3_log_sigmoid", run_GNN, False) # ReLU is much better save_path = os.path.join(save_path, machine_learning_model[0]) - # if os.path.exists(save_path): - # shutil.rmtree(save_path) + if os.path.exists(save_path): + shutil.rmtree(save_path) print("save path ", save_path) - graph_creation_time = 25#11#22#159#0#159#66 + + + # graph creation time: 2 features: 61 sec + # conductivity: 43 sec + # porosity: 40 sec + graph_creation_time = 43#25#11#22#159#0#159#66 config = {'machine_learning_model': machine_learning_model, 'save_path': save_path, @@ -605,14 +667,15 @@ def get_arguments(arguments): 'corr_field_config': corr_field_config, 'n_train_samples': 2000, 'val_samples_ratio': 0.2, - 'batch_size': 200, - 'epochs': 5, + 'batch_size': 20, + 'epochs': 2, 'learning_rate': 0.001, 'graph_creation_time': graph_creation_time, - 'save_model': True + 'save_model': True, + 'feature_names': feature_names } - #statistics(config) + statistics(config) analyze_statistics(config) From 7b946fc6b28b0d9a312553fe264941a517fcaf53 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Fri, 5 Nov 2021 16:53:50 +0100 Subject: [PATCH 40/67] graph features fix --- mlmc/metamodel/analyze_nn.py | 32 +- mlmc/metamodel/create_graph.py | 2 +- mlmc/metamodel/own_cheb_conv.py | 4 +- mlmc/metamodel/random_field_time.py | 29 ++ mlmc/random/correlated_field.py | 7 +- mlmc/tool/flow_mc_2.py | 518 ++++++++++++++++++++++++++++ test/metamodels/metamodel_test.py | 10 +- 7 files changed, 581 insertions(+), 21 deletions(-) create mode 100644 mlmc/tool/flow_mc_2.py diff --git a/mlmc/metamodel/analyze_nn.py b/mlmc/metamodel/analyze_nn.py index 443f5c47..befe1d1e 100644 --- a/mlmc/metamodel/analyze_nn.py +++ b/mlmc/metamodel/analyze_nn.py @@ -565,14 +565,21 @@ def predict_data(config, model, mesh_file, log=True): inputs, target = batch x, a = inputs + print("x ", x) + for conv_index, conv_layer in enumerate(model._conv_layers): if conv_index not in conv_layers: conv_layers[conv_index] = [[], [], []] conv_layers[conv_index][0].extend(x) # inputs + print("conv_layer.kernel.numpy().shape", conv_layer.kernel.numpy().shape) conv_layers[conv_index][1].extend(conv_layer.kernel.numpy()) # weights (kernel) conv_out = conv_layer([x, a]) + + print("conv out ", conv_out) conv_layers[conv_index][2].extend(conv_out) # outputs + exit() + flatten_input = conv_layers[conv_index][2][-1] # flatten_output = model.flatten(conv_out) # @@ -679,9 +686,9 @@ def analyze_statistics(config): data_dict = process_data(data_dict) - print("train predictions type ", type(data_dict["train_predictions"])) - print("train predictions type ", type(data_dict["train_predictions"][0])) - print("train predictions shape ", np.array(data_dict["train_predictions"]).shape) + # print("train predictions type ", type(data_dict["train_predictions"])) + # print("train predictions type ", type(data_dict["train_predictions"][0])) + # print("train predictions shape ", np.array(data_dict["train_predictions"]).shape) # print("train predictions ", data_dict["train_predictions"]) # print("train predictions as matrix shape", np.asmatrix(np.array(data_dict["train_predictions"])).shape) @@ -728,7 +735,7 @@ def analyze_statistics(config): # if i == 3: # break - print("index ", i) + #print("index ", i) # if i not in [2, 5, 7]: # continue @@ -1203,7 +1210,7 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): if graph_creation_time == 0: graph_creator_preproces_time = time.process_time() graph_creator(config['output_dir'], config['hdf_path'], config['mesh'], level=config['level'], - feature_names=config['feature_names']) + feature_names=config.get('feature_names', [['conductivity']])) graph_creation_time = time.process_time() - graph_creator_preproces_time print("graph creation time ", graph_creation_time) exit() @@ -1213,6 +1220,11 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): data = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log) data = data#[:10000] + # print("n node features ", data.graphs[0].n_node_features) + # print("graph x", data.graphs[0].x) + # print("graphs[0] ", repr(data.graphs[0])) + # exit() + #print("len data ", len(data)) #data.shuffle(seed=seed) preprocess_time = time.process_time() - preprocess_start_time @@ -1320,7 +1332,7 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): stats=stats, corr_field_config=config['corr_field_config'], seed=seed, - feature_names=config['feature_names'] + feature_names=config.get('feature_names', [['conductivity']]) ) #predict_l_0_time = time.process_time() - predict_l_0_start_time @@ -1347,7 +1359,7 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): def predict_level_zero(nn, output_dir, hdf_path, mesh, conv_layer, batch_size=1000, log=False, stats=False, corr_field_config=None, seed=1234, feature_names=[]): - graph_creator(output_dir, hdf_path, mesh, level=0, feature_names=feature_names) + #graph_creator(output_dir, hdf_path, mesh, level=0, feature_names=feature_names) # Load data sample_time = 0 if corr_field_config: @@ -1357,9 +1369,9 @@ def predict_level_zero(nn, output_dir, hdf_path, mesh, conv_layer, batch_size=10 #data = data # [:10000] data.shuffle(seed=seed) - print("output_dir ", output_dir) - print("len(data) ", len(data)) - print("data[0] ", data[0]) + # print("output_dir ", output_dir) + # print("len(data) ", len(data)) + # print("data[0] ", data[0]) predict_time_start = time.process_time() data.a = conv_layer.preprocess(data.a) diff --git a/mlmc/metamodel/create_graph.py b/mlmc/metamodel/create_graph.py index 3c46a949..a6df5c91 100644 --- a/mlmc/metamodel/create_graph.py +++ b/mlmc/metamodel/create_graph.py @@ -94,7 +94,7 @@ def get_node_features(fields_mesh, feature_names): features.append(list(joint_features.values())) - return features + return np.array(features).T def join_fields(fields, f_names): diff --git a/mlmc/metamodel/own_cheb_conv.py b/mlmc/metamodel/own_cheb_conv.py index 1d830e1a..de54ded7 100644 --- a/mlmc/metamodel/own_cheb_conv.py +++ b/mlmc/metamodel/own_cheb_conv.py @@ -204,9 +204,9 @@ def normalized_adjacency(A, symmetric=True): :return: the normalized adjacency matrix. """ if symmetric: - print("symmetric") + #print("symmetric") normalized_D = degree_power(A, -0.5) - print("normalized D") + #print("normalized D") return normalized_D.dot(A).dot(normalized_D) else: normalized_D = degree_power(A, -1.0) diff --git a/mlmc/metamodel/random_field_time.py b/mlmc/metamodel/random_field_time.py index 1eade569..8bf2a27e 100644 --- a/mlmc/metamodel/random_field_time.py +++ b/mlmc/metamodel/random_field_time.py @@ -2,12 +2,41 @@ import numpy as np from mlmc.tool import gmsh_io from mlmc.tool.flow_mc import FlowSim, create_corr_field +from mlmc.tool.flow_mc_2 import FlowSimProcConc +from mlmc.tool.flow_mc_2 import create_corr_field as conc_create_corr_fields +import gstools +from mlmc.random import correlated_field as cf + + +def conc_rnd_sample_time(mesh_file, corr_field_config): + + start_time = time.process_time() + mesh_data = FlowSim.extract_mesh(mesh_file) + + fields = conc_create_corr_fields(dim=2, log=corr_field_config["log"], mode_no=10000) + + n_samples = 200 + for i in range(n_samples): + fields.set_points(mesh_data['points'], mesh_data['point_region_ids'], + mesh_data['region_map']) + + fine_input_sample, coarse_input_sample = FlowSimProcConc.generate_random_sample(fields, coarse_step=0, + n_fine_elements=len( + mesh_data['points'])) + + rnd_time = time.process_time() - start_time + print("rnd_time / n_samples ", rnd_time / n_samples) + return rnd_time / n_samples def corr_field_sample_time(mesh_file=None, corr_length_config=None): # import matplotlib # from matplotlib import ticker, cm #matplotlib.rcParams.update({'font.size': 22}) + + if corr_length_config['02_conc']: + return conc_rnd_sample_time(mesh_file, corr_length_config) + dim = 2 log = True cl = 0.1 diff --git a/mlmc/random/correlated_field.py b/mlmc/random/correlated_field.py index ba83e15b..5309f84d 100644 --- a/mlmc/random/correlated_field.py +++ b/mlmc/random/correlated_field.py @@ -62,7 +62,7 @@ def __init__(self, name, field=None, param_fields=[], regions=[]): if type(field) in [float, int]: self.const = field assert len(param_fields) == 0 - elif isinstance(field, RandomFieldBase): + elif isinstance(field, RandomFieldBase) or isinstance(field, gstools.covmodel.models.CovModel): self.correlated_field = field assert len(param_fields) == 0 else: @@ -372,7 +372,6 @@ def _initialize(self, **kwargs): """ Called after initialization in common constructor. """ - ### Attributes computed in precalculation. self.cov_mat = None # Covariance matrix (dense). @@ -500,14 +499,14 @@ def _sample(self): class GSToolsSpatialCorrelatedField(RandomFieldBase): - def __init__(self, model, mode_no=1000, log=False, sigma=1): + def __init__(self, model, mode_no=1000, log=False, sigma=1, mean=0): """ :param model: instance of covariance model class, which parent is gstools.covmodel.CovModel :param mode_no: number of Fourier modes, default: 1000 as in gstools package """ self.model = model self.mode_no = mode_no - self.srf = gstools.SRF(model, mode_no=mode_no) + self.srf = gstools.SRF(model, mean=mean, mode_no=mode_no) self.mu = self.srf.mean self.sigma = sigma self.dim = model.dim diff --git a/mlmc/tool/flow_mc_2.py b/mlmc/tool/flow_mc_2.py new file mode 100644 index 00000000..0219997a --- /dev/null +++ b/mlmc/tool/flow_mc_2.py @@ -0,0 +1,518 @@ +import os +import os.path +import subprocess +import numpy as np +import shutil +import ruamel.yaml as yaml +from typing import List +import gstools +from mlmc.level_simulation import LevelSimulation +from mlmc.tool import gmsh_io +from mlmc.sim.simulation import Simulation +from mlmc.quantity.quantity_spec import QuantitySpec +from mlmc.random import correlated_field as cf + + +def create_corr_field(model='gauss', corr_length=0.125, dim=2, log=True, sigma=1, mode_no=1000): + """ + Create random fields + :return: + """ + # por_top = cf.SpatialCorrelatedField( + # corr_exp='gauss', + # dim=2, + # corr_length=0.2, + # mu=-1.0, + # sigma=1.0, + # log=True + # ) + # + # print("por top ", por_top) + + por_top = cf.GSToolsSpatialCorrelatedField(gstools.Gaussian(dim=2, len_scale=0.2), + log=log, mean=-1.0, sigma=1.0, mode_no=mode_no) + + #print("por top gstools ", por_top_gstools) + + # por_bot = cf.SpatialCorrelatedField( + # corr_exp='gauss', + # dim=2, + # corr_length=0.2, + # mu=-1.0, + # sigma=1.0, + # log=True + # ) + + por_bot = cf.GSToolsSpatialCorrelatedField(gstools.Gaussian(dim=2, len_scale=0.2), + log=log, mean=-1.0, sigma=1.0, mode_no=mode_no) + + #por_bot = gstools.Gaussian(dim=dim, len_scale=0.2, mu=-1.0, sigma=1.0, log=True) + + water_viscosity = 8.90e-4 + + factor_top_model = gstools.Gaussian(dim=dim, len_scale=1) + factor_bot_model = gstools.Gaussian(dim=dim, len_scale=1) + + fields = cf.Fields([ + cf.Field('por_top', por_top, regions='ground_0'), + cf.Field('porosity_top', cf.positive_to_range, ['por_top', 0.02, 0.1], regions='ground_0'), + cf.Field('por_bot', por_bot, regions='ground_1'), + cf.Field('porosity_bot', cf.positive_to_range, ['por_bot', 0.01, 0.05], regions='ground_1'), + cf.Field('porosity_repo', 0.5, regions='repo'), + #cf.Field('factor_top', cf.SpatialCorrelatedField('gauss', mu=1e-8, sigma=1, log=True), regions='ground_0'), + + cf.Field('factor_top', cf.GSToolsSpatialCorrelatedField(factor_top_model, log=log, mean=1e-8, sigma=1.0, mode_no=mode_no), + regions='ground_0'), + + #cf.Field('factor_top', gstools.Gaussian(len_scale=1, mu=1e-8, sigma=1.0, log=True), regions='ground_0'), + # conductivity about + #cf.Field('factor_bot', cf.SpatialCorrelatedField('gauss', mu=1e-8, sigma=1, log=True), regions='ground_1'), + #cf.Field('factor_bot', gstools.Gaussian(len_scale=1, mu=1e-8, sigma=1, log=True), regions='ground_1'), + cf.Field('factor_bot', + cf.GSToolsSpatialCorrelatedField(factor_bot_model, log=log, mean=1e-8, sigma=1.0, mode_no=mode_no), + regions='ground_1'), + + # cf.Field('factor_repo', cf.SpatialCorrelatedField('gauss', mu=1e-10, sigma=1, log=True), regions='repo'), + cf.Field('conductivity_top', cf.kozeny_carman, ['porosity_top', 1, 'factor_top', water_viscosity], + regions='ground_0'), + cf.Field('conductivity_bot', cf.kozeny_carman, ['porosity_bot', 1, 'factor_bot', water_viscosity], + regions='ground_1'), + # cf.Field('conductivity_repo', cf.kozeny_carman, ['porosity_repo', 1, 'factor_repo', water_viscosity], regions='repo') + cf.Field('conductivity_repo', 0.001, regions='repo') + ]) + + return fields + + +def substitute_placeholders(file_in, file_out, params): + """ + Substitute for placeholders of format '' from the dict 'params'. + :param file_in: Template file. + :param file_out: Values substituted. + :param params: { 'name': value, ...} + """ + used_params = [] + with open(file_in, 'r') as src: + text = src.read() + for name, value in params.items(): + placeholder = '<%s>' % name + n_repl = text.count(placeholder) + if n_repl > 0: + used_params.append(name) + text = text.replace(placeholder, str(value)) + with open(file_out, 'w') as dst: + dst.write(text) + return used_params + + +def force_mkdir(path, force=False): + """ + Make directory 'path' with all parents, + remove the leaf dir recursively if it already exists. + :param path: path to directory + :param force: if dir already exists then remove it and create new one + :return: None + """ + if force: + if os.path.isdir(path): + shutil.rmtree(path) + os.makedirs(path, mode=0o775, exist_ok=True) + + +class FlowSimProcConc(Simulation): + # placeholders in YAML + total_sim_id = 0 + MESH_FILE_VAR = 'mesh_file' + # Timestep placeholder given as O(h), h = mesh step + TIMESTEP_H1_VAR = 'timestep_h1' + # Timestep placeholder given as O(h^2), h = mesh step + TIMESTEP_H2_VAR = 'timestep_h2' + + # files + GEO_FILE = 'repo.geo' + MESH_FILE = 'repo.msh' + YAML_TEMPLATE = '02_conc_tmpl.yaml' + YAML_FILE = '02_conc.yaml' + FIELDS_FILE = 'fields_sample.msh' + + """ + Gather data for single flow call (coarse/fine) + + Usage: + mlmc.sampler.Sampler uses instance of FlowSimProcConc, it calls once level_instance() for each level step (The level_instance() method + is called as many times as the number of levels), it takes place in main process + + mlmc.tool.pbs_job.PbsJob uses static methods in FlowSimProcConc, it calls calculate(). That's where the calculation actually runs, + it takes place in PBS process + It also extracts results and passes them back to PbsJob, which handles the rest + + """ + + def __init__(self, config=None, clean=None): + """ + Simple simulation using flow123d + :param config: configuration of the simulation, processed keys: + env - Environment object. + fields - FieldSet object + yaml_file: Template for main input file. Placeholders: + - replaced by generated mesh + - for FIELD be name of any of `fields`, replaced by the FieldElementwise field with generated + field input file and the field name for the component. + geo_file: Path to the geometry file. + :param clean: bool, if True remove existing simulation files - mesh files, ... + """ + self.need_workspace = True + # This simulation requires workspace + self.env = config['env'] + # Environment variables, flow123d, gmsh, ... + self._fields_params = config['fields_params'] + self._fields = create_corr_field(**config['fields_params']) + self._fields_used_params = None + # Random fields instance + self.time_factor = config.get('time_factor', 1.0) + # It is used for minimal element from mesh determination (see level_instance method) + + self.base_yaml_file = config['yaml_file'] + self.base_geo_file = config['geo_file'] + self.field_template = config.get('field_template', + "!FieldElementwise {mesh_data_file: $INPUT_DIR$/%s, field_name: %s}") + self.work_dir = config['work_dir'] + self.clean = clean + + super(Simulation, self).__init__() + + def level_instance(self, fine_level_params: List[float], coarse_level_params: List[float]) -> LevelSimulation: + """ + Called from mlmc.Sampler, it creates single instance of LevelSimulation (mlmc.) + :param fine_level_params: in this version, it is just fine simulation step + :param coarse_level_params: in this version, it is just coarse simulation step + :return: mlmc.LevelSimulation object, this object is serialized in SamplingPoolPbs and deserialized in PbsJob, + so it allows pass simulation data from main process to PBS process + """ + fine_step = fine_level_params[0] + coarse_step = coarse_level_params[0] + + # TODO: determine minimal element from mesh + self.time_step_h1 = self.time_factor * fine_step + self.time_step_h2 = self.time_factor * fine_step * fine_step + + # Set fine simulation common files directory + # Files in the directory are used by each simulation at that level + common_files_dir = os.path.join(self.work_dir, "l_step_{}_common_files".format(fine_step)) + force_mkdir(common_files_dir, force=self.clean) + + self.mesh_file = os.path.join(common_files_dir, self.MESH_FILE) + + if self.clean: + # Prepare mesh + geo_file = os.path.join(common_files_dir, self.GEO_FILE) + shutil.copyfile(self.base_geo_file, geo_file) + self._make_mesh(geo_file, self.mesh_file, fine_step) # Common computational mesh for all samples. + + # Prepare main input YAML + yaml_template = os.path.join(common_files_dir, self.YAML_TEMPLATE) + shutil.copyfile(self.base_yaml_file, yaml_template) + yaml_file = os.path.join(common_files_dir, self.YAML_FILE) + self._substitute_yaml(yaml_template, yaml_file) + + # Mesh is extracted because we need number of mesh points to determine task_size parameter (see return value) + fine_mesh_data = self.extract_mesh(self.mesh_file) + + # Set coarse simulation common files directory + # Files in the directory are used by each simulation at that level + coarse_sim_common_files_dir = None + if coarse_step != 0: + coarse_sim_common_files_dir = os.path.join(self.work_dir, "l_step_{}_common_files".format(coarse_step)) + + # Simulation config + # Configuration is used in mlmc.tool.pbs_job.PbsJob instance which is run from PBS process + # It is part of LevelSimulation which is serialized and then deserialized in mlmc.tool.pbs_job.PbsJob + config = dict() + config["fine"] = {} + config["coarse"] = {} + config["fine"]["step"] = fine_step + config["coarse"]["step"] = coarse_step + config["fine"]["common_files_dir"] = common_files_dir + config["coarse"]["common_files_dir"] = coarse_sim_common_files_dir + + config[ + "fields_used_params"] = self._fields_used_params # Params for Fields instance, which is createed in PbsJob + config["gmsh"] = self.env['gmsh'] + config["flow123d"] = self.env['flow123d'] + config['fields_params'] = self._fields_params + + # Auxiliary parameter which I use to determine task_size (should be from 0 to 1, if task_size is above 1 then pbs job is scheduled) + job_weight = 17000000 # 4000000 - 20 min, 2000000 - cca 10 min + + return LevelSimulation(config_dict=config, + task_size=len(fine_mesh_data['points']) / job_weight, + calculate=FlowSimProcConc.calculate, + # method which carries out the calculation, will be called from PBS processs + need_sample_workspace=True # If True, a sample directory is created + ) + + @staticmethod + def calculate(config, seed): + """ + Method that actually run the calculation, it's called from mlmc.tool.pbs_job.PbsJob.calculate_samples() + Calculate fine and coarse sample and also extract their results + :param config: dictionary containing simulation configuration, LevelSimulation.config_dict (set in level_instance) + :param seed: random seed, int + :return: List[fine result, coarse result], both flatten arrays (see mlmc.sim.synth_simulation.calculate()) + """ + # Init correlation field objects + fields = create_corr_field(**config['fields_params']) # correlated_field.Fields instance + fields.set_outer_fields(config["fields_used_params"]) + + coarse_step = config["coarse"]["step"] # Coarse simulation step, zero if one level MC + flow123d = config["flow123d"] # Flow123d command + + # Extract fine mesh + fine_common_files_dir = config["fine"]["common_files_dir"] # Directory with fine simulation common files + fine_mesh_data = FlowSimProcConc.extract_mesh(os.path.join(fine_common_files_dir, FlowSimProcConc.MESH_FILE)) + + # Extract coarse mesh + coarse_mesh_data = None + coarse_common_files_dir = None + if coarse_step != 0: + coarse_common_files_dir = config["coarse"][ + "common_files_dir"] # Directory with coarse simulation common files + coarse_mesh_data = FlowSimProcConc.extract_mesh(os.path.join(coarse_common_files_dir, FlowSimProcConc.MESH_FILE)) + + # Create fields both fine and coarse + fields = FlowSimProcConc.make_fields(fields, fine_mesh_data, coarse_mesh_data) + + # Set random seed, seed is calculated from sample id, so it is not user defined + np.random.seed(seed) + # Generate random samples + fine_input_sample, coarse_input_sample = FlowSimProcConc.generate_random_sample(fields, coarse_step=coarse_step, + n_fine_elements=len( + fine_mesh_data['points'])) + + # Run fine sample + fields_file = os.path.join(os.getcwd(), FlowSimProcConc.FIELDS_FILE) + fine_res = FlowSimProcConc._run_sample(fields_file, fine_mesh_data['ele_ids'], fine_input_sample, flow123d, + fine_common_files_dir) + + # Rename fields_sample.msh to fine_fields_sample.msh, we might remove it + for filename in os.listdir(os.getcwd()): + if not filename.startswith("fine"): + shutil.move(os.path.join(os.getcwd(), filename), os.path.join(os.getcwd(), "fine_" + filename)) + + # Run coarse sample + coarse_res = np.zeros(len(fine_res)) + if coarse_input_sample: + coarse_res = FlowSimProcConc._run_sample(fields_file, coarse_mesh_data['ele_ids'], coarse_input_sample, flow123d, + coarse_common_files_dir) + + return fine_res, coarse_res + + @staticmethod + def make_fields(fields, fine_mesh_data, coarse_mesh_data): + """ + Create random fields that are used by both coarse and fine simulation + :param fields: correlated_field.Fields instance + :param fine_mesh_data: Dict contains data extracted from fine mesh file (points, point_region_ids, region_map) + :param coarse_mesh_data: Dict contains data extracted from coarse mesh file (points, point_region_ids, region_map) + :return: correlated_field.Fields + """ + # One level MC has no coarse_mesh_data + if coarse_mesh_data is None: + fields.set_points(fine_mesh_data['points'], fine_mesh_data['point_region_ids'], + fine_mesh_data['region_map']) + else: + coarse_centers = coarse_mesh_data['points'] + both_centers = np.concatenate((fine_mesh_data['points'], coarse_centers), axis=0) + both_regions_ids = np.concatenate( + (fine_mesh_data['point_region_ids'], coarse_mesh_data['point_region_ids'])) + assert fine_mesh_data['region_map'] == coarse_mesh_data['region_map'] + fields.set_points(both_centers, both_regions_ids, fine_mesh_data['region_map']) + + return fields + + @staticmethod + def _run_sample(fields_file, ele_ids, fine_input_sample, flow123d, common_files_dir): + """ + Create random fields file, call Flow123d and extract results + :param fields_file: Path to file with random fields + :param ele_ids: Element IDs in computational mesh + :param fine_input_sample: fields: {'field_name' : values_array, ..} + :param flow123d: Flow123d command + :param common_files_dir: Directory with simulations common files (flow_input.yaml, ) + :return: simulation result, ndarray + """ + gmsh_io.GmshIO().write_fields(fields_file, ele_ids, fine_input_sample) + + # x = [*flow123d, "--yaml_balance", '-i', os.getcwd(), '-s', "{}/flow_input.yaml".format(common_files_dir), + # "-o", os.getcwd(), ">{}/flow.out".format(os.getcwd())] + + #try: + subprocess.call( + [flow123d, "--yaml_balance", '-i', os.getcwd(), '-s', "{}/02_conc.yaml".format(common_files_dir), + "-o", os.getcwd(), ">{}/flow.out".format(os.getcwd())]) + # except: + # import sys + # print(sys.exc_info()) + + return FlowSimProcConc._extract_result(os.getcwd()) + + @staticmethod + def generate_random_sample(fields, coarse_step, n_fine_elements): + """ + Generate random field, both fine and coarse part. + Store them separeted. + :return: Dict, Dict + """ + fields_sample = fields.sample() + fine_input_sample = {name: values[:n_fine_elements, None] for name, values in fields_sample.items()} + coarse_input_sample = {} + if coarse_step != 0: + coarse_input_sample = {name: values[n_fine_elements:, None] for name, values in + fields_sample.items()} + + return fine_input_sample, coarse_input_sample + + def _make_mesh(self, geo_file, mesh_file, fine_step): + """ + Make the mesh, mesh_file: _step.msh. + Make substituted yaml: _step.yaml, + using common fields_step.msh file for generated fields. + :return: + """ + if self.env['gmsh_version'] == 2: + subprocess.call( + [self.env['gmsh'], "-2", '-format', 'msh2', '-clscale', str(fine_step), '-o', mesh_file, geo_file]) + else: + subprocess.call([self.env['gmsh'], "-2", '-clscale', str(fine_step), '-o', mesh_file, geo_file]) + + @staticmethod + def extract_mesh(mesh_file): + """ + Extract mesh from file + :param mesh_file: Mesh file path + :return: Dict + """ + mesh = gmsh_io.GmshIO(mesh_file) + is_bc_region = {} + region_map = {} + for name, (id, _) in mesh.physical.items(): + unquoted_name = name.strip("\"'") + is_bc_region[id] = (unquoted_name[0] == '.') + region_map[unquoted_name] = id + + bulk_elements = [] + for id, el in mesh.elements.items(): + _, tags, i_nodes = el + region_id = tags[0] + if not is_bc_region[region_id]: + bulk_elements.append(id) + + n_bulk = len(bulk_elements) + centers = np.empty((n_bulk, 3)) + ele_ids = np.zeros(n_bulk, dtype=int) + point_region_ids = np.zeros(n_bulk, dtype=int) + + for i, id_bulk in enumerate(bulk_elements): + _, tags, i_nodes = mesh.elements[id_bulk] + region_id = tags[0] + centers[i] = np.average(np.array([mesh.nodes[i_node] for i_node in i_nodes]), axis=0) + point_region_ids[i] = region_id + ele_ids[i] = id_bulk + + min_pt = np.min(centers, axis=0) + max_pt = np.max(centers, axis=0) + diff = max_pt - min_pt + min_axis = np.argmin(diff) + non_zero_axes = [0, 1, 2] + # TODO: be able to use this mesh_dimension in fields + if diff[min_axis] < 1e-10: + non_zero_axes.pop(min_axis) + points = centers[:, non_zero_axes] + + return {'points': points, 'point_region_ids': point_region_ids, 'ele_ids': ele_ids, 'region_map': region_map} + + def _substitute_yaml(self, yaml_tmpl, yaml_out): + """ + Create substituted YAML file from the tamplate. + :return: + """ + param_dict = {} + field_tmpl = self.field_template + for field_name in self._fields.names: + param_dict[field_name] = field_tmpl % (self.FIELDS_FILE, field_name) + param_dict[self.MESH_FILE_VAR] = self.mesh_file + param_dict[self.TIMESTEP_H1_VAR] = self.time_step_h1 + param_dict[self.TIMESTEP_H2_VAR] = self.time_step_h2 + used_params = substitute_placeholders(yaml_tmpl, yaml_out, param_dict) + + self._fields_used_params = used_params + + @staticmethod + def _extract_result(sample_dir): + """ + Extract the observed value from the Flow123d output. + :param sample_dir: str, path to sample directory + :return: None, inf or water balance result (float) and overall sample time + """ + # extract the flux + balance_file = os.path.join(sample_dir, "mass_balance.yaml") + + with open(balance_file, "r") as f: + balance = yaml.load(f) + + flux_regions = ['.surface'] + max_flux = 0.0 + found = False + for flux_item in balance['data']: + if 'region' not in flux_item: + os.remove(os.path.join(sample_dir, "mass_balance.yaml")) + break + + if flux_item['region'] in flux_regions: + out_flux = -float(flux_item['data'][0]) + if not np.isfinite(out_flux): + return np.inf + # flux_in = float(flux_item['data'][1]) + # if flux_in > 1e-10: + # raise Exception("Possitive inflow at outlet region.") + max_flux = max(max_flux, out_flux) # flux field + found = True + + # Get flow123d computing time + # run_time = FlowSimProcConc.get_run_time(sample_dir) + + if not found: + raise Exception + return np.array([max_flux]) + + @staticmethod + def result_format() -> List[QuantitySpec]: + """ + Define simulation result format + :return: List[QuantitySpec, ...] + """ + spec1 = QuantitySpec(name="conductivity", unit="m", shape=(1, 1), times=[1], locations=['0']) + # spec2 = QuantitySpec(name="width", unit="mm", shape=(2, 1), times=[1, 2, 3], locations=['30', '40']) + return [spec1] + + # @staticmethod + # def get_run_time(sample_dir): + # """ + # Get flow123d sample running time from profiler + # :param sample_dir: Sample directory + # :return: float + # """ + # profiler_file = os.path.join(sample_dir, "profiler_info_*.json") + # profiler = glob.glob(profiler_file)[0] + # + # try: + # with open(profiler, "r") as f: + # prof_content = json.load(f) + # + # run_time = float(prof_content['children'][0]['cumul-time-sum']) + # except: + # print("Extract run time failed") + # + # return run_time + + diff --git a/test/metamodels/metamodel_test.py b/test/metamodels/metamodel_test.py index 44a74d94..6c85869c 100644 --- a/test/metamodels/metamodel_test.py +++ b/test/metamodels/metamodel_test.py @@ -63,7 +63,7 @@ def get_gnn(): "net_model_config": net_model_config, "verbose": True} - corr_field_config = {'corr_length': 0.1, 'sigma': 1, 'log': True} + corr_field_config = {'02_conc': True, 'corr_length': 0.1, 'sigma': 1, 'log': True} return GNN, conv_layer, corr_field_config, model_config @@ -519,7 +519,7 @@ def get_arguments(arguments): args = get_arguments(sys.argv[1:]) data_dir = args.data_dir work_dir = args.work_dir - case = 12 + case = 14 #data_dir = "/home/martin/Documents/metamodels/data/1000_ele/" output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file,\ replace_level, nn_level, mlmc_hdf_path, feature_names = get_config(data_dir, case) @@ -632,6 +632,8 @@ def get_arguments(arguments): machine_learning_model = ("test_02_conc", run_GNN, False) + #machine_learning_model = ("L1_1_02_conc_cond", run_GNN, False) + #machine_learning_model = ("mesh_L3_log_50k_weights_test_dense", run_GNN, False) #machine_learning_model = ("mesh_L3_log_sigmoid", run_GNN, False) # ReLU is much better @@ -643,11 +645,11 @@ def get_arguments(arguments): print("save path ", save_path) - + # 02 proc times # graph creation time: 2 features: 61 sec # conductivity: 43 sec # porosity: 40 sec - graph_creation_time = 43#25#11#22#159#0#159#66 + graph_creation_time = 60#25#11#22#159#0#159#66 config = {'machine_learning_model': machine_learning_model, 'save_path': save_path, From 0ceaa84c1599ded2b672f8f2bb5a2a0341055b44 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Fri, 5 Nov 2021 16:57:21 +0100 Subject: [PATCH 41/67] rnd field dict get --- mlmc/metamodel/random_field_time.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlmc/metamodel/random_field_time.py b/mlmc/metamodel/random_field_time.py index 8bf2a27e..f749abe9 100644 --- a/mlmc/metamodel/random_field_time.py +++ b/mlmc/metamodel/random_field_time.py @@ -34,7 +34,7 @@ def corr_field_sample_time(mesh_file=None, corr_length_config=None): # from matplotlib import ticker, cm #matplotlib.rcParams.update({'font.size': 22}) - if corr_length_config['02_conc']: + if corr_length_config.get('02_conc', False): return conc_rnd_sample_time(mesh_file, corr_length_config) dim = 2 From 949525a37802df9367a71b7a8a03cf3d00322393 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Mon, 8 Nov 2021 13:03:30 +0100 Subject: [PATCH 42/67] set cuda device --- mlmc/metamodel/analyze_nn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mlmc/metamodel/analyze_nn.py b/mlmc/metamodel/analyze_nn.py index befe1d1e..0c7f00be 100644 --- a/mlmc/metamodel/analyze_nn.py +++ b/mlmc/metamodel/analyze_nn.py @@ -4,8 +4,7 @@ import glob import copy import pickle - -os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only +#os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only from mlmc.metamodel.flow_dataset import FlowDataset from mlmc.metamodel.create_graph import graph_creator from mlmc.moments import Legendre_tf, Monomial @@ -32,6 +31,7 @@ print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU'))) epochs = 100 + def prepare_data(data): data = np.squeeze(np.stack(data.to_numpy(), axis=0)) return np.asarray(data).astype('float64') From 19047b36b6adfa11c8932d097cae3d297863f1fb Mon Sep 17 00:00:00 2001 From: Martin Spetlik Date: Mon, 8 Nov 2021 13:06:02 +0100 Subject: [PATCH 43/67] n samples --- mlmc/metamodel/analyze_nn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlmc/metamodel/analyze_nn.py b/mlmc/metamodel/analyze_nn.py index befe1d1e..dec16940 100644 --- a/mlmc/metamodel/analyze_nn.py +++ b/mlmc/metamodel/analyze_nn.py @@ -349,7 +349,7 @@ def predict_level_zero_SVR(nn, output_dir, hdf_path, mesh, batch_size=1000, log= def statistics(config): - n_subsamples = 2 + n_subsamples = 15 model_title, mch_l_model, log = config['machine_learning_model'] model_data = {} From d3e17b069804b4c0731ea2cdc7c9dff50e344f41 Mon Sep 17 00:00:00 2001 From: Martin Spetlik Date: Wed, 22 Dec 2021 15:04:39 +0100 Subject: [PATCH 44/67] data rescale --- mlmc/metamodel/analyze_nn.py | 15 +++- mlmc/metamodel/flow_dataset.py | 137 ++++++++++++++++++++++----------- 2 files changed, 101 insertions(+), 51 deletions(-) diff --git a/mlmc/metamodel/analyze_nn.py b/mlmc/metamodel/analyze_nn.py index eb97ce24..1cd2520c 100644 --- a/mlmc/metamodel/analyze_nn.py +++ b/mlmc/metamodel/analyze_nn.py @@ -361,6 +361,12 @@ def statistics(config): if not os.path.isdir(config['save_path']): os.makedirs(config['save_path']) + + # Save config to Pickle + import pickle + # create a binary pickle file + with open(os.path.join(config['save_path'], "dataset_config.pkl"), "wb") as writer: + pickle.dump(config.get("dataset_config", {}), writer) else: print("dir exists {}".format(config['save_path'])) exit() @@ -1217,7 +1223,7 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): preprocess_start_time = time.process_time() # Load data - data = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log) + data = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log, config=config) data = data#[:10000] # print("n node features ", data.graphs[0].n_node_features) @@ -1332,7 +1338,8 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): stats=stats, corr_field_config=config['corr_field_config'], seed=seed, - feature_names=config.get('feature_names', [['conductivity']]) + feature_names=config.get('feature_names', [['conductivity']]), + config=config ) #predict_l_0_time = time.process_time() - predict_l_0_start_time @@ -1358,14 +1365,14 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): def predict_level_zero(nn, output_dir, hdf_path, mesh, conv_layer, batch_size=1000, log=False, stats=False, - corr_field_config=None, seed=1234, feature_names=[]): + corr_field_config=None, seed=1234, feature_names=[], config=None): #graph_creator(output_dir, hdf_path, mesh, level=0, feature_names=feature_names) # Load data sample_time = 0 if corr_field_config: sample_time = corr_field_sample_time(mesh, corr_field_config) - data = FlowDataset(output_dir=output_dir, log=log)#, mesh=mesh, corr_field_config=corr_field_config) + data = FlowDataset(output_dir=output_dir, log=log, config=config)#, mesh=mesh, corr_field_config=corr_field_config) #data = data # [:10000] data.shuffle(seed=seed) diff --git a/mlmc/metamodel/flow_dataset.py b/mlmc/metamodel/flow_dataset.py index ed025700..dd4ded7b 100644 --- a/mlmc/metamodel/flow_dataset.py +++ b/mlmc/metamodel/flow_dataset.py @@ -21,7 +21,7 @@ class FlowDataset(Dataset): GRAPHS_FILE = "graphs" DATA_FILE = "data" - def __init__(self, output_dir=None, level=0, log=False, mesh=None, corr_field_config=None, **kwargs): + def __init__(self, output_dir=None, level=0, log=False, mesh=None, corr_field_config=None, config={}, **kwargs): self._output_dir = output_dir # if self._output_dir is None: # self._output_dir = OUTPUT_DIR @@ -31,7 +31,12 @@ def __init__(self, output_dir=None, level=0, log=False, mesh=None, corr_field_co self._corr_field_config = corr_field_config self.adjacency_matrix = np.load(os.path.join(self._output_dir, "adjacency_matrix.npy"), allow_pickle=True) # adjacency matrix self.data = [] + self._config = config + self._dataset_config = config.get('dataset_config', {}) + self._min_feature = None + self._max_feature = None super().__init__(**kwargs) + #self.a = self.adjacency_matrix self.dataset = pd.DataFrame(self.data) @@ -101,40 +106,55 @@ def read(self): # # return graphs - # i = 0 - all_outputs = [] - all_features = [] - # - # for s_dir in os.listdir(self._output_dir): - # try: - # l = re.findall(r'L(\d+)_S', s_dir)[0] - # if int(l) != self.level: - # continue - # except IndexError: - # continue - # if os.path.isdir(os.path.join(self._output_dir, s_dir)): - # sample_dir = os.path.join(self._output_dir, s_dir) - # if os.path.exists(os.path.join(sample_dir, "nodes_features.npy")): - # features = np.load(os.path.join(sample_dir, "nodes_features.npy")) - # output = np.load(os.path.join(sample_dir, "output.npy")) - # all_outputs.append(output) - # all_features.append(features) - # - # #print("all outputs ", np.array(all_outputs).shape) - # min_output = np.min(all_outputs) - # max_output = np.max(all_outputs) - # - # maximum = np.max(all_features) - # minimum = np.min(all_features) - # - # if self._log: - # minimum = np.log(minimum) - # maximum = np.log(maximum) - # - # self.min_output = min_output - # self.max_output = max_output - # self.min_feature = minimum - # self.max_feature = maximum + if self._dataset_config.get("features_normalization", False) or self._dataset_config.get("calc_output_mult_factor", False): + # i = 0 + all_outputs = [] + all_features = [] + + for s_dir in os.listdir(self._output_dir): + try: + l = re.findall(r'L(\d+)_S', s_dir)[0] + if int(l) != self.level: + continue + except IndexError: + continue + if os.path.isdir(os.path.join(self._output_dir, s_dir)): + sample_dir = os.path.join(self._output_dir, s_dir) + if os.path.exists(os.path.join(sample_dir, "nodes_features.npy")): + features = np.load(os.path.join(sample_dir, "nodes_features.npy")) + all_features.extend(features) + + if self._dataset_config.get("calc_output_mult_factor", False) is True: + output = np.load(os.path.join(sample_dir, "output.npy")) + all_outputs.append(output) + + #print("all outputs ", np.array(all_outputs).shape) + # min_output = np.min(all_outputs) + # max_output = np.max(all_outputs) + + if self._dataset_config.get("calc_output_mult_factor", False) is True: + self._dataset_config["output_mult_factor"] = 1/np.mean(all_outputs) + self._save_output_mult_factor() + print("output mult factor ", self._dataset_config["output_mult_factor"]) + + if self._dataset_config.get("features_normalization", False): + self._min_feature = np.min(all_features) + self._max_feature = np.max(all_features) + + # if self._log and self._dataset_config.get("features_log", False) is False and self._dataset_config.get("output_log", + # False) is False: + # all_features = np.log(all_features) + # all_outputs = np.log(all_outputs) + # + # if self._dataset_config.get("features_log", False): + # all_features = np.log(all_features) + # + # if self._dataset_config.get("output_log", False): + # all_outputs = np.log(all_outputs) + + + # self.min_output = min_output + # self.max_output = max_output graphs = [] for s_dir in os.listdir(self._output_dir): @@ -151,21 +171,32 @@ def read(self): features = np.load(os.path.join(sample_dir, "nodes_features.npy")) output = np.load(os.path.join(sample_dir, "output.npy")) - #features = (features - minimum) / (maximum - minimum) - # - # output = (output - min_output) / (max_output - min_output) - # print("max ", maximum) - # print("max ", minimum) - # - # print("new featuers max ", np.max(new_features)) - # print("new featuers min ", np.min(new_features)) - # exit() - - if self._log: + if self._min_feature is not None and self._max_feature is not None: + features = (features - self._min_feature) / (self._max_feature - self._min_feature) + + # output = (output - min_output) / (max_output - min_output) + # print("max ", maximum) + # print("max ", minimum) + # + # print("new featuers max ", np.max(new_features)) + # print("new featuers min ", np.min(new_features)) + # exit() + + output_mult_factor = self._dataset_config.get("output_mult_factor", 1) + features_mult_factor = self._dataset_config.get("features_mult_factor", 1) + + features *= features_mult_factor + output *= output_mult_factor + + if self._log and self._dataset_config.get("features_log", False) is False and self._dataset_config.get("output_log", False) is False: features = np.log(features) output = np.log(output) - #features = (features - minimum) / (maximum - minimum) + if self._dataset_config.get("features_log", False): + features = np.log(features) + + if self._dataset_config.get("output_log", False): + output = np.log(output) graphs.append(Graph(x=features, y=output))#, a=self.adjacency_matrix)) @@ -175,6 +206,18 @@ def read(self): self.a = self.adjacency_matrix return graphs + def _save_output_mult_factor(self): + # Save config to Pickle + import pickle + import shutil + + if os.path.exists(os.path.join(self._config['save_path'], "dataset_config.pkl")): + os.remove(os.path.join(self._config['save_path'], "dataset_config.pkl")) + + # create a binary pickle file + with open(os.path.join(self._config['save_path'], "dataset_config.pkl"), "wb") as writer: + pickle.dump(self._dataset_config, writer) + @staticmethod def pickle_data(data, output_dir, file_path): with open(os.path.join(output_dir, file_path), 'wb') as writer: From 923f360e0e207fde8ee804f21cc1d7e35eda9eaa Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Wed, 22 Dec 2021 15:11:10 +0100 Subject: [PATCH 45/67] postprocessing update --- mlmc/metamodel/analyze_nn.py | 44 ++++- mlmc/metamodel/postprocessing.py | 277 +++++++++++++++++++++------- mlmc/metamodel/random_field_time.py | 14 +- mlmc/plot/plots.py | 13 +- mlmc/tool/flow_mc_2.py | 13 +- mlmc/tool/simple_distribution.py | 69 ++++--- test/metamodels/metamodel_test.py | 153 +++++++++++---- 7 files changed, 426 insertions(+), 157 deletions(-) diff --git a/mlmc/metamodel/analyze_nn.py b/mlmc/metamodel/analyze_nn.py index 0c7f00be..8a2dd1c4 100644 --- a/mlmc/metamodel/analyze_nn.py +++ b/mlmc/metamodel/analyze_nn.py @@ -349,7 +349,7 @@ def predict_level_zero_SVR(nn, output_dir, hdf_path, mesh, batch_size=1000, log= def statistics(config): - n_subsamples = 2 + n_subsamples = 1 model_title, mch_l_model, log = config['machine_learning_model'] model_data = {} @@ -361,6 +361,12 @@ def statistics(config): if not os.path.isdir(config['save_path']): os.makedirs(config['save_path']) + + # Save config to Pickle + import pickle + # create a binary pickle file + with open(os.path.join(config['save_path'], "dataset_config.pkl"), "wb") as writer: + pickle.dump(config.get("dataset_config", {}), writer) else: print("dir exists {}".format(config['save_path'])) exit() @@ -457,6 +463,14 @@ def load_statistics(dir_path): models_data[file_name].append(np.load(file, allow_pickle=True)) + if os.path.exists(os.path.join(dir_path, "dataset_config.pkl")): + # Save config to Pickle + import pickle + # create a binary pickle file + with open(os.path.join(dir_path, "dataset_config.pkl"), "rb") as reader: + dataset_config = pickle.load(reader) + models_data["dataset_config"] = dataset_config + return models_data @@ -578,8 +592,6 @@ def predict_data(config, model, mesh_file, log=True): print("conv out ", conv_out) conv_layers[conv_index][2].extend(conv_out) # outputs - exit() - flatten_input = conv_layers[conv_index][2][-1] # flatten_output = model.flatten(conv_out) # @@ -730,14 +742,16 @@ def analyze_statistics(config): orth_nn_means_mse = [] limit = 100 # 0.008#0.01#0.0009 + #limit = 0.37 for i in range(len(data_dict["test_targets"])): - # if i == 3: + print("index i ", i) + # if i == 1: # break #print("index ", i) - # if i not in [2, 5, 7]: + # if i not in [2, 3, 4]: # continue # if i in [2, 11, 12]: @@ -800,7 +814,8 @@ def analyze_statistics(config): replace_level=config['replace_level'], mlmc_hdf_file=config['mlmc_hdf_path'], stats=True, - learning_time=learning_time) + learning_time=learning_time, + dataset_config=config.get("dataset_config", {})) # except: # continue @@ -863,6 +878,9 @@ def analyze_statistics(config): display_vars(mlmc_vars, nn_vars, target_variance=target_variance) + + print("mlmc l vars list ", mlmc_l_vars) + print("mlmc l vars ", np.mean(mlmc_l_vars, axis=0)) print("nn l vars ", np.mean(nn_l_vars, axis=0)) @@ -1107,6 +1125,13 @@ def analyze_statistics(config): print("train MSE ", np.mean(train_MSE)) # print("train MSE sqrt var", np.sqrt(np.var(train_MSE))) # print("train MSE std", np.std(train_MSE)) + + # output_mult_factor = 1437603411 + # print("orig train MSE ", train_MSE) + # train_MSE = np.array(train_MSE) * output_mult_factor + # print("train MSE ", train_MSE) + # test_MSE = np.array(test_MSE) * output_mult_factor + print("train MSE ", train_MSE) print("stats.sem(train_MSE) ", stats.sem(train_MSE)) print("test MSE ", np.mean(test_MSE)) @@ -1119,9 +1144,16 @@ def analyze_statistics(config): print("nn total time ", nn_total_time) print("mlmc total time ", mlmc_total_time) + print("KL mlmc ", np.mean(kl_mlmc_all)) + print("KL nn ", np.mean(kl_nn_all)) + + print("mean learning time ", np.mean(learning_times)) + print("max learning time ", np.max(learning_times)) + print("######################################") return train_MSE, test_MSE, np.mean(all_train_RSE), np.mean(all_test_RSE) + def plot_sse(data_nn, data_mlmc, x_label="ith moment", y_label="MSE", title=""): import matplotlib matplotlib.rcParams.update({'font.size': 16}) diff --git a/mlmc/metamodel/postprocessing.py b/mlmc/metamodel/postprocessing.py index da36d73f..96ab5e23 100644 --- a/mlmc/metamodel/postprocessing.py +++ b/mlmc/metamodel/postprocessing.py @@ -20,33 +20,85 @@ TARGET_VAR = 1e-5 +def cut_original_test(mlmc_hdf_file): + sample_storage = SampleStorageHDF(file_path=mlmc_hdf_file) + + print("mlmc sample storage get N collected ", sample_storage.get_n_collected()) + + n_levels = len(sample_storage.get_level_ids()) + original_moments, estimator, original_true_domain, _ = estimate_moments(sample_storage) + + n_ops, field_times, coarse_flow, fine_flow = get_sample_times_mlmc(mlmc_hdf_file) + n_ops_est = n_ops + + data_mlmc = [] + mlmc_n_collected = estimator._sample_storage.get_n_collected() + for l_id, l_n_collected in zip(range(n_levels), mlmc_n_collected): + level_samples = estimator.get_level_samples(level_id=l_id, n_samples=l_n_collected) + data_mlmc.append(level_samples) + + print("original level params", sample_storage.get_level_parameters()) + sample_storage = create_quantity_mlmc(data_mlmc, level_parameters=sample_storage.get_level_parameters()) + + n0 = 2000 + nL = 100 + n_levels = sample_storage.get_n_levels() + n_samples = np.round(np.exp2(np.linspace(np.log2(n0), np.log2(nL), n_levels))).astype(int) + print('n samples ', n_samples) + + sample_storage_for_estimated = cut_samples(data_mlmc, sample_storage, n_samples) # [2300]) + + original_q_estimator_est = get_quantity_estimator(sample_storage_for_estimated) + + n_estimated_orig, l_vars_orig, n_samples_orig = get_n_estimated(sample_storage_for_estimated, + original_q_estimator_est, n_ops=n_ops_est) + print("n estimated orig ", n_estimated_orig) + + sample_storage_for_estimated = cut_samples(data_mlmc, sample_storage_for_estimated, n_estimated_orig, + bootstrap=True) + + def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predictions, train_targets, train_predictions, val_targets, l_0_targets=None, l_0_predictions=None, l1_sample_time=None, l0_sample_time=None, nn_level=0, replace_level=False, stats=False, mlmc_hdf_file=None, - learning_time=0): + learning_time=0, dataset_config={}): # level_zero = False cut_est = True + domain_largest = False # If False than domain=None - domain is determined given the simulation samples + distr_domain_largest = False + + # cut_original_test(nn_hdf_file) + # exit() + + output_mult_factor = 1 + output_mult_factor = 1 + #output_mult_factor = 1437603411 # 02_conc_cond + output_mult_factor = 1521839229.4794111 # rascale log (case_1) + #output_mult_factor = -15.580288536 # cl_0_1_s_1 + #output_mult_factor = 1.0386 # cl_0_1_s_1 rescale log (case_1) + + targets = np.exp(targets) + predictions = np.exp(predictions) + l_0_predictions = np.exp(l_0_predictions) + l_0_targets = np.exp(l_0_targets) if mlmc_hdf_file is None: mlmc_hdf_file = nn_hdf_file - if not stats: - print("nn_level ", nn_level) - print("replace level ", replace_level) - if not stats: # print("nn_level ", nn_level) # print("replace level ", replace_level) - targets = np.exp(targets) - predictions = np.exp(predictions) - l_0_predictions = np.exp(l_0_predictions) - l_0_targets = np.exp(l_0_targets) + # targets = np.exp(targets) + # predictions = np.exp(predictions) + # l_0_predictions = np.exp(l_0_predictions) + # l_0_targets = np.exp(l_0_targets) # print("targets ", targets) # print("predictions ", predictions) + print("targets ", targets) plt.hist(targets, bins=50, alpha=0.5, label='target', density=True) - plt.hist(predictions, bins=50, alpha=0.5, label='predictions', density=True) + #plt.hist(predictions, bins=50, alpha=0.5, label='predictions', density=True) # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) plt.legend(loc='upper right') @@ -57,21 +109,25 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic plt.hist(l_0_targets, bins=50, alpha=0.5, label='l_0_target', density=True) plt.hist(l_0_predictions, bins=50, alpha=0.5, label='l_0_predictions', density=True) + # print("lo targets ", l_0_targets) + # print("l0 predictions ", l_0_predictions) + # exit() + # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) plt.legend(loc='upper right') # plt.xlim(-0.5, 1000) plt.yscale('log') plt.show() - # targets = np.exp(targets) - # predictions = np.exp(predictions) + # targets = np.exp(targets) + # predictions = np.exp(predictions) - # # Creating plot - # plt.boxplot(targets) - # plt.boxplot(predictions) - # # show plot - # plt.show() - #exit() + # # Creating plot + # plt.boxplot(targets) + # plt.boxplot(predictions) + # # show plot + # plt.show() + #exit() ####### ### Create storage fromm original MLMC data @@ -83,9 +139,15 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic n_levels = len(sample_storage.get_level_ids()) original_moments, estimator, original_true_domain, _ = estimate_moments(sample_storage) + print("moments.mean ", original_moments.mean) + print("moments.var ", original_moments.var) + sample_storage_nn = SampleStorageHDF(file_path=nn_hdf_file) original_moments_nn, estimator_nn, original_true_domain_nn, _ = estimate_moments(sample_storage_nn) + print("nn moments.mean ", original_moments_nn.mean) + print("nn moments.var ", original_moments_nn.var) + orig_max_vars = np.max(original_moments.l_vars, axis=1) # print("orig max vars ", orig_max_vars) @@ -104,10 +166,7 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic data_mlmc = [] mlmc_n_collected = estimator._sample_storage.get_n_collected() for l_id, l_n_collected in zip(range(n_levels), mlmc_n_collected): - # if l_id == 1: - # continue level_samples = estimator.get_level_samples(level_id=l_id, n_samples=l_n_collected) - # print("len level samples ", len(np.squeeze(level_samples))) data_mlmc.append(level_samples) print("original level params", sample_storage.get_level_parameters()) @@ -180,9 +239,14 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic level_samples = l_0_predictions.reshape(1, len(l_0_predictions), 1) # level_samples = np.ones((1, len(l_0_predictions), 1)) ft_index = nn_level + if nn_level > 0: ft_index = nn_level - 1 n_ops_predict.append(l0_sample_time) # + field_times[ft_index] / 2) + + # print("l0_sample_time ", l0_sample_time) + # print("len l0 predictions ", len(l_0_predictions)) + # exit() else: if replace_level: level_id = l_id @@ -195,6 +259,10 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic coarse_level_samples = predictions.reshape(1, len(predictions), 1) fine_level_samples = targets.reshape(1, len(targets), 1) + # print("coarse level samples ", coarse_level_samples) + # print("fine level sampels ", fine_level_samples) + # exit() + # coarse_level_samples = np.concatenate((coarse_level_samples, # train_predictions.reshape(1, len(train_predictions), 1)), axis=1) # @@ -206,6 +274,13 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic level_samples = np.concatenate((fine_level_samples, coarse_level_samples), axis=2) print("nn level ", nn_level) + print("level sampels ", level_samples) + print("fine - coarse ", fine_level_samples - coarse_level_samples) + print("fine - coarse ", np.var(fine_level_samples - coarse_level_samples)) + + # if output_mult_factor != 1: + # level_samples /= output_mult_factor + n_ops_predict.append(n_ops[mlmc_nn_diff_level - 1] - nn_lev_sim_time + l1_sample_time) else: if replace_level: @@ -220,9 +295,27 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic n_ops_predict.append(n_ops[level_id]) # print("n ops predict append", n_ops_predict) + # if output_mult_factor != 1: + # level_samples /= output_mult_factor + + if output_mult_factor != 1: + level_samples /= output_mult_factor + + print("level samples rescaled ", level_samples) + + #level_samples = np.log(level_samples) + + #level_samples /= output_mult_factor + + print("leel samples exp ", level_samples) + + #print("level samples exp ", level_samples) + + data_nn.append(level_samples) print("n ops predict ", n_ops_predict) + # n_ops_predict[1] += n_ops_predict[1]*0.2 print("level params ", level_params) @@ -234,9 +327,11 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic n_samples = np.round(np.exp2(np.linspace(np.log2(n0), np.log2(nL), n_levels))).astype(int) print('n samples predict', n_samples) sample_storage_predict_for_estimate = cut_samples(data_nn, sample_storage_predict, n_samples) # [4500, 1500]) - # print("n ops predict ", n_ops_predict) - print("Storage predict info") - predict_storage_n_collected, predict_storage_max_vars = get_storage_info(sample_storage_predict) + # print("n ops predict ", n_ops_predict) + print("Storage predict info") + predict_storage_n_collected, predict_storage_max_vars = get_storage_info(sample_storage_predict_for_estimate) + print("predict storage n collected ", predict_storage_n_collected) + print("predict storage max vars ", predict_storage_max_vars) # if stats: # return orig_storage_max_vars, predict_storage_max_vars @@ -245,8 +340,10 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic ### Create estimators ###### ref_sample_storage = ref_storage(ref_mlmc_file) - domain = get_largest_domain([sample_storage, sample_storage_predict, ref_sample_storage]) - domain = None + if domain_largest: + domain = get_largest_domain([sample_storage, sample_storage_predict, ref_sample_storage]) + else: + domain = None original_q_estimator = get_quantity_estimator(sample_storage, true_domain=domain) predict_q_estimator = get_quantity_estimator(sample_storage_predict, true_domain=domain) @@ -274,17 +371,22 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic #### Original data n_ops_est = copy.deepcopy(n_ops) # n_ops_est[0] = n_ops_est[0] / 1000 + print("sample_storage.get_n_collected() ", sample_storage.get_n_collected()) n_estimated_orig, l_vars_orig, n_samples_orig = get_n_estimated(sample_storage, original_q_estimator, n_ops=n_ops_est) - print("n estimted orig ", n_estimated_orig) + print("n estimated orig ", n_estimated_orig) print("n ops est ", n_ops_est) + print("sample storage for estimated n collected ", sample_storage_for_estimated.get_n_collected()) ###### ## initial N geuss if cut_est: + print("n ops est ", n_ops_est) n_estimated_orig, l_vars_orig, n_samples_orig = get_n_estimated(sample_storage_for_estimated, original_q_estimator_est, n_ops=n_ops_est) + print("n estimated orig ", n_estimated_orig) + sample_storage_for_estimated = cut_samples(data_mlmc, sample_storage_for_estimated, n_estimated_orig, bootstrap=True) @@ -396,6 +498,7 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic NN_time_levels = n_ops_predict_orig * np.array(n_estimated_nn) n_collected_times = n_ops * np.array(n_estimated_orig) + print("NN time levels ", NN_time_levels) print("MLMC time levels", n_collected_times) nn_total_time = np.sum(NN_time_levels) @@ -448,9 +551,14 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic # for l_id, (l_mom, l_mom_pred) in enumerate(zip(moments_nn.l_means, moments_predict.l_means)): # print("L id: {}, mom diff: {}".format(l_id, l_mom - l_mom_pred)) - domain = get_largest_domain([sample_storage, sample_storage_predict, ref_sample_storage]) - common_domain = get_largest_domain([sample_storage, sample_storage_predict, ref_sample_storage]) - domain = None + if domain_largest: + domain = get_largest_domain([sample_storage, sample_storage_predict, ref_sample_storage]) + common_domain = get_largest_domain([sample_storage, sample_storage_predict, ref_sample_storage]) + else: + domain = None + + #domain = get_largest_domain([sample_storage, sample_storage_predict, ref_sample_storage]) + original_q_estimator = get_quantity_estimator(sample_storage, true_domain=domain) predict_q_estimator = get_quantity_estimator(sample_storage_predict, true_domain=domain) @@ -458,8 +566,19 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic original_q_estimator = get_quantity_estimator(sample_storage_for_estimated, true_domain=domain) predict_q_estimator = get_quantity_estimator(sample_storage_predict_for_estimate, true_domain=domain) - ref_estimator = get_quantity_estimator(ref_sample_storage, true_domain=domain) + print("ref samples ", ref_sample_storage) + print("domain ", domain) + + if distr_domain_largest: + domain = get_largest_domain([sample_storage, sample_storage_predict, ref_sample_storage]) + common_domain = get_largest_domain([sample_storage, sample_storage_predict, ref_sample_storage]) + original_q_estimator = get_quantity_estimator(sample_storage, true_domain=domain) + predict_q_estimator = get_quantity_estimator(sample_storage_predict, true_domain=domain) + else: + domain = None + ref_estimator = get_quantity_estimator(ref_sample_storage, true_domain=domain) + #ref_estimator = None orig_moments_mean, predict_moments_mean, ref_moments_mean = compare_moments(original_q_estimator, predict_q_estimator, ref_estimator) @@ -478,7 +597,6 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic plot_moments({"ref": ref_estimator, "orig": original_q_estimator, "nn": predict_q_estimator}) - def plot_loss(train_loss, val_loss): plt.plot(train_loss, label='loss') plt.plot(val_loss, label='val_loss') @@ -711,6 +829,7 @@ def estimate_moments(sample_storage, true_domain=None): quantile = QUANTILE true_domain = mlmc.estimator.Estimate.estimate_domain(q_value, sample_storage, quantile=quantile) moments_fn = Legendre(n_moments, true_domain) + print("true domain ", true_domain) estimator = mlmc.estimator.Estimate(quantity=q_value, sample_storage=sample_storage, moments_fn=moments_fn) #means, vars = estimator.estimate_moments(moments_fn) @@ -743,49 +862,60 @@ def get_largest_domain(storages): true_domains = np.array(true_domains) - #print("true domains ", true_domains) - true_domain = [np.min(true_domains[:, 0]), np.max(true_domains[:, 1])] - #true_domain = [np.max(true_domains[:, 0]), np.min(true_domains[:, 1])] + #true_domain = [np.min(true_domains[:, 0]), np.max(true_domains[:, 1])] + true_domain = [np.max(true_domains[:, 0]), np.min(true_domains[:, 1])] #true_domain = [np.mean(true_domains[:, 0]), np.mean(true_domains[:, 1])] - - - #true_domain = true_domain[-1] - return true_domain -def compare_moments(original_q_estimator, predict_q_estimator, ref_estimator): +def compare_moments(original_q_estimator, predict_q_estimator, ref_estimator=None): + print("############################ COMPARE MOMENTS ####################################") original_q_estimator.estimate_moments() orig_moments_mean = original_q_estimator.moments_mean predict_q_estimator.estimate_moments() predict_moments_mean = predict_q_estimator.moments_mean - ref_estimator.estimate_moments() - ref_moments_mean = ref_estimator.moments_mean + ref_moments_mean = None + if ref_estimator is not None: + ref_estimator.estimate_moments() + ref_moments_mean = ref_estimator.moments_mean - #print("ref moments mean ", ref_moments_mean.mean) - print("orig moments mean ", orig_moments_mean.mean) - print("predict moments mean ", predict_moments_mean.mean) + #print("ref moments mean ", ref_moments_mean.mean) + print("orig moments mean ", orig_moments_mean.mean) + print("predict moments mean ", predict_moments_mean.mean) - print("ref orig mean SSE ", np.sum((ref_moments_mean.mean - orig_moments_mean.mean)**2)) - print("ref predict mean SSE ", np.sum((ref_moments_mean.mean - predict_moments_mean.mean) ** 2)) - # - print("ref orig mean SE ", np.sum(np.abs((ref_moments_mean.mean - orig_moments_mean.mean)))) - print("ref predict mean SE ", np.sum(np.abs((ref_moments_mean.mean - predict_moments_mean.mean)))) + print("ref orig mean SSE ", np.sum((ref_moments_mean.mean - orig_moments_mean.mean)**2)) + print("ref predict mean SSE ", np.sum((ref_moments_mean.mean - predict_moments_mean.mean) ** 2)) + # + print("ref orig mean SE ", np.sum(np.abs((ref_moments_mean.mean - orig_moments_mean.mean)))) + print("ref predict mean SE ", np.sum(np.abs((ref_moments_mean.mean - predict_moments_mean.mean)))) - #print("ref moments var ", ref_moments_mean.var) - print("orig moments var ", orig_moments_mean.var) - print("predict moments var ", predict_moments_mean.var) + orig_diff = ref_moments_mean.mean - orig_moments_mean.mean + predict_diff = ref_moments_mean.mean - predict_moments_mean.mean + orig_diff[0] = 1 + predict_diff[0] = 1 - # print("MAX orig moments var ", np.max(orig_moments_mean.l_vars, axis=1)) - # print("MAX predict moments var ", np.max(predict_moments_mean.l_vars, axis=1)) + # print("np.abs((ref_moments_mean.mean - orig_moments_mean.mean))/orig_diff ", np.abs((ref_moments_mean.mean - orig_moments_mean.mean))/orig_diff) + # print("np.sum(np.abs((ref_moments_mean.mean - predict_moments_mean.mean))/predict_diff ", np.abs((ref_moments_mean.mean - predict_moments_mean.mean))/predict_diff) + # + # print("ref orig mean SE relative", np.sum(np.abs((ref_moments_mean.mean - orig_moments_mean.mean))/orig_diff)) + # print("ref predict mean SE relative", np.sum(np.abs((ref_moments_mean.mean - predict_moments_mean.mean))/predict_diff)) - print("ref orig var SSE ", np.sum((ref_moments_mean.var - orig_moments_mean.var) ** 2)) - print("ref predict var SSE ", np.sum((ref_moments_mean.var - predict_moments_mean.var) ** 2)) - # - print("ref orig var SE ", np.sum(np.abs((ref_moments_mean.var - orig_moments_mean.var)))) - print("ref predict var SE ", np.sum(np.abs((ref_moments_mean.var - predict_moments_mean.var)))) + #print("ref moments var ", ref_moments_mean.var) + print("orig moments var ", orig_moments_mean.var) + print("predict moments var ", predict_moments_mean.var) + + # print("MAX orig moments var ", np.max(orig_moments_mean.l_vars, axis=1)) + # print("MAX predict moments var ", np.max(predict_moments_mean.l_vars, axis=1)) + + print("ref orig var SSE ", np.sum((ref_moments_mean.var - orig_moments_mean.var) ** 2)) + print("ref predict var SSE ", np.sum((ref_moments_mean.var - predict_moments_mean.var) ** 2)) + # + print("ref orig var SE ", np.sum(np.abs((ref_moments_mean.var - orig_moments_mean.var)))) + print("ref predict var SE ", np.sum(np.abs((ref_moments_mean.var - predict_moments_mean.var)))) + + print("##############################################################") return orig_moments_mean, predict_moments_mean, ref_moments_mean @@ -809,7 +939,7 @@ def compare_moments(original_q_estimator, predict_q_estimator, ref_estimator): def compare_densities(estimator_1, estimator_2, ref_estimator, label_1="", label_2=""): - distr_plot = plots.ArticleDistributionPDF(title="densities", log_density=True) + distr_plot = plots.ArticleDistributionPDF(title="densities", log_density=True, set_x_lim=False) tol = 1e-7 reg_param = 0 @@ -832,14 +962,16 @@ def compare_densities(estimator_1, estimator_2, ref_estimator, label_1="", label reg_param=reg_param, orth_moments_tol=TARGET_VAR) - domain = [np.max([ref_distr_obj.domain[0], distr_obj_1.domain[0], distr_obj_2.domain[0]]), - np.min([ref_distr_obj.domain[1], distr_obj_1.domain[1], distr_obj_2.domain[1]])] + # domain = [np.max([ref_distr_obj.domain[0], distr_obj_1.domain[0], distr_obj_2.domain[0]]), + # np.min([ref_distr_obj.domain[1], distr_obj_1.domain[1], distr_obj_2.domain[1]])] + domain = [np.max([ref_distr_obj.domain[0], distr_obj_1.domain[0]]), + np.min([ref_distr_obj.domain[1], distr_obj_1.domain[1]])] kl_div_ref_mlmc = mlmc.tool.simple_distribution.KL_divergence(ref_distr_obj.density, distr_obj_1.density, domain[0], domain[1]) print("KL div ref|mlmc: {}".format(kl_div_ref_mlmc)) - # domain = [np.max([ref_distr_obj.domain[0], distr_obj_2.domain[0]]), - # np.min([ref_distr_obj.domain[1], distr_obj_2.domain[1]])] + domain = [np.max([ref_distr_obj.domain[0], distr_obj_2.domain[0]]), + np.min([ref_distr_obj.domain[1], distr_obj_2.domain[1]])] kl_div_ref_gnn = mlmc.tool.simple_distribution.KL_divergence(ref_distr_obj.density, distr_obj_2.density, domain[0], domain[1]) @@ -849,7 +981,7 @@ def compare_densities(estimator_1, estimator_2, ref_estimator, label_1="", label #distr_plot.add_distribution(distr_obj_2, label=label_2 + ", KL(ref|gnn):{:0.4g}".format(kl_div_ref_gnn), color="red", line_style="--") distr_plot.add_distribution(distr_obj_1, label=r"$D_{MC}:$" + "{:0.4g}".format(kl_div_ref_mlmc), color="blue") distr_plot.add_distribution(distr_obj_2, label=r"$D_{meta}:$" + "{:0.4g}".format(kl_div_ref_gnn), color="red", line_style="--") - distr_plot.add_distribution(ref_distr_obj, label="MC ref", color="black", line_style=":") + #distr_plot.add_distribution(ref_distr_obj, label="MC ref", color="black", line_style=":") distr_plot.show(file="densities.pdf") distr_plot.show(file=None) @@ -872,12 +1004,12 @@ def get_quantity_estimator(sample_storage, true_domain=None, quantity=None, n_mo quantile = QUANTILE true_domain = mlmc.estimator.Estimate.estimate_domain(quantity, sample_storage, quantile=quantile) + print("true domain") moments_fn = Legendre(n_moments, true_domain) return mlmc.estimator.Estimate(quantity=quantity, sample_storage=sample_storage, moments_fn=moments_fn) - def get_n_estimated(sample_storage, estimator, n_ops=None): target_var = TARGET_VAR #moments, estimator, _, quantity = estimate_moments(sample_storage, true_domain=true_domain) @@ -887,11 +1019,13 @@ def get_n_estimated(sample_storage, estimator, n_ops=None): print("n level samples ", n_level_samples) variances, n_samples = estimator.estimate_diff_vars() + print("n samples ", n_samples) #variances, est_n_ops = estimator.estimate_diff_vars_regression(n_level_samples) if n_ops is None: n_ops = n_samples print("get n estimated n ops ", n_ops) + print("variances ", variances) n_estimated = mlmc.estimator.estimate_n_samples_for_target_variance(target_var, variances, n_ops, n_levels=len(n_level_samples)) return n_estimated, variances, n_samples @@ -903,6 +1037,10 @@ def get_storage_info(sample_storage): max_vars = np.max(np.array(moments.l_vars) / np.array(sample_storage.get_n_collected())[:, np.newaxis], axis=1) print("n collected ", n_collected) print("moments.l_vars max ", max_vars) + + + + print('moments l vars ', moments.l_vars) return n_collected, max_vars @@ -928,6 +1066,11 @@ def cut_samples(data, sample_storage, new_n_collected, new_l_0=0, bootstrap=Fals else: new_data.append(d[:, :np.min([d.shape[1], n_est]), :]) + # print("new data ", new_data) + # print("new data shape ", np.array(new_data).shape) + # + # print("var new data ", np.var(new_data, axis=-2)) + sample_storage = create_quantity_mlmc(new_data, level_parameters=sample_storage.get_level_parameters()) return sample_storage diff --git a/mlmc/metamodel/random_field_time.py b/mlmc/metamodel/random_field_time.py index f749abe9..fc29f9c6 100644 --- a/mlmc/metamodel/random_field_time.py +++ b/mlmc/metamodel/random_field_time.py @@ -23,9 +23,12 @@ def conc_rnd_sample_time(mesh_file, corr_field_config): fine_input_sample, coarse_input_sample = FlowSimProcConc.generate_random_sample(fields, coarse_step=0, n_fine_elements=len( mesh_data['points'])) + #print("fine input sample ", fine_input_sample) + rnd_time = time.process_time() - start_time print("rnd_time / n_samples ", rnd_time / n_samples) + return rnd_time / n_samples @@ -72,6 +75,9 @@ def corr_field_sample_time(mesh_file=None, corr_length_config=None): len(fine_input_sample["conductivity"]) features_log = np.log(fine_input_sample["conductivity"]) + + # print("conductivity mean ", np.mean(fine_input_sample["conductivity"])) + # print("conductivity var ", np.var(fine_input_sample["conductivity"])) output = 1 # # print("fine input sample ", fine_input_sample["conductivity"].shape) @@ -114,7 +120,13 @@ def corr_field_sample_time(mesh_file=None, corr_length_config=None): pr = cProfile.Profile() pr.enable() - my_result = corr_field_sample_time() + corr_file_config = {"02_conc": True, 'log': True} + mesh_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_por/l_step_1.0_common_files/repo.msh" + + # corr_file_config = {"02_conc": False, 'log': True, 'corr_length':0.1, 'sigma':1} + # mesh_file = "/home/martin/Documents/metamodels/data/1000_ele/l_step_0.055_common_files/mesh.msh" + + my_result = corr_field_sample_time(mesh_file, corr_file_config) pr.disable() ps = pstats.Stats(pr).sort_stats('cumtime') diff --git a/mlmc/plot/plots.py b/mlmc/plot/plots.py index 08f766db..71e0ba06 100644 --- a/mlmc/plot/plots.py +++ b/mlmc/plot/plots.py @@ -1791,7 +1791,8 @@ class ArticleDistributionPDF(Distribution): Provides methods to: add more plots, add exact PDF, add ECDF/histogram from single level MC """ def __init__(self, exact_distr=None, title="", quantity_name="Y", legend_title="", - log_density=False, cdf_plot=False, log_x=False, error_plot='l2', reg_plot=False, multipliers_plot=True): + log_density=False, cdf_plot=False, log_x=False, error_plot='l2', reg_plot=False, multipliers_plot=True, + set_x_lim=True): """ Plot configuration :param exact_distr: Optional exact domain (for adding to plot and computing error) @@ -1853,12 +1854,12 @@ def __init__(self, exact_distr=None, title="", quantity_name="Y", legend_title=" # if self._log_x: # self.ax_cdf.set_xscale('log') - self.x_lim = [0, 2.6] + if set_x_lim: + self.x_lim = [0, 2.6] + #self.x_lim = [0, 5] + self.x_lim = [0, 2.5] - #self.x_lim = [0, 5] - self.x_lim = [0, 2.5] - - self.ax_pdf.set_xlim(*self.x_lim) + self.ax_pdf.set_xlim(*self.x_lim) #self.ax_cdf.set_xlim(*self.x_lim) # """adjust ax2 ylimit so that v2 in ax2 is aligned to v1 in ax1""" diff --git a/mlmc/tool/flow_mc_2.py b/mlmc/tool/flow_mc_2.py index 0219997a..ea9e27c1 100644 --- a/mlmc/tool/flow_mc_2.py +++ b/mlmc/tool/flow_mc_2.py @@ -13,11 +13,11 @@ from mlmc.random import correlated_field as cf -def create_corr_field(model='gauss', corr_length=0.125, dim=2, log=True, sigma=1, mode_no=1000): +def create_corr_field(model='gauss', corr_length=0.125, dim=2, log=True, por_sigma=1, mode_no=1000): """ Create random fields :return: - """ + # """ # por_top = cf.SpatialCorrelatedField( # corr_exp='gauss', # dim=2, @@ -30,7 +30,7 @@ def create_corr_field(model='gauss', corr_length=0.125, dim=2, log=True, sigma=1 # print("por top ", por_top) por_top = cf.GSToolsSpatialCorrelatedField(gstools.Gaussian(dim=2, len_scale=0.2), - log=log, mean=-1.0, sigma=1.0, mode_no=mode_no) + log=log, mean=-1.0, sigma=por_sigma, mode_no=mode_no) #print("por top gstools ", por_top_gstools) @@ -44,14 +44,15 @@ def create_corr_field(model='gauss', corr_length=0.125, dim=2, log=True, sigma=1 # ) por_bot = cf.GSToolsSpatialCorrelatedField(gstools.Gaussian(dim=2, len_scale=0.2), - log=log, mean=-1.0, sigma=1.0, mode_no=mode_no) + log=log, mean=-1.0, sigma=por_sigma, mode_no=mode_no) + #por_bot = gstools.Gaussian(dim=dim, len_scale=0.2, mu=-1.0, sigma=1.0, log=True) water_viscosity = 8.90e-4 - factor_top_model = gstools.Gaussian(dim=dim, len_scale=1) - factor_bot_model = gstools.Gaussian(dim=dim, len_scale=1) + factor_top_model = gstools.Gaussian(dim=dim) + factor_bot_model = gstools.Gaussian(dim=dim) fields = cf.Fields([ cf.Field('por_top', por_top, regions='ground_0'), diff --git a/mlmc/tool/simple_distribution.py b/mlmc/tool/simple_distribution.py index 77513f6a..df3f55cb 100644 --- a/mlmc/tool/simple_distribution.py +++ b/mlmc/tool/simple_distribution.py @@ -104,7 +104,6 @@ def density(self, value): power = np.minimum(np.maximum(power, -200), 200) return np.exp(power) - def cdf(self, values): values = np.atleast_1d(values) np.sort(values) @@ -267,10 +266,11 @@ def _calculate_functional(self, multipliers): integral = np.dot(q_density, self._quad_weights) sum = np.sum(self.moment_means * multipliers / self._moment_errs) - end_diff = np.dot(self._end_point_diff, multipliers) - penalty = np.sum(np.maximum(end_diff, 0)**2) fun = sum + integral - fun = fun + np.abs(fun) * self._penalty_coef * penalty + if self._penalty_coef != 0: + end_diff = np.dot(self._end_point_diff, multipliers) + penalty = np.sum(np.maximum(end_diff, 0) ** 2) + fun = fun + np.abs(fun) * self._penalty_coef * penalty return fun @@ -284,46 +284,41 @@ def _calculate_gradient(self, multipliers): q_gradient = self._quad_moments.T * q_density integral = np.dot(q_gradient, self._quad_weights) / self._moment_errs - end_diff = np.dot(self._end_point_diff, multipliers) - penalty = 2 * np.dot( np.maximum(end_diff, 0), self._end_point_diff) - fun = np.sum(self.moment_means * multipliers / self._moment_errs) + integral[0] * self._moment_errs[0] - gradient = self.moment_means / self._moment_errs - integral + np.abs(fun) * self._penalty_coef * penalty + if self._penalty_coef != 0: + end_diff = np.dot(self._end_point_diff, multipliers) + penalty = 2 * np.dot(np.maximum(end_diff, 0), self._end_point_diff) + fun = np.sum(self.moment_means * multipliers / self._moment_errs) + integral[0] * self._moment_errs[0] + + gradient = self.moment_means / self._moment_errs - integral + np.abs(fun) * self._penalty_coef * penalty + else: + gradient = self.moment_means / self._moment_errs - integral # + np.abs(fun) * self._penalty_coef * penalty + return gradient + def _calc_jac(self): + q_density = self.density(self._quad_points) + q_density_w = q_density * self._quad_weights + + jacobian_matrix = (self._quad_moments.T * q_density_w) @ self._quad_moments + return jacobian_matrix + def _calculate_jacobian_matrix(self, multipliers): """ :return: jacobian matrix, symmetric, (n_moments, n_moments) """ - self._update_quadrature(multipliers) - q_density = self._density_in_quads(multipliers) - q_density_w = q_density * self._quad_weights - q_mom = self._quad_moments / self._moment_errs - - jacobian_matrix = (q_mom.T * q_density_w) @ q_mom - - # Compute just triangle use lot of memory (possibly faster) - # moment_outer = np.einsum('ki,kj->ijk', q_mom, q_mom) - # triu_idx = np.triu_indices(self.approx_size) - # triu_outer = moment_outer[triu_idx[0], triu_idx[1], :] - # integral = np.dot(triu_outer, q_density_w) - # jacobian_matrix = np.empty(shape=(self.approx_size, self.approx_size)) - # jacobian_matrix[triu_idx[0], triu_idx[1]] = integral - # jacobian_matrix[triu_idx[1], triu_idx[0]] = integral - - end_diff = np.dot(self._end_point_diff, multipliers) - fun = np.sum(self.moment_means * multipliers / self._moment_errs) + jacobian_matrix[0,0] * self._moment_errs[0]**2 - for side in [0, 1]: - if end_diff[side] > 0: - penalty = 2 * np.outer(self._end_point_diff[side], self._end_point_diff[side]) - jacobian_matrix += np.abs(fun) * self._penalty_coef * penalty - - - #e_vals = np.linalg.eigvalsh(jacobian_matrix) + # jacobian_matrix_hess = hessian(self._calculate_functional)(multipliers) + # print(pd.DataFrame(jacobian_matrix_hess)) + jacobian_matrix = self._calc_jac() + + if self._penalty_coef != 0: + end_diff = np.dot(self._end_point_diff, multipliers) + fun = np.sum(self.moment_means * multipliers / self._moment_errs) + jacobian_matrix[0, 0] * \ + self._moment_errs[0] ** 2 + for side in [0, 1]: + if end_diff[side] > 0: + penalty = 2 * np.outer(self._end_point_diff[side], self._end_point_diff[side]) + jacobian_matrix += np.abs(fun) * self._penalty_coef * penalty - #print(multipliers) - #print("jac spectra: ", e_vals) - #print("means:", self.moment_means) - #print("\n jac:", np.diag(jacobian_matrix)) return jacobian_matrix diff --git a/test/metamodels/metamodel_test.py b/test/metamodels/metamodel_test.py index 6c85869c..5864edfb 100644 --- a/test/metamodels/metamodel_test.py +++ b/test/metamodels/metamodel_test.py @@ -1,7 +1,7 @@ import os import numpy as np import warnings -#os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only +os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only import sys import shutil import subprocess @@ -16,6 +16,7 @@ from tensorflow.keras import Model from tensorflow.keras.models import Sequential from tensorflow.keras.layers import Dense +from tensorflow.keras import regularizers from spektral.layers import GlobalSumPool, GlobalMaxPool, GlobalAvgPool import tensorflow as tf from mlmc.plot import plots @@ -75,7 +76,7 @@ def __init__(self, conv_layer, hidden_activation, output_activation, kernel_regu # self.normalizer = normalizer # self.norm_layer = tf.keras.layers.LayerNormalization(axis=1) - self._conv_layers = [conv_layer(8, K=1, activation=hidden_activation, kernel_regularizer=kernel_regularization)]#, + self._conv_layers = [conv_layer(8, K=4, activation=hidden_activation, kernel_regularizer=kernel_regularization)]#, #conv_layer(64, K=1, activation=hidden_activation, kernel_regularizer=kernel_regularization)] # self.conv3 = conv_layer(32, K=1, activation=hidden_activation, kernel_regularizer=kernel_regularization) # self.conv4 = conv_layer(32, K=1, activation=hidden_activation, kernel_regularizer=kernel_regularization) @@ -257,38 +258,39 @@ def get_config(data_dir, case=0): elif case == 12: # mesh size comparison data_dir = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/" # cl = "cl_0_1_s_1" - level = 1 + level = 3 nn_level = 0 replace_level = False mesh = os.path.join(data_dir, "l_step_1.0_common_files/repo.msh") # L1, 7s - # mesh = os.path.join(data_dir, "l_step_0.27232698153315_common_files/repo.msh".format(cl)) #L2 10.5 s - # mesh = os.path.join(data_dir, "l_step_0.07416198487095663_common_files/mesh.msh".format(cl)) #L3 12s - # mesh = os.path.join(data_dir, "l_step_0.020196309484414757_common_files/mesh.msh".format(cl)) #L4 22s - # mesh = os.path.join(data_dir, "l_step_0.0055_common_files/mesh.msh".format(cl)) #L5 + #mesh = os.path.join(data_dir, "l_step_0.3760603093086394_common_files/repo.msh") #L2 10.5 s + mesh = os.path.join(data_dir, "l_step_0.1414213562373095_common_files/repo.msh") #L3 12s + #mesh = os.path.join(data_dir, "l_step_0.053182958969449884_common_files/repo.msh") # L4 output_dir = os.path.join(data_dir, "L1_{}/test/02_conc/output/".format(level)) hdf_path = os.path.join(data_dir, "L1_{}/mlmc_1.hdf5".format(level)) - mlmc_hdf_path = os.path.join(data_dir, "/mlmc_hdf/L1_{}/mlmc_1.hdf5".format(level)) + mlmc_hdf_path = os.path.join(data_dir, "mlmc_hdf/L1_{}/mlmc_1.hdf5".format(level)) save_path = data_dir l_0_output_dir = os.path.join(data_dir, "L0_MC/L1_{}/test/02_conc/output/".format(level)) l_0_hdf_path = os.path.join(data_dir, "L0_MC/L1_{}/mlmc_1.hdf5".format(level)) sampling_info_path = os.path.join(data_dir, "sampling_info") + + #ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_3/mlmc_1.hdf5" + #ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_1/mlmc_1.hdf5" ref_mlmc_file = os.path.join(data_dir, "L1_benchmark/mlmc_1.hdf5") + feature_names = [['conductivity_top', 'conductivity_bot', 'conductivity_repo']] elif case == 13: # mesh size comparison data_dir = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_por/" # cl = "cl_0_1_s_1" - level = 1 + level = 3 nn_level = 0 replace_level = False mesh = os.path.join(data_dir, "l_step_1.0_common_files/repo.msh") # L1, 7s - # mesh = os.path.join(data_dir, "l_step_0.27232698153315_common_files/repo.msh".format(cl)) #L2 10.5 s - # mesh = os.path.join(data_dir, "l_step_0.07416198487095663_common_files/mesh.msh".format(cl)) #L3 12s - # mesh = os.path.join(data_dir, "l_step_0.020196309484414757_common_files/mesh.msh".format(cl)) #L4 22s - # mesh = os.path.join(data_dir, "l_step_0.0055_common_files/mesh.msh".format(cl)) #L5 + #mesh = os.path.join(data_dir, "l_step_0.3760603093086394_common_files/repo.msh") # L2 10.5 s + mesh = os.path.join(data_dir, "l_step_0.1414213562373095_common_files/repo.msh") #L3 12s output_dir = os.path.join(data_dir, "L1_{}/test/02_conc/output/".format(level)) hdf_path = os.path.join(data_dir, "L1_{}/mlmc_1.hdf5".format(level)) - mlmc_hdf_path = os.path.join(data_dir, "/mlmc_hdf/L1_{}/mlmc_1.hdf5".format(level)) + mlmc_hdf_path = os.path.join(data_dir, "mlmc_hdf/L1_{}/mlmc_1.hdf5".format(level)) save_path = data_dir l_0_output_dir = os.path.join(data_dir, "L0_MC/L1_{}/test/02_conc/output/".format(level)) l_0_hdf_path = os.path.join(data_dir, "L0_MC/L1_{}/mlmc_1.hdf5".format(level)) @@ -299,28 +301,27 @@ def get_config(data_dir, case=0): elif case == 14: # mesh size comparison data_dir = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_2_features/" # cl = "cl_0_1_s_1" - level = 1 + level = 3 nn_level = 0 replace_level = False mesh = os.path.join(data_dir, "l_step_1.0_common_files/repo.msh") # L1, 7s - # mesh = os.path.join(data_dir, "l_step_0.27232698153315_common_files/repo.msh".format(cl)) #L2 10.5 s - # mesh = os.path.join(data_dir, "l_step_0.07416198487095663_common_files/mesh.msh".format(cl)) #L3 12s - # mesh = os.path.join(data_dir, "l_step_0.020196309484414757_common_files/mesh.msh".format(cl)) #L4 22s - # mesh = os.path.join(data_dir, "l_step_0.0055_common_files/mesh.msh".format(cl)) #L5 + #mesh = os.path.join(data_dir, "l_step_0.3760603093086394_common_files/repo.msh") # L2 10.5 s + #mesh = os.path.join(data_dir, "l_step_0.1414213562373095_common_files/repo.msh") #L3 12s + mesh = os.path.join(data_dir, "l_step_0.053182958969449884_common_files/repo.msh") # L3 12s output_dir = os.path.join(data_dir, "L1_{}/test/02_conc/output/".format(level)) hdf_path = os.path.join(data_dir, "L1_{}/mlmc_1.hdf5".format(level)) - mlmc_hdf_path = os.path.join(data_dir, "/mlmc_hdf/L1_{}/mlmc_1.hdf5".format(level)) + mlmc_hdf_path = os.path.join(data_dir, "mlmc_hdf/L1_{}/mlmc_1.hdf5".format(level)) save_path = data_dir l_0_output_dir = os.path.join(data_dir, "L0_MC/L1_{}/test/02_conc/output/".format(level)) l_0_hdf_path = os.path.join(data_dir, "L0_MC/L1_{}/mlmc_1.hdf5".format(level)) sampling_info_path = os.path.join(data_dir, "sampling_info") ref_mlmc_file = os.path.join(data_dir, "L1_benchmark/mlmc_1.hdf5") - feature_names = [['conductivity_top', 'conductivity_bot', 'conductivity_repo'], ['porosity_top', 'porosity_bot', 'porosity_repo']] - return output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file, replace_level, nn_level, mlmc_hdf_path, feature_names + return output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file,\ + replace_level, nn_level, mlmc_hdf_path, feature_names # def plot_results_corr_length(): @@ -519,7 +520,7 @@ def get_arguments(arguments): args = get_arguments(sys.argv[1:]) data_dir = args.data_dir work_dir = args.work_dir - case = 14 + case = 12 #data_dir = "/home/martin/Documents/metamodels/data/1000_ele/" output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file,\ replace_level, nn_level, mlmc_hdf_path, feature_names = get_config(data_dir, case) @@ -628,28 +629,112 @@ def get_arguments(arguments): #machine_learning_model = ("DNN_mesh_L3_6", run_DNN, True) machine_learning_model = ("GCN_mesh_L3_log_16", run_GNN, True) machine_learning_model = ("mesh_L3_log_test_saved_model", run_GNN, True) - machine_learning_model = ("mesh_L3_log_50k_weights_5", run_GNN, True) + if case == 7: + #machine_learning_model = ("mesh_L3_log_50k_weights", run_GNN, True) + machine_learning_model = ("mesh_L3_log_50k", run_GNN, True) + machine_learning_model = ("L1_3_cl_0_1_s_1_all_log_output_mult", run_GNN, False) + #machine_learning_model = ("L1_3_cl_0_1_s_1_all_log_output_mult_case_1", run_GNN, False) + + # machine_learning_model = ("test_02_conc", run_GNN, False) + # + # machine_learning_model = ("L1_1_02_conc_cond_5", run_GNN, False) + # # # + # machine_learning_model = ("L1_1_02_conc_cond", run_GNN, False) + # machine_learning_model = ("L1_1_02_conc_cond_norm", run_GNN, False) + # machine_learning_model = ("L1_1_02_conc_cond_norm_output_mult", run_GNN, False) + if case == 12: + machine_learning_model = ("L1_1_02_conc_cond_norm_output_mult", run_GNN, False) + machine_learning_model = ("L1_1_02_conc_cond_output_mult", run_GNN, False) + machine_learning_model = ("L1_1_02_conc_cond_norm", run_GNN, False) + + + #machine_learning_model = ("L1_1_02_conc_cond", run_GNN, False) + # + # #machine_learning_model = ("L1_1_02_conc_cond_all_log_output_mult", run_GNN, False) + # #machine_learning_model = ("L1_1_02_conc_cond_features_log_output_mult", run_GNN, False) + # machine_learning_model = ("L1_1_02_conc_cond_output_log_output_mult", run_GNN, False) + # + # #machine_learning_model = ("L1_3_02_conc_cond_test", run_GNN, False) + # machine_learning_model = ("L1_3_02_conc_cond_all_log_output_mult", run_GNN, False) + machine_learning_model = ("L1_3_02_conc_cond_all_log_output_mult_case_1", run_GNN, False) + + + #### CASE 1 #### + machine_learning_model = ("L1_3_02_conc_cond_features_norm_case_1", run_GNN, False) + machine_learning_model = ("L1_3_02_conc_cond_features_norm_mult_output_case_1", run_GNN, False) + + machine_learning_model = ("L1_3_02_conc_cond_output_mult_case_1", run_GNN, False) + ### log + machine_learning_model = ("L1_3_02_conc_cond_log_features_norm_case_1", run_GNN, False) + #machine_learning_model = ("L1_3_02_conc_cond_true_features_norm_mult_output_case_1", run_GNN, False) + #machine_learning_model = ("L1_3_02_conc_cond_log_output_mult_case_1", run_GNN, False) + + #machine_learning_model = ("L1_3_02_conc_cond_all_log_output_mult_T1_case_1", run_GNN, False) + + machine_learning_model = ("L1_3_02_conc_cond_log_output_mult_T28_case_1", run_GNN, False) + + #machine_learning_model = ("L1_3_02_conc_cond_log_output_mult_T34_AOF6_case_1", run_GNN, False) + + #machine_learning_model = ("L1_3_test", run_GNN, False) + + if case == 13: + machine_learning_model = ("L1_1_02_conc_por_all_log_output_mult", run_GNN, False) + machine_learning_model = ("L1_1_02_conc_por_features_log_output_mult", run_GNN, False) + machine_learning_model = ("L1_1_02_conc_por_output_log_output_mult", run_GNN, False) + + machine_learning_model = ("L1_1_02_conc_por_test", run_GNN, False) + + machine_learning_model = ("L1_3_02_conc_por_all_log_output_mult", run_GNN, False) + + if case == 14: + machine_learning_model = ("L1_1_02_conc_2_features", run_GNN, False) + #machine_learning_model = ("L1_1_02_conc_2_features_log", run_GNN, True) + + #machine_learning_model = ("L1_1_02_conc_2_features_log_output_mult", run_GNN, False) + machine_learning_model = ("L1_1_02_conc_2_features_all_log_mult_output", run_GNN, False) + #machine_learning_model = ("L1_1_02_conc_2_features_output_log_mult_output", run_GNN, False) + #machine_learning_model = ("L1_1_02_conc_2_features_features_log_mult_output", run_GNN, False) + + machine_learning_model = ("L1_1_02_conc_2_features_test", run_GNN, False) + + machine_learning_model = ("L1_3_02_conc_2_features_all_log_output_mult", run_GNN, False) + + machine_learning_model = ("L1_3_02_conc_2_features_log_output_mult_T19_case_1", run_GNN, False) - machine_learning_model = ("test_02_conc", run_GNN, False) - #machine_learning_model = ("L1_1_02_conc_cond", run_GNN, False) + #machine_learning_model = ("L1_1_02_conc_2_features_test", run_GNN, False) - #machine_learning_model = ("mesh_L3_log_50k_weights_test_dense", run_GNN, False) + #machine_learning_model = ("mesh_L3_log_50k_weights_5", run_GNN, False) #machine_learning_model = ("mesh_L3_log_sigmoid", run_GNN, False) # ReLU is much better save_path = os.path.join(save_path, machine_learning_model[0]) - if os.path.exists(save_path): - shutil.rmtree(save_path) + # if os.path.exists(save_path): + # shutil.rmtree(save_path) print("save path ", save_path) # 02 proc times - # graph creation time: 2 features: 61 sec - # conductivity: 43 sec - # porosity: 40 sec - graph_creation_time = 60#25#11#22#159#0#159#66 + # graph creation time: 2 features: 53 sec + # conductivity: 35 sec + # porosity: 35 sec + + # L2 + # graph creation time: 2 features: 104 sec + # conductivity: 68 sec + # porosity: 66 sec + + # L3 + # graph creation time: 2 features: 396 sec + # conductivity: 251 sec + # porosity: 250 sec + + # L4 + # graph creation time: 2 features: + # conductivity: 1670 + # porosity: + graph_creation_time = 250#25#11#22#159#0#159#66 config = {'machine_learning_model': machine_learning_model, 'save_path': save_path, @@ -677,7 +762,7 @@ def get_arguments(arguments): 'feature_names': feature_names } - statistics(config) + #statistics(config) analyze_statistics(config) From 6500f4660e6ded12151e0c5eda7afe43f8920895 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Wed, 22 Dec 2021 16:12:31 +0100 Subject: [PATCH 46/67] dataset config postprocessing --- mlmc/metamodel/analyze_nn.py | 4 +++- mlmc/metamodel/postprocessing.py | 17 ++++++----------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/mlmc/metamodel/analyze_nn.py b/mlmc/metamodel/analyze_nn.py index 7873bb43..30aa989f 100644 --- a/mlmc/metamodel/analyze_nn.py +++ b/mlmc/metamodel/analyze_nn.py @@ -698,6 +698,7 @@ def analyze_statistics(config): data_dict = process_data(data_dict) + # print("train predictions type ", type(data_dict["train_predictions"])) # print("train predictions type ", type(data_dict["train_predictions"][0])) # print("train predictions shape ", np.array(data_dict["train_predictions"]).shape) @@ -815,10 +816,11 @@ def analyze_statistics(config): mlmc_hdf_file=config['mlmc_hdf_path'], stats=True, learning_time=learning_time, - dataset_config=config.get("dataset_config", {})) + dataset_config=data_dict.get("dataset_config", {})) # except: # continue + mlmc_n_collected_all.append(mlmc_n_collected) nn_n_collected_all.append(nn_mlmc_n_collected) n_ops_all.append(n_ops) diff --git a/mlmc/metamodel/postprocessing.py b/mlmc/metamodel/postprocessing.py index 96ab5e23..4728392c 100644 --- a/mlmc/metamodel/postprocessing.py +++ b/mlmc/metamodel/postprocessing.py @@ -70,17 +70,12 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic # cut_original_test(nn_hdf_file) # exit() - output_mult_factor = 1 - output_mult_factor = 1 - #output_mult_factor = 1437603411 # 02_conc_cond - output_mult_factor = 1521839229.4794111 # rascale log (case_1) - #output_mult_factor = -15.580288536 # cl_0_1_s_1 - #output_mult_factor = 1.0386 # cl_0_1_s_1 rescale log (case_1) - - targets = np.exp(targets) - predictions = np.exp(predictions) - l_0_predictions = np.exp(l_0_predictions) - l_0_targets = np.exp(l_0_targets) + output_mult_factor = dataset_config.get('output_mult_factor', 1) + if dataset_config.get('output_log', False): + targets = np.exp(targets) + predictions = np.exp(predictions) + l_0_predictions = np.exp(l_0_predictions) + l_0_targets = np.exp(l_0_targets) if mlmc_hdf_file is None: mlmc_hdf_file = nn_hdf_file From ea716c6ffe81c2de5a022a2407b98ca0d310bad8 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Mon, 3 Jan 2022 11:08:51 +0100 Subject: [PATCH 47/67] predict on different mesh --- mlmc/metamodel/analyze_nn.py | 127 +++++- mlmc/metamodel/flow_dataset.py | 8 +- test/metamodels/predict_on_different_mesh.py | 392 +++++++++++++++++++ 3 files changed, 511 insertions(+), 16 deletions(-) create mode 100644 test/metamodels/predict_on_different_mesh.py diff --git a/mlmc/metamodel/analyze_nn.py b/mlmc/metamodel/analyze_nn.py index 30aa989f..ded4e02d 100644 --- a/mlmc/metamodel/analyze_nn.py +++ b/mlmc/metamodel/analyze_nn.py @@ -378,7 +378,7 @@ def statistics(config): model, targets, predictions, learning_time, train_targets, train_predictions, \ val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, total_steps = \ - mch_l_model(config, stats=True, train=True, log=log, seed=i) + mch_l_model(config, stats=True, train=config.get('train_model', True), log=log, seed=i) if config['save_model']: model_data["model"] = model @@ -491,11 +491,21 @@ def plot_sse(data_nn, data_mlmc, x_label="ith moment", y_label="MSE", title=""): fig.show() -def check_loss(config, model, log=True): +def compare_models(model_1, model_2, config): + check_loss(config, model_1, dataset_config=config["dataset_config"]) + check_loss(config, model_2, dataset_config=config["dataset_config"]) + + exit() + + +def check_loss(config, model, log=True, dataset_config={}): if model is None: return batch_size = config['batch_size'] - data = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log) + + config['dataset_config'] = dataset_config + + data = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log, config=config) data = data # [:10000] data.a = config['conv_layer'].preprocess(data.a) @@ -503,7 +513,7 @@ def check_loss(config, model, log=True): train_data_len = config["n_train_samples"] - idx = 1 + idx = 0 data_tr = data[idx * train_data_len: idx * train_data_len + train_data_len] data_te = data.get_test_data(idx, train_data_len) @@ -513,7 +523,6 @@ def check_loss(config, model, log=True): loader_tr = MixedLoader(data_tr, batch_size=batch_size) loader_te = MixedLoader(data_te, batch_size=batch_size) - train_targets, train_predictions = model_predict(model, loader_tr) train_predictions = np.squeeze(train_predictions) @@ -523,8 +532,13 @@ def check_loss(config, model, log=True): train_MSE = np.mean((train_predictions - train_targets) ** 2) test_MSE = np.mean((test_predictions - test_targets) ** 2) + print("test targets ", np.sort(test_targets)[:10]) + print("test predictions ", test_predictions) + + print("train MSE: {}, test MSE: {}".format(train_MSE, test_MSE)) + conv_layers = {} dense_layers = {} flatten_input = [] @@ -689,7 +703,7 @@ def process_data(data_dict): return new_dict -def analyze_statistics(config): +def analyze_statistics(config, get_model=True): if not os.path.isdir(config['save_path']): print("dir not exists") exit() @@ -698,7 +712,6 @@ def analyze_statistics(config): data_dict = process_data(data_dict) - # print("train predictions type ", type(data_dict["train_predictions"])) # print("train predictions type ", type(data_dict["train_predictions"][0])) # print("train predictions shape ", np.array(data_dict["train_predictions"]).shape) @@ -747,8 +760,8 @@ def analyze_statistics(config): for i in range(len(data_dict["test_targets"])): print("index i ", i) - # if i == 1: - # break + if i == 1: + break #print("index ", i) @@ -783,7 +796,7 @@ def analyze_statistics(config): except: model = None - #check_loss(config, model) + check_loss(config, model, dataset_config=data_dict.get("dataset_config", {})) #predict_data(config, model, mesh_file=config["mesh"]) iter_test_MSE = np.mean((predictions - targets) ** 2) @@ -1226,6 +1239,80 @@ def display_vars(mlmc_vars, nn_vars, target_variance, title=""): fig.show() +def set_model_weights(new_model, old_model): + for new_conv_layer, old_conv_layer in zip(new_model._conv_layers, old_model._conv_layers): + new_conv_layer.kernel = old_conv_layer.kernel + new_conv_layer.bias = old_conv_layer.bias + + # print("input data shape ", input_data.dataset[0].shape) + # + # print(old_conv_layer.kernel.numpy().shape) + # input_imgs = Input(shape=(None, 108, 1)) + # print("old_model.flatten.weights", old_model.flatten().weights) + + for new_dense_layer, old_dense_layer in zip(new_model._dense_layers, old_model._dense_layers): + + # print("old_dense_layer.get_weights() shape ", old_dense_layer.get_weights()[0].shape) + # print("old_dense_layer.get_weights() shape ", old_dense_layer.get_weights()[1].shape) + # input_imgs = Input(shape=(None, 108, 1)) + # new_dense_layer(input_imgs) + # # model = Model(inputs=input_imgs, outputs=encoded) + # # dense_layer.set_weights(weights) + # + # print("new dense layer weights ", new_dense_layer.weights) + new_dense_layer.set_weights(old_dense_layer.get_weights()) + + # print("old_dense_layer.get_weights() ", old_dense_layer.get_weights()) + # print("new_model._dense_layers[-1].weights ", new_model._dense_layers[-1].weights) + # exit() + + +def set_model_layers(new_model, old_model): + for new_conv_layer, old_conv_layer in zip(new_model._conv_layers, old_model._conv_layers): + new_conv_layer.kernel = old_conv_layer.kernel + new_conv_layer.bias = old_conv_layer.bias + + # print("old conv layer get config ", old_conv_layer.get_config()) + # print("new conv layer get config ", new_conv_layer.get_config()) + + #exit() + + # print("input data shape ", input_data.dataset[0].shape) + # + # print(old_conv_layer.kernel.numpy().shape) + # input_imgs = Input(shape=(None, 108, 1)) + # print("old_model.flatten.weights", old_model.flatten().weights) + + for new_dense_layer, old_dense_layer in zip(new_model._dense_layers, old_model._dense_layers): + # config = layer.get_config() + # weights = layer.get_weights() + # cloned_layer = type(layer).from_config(config) + # cloned_layer.build(layer.input_shape) + # cloned_layer.set_weights(weights) + + # print("old_dense_layer.get_weights() shape ", old_dense_layer.get_weights()[0].shape) + # print("old_dense_layer.get_weights() shape ", old_dense_layer.get_weights()[1].shape) + # input_imgs = Input(shape=(None, 108, 1)) + # new_dense_layer(input_imgs) + # # model = Model(inputs=input_imgs, outputs=encoded) + # # dense_layer.set_weights(weights) + # + # print("new dense layer weights ", new_dense_layer.weights) + #new_dense_layer.set_weights(old_dense_layer.get_weights()) + + new_dense_layer.set_weights(old_dense_layer.get_weights()) + + # print("new_model._dense_layers[0].get_config() ", new_model._dense_layers[0].get_config()) + # print("old_model._dense_layers[0].get_config() ", old_model._dense_layers[0].get_config()) + # print("new_model._dense_layers[0].get_weights() ", new_model._dense_layers[0].get_weights()) + # print("old_model._dense_layers[0].get_weights() ", old_model._dense_layers[0].get_weights()) + + + # print("old_dense_layer.get_weights() ", old_dense_layer.get_weights()) + # print("new_model._dense_layers[-1].weights ", new_model._dense_layers[-1].weights) + # exit() + + def run_GNN(config, stats=True, train=True, log=False, seed=0): print("seed ", seed) @@ -1273,8 +1360,13 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): train_data_len = config['n_train_samples'] # Train/valid/test split - data_tr = data[seed*train_data_len: seed*train_data_len + train_data_len] - data_te = data.get_test_data(seed, train_data_len) + if not train: + data_tr = data + data_te = data + else: + data_tr = data[seed*train_data_len: seed*train_data_len + train_data_len] + print("data tr ", data_tr) + data_te = data.get_test_data(seed, train_data_len) #data_tr, data_te = data[:train_data_len], data[train_data_len:] gnn = config['gnn'](**config['model_config']) @@ -1300,6 +1392,17 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): loader_tr = MixedLoader(data_tr, batch_size=batch_size, epochs=epochs) loader_va = MixedLoader(data_va, batch_size=batch_size) loader_te = MixedLoader(data_te, batch_size=batch_size) + + if not train: + gnn.fit(MixedLoader(data_tr[:10], batch_size=batch_size, epochs=epochs), + MixedLoader(data_tr[10:20], batch_size=batch_size), MixedLoader(data_tr[20:30], batch_size=batch_size)) + set_model_weights(gnn._model, config["set_model"]) + + #set_model_layers(gnn._model, config["set_model"]) + + #gnn._model = config["set_model"] + #compare_models(gnn._model, config["set_model"], config) + # if gnn is None: gnn = GNN(loss=loss, optimizer=optimizer, conv_layer=config['conv_layer'], output_activation=abs_activation, diff --git a/mlmc/metamodel/flow_dataset.py b/mlmc/metamodel/flow_dataset.py index dd4ded7b..a7547cca 100644 --- a/mlmc/metamodel/flow_dataset.py +++ b/mlmc/metamodel/flow_dataset.py @@ -31,7 +31,7 @@ def __init__(self, output_dir=None, level=0, log=False, mesh=None, corr_field_co self._corr_field_config = corr_field_config self.adjacency_matrix = np.load(os.path.join(self._output_dir, "adjacency_matrix.npy"), allow_pickle=True) # adjacency matrix self.data = [] - self._config = config + self._save_path = config['save_path'] self._dataset_config = config.get('dataset_config', {}) self._min_feature = None self._max_feature = None @@ -211,11 +211,11 @@ def _save_output_mult_factor(self): import pickle import shutil - if os.path.exists(os.path.join(self._config['save_path'], "dataset_config.pkl")): - os.remove(os.path.join(self._config['save_path'], "dataset_config.pkl")) + if os.path.exists(os.path.join(self._save_path, "dataset_config.pkl")): + os.remove(os.path.join(self._save_path, "dataset_config.pkl")) # create a binary pickle file - with open(os.path.join(self._config['save_path'], "dataset_config.pkl"), "wb") as writer: + with open(os.path.join(self._save_path, "dataset_config.pkl"), "wb") as writer: pickle.dump(self._dataset_config, writer) @staticmethod diff --git a/test/metamodels/predict_on_different_mesh.py b/test/metamodels/predict_on_different_mesh.py new file mode 100644 index 00000000..2fd75905 --- /dev/null +++ b/test/metamodels/predict_on_different_mesh.py @@ -0,0 +1,392 @@ +import os +import numpy as np +import warnings +os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only +import sys +import shutil +import subprocess +from mlmc.metamodel.analyze_nn import run_GNN, run_SVR, statistics, analyze_statistics, process_results +from mlmc.moments import Legendre_tf, Monomial +from keras.layers import Input +from mlmc.metamodel.flow_task_GNN_2 import GNN +from spektral.layers import GCNConv, GlobalSumPool, ChebConv, GraphSageConv, ARMAConv, GATConv, APPNPConv, GINConv, GeneralConv +from mlmc.metamodel.own_cheb_conv import OwnChebConv +from tensorflow.keras.losses import MeanSquaredError, KLDivergence, MeanAbsoluteError +from mlmc.metamodel.custom_methods import abs_activation, MSE_moments +from tensorflow.keras import Model +from tensorflow.keras.models import Sequential +from tensorflow.keras.layers import Dense +from tensorflow.keras import regularizers +from spektral.layers import GlobalSumPool, GlobalMaxPool, GlobalAvgPool +import tensorflow as tf +from mlmc.plot import plots +from tensorflow.keras.layers.experimental import preprocessing +from mlmc.metamodel.analyze_nn import load_statistics, process_data +warnings.filterwarnings("ignore", category=DeprecationWarning) + + +def get_gnn(): + # Parameters + # conv_layer = GCNConv + conv_layer = ChebConv # Seems better than GCNConv, good distribution of predictions + conv_layer = OwnChebConv + # conv_layer = GraphSageConv # Seems better than ChebConv, good loss but very narrow distribution of predictions + # # conv_layer = ARMAConv # Seems worse than GraphSageConv + # conv_layer = GATConv # Slow and not better than GraphSageConv + # # conv_layer = APPNPConv # Not bad but worse than GraphSageConv + # # conv_layer = GINConv # it is comparable to APPNPConv + # act_func = "relu" # "tanh"#"elu" + + loss = MeanSquaredError() # var_loss_function# + #loss = MSE_moments + # loss = MeanAbsoluteError() + # loss = MeanSquaredLogarithmicError() + # loss = KLDivergence() + # loss = total_loss_function + optimizer = tf.optimizers.Adam(learning_rate=0.001) + patience = 150 + hidden_regularization = None # l2(2e-10) + + net_model_config = { + "conv_layer": conv_layer, + "hidden_activation": 'relu', + "output_activation": abs_activation, + #"output_activation": 'linear', + "kernel_regularization": hidden_regularization, + "normalizer": preprocessing.Normalization() + } + + #model = Net(**net_model_config) + + model_config = {"loss": loss, + "optimizer": optimizer, + "patience": patience, + "model_class": Net, + "net_model_config": net_model_config, + "verbose": True} + + corr_field_config = {'02_conc': True, 'corr_length': 0.1, 'sigma': 1, 'log': True} + + return GNN, conv_layer, corr_field_config, model_config + + +class Net(Model): + def __init__(self, conv_layer, hidden_activation, output_activation, kernel_regularization, normalizer, + **kwargs): + super().__init__(**kwargs) + + # T19 + self._conv_layers = [ + conv_layer(8, K=4, activation=hidden_activation, kernel_regularizer=kernel_regularization)] + self.flatten = GlobalSumPool() + + self._dense_layers = [Dense(32, activation=hidden_activation), Dense(16, activation=hidden_activation), + Dense(1)] + + # T34 + # self._conv_layers = [ + # conv_layer(8, K=4, activation=hidden_activation, kernel_regularizer=kernel_regularization)] # ,n) + # self.flatten = GlobalSumPool() + # + # self._dense_layers = [Dense(256, activation=hidden_activation), Dense(128, activation=hidden_activation), + # Dense(1)] + + + def call(self, inputs): + x, a = inputs + + for c_layer in self._conv_layers: + x = c_layer([x, a]) + + output = self.flatten(x) + + # print("output shape ", output.shape) + # exit()1 + + for d_layer in self._dense_layers: + output = d_layer(output) + + return output + + +def get_config(data_dir, case=0): + + if case == 12: # mesh size comparison + data_dir = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/" + # cl = "cl_0_1_s_1" + level = 3 + nn_level = 0 + replace_level = False + mesh = os.path.join(data_dir, "l_step_1.0_common_files/repo.msh") # L1, 7s + #mesh = os.path.join(data_dir, "l_step_0.3760603093086394_common_files/repo.msh") #L2 10.5 s + mesh = os.path.join(data_dir, "l_step_0.1414213562373095_common_files/repo.msh") #L3 12s + #mesh = os.path.join(data_dir, "l_step_0.053182958969449884_common_files/repo.msh") # L4 + output_dir = os.path.join(data_dir, "L1_{}/test/02_conc/output/".format(level)) + hdf_path = os.path.join(data_dir, "L1_{}/mlmc_1.hdf5".format(level)) + mlmc_hdf_path = os.path.join(data_dir, "mlmc_hdf/L1_{}/mlmc_1.hdf5".format(level)) + save_path = data_dir + l_0_output_dir = os.path.join(data_dir, "L0_MC/L1_{}/test/02_conc/output/".format(level)) + l_0_hdf_path = os.path.join(data_dir, "L0_MC/L1_{}/mlmc_1.hdf5".format(level)) + sampling_info_path = os.path.join(data_dir, "sampling_info") + + #ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_3/mlmc_1.hdf5" + #ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_1/mlmc_1.hdf5" + ref_mlmc_file = os.path.join(data_dir, "L1_benchmark/mlmc_1.hdf5") + + feature_names = [['conductivity_top', 'conductivity_bot', 'conductivity_repo']] + + if case == "L2": # mesh size comparison + data_dir = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/" + level = 2 + nn_level = 0 + replace_level = False + mesh = os.path.join(data_dir, "l_step_0.3760603093086394_common_files/repo.msh") + graph_creation_time = 35 + + output_dir = os.path.join(data_dir, "L1_{}/test/02_conc/output/".format(level)) + hdf_path = os.path.join(data_dir, "L1_{}/mlmc_1.hdf5".format(level)) + mlmc_hdf_path = os.path.join(data_dir, "mlmc_hdf/L1_{}/mlmc_1.hdf5".format(level)) + save_path = data_dir + l_0_output_dir = os.path.join(data_dir, "L0_MC/L1_{}/test/02_conc/output/".format(level)) + l_0_hdf_path = os.path.join(data_dir, "L0_MC/L1_{}/mlmc_1.hdf5".format(level)) + sampling_info_path = os.path.join(data_dir, "sampling_info") + + #ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_3/mlmc_1.hdf5" + #ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_1/mlmc_1.hdf5" + ref_mlmc_file = os.path.join(data_dir, "L1_benchmark/mlmc_1.hdf5") + + feature_names = [['conductivity_top', 'conductivity_bot', 'conductivity_repo']] + + if case == 'L1': # mesh size comparison + data_dir = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/" + level = 1 + nn_level = 0 + replace_level = False + mesh = os.path.join(data_dir, "l_step_1.0_common_files/repo.msh") + graph_creation_time = 68 + output_dir = os.path.join(data_dir, "L1_{}/test/02_conc/output/".format(level)) + hdf_path = os.path.join(data_dir, "L1_{}/mlmc_1.hdf5".format(level)) + mlmc_hdf_path = os.path.join(data_dir, "mlmc_hdf/L1_{}/mlmc_1.hdf5".format(level)) + save_path = data_dir + l_0_output_dir = os.path.join(data_dir, "L0_MC/L1_{}/test/02_conc/output/".format(level)) + l_0_hdf_path = os.path.join(data_dir, "L0_MC/L1_{}/mlmc_1.hdf5".format(level)) + sampling_info_path = os.path.join(data_dir, "sampling_info") + + #ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_3/mlmc_1.hdf5" + #ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_1/mlmc_1.hdf5" + ref_mlmc_file = os.path.join(data_dir, "L1_benchmark/mlmc_1.hdf5") + + feature_names = [['conductivity_top', 'conductivity_bot', 'conductivity_repo']] + + if case == 'L3': # mesh size comparison + data_dir = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/" + level = 3 + nn_level = 0 + replace_level = False + mesh = os.path.join(data_dir, "l_step_0.1414213562373095_common_files/repo.msh") # L3 12s + graph_creation_time = 250 + output_dir = os.path.join(data_dir, "L1_{}/test/02_conc/output/".format(level)) + hdf_path = os.path.join(data_dir, "L1_{}/mlmc_1.hdf5".format(level)) + mlmc_hdf_path = os.path.join(data_dir, "mlmc_hdf/L1_{}/mlmc_1.hdf5".format(level)) + save_path = data_dir + l_0_output_dir = os.path.join(data_dir, "L0_MC/L1_{}/test/02_conc/output/".format(level)) + l_0_hdf_path = os.path.join(data_dir, "L0_MC/L1_{}/mlmc_1.hdf5".format(level)) + sampling_info_path = os.path.join(data_dir, "sampling_info") + + # output_dir = l_0_output_dir + # hdf_path = l_0_hdf_path + + #ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_3/mlmc_1.hdf5" + #ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_1/mlmc_1.hdf5" + ref_mlmc_file = os.path.join(data_dir, "L1_benchmark/mlmc_1.hdf5") + + feature_names = [['conductivity_top', 'conductivity_bot', 'conductivity_repo']] + + elif case == 429: # mesh size comparison + data_dir = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/" + # cl = "cl_0_1_s_1" + level = "3_429" + nn_level = 0 + replace_level = False + #mesh = os.path.join(data_dir, "l_step_1.0_common_files/repo.msh") # L1, 7s + #mesh = os.path.join(data_dir, "l_step_0.3760603093086394_common_files/repo.msh") #L2 10.5 s + mesh = os.path.join(data_dir, "l_step_0.15_common_files/repo.msh") #L3 12s + #mesh = os.path.join(data_dir, "l_step_0.053182958969449884_common_files/repo.msh") # L4 + output_dir = os.path.join(data_dir, "L1_{}/test/02_conc/output/".format(level)) + hdf_path = os.path.join(data_dir, "L1_{}/mlmc_1.hdf5".format(level)) + mlmc_hdf_path = None#os.path.join(data_dir, "mlmc_hdf/L1_{}/mlmc_1.hdf5".format(level)) + save_path = data_dir + l_0_output_dir = os.path.join(data_dir, "L0_MC/L1_{}/test/02_conc/output/".format(level)) + l_0_hdf_path = os.path.join(data_dir, "L0_MC/L1_{}/mlmc_1.hdf5".format(level)) + sampling_info_path = os.path.join(data_dir, "sampling_info") + + graph_creation_time = 241 + + #ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_3/mlmc_1.hdf5" + #ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_1/mlmc_1.hdf5" + ref_mlmc_file = os.path.join(data_dir, "L1_benchmark/mlmc_1.hdf5") + + feature_names = [['conductivity_top', 'conductivity_bot', 'conductivity_repo']] + + elif case == 521: # mesh size comparison + data_dir = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/" + # cl = "cl_0_1_s_1" + level = "3_521" + nn_level = 0 + replace_level = False + #mesh = os.path.join(data_dir, "l_step_1.0_common_files/repo.msh") # L1, 7s + #mesh = os.path.join(data_dir, "l_step_0.3760603093086394_common_files/repo.msh") #L2 10.5 s + mesh = os.path.join(data_dir, "l_step_0.13_common_files/repo.msh") #L3 12s + #mesh = os.path.join(data_dir, "l_step_0.053182958969449884_common_files/repo.msh") # L4 + output_dir = os.path.join(data_dir, "L1_{}/test/02_conc/output/".format(level)) + hdf_path = os.path.join(data_dir, "L1_{}/mlmc_1.hdf5".format(level)) + mlmc_hdf_path = None#os.path.join(data_dir, "mlmc_hdf/L1_{}/mlmc_1.hdf5".format(level)) + save_path = data_dir + l_0_output_dir = os.path.join(data_dir, "L0_MC/L1_{}/test/02_conc/output/".format(level)) + l_0_hdf_path = os.path.join(data_dir, "L0_MC/L1_{}/mlmc_1.hdf5".format(level)) + sampling_info_path = os.path.join(data_dir, "sampling_info") + + #ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_3/mlmc_1.hdf5" + #ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_1/mlmc_1.hdf5" + ref_mlmc_file = os.path.join(data_dir, "L1_benchmark/mlmc_1.hdf5") + + feature_names = [['conductivity_top', 'conductivity_bot', 'conductivity_repo']] + + graph_creation_time = 285 + + return output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file,\ + replace_level, nn_level, mlmc_hdf_path, feature_names, graph_creation_time + + +def get_arguments(arguments): + """ + Getting arguments from console + :param arguments: list of arguments + :return: namespace + """ + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('work_dir', help='work directory') + parser.add_argument('data_dir', help='data directory') + args = parser.parse_args(arguments) + return args + + +# def set_weights(new_model, old_model): +# for new_conv_layer, old_conv_layer in zip(new_model._conv_layers, old_model._conv_layers): +# new_conv_layer.kernel = old_conv_layer.kernel +# +# print(old_conv_layer.kernel.numpy().shape) +# input_imgs = Input(shape=(None, 108, 1)) +# print("old_model.flatten.weights", old_model.flatten().weights) +# +# for new_dense_layer, old_dense_layer in zip(new_model._dense_layers, old_model._dense_layers): +# +# print("old_dense_layer.get_weights() shape ", old_dense_layer.get_weights()[0].shape) +# print("old_dense_layer.get_weights() shape ", old_dense_layer.get_weights()[1].shape) +# input_imgs = Input(shape=(None, 108, 1)) +# new_dense_layer(input_imgs) +# # model = Model(inputs=input_imgs, outputs=encoded) +# # dense_layer.set_weights(weights) +# +# print("new dense layer weights ", new_dense_layer.weights) +# new_dense_layer.set_weights(old_dense_layer.get_weights()) + + +if __name__ == "__main__": + ####################### + # Load trained model # + ####################### + machine_learning_model = ("L1_3_02_conc_cond_log_output_mult_T19_case_1", run_GNN, False) + save_path = os.path.join("/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/", machine_learning_model[0]) + data_dict = load_statistics(save_path) + #data_dict = process_data(data_dict) + #for i in range(len(data_dict["test_targets"])): + model = data_dict["model"][0] + + # newInput = Input(batch_shape=(None, 108, 1)) + # newOutputs = model(newInput) + # newModel = Model(newInput, newOutputs) + # + # if type(model._conv_layers[0]).__name__ == 'OwnChebConv': + # conv_layer = OwnChebConv + # + # print("conv layer ", conv_layer) + + + + ##################### + ## New case config ## + ##################### + args = get_arguments(sys.argv[1:]) + data_dir = args.data_dir + work_dir = args.work_dir + case = "L2" + case = 521 + #data_dir = "/home/martin/Documents/metamodels/data/1000_ele/" + output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file,\ + replace_level, nn_level, mlmc_hdf_path, feature_names, graph_creation_time = get_config(data_dir, case) + + machine_learning_model = ("L1_2_02_conc_cond_log_output_mult_T19_case_1_trained_L1_3", run_GNN, False) + save_path = os.path.join(save_path, machine_learning_model[0]) + + if os.path.exists(save_path): + shutil.rmtree(save_path) + + corr_field_config = {'02_conc': True, 'corr_length': 0.1, 'sigma': 1, 'log': True} + + gnn, conv_layer, corr_field_config, model_config = get_gnn() + + #gnn_au = gnn(**model_config) + # + # print("gnn au model ", gnn_au._model) + # print("model ", model) + + #set_weights(gnn_au._model, model) + + dataset_config = {"features_normalization": False, + "calc_output_mult_factor": True, + "output_mult_factor": 1, + "features_mult_factor": 1, + "features_log": False, + "output_log": True + } + + config = {'machine_learning_model': machine_learning_model, + 'save_path': save_path, + 'output_dir': output_dir, + 'hdf_path': hdf_path, + 'mlmc_hdf_path': mlmc_hdf_path, + 'mesh': mesh, + 'l_0_output_dir': l_0_output_dir, + 'l_0_hdf_path': l_0_hdf_path, + 'sampling_info_path': sampling_info_path, + 'ref_mlmc_file': ref_mlmc_file, + 'level': nn_level, + 'conv_layer': conv_layer, + 'gnn': gnn, + 'model_config': model_config, + 'replace_level': replace_level, + 'corr_field_config': corr_field_config, + 'n_train_samples': 2000, + 'val_samples_ratio': 0.2, + 'batch_size': 200, + 'epochs': 1, + 'learning_rate': 0.001, + 'graph_creation_time': graph_creation_time, + 'save_model': True, + 'feature_names': feature_names, + "train_model": False, + "set_model": model, + "dataset_config": dataset_config + } + + model_title, mch_l_model, log = config['machine_learning_model'] + + statistics(config) + + #analyze_statistics(config) + + # save_path = os.path.join(save_path, "SVR") + # statistics(run_SVR, output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, level=nn_level, log=True) + From 4c402b176153c341b64f954e4448880d5fb6589b Mon Sep 17 00:00:00 2001 From: Martin Spetlik Date: Wed, 19 Jan 2022 16:12:48 +0100 Subject: [PATCH 48/67] flow dataset preprocess --- mlmc/metamodel/analyze_nn.py | 21 ++-- mlmc/metamodel/flow_dataset.py | 217 +++++++++++++++++++-------------- 2 files changed, 137 insertions(+), 101 deletions(-) diff --git a/mlmc/metamodel/analyze_nn.py b/mlmc/metamodel/analyze_nn.py index 1cd2520c..c3a7ecb0 100644 --- a/mlmc/metamodel/analyze_nn.py +++ b/mlmc/metamodel/analyze_nn.py @@ -362,11 +362,13 @@ def statistics(config): if not os.path.isdir(config['save_path']): os.makedirs(config['save_path']) - # Save config to Pickle - import pickle - # create a binary pickle file + if os.path.exists(os.path.join(config['save_path'], "dataset_config.pkl")): + os.remove(os.path.join(config['save_path'], "dataset_config.pkl")) + + # create a binary pickle file with open(os.path.join(config['save_path'], "dataset_config.pkl"), "wb") as writer: - pickle.dump(config.get("dataset_config", {}), writer) + pickle.dump(config["dataset_config"], writer) + else: print("dir exists {}".format(config['save_path'])) exit() @@ -376,6 +378,8 @@ def statistics(config): if not os.path.isdir(iter_dir): os.makedirs(iter_dir) + config['iter_dir'] = iter_dir + model, targets, predictions, learning_time, train_targets, train_predictions, \ val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, total_steps = \ mch_l_model(config, stats=True, train=True, log=log, seed=i) @@ -1223,9 +1227,12 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): preprocess_start_time = time.process_time() # Load data - data = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log, config=config) + data = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log, config=config, index=seed) data = data#[:10000] + # Dataset preprocess config + config['dataset_config'] = data._dataset_config + # print("n node features ", data.graphs[0].n_node_features) # print("graph x", data.graphs[0].x) # print("graphs[0] ", repr(data.graphs[0])) @@ -1292,7 +1299,7 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): # min_key = np.min(list(states.keys())) # gnn = states[min_key] - train_targets, train_predictions = gnn.predict(loader_tr) + train_targets, train_predictions = gnn.predict(MixedLoader(data_tr, batch_size=batch_size, epochs=1)) train_predictions = np.squeeze(train_predictions) val_targets, val_predictions = gnn.predict(loader_va) @@ -1372,7 +1379,7 @@ def predict_level_zero(nn, output_dir, hdf_path, mesh, conv_layer, batch_size=10 if corr_field_config: sample_time = corr_field_sample_time(mesh, corr_field_config) - data = FlowDataset(output_dir=output_dir, log=log, config=config)#, mesh=mesh, corr_field_config=corr_field_config) + data = FlowDataset(output_dir=output_dir, log=log, config=config, predict=True)#, mesh=mesh, corr_field_config=corr_field_config) #data = data # [:10000] data.shuffle(seed=seed) diff --git a/mlmc/metamodel/flow_dataset.py b/mlmc/metamodel/flow_dataset.py index dd4ded7b..5aba0c4b 100644 --- a/mlmc/metamodel/flow_dataset.py +++ b/mlmc/metamodel/flow_dataset.py @@ -21,7 +21,8 @@ class FlowDataset(Dataset): GRAPHS_FILE = "graphs" DATA_FILE = "data" - def __init__(self, output_dir=None, level=0, log=False, mesh=None, corr_field_config=None, config={}, **kwargs): + def __init__(self, output_dir=None, level=0, log=False, mesh=None, corr_field_config=None, config={}, + index=None, predict=False, **kwargs): self._output_dir = output_dir # if self._output_dir is None: # self._output_dir = OUTPUT_DIR @@ -32,9 +33,31 @@ def __init__(self, output_dir=None, level=0, log=False, mesh=None, corr_field_co self.adjacency_matrix = np.load(os.path.join(self._output_dir, "adjacency_matrix.npy"), allow_pickle=True) # adjacency matrix self.data = [] self._config = config + self._index = index + self._predict = predict self._dataset_config = config.get('dataset_config', {}) - self._min_feature = None - self._max_feature = None + + if predict: + self._min_features = self._dataset_config.get('min_features', None) + self._max_features = self._dataset_config.get('max_features', None) + self._mean_features = self._dataset_config.get('mean_features', None) + self._var_features = self._dataset_config.get('var_features', None) + self._min_output = self._dataset_config.get('min_output', None) + self._max_output = self._dataset_config.get('max_output', None) + self._mean_output = self._dataset_config.get('mean_output', None) + self._var_output = self._dataset_config.get('var_output', None) + self._output_mult_factor = self._dataset_config.get('output_mult_factor', 1) + else: + self._min_features = None + self._max_features = None + self._mean_features = None + self._var_features = None + self._min_output = None + self._max_output = None + self._mean_output = None + self._var_output = None + self._output_mult_factor = 1 + super().__init__(**kwargs) #self.a = self.adjacency_matrix @@ -94,128 +117,134 @@ def shuffle(self, seed=None): # return graphs def read(self): - # if self._mesh is not None: - # return self.generate_data() - - # with open(os.path.join(OUTPUT_DIR, FlowDataset.GRAPHS_FILE), 'rb') as reader: - # graphs = pickle.loads(reader) - # - # if os.path.exists(os.path.join(OUTPUT_DIR, FlowDataset.DATA_FILE)): - # with open(os.path.join(OUTPUT_DIR, FlowDataset.DATA_FILE), 'rb') as reader: - # self.data = pickle.loads(reader) - # - # return graphs - - if self._dataset_config.get("features_normalization", False) or self._dataset_config.get("calc_output_mult_factor", False): - # i = 0 - all_outputs = [] - all_features = [] - - for s_dir in os.listdir(self._output_dir): - try: - l = re.findall(r'L(\d+)_S', s_dir)[0] - if int(l) != self.level: - continue - except IndexError: - continue - if os.path.isdir(os.path.join(self._output_dir, s_dir)): - sample_dir = os.path.join(self._output_dir, s_dir) - if os.path.exists(os.path.join(sample_dir, "nodes_features.npy")): - features = np.load(os.path.join(sample_dir, "nodes_features.npy")) - all_features.extend(features) - - if self._dataset_config.get("calc_output_mult_factor", False) is True: - output = np.load(os.path.join(sample_dir, "output.npy")) - all_outputs.append(output) - - #print("all outputs ", np.array(all_outputs).shape) - # min_output = np.min(all_outputs) - # max_output = np.max(all_outputs) - - if self._dataset_config.get("calc_output_mult_factor", False) is True: - self._dataset_config["output_mult_factor"] = 1/np.mean(all_outputs) - self._save_output_mult_factor() - print("output mult factor ", self._dataset_config["output_mult_factor"]) - - if self._dataset_config.get("features_normalization", False): - self._min_feature = np.min(all_features) - self._max_feature = np.max(all_features) + all_outputs = [] + all_features = [] - # if self._log and self._dataset_config.get("features_log", False) is False and self._dataset_config.get("output_log", - # False) is False: - # all_features = np.log(all_features) - # all_outputs = np.log(all_outputs) - # - # if self._dataset_config.get("features_log", False): - # all_features = np.log(all_features) - # - # if self._dataset_config.get("output_log", False): - # all_outputs = np.log(all_outputs) - - - # self.min_output = min_output - # self.max_output = max_output - - graphs = [] for s_dir in os.listdir(self._output_dir): try: l = re.findall(r'L(\d+)_S', s_dir)[0] if int(l) != self.level: continue except IndexError: - continue - + continue if os.path.isdir(os.path.join(self._output_dir, s_dir)): sample_dir = os.path.join(self._output_dir, s_dir) if os.path.exists(os.path.join(sample_dir, "nodes_features.npy")): features = np.load(os.path.join(sample_dir, "nodes_features.npy")) + all_features.append(features) + output = np.load(os.path.join(sample_dir, "output.npy")) + all_outputs.append(output) + + if self._dataset_config.get("first_log_features", False): + all_features = np.log(all_features) + + if self._dataset_config.get("first_log_output", False): + all_outputs = np.log(all_outputs) + + #print("all outputs ", np.array(all_outputs).shape) + # min_output = np.min(all_outputs) + # max_output = np.max(all_outputs) + + if not self._predict: + train_outputs = all_outputs[self._index * self._config['n_train_samples']: + self._index * self._config['n_train_samples'] + self._config['n_train_samples']] + train_features = all_features[self._index * self._config['n_train_samples']: + self._index * self._config['n_train_samples'] + self._config['n_train_samples']] + + if self._dataset_config.get("calc_output_mult_factor", False) is True: + self._output_mult_factor = 1/np.mean(train_outputs) + self._dataset_config["output_mult_factor"] = self._output_mult_factor + self._save_data_config() + print("output mult factor ", self._dataset_config["output_mult_factor"]) + + if self._dataset_config.get("features_normalization", False): + self._min_features = np.min(train_features, axis=0) + self._max_features = np.max(train_features, axis=0) + self._dataset_config["min_features"] = self._min_features + self._dataset_config["max_features"] = self._max_features + + if self._dataset_config.get("features_scale", False): + self._mean_features = np.mean(train_features, axis=0) + self._var_features = np.var(train_features, axis=0) + self._dataset_config["mean_features"] = self._mean_features + self._dataset_config["var_features"] = self._var_features + + if self._dataset_config.get("output_normalization", False): + self._min_output = np.min(train_outputs, axis=0) + self._max_output = np.max(train_outputs, axis=0) + self._dataset_config["min_output"] = self._min_output + self._dataset_config["max_output"] = self._max_output + + if self._dataset_config.get("output_scale", False): + self._mean_output = np.mean(train_outputs, axis=0) + self._var_output = np.var(train_outputs, axis=0) + self._dataset_config["mean_output"] = self._mean_output + self._dataset_config["var_output"] = self._var_output + + self._save_data_config() + + graphs = [] + for features, output in zip(all_features, all_outputs): + if self._dataset_config.get("features_normalization", False): + features = (features - self._min_features) / (self._max_features - self._min_features) + features = np.nan_to_num(features) + + if self._dataset_config.get("output_normalization", False): + output = (output - self._min_output) / (self._max_output - self._min_output) + output = np.nan_to_num(output) + + if self._dataset_config.get("features_scale", False): + features -= self._mean_features + features /= self._var_features - if self._min_feature is not None and self._max_feature is not None: - features = (features - self._min_feature) / (self._max_feature - self._min_feature) + if self._dataset_config.get("output_scale", False): + output -= self._mean_output + output /= self._var_output - # output = (output - min_output) / (max_output - min_output) - # print("max ", maximum) - # print("max ", minimum) - # - # print("new featuers max ", np.max(new_features)) - # print("new featuers min ", np.min(new_features)) - # exit() + # output = (output - min_output) / (max_output - min_output) + # print("max ", maximum) + # print("max ", minimum) + # + # print("new featuers max ", np.max(new_features)) + # print("new featuers min ", np.min(new_features)) + # exit() - output_mult_factor = self._dataset_config.get("output_mult_factor", 1) - features_mult_factor = self._dataset_config.get("features_mult_factor", 1) + output_mult_factor = self._output_mult_factor + features_mult_factor = self._dataset_config.get("features_mult_factor", 1) - features *= features_mult_factor - output *= output_mult_factor + features *= features_mult_factor + output *= output_mult_factor - if self._log and self._dataset_config.get("features_log", False) is False and self._dataset_config.get("output_log", False) is False: - features = np.log(features) - output = np.log(output) + if self._log and self._dataset_config.get("features_log", False) is False and self._dataset_config.get("output_log", False) is False: + features = np.log(features) + output = np.log(output) + print("self._log SET") - if self._dataset_config.get("features_log", False): - features = np.log(features) + if self._dataset_config.get("last_log_features", False): + features = np.log(features) - if self._dataset_config.get("output_log", False): - output = np.log(output) + if self._dataset_config.get("last_log_output", False): + output = np.log(output) - graphs.append(Graph(x=features, y=output))#, a=self.adjacency_matrix)) + graphs.append(Graph(x=features, y=output))#, a=self.adjacency_matrix)) - # Save data for pandas dataframe creation, not used with Graph neural network - self.data.append({'x': features, 'y': output}) + # Save data for pandas dataframe creation, not used with Graph neural network + self.data.append({'x': features, 'y': output}) self.a = self.adjacency_matrix return graphs - def _save_output_mult_factor(self): + def _save_data_config(self): # Save config to Pickle import pickle import shutil - if os.path.exists(os.path.join(self._config['save_path'], "dataset_config.pkl")): - os.remove(os.path.join(self._config['save_path'], "dataset_config.pkl")) + if os.path.exists(os.path.join(self._config['iter_dir'], "dataset_config.pkl")): + os.remove(os.path.join(self._config['iter_dir'], "dataset_config.pkl")) # create a binary pickle file - with open(os.path.join(self._config['save_path'], "dataset_config.pkl"), "wb") as writer: + with open(os.path.join(self._config['iter_dir'], "dataset_config.pkl"), "wb") as writer: pickle.dump(self._dataset_config, writer) @staticmethod From 06ed2daaec2269a03010a64ff808ce4822a053ee Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Wed, 19 Jan 2022 16:18:24 +0100 Subject: [PATCH 49/67] postprocessing --- mlmc/metamodel/02_conc_rnd_field_tests.py | 394 ++++++++++++++++++++++ mlmc/metamodel/analyze_nn.py | 40 ++- mlmc/metamodel/postprocessing.py | 286 +++++++++++++--- mlmc/metamodel/random_field_time.py | 64 +++- 4 files changed, 710 insertions(+), 74 deletions(-) create mode 100644 mlmc/metamodel/02_conc_rnd_field_tests.py diff --git a/mlmc/metamodel/02_conc_rnd_field_tests.py b/mlmc/metamodel/02_conc_rnd_field_tests.py new file mode 100644 index 00000000..d00e92ab --- /dev/null +++ b/mlmc/metamodel/02_conc_rnd_field_tests.py @@ -0,0 +1,394 @@ +import time +import numpy as np +from mlmc.tool import gmsh_io +from mlmc.tool.flow_mc import FlowSim, create_corr_field +from mlmc.tool.flow_mc_2 import FlowSimProcConc +import gstools +from mlmc.random import correlated_field as cf +import matplotlib.pyplot as plt +from mlmc.metamodel.create_graph import get_node_features, extract_mesh_gmsh_io + + +def create_corr_fields(model='gauss', corr_length=0.125, dim=2, log=True, factor_sigma=1, sigma=1, mode_no=1000, + por_top_mean=-1.0, por_bot_mean=-1.0, por_top_sigma=1, por_bot_sigma=1, + por_top_len_scale=0.2, por_bot_len_scale=0.2, factor_top_mean=1e-8, factor_bot_mean=1e-8): + """ + Create random fields + :return: + """ + # por_top = cf.SpatialCorrelatedField( + # corr_exp='gauss', + # dim=2, + # corr_length=0.2, + # mu=-1.0, + # sigma=1.0, + # log=True + # ) + # + # print("por top ", por_top) + + + por_top = cf.GSToolsSpatialCorrelatedField(gstools.Gaussian(dim=2, len_scale=por_top_len_scale), + log=log, mean=por_top_mean, sigma=por_top_sigma, mode_no=mode_no) + + #print("por top gstools ", por_top_gstools) + + # por_bot = cf.SpatialCorrelatedField( + # corr_exp='gauss', + # dim=2, + # corr_length=0.2, + # mu=-1.0, + # sigma=1.0, + # log=True + # ) + + por_bot = cf.GSToolsSpatialCorrelatedField(gstools.Gaussian(dim=2, len_scale=por_bot_len_scale), + log=log, mean=por_bot_mean, sigma=por_bot_sigma, mode_no=mode_no) + + #por_bot = gstools.Gaussian(dim=dim, len_scale=0.2, mu=-1.0, sigma=1.0, log=True) + + water_viscosity = 8.90e-4 + + factor_top_model = gstools.Gaussian(dim=dim, len_scale=1) + factor_bot_model = gstools.Gaussian(dim=dim, len_scale=1) + + fields = cf.Fields([ + cf.Field('por_top', por_top, regions='ground_0'), + cf.Field('porosity_top', cf.positive_to_range, ['por_top', 0.02, 0.1], regions='ground_0'), + cf.Field('por_bot', por_bot, regions='ground_1'), + cf.Field('porosity_bot', cf.positive_to_range, ['por_bot', 0.01, 0.05], regions='ground_1'), + cf.Field('porosity_repo', 0.5, regions='repo'), + #cf.Field('factor_top', cf.SpatialCorrelatedField('gauss', mu=1e-8, sigma=1, log=True), regions='ground_0'), + + cf.Field('factor_top', cf.GSToolsSpatialCorrelatedField(factor_top_model, log=log, mean=factor_top_mean, sigma=factor_sigma, mode_no=mode_no), + regions='ground_0'), + + #cf.Field('factor_top', gstools.Gaussian(len_scale=1, mu=1e-8, sigma=1.0, log=True), regions='ground_0'), + # conductivity about + #cf.Field('factor_bot', cf.SpatialCorrelatedField('gauss', mu=1e-8, sigma=1, log=True), regions='ground_1'), + #cf.Field('factor_bot', gstools.Gaussian(len_scale=1, mu=1e-8, sigma=1, log=True), regions='ground_1'), + cf.Field('factor_bot', + cf.GSToolsSpatialCorrelatedField(factor_bot_model, log=log, mean=factor_bot_mean, sigma=factor_sigma, mode_no=mode_no), + regions='ground_1'), + + # cf.Field('factor_repo', cf.SpatialCorrelatedField('gauss', mu=1e-10, sigma=1, log=True), regions='repo'), + cf.Field('conductivity_top', cf.kozeny_carman, ['porosity_top', 1, 'factor_top', water_viscosity], + regions='ground_0'), + cf.Field('conductivity_bot', cf.kozeny_carman, ['porosity_bot', 1, 'factor_bot', water_viscosity], + regions='ground_1'), + # cf.Field('conductivity_repo', cf.kozeny_carman, ['porosity_repo', 1, 'factor_repo', water_viscosity], regions='repo') + cf.Field('conductivity_repo', 0.001, regions='repo') + ]) + + return fields + + + +# def corr_field_sample_time(mesh_file=None, corr_length_config=None): +# # import matplotlib +# # from matplotlib import ticker, cm +# #matplotlib.rcParams.update({'font.size': 22}) +# +# if corr_length_config.get('02_conc', False): +# return conc_rnd_sample_time(mesh_file, corr_length_config) +# +# dim = 2 +# log = True +# cl = 0.1 +# s = 1 +# +# if mesh_file is None: +# #mesh_file = "/home/martin/Sync/Documents/flow123d_results/flow_experiments/Exponential/corr_length_0_01/l_step_0.0055_common_files/mesh.msh" +# #mesh_file = "/home/martin/Documents/metamodels/data/5_ele/cl_0_1_s_1/L5/l_step_0.020196309484414757_common_files/mesh.msh" +# mesh_file = "/home/martin/Documents/metamodels/data/1000_ele/l_step_0.055_common_files/mesh.msh" +# +# +# start_time = time.process_time() +# mesh_data = FlowSim.extract_mesh(mesh_file) +# if corr_length_config is not None: +# fields = create_corr_field(model="exp", dim=dim, +# sigma=corr_length_config['sigma'], +# corr_length=corr_length_config['corr_length'], +# log=corr_length_config['log']) +# else: +# fields = create_corr_field(model="exp", dim=dim, +# sigma=s, +# corr_length=cl, +# log=log) +# # # Create fields both fine and coarse +# fields = FlowSim.make_fields(fields, mesh_data, None) +# +# n_samples = 200 +# for i in range(n_samples): +# +# fine_input_sample, coarse_input_sample = FlowSim.generate_random_sample(fields, coarse_step=0, +# n_fine_elements=len( +# mesh_data['points'])) +# +# len(fine_input_sample["conductivity"]) +# features_log = np.log(fine_input_sample["conductivity"]) +# +# # print("conductivity mean ", np.mean(fine_input_sample["conductivity"])) +# # print("conductivity var ", np.var(fine_input_sample["conductivity"])) +# output = 1 +# # +# # print("fine input sample ", fine_input_sample["conductivity"].shape) +# # +# # gmsh_io.GmshIO().write_fields('fields_sample.msh', mesh_data['ele_ids'], fine_input_sample) +# # +# # mesh = gmsh_io.GmshIO('fields_sample.msh') +# # element_data = mesh.current_elem_data +# # features = list(element_data.values()) +# # print("features ", np.array(features).shape) +# +# rnd_time = time.process_time() - start_time +# print("rnd_time / n_samples ", rnd_time / n_samples) +# return rnd_time / n_samples +# +# #Xfinal, Yfinal = fields.fields[0].correlated_field.points[:, 0], fields.fields[0].correlated_field.points[:, 1] +# +# # cont = ax.tricontourf(Xfinal, +# # Yfinal, +# # fine_input_sample['conductivity'].ravel())#, locator=ticker.LogLocator()) +# +# # fig.colorbar(cont) +# # fig.savefig("cl_{}_var_{}.pdf".format(cl, s ** 2)) +# # plt.show() +# +# # print("fields ", fields) +# # model = gs.Exponential(dim=2, len_scale=cl) +# # srf = gs.SRF(model, mesh_type="unstructed", seed=20170519, mode_no=1000, generator='RandMeth') +# # print("model.var ", model.var) +# # field = srf( +# # (fields.fields[0].correlated_field.points[:, 0], fields.fields[0].correlated_field.points[:, 1])) +# # srf.vtk_export("field") +# # ax = srf.plot() +# # ax.set_aspect("equal") + + +def conc_corr_field(mesh_file, corr_field_config): + mesh_data = FlowSim.extract_mesh(mesh_file) + feature_names = [['conductivity_top', 'conductivity_bot', 'conductivity_repo']] + + por_top_mean = -1.0 + por_bot_mean = -1.0 + por_top_sigma = 1 + por_bot_sigma = 1 + factor_sigma = 1 + por_top_len_scale = 1 + por_bot_len_scale = 1 + factor_top_mean = 1e-8 + factor_bot_mean = 1e-8 + + all_features = [] + n_samples = 50 + for i in range(n_samples): + fields = create_corr_fields(dim=2, log=corr_field_config["log"], + por_top_mean=por_top_mean, + por_bot_mean=por_bot_mean, + por_top_sigma=por_top_sigma, + por_bot_sigma=por_bot_sigma, + factor_sigma=factor_sigma, + mode_no=1000, + por_top_len_scale=por_top_len_scale, + por_bot_len_scale=por_bot_len_scale, + factor_top_mean=factor_top_mean, + factor_bot_mean=factor_bot_mean + ) + + fields.set_points(mesh_data['points'], mesh_data['point_region_ids'], + mesh_data['region_map']) + + fine_input_sample, coarse_input_sample = FlowSimProcConc.generate_random_sample(fields, coarse_step=0, + n_fine_elements=len( + mesh_data['points'])) + fields_file = 'fields_sample.msh' + gmsh_io.GmshIO().write_fields(fields_file, mesh_data['ele_ids'], fine_input_sample) + + + features = get_node_features(fields_file, feature_names) + + #features = np.log(features) + all_features.append(features) + + plot_rescale(all_features, mesh_file) + + # mesh_data = extract_mesh_gmsh_io(mesh_file, get_points=True) + # points = mesh_data['points'] + # X = points[:, 0] + # Y = points[:, 1] + # fig, ax = plt.subplots(1, 1, figsize=(15, 10)) + # + # cont = ax.tricontourf(X, Y, features.ravel(), levels=32) + # fig.colorbar(cont) + # plt.title("input") + # plt.show() + + +def cond_corr_field(mesh_file=None, corr_length_config=None): + # import matplotlib + # from matplotlib import ticker, cm + # matplotlib.rcParams.update({'font.size': 22}) + dim = 2 + log = True + cl = 0.1 + s = 1 + + feature_names = [['conductivity']] + + if mesh_file is None: + # mesh_file = "/home/martin/Sync/Documents/flow123d_results/flow_experiments/Exponential/corr_length_0_01/l_step_0.0055_common_files/mesh.msh" + # mesh_file = "/home/martin/Documents/metamodels/data/5_ele/cl_0_1_s_1/L5/l_step_0.020196309484414757_common_files/mesh.msh" + mesh_file = "/home/martin/Documents/metamodels/data/1000_ele/l_step_0.055_common_files/mesh.msh" + mesh_file = "/home/martin/Documents/metamodels/data/cl_0_1_s_1/l_step_0.27232698153315_common_files/mesh.msh" # L2 10.5 s + mesh_file = "/home/martin/Documents/metamodels/data/cl_0_1_s_1/l_step_0.07416198487095663_common_files/mesh.msh" # L3 12s + + start_time = time.process_time() + mesh_data = FlowSim.extract_mesh(mesh_file) + + all_features = [] + n_samples = 500 + for i in range(n_samples): + if corr_length_config is not None: + fields = create_corr_field(model="exp", dim=dim, + sigma=corr_length_config['sigma'], + corr_length=corr_length_config['corr_length'], + log=corr_length_config['log']) + else: + fields = create_corr_field(model="exp", dim=dim, + sigma=s, + corr_length=cl, + log=log) + + # # Create fields both fine and coarse + fields = FlowSim.make_fields(fields, mesh_data, None) + + # len(fine_input_sample["conductivity"]) + # features_log = np.log(fine_input_sample["conductivity"]) + + fields.set_points(mesh_data['points'], mesh_data['point_region_ids'], + mesh_data['region_map']) + + fine_input_sample, coarse_input_sample = FlowSim.generate_random_sample(fields, coarse_step=0, + n_fine_elements=len( + mesh_data['points'])) + + fields_file = 'fields_sample.msh' + gmsh_io.GmshIO().write_fields(fields_file, mesh_data['ele_ids'], fine_input_sample) + + + features = get_node_features(fields_file, feature_names) + + #features = np.log(features) + all_features.append(features) + + + #### + # Plot random field + #### + # fig, ax = plt.subplots(1, 1, figsize=(15, 10)) + # mesh_data = extract_mesh_gmsh_io(mesh_file, get_points=True) + # points = mesh_data['points'] + # X = points[:, 0] + # Y = points[:, 1] + # cont = ax.tricontourf(X, Y, features.ravel(), levels=32) + # fig.colorbar(cont) + # plt.title("input") + # plt.show() + + + plot_rescale(all_features, mesh_file) + + # print("conductivity mean ", np.mean(fine_input_sample["conductivity"])) + # print("conductivity var ", np.var(fine_input_sample["conductivity"])) + output = 1 + # + # print("fine input sample ", fine_input_sample["conductivity"].shape) + # + # gmsh_io.GmshIO().write_fields('fields_sample.msh', mesh_data['ele_ids'], fine_input_sample) + # + # mesh = gmsh_io.GmshIO('fields_sample.msh') + # element_data = mesh.current_elem_data + # features = list(element_data.values()) + # print("features ", np.array(features).shape) + + rnd_time = time.process_time() - start_time + print("rnd_time / n_samples ", rnd_time / n_samples) + return rnd_time / n_samples + + +def plot_rescale(all_features, mesh_file): + mean_features = np.mean(all_features, axis=0) + variance_features = np.var(all_features, axis=0) + print('mean features ', mean_features) + print("variance features ", variance_features) + + min_features = np.min(all_features, axis=0) + max_features = np.max(all_features, axis=0) + + print("min features ", min_features) + print("max features ", max_features) + + for features in all_features[:3]: + print("features[:10] ", features[:10]) + fig, ax = plt.subplots(1, 1, figsize=(15, 10)) + mesh_data = extract_mesh_gmsh_io(mesh_file, get_points=True) + points = mesh_data['points'] + X = points[:, 0] + Y = points[:, 1] + print("features shape ", features.shape) + cont = ax.tricontourf(X, Y, features.ravel(), levels=32) + fig.colorbar(cont) + plt.title("input") + plt.show() + + # features -= mean_features + # print("features - mean ", features) + # features /= variance_features + + + #features = np.log(features) + + # print("features - min_features ", features - min_features) + features = (features - min_features) / (max_features - min_features) + # print("final features ", features) + + features = np.nan_to_num(features) + print("final features ", features) + + # print("final features nan to num ", features) + # + # print("final features ", features.shape) + + fig, ax = plt.subplots(1, 1, figsize=(15, 10)) + mesh_data = extract_mesh_gmsh_io(mesh_file, get_points=True) + points = mesh_data['points'] + X = points[:, 0] + Y = points[:, 1] + cont = ax.tricontourf(X, Y, features.ravel(), levels=32) + fig.colorbar(cont) + plt.title("input") + plt.show() + + +if __name__ == "__main__": + corr_file_config = {"02_conc": True, 'log': True} + #mesh_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_conr/l_step_1.0_common_files/repo.msh" + mesh_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/l_step_0.1414213562373095_common_files/repo.msh" + + # corr_file_config = {"02_conc": False, 'log': True, 'corr_length':0.1, 'sigma':1} + # mesh_file = "/home/martin/Documents/metamodels/data/1000_ele/l_step_0.055_common_files/mesh.msh" + + + ############################### + ### generate conc random sample + #### 02_conc + #conc_corr_field(mesh_file, corr_file_config) + + ## + #### 01_cond_field + corr_file_config = {"02_conc": False, 'sigma': 1, 'corr_length': 0.1, 'log': True} + mesh_file = "/home/martin/Documents/metamodels/data/mesh_size/l_step_0.07416198487095663_common_files/mesh.msh" + + cond_corr_field(mesh_file, corr_file_config) + diff --git a/mlmc/metamodel/analyze_nn.py b/mlmc/metamodel/analyze_nn.py index ded4e02d..4bd1e68c 100644 --- a/mlmc/metamodel/analyze_nn.py +++ b/mlmc/metamodel/analyze_nn.py @@ -442,6 +442,7 @@ def load_statistics(dir_path): models_data["total_steps"] = [] models_data["learning_times"] = [] models_data["log"] = [] + models_data["dataset_config"] = [] #dirs = (os.path.split(dir_path)[-1]).split("_") n_iters = 25 @@ -450,7 +451,7 @@ def load_statistics(dir_path): if not os.path.isdir(data_dir_path): print("data dir not exists {}".format(data_dir_path)) break - if os.path.exists(os.path.join(data_dir_path,'model')): + if os.path.exists(os.path.join(data_dir_path, 'model')): models_data['model'].append(keras.models.load_model(os.path.join(data_dir_path, 'model'))) for file in glob.glob(os.path.join(data_dir_path, "*.npy")): file_name = os.path.split(file)[-1] @@ -460,16 +461,15 @@ def load_statistics(dir_path): # models_data[file_name] = [] # print("np.load(file, allow_pickle=True) ", np.load(file, allow_pickle=True)) # exit() - models_data[file_name].append(np.load(file, allow_pickle=True)) - if os.path.exists(os.path.join(dir_path, "dataset_config.pkl")): - # Save config to Pickle - import pickle - # create a binary pickle file - with open(os.path.join(dir_path, "dataset_config.pkl"), "rb") as reader: - dataset_config = pickle.load(reader) - models_data["dataset_config"] = dataset_config + if os.path.exists(os.path.join(data_dir_path, "dataset_config.pkl")): + # Save config to Pickle + import pickle + # create a binary pickle file + with open(os.path.join(data_dir_path, "dataset_config.pkl"), "rb") as reader: + dataset_config = pickle.load(reader) + models_data["dataset_config"].append(dataset_config) return models_data @@ -755,17 +755,17 @@ def analyze_statistics(config, get_model=True): orth_mlmc_means_mse = [] orth_nn_means_mse = [] - limit = 100 # 0.008#0.01#0.0009 + limit = 1e10 # 0.008#0.01#0.0009 #limit = 0.37 for i in range(len(data_dict["test_targets"])): print("index i ", i) - if i == 1: - break + # if i == 4: + # break #print("index ", i) - # if i not in [2, 3, 4]: + # if i not in [0,2]: # continue # if i in [2, 11, 12]: @@ -796,7 +796,7 @@ def analyze_statistics(config, get_model=True): except: model = None - check_loss(config, model, dataset_config=data_dict.get("dataset_config", {})) + #check_loss(config, model, dataset_config=data_dict.get("dataset_config", {})) #predict_data(config, model, mesh_file=config["mesh"]) iter_test_MSE = np.mean((predictions - targets) ** 2) @@ -812,6 +812,11 @@ def analyze_statistics(config, get_model=True): # current_patience = data_dict["current_patience"][i] # print("current patience ", current_patience) + if 'dataset_config' in data_dict: + dataset_config = data_dict.get("dataset_config")[i] + else: + dataset_config = {} + print("total steps ", total_steps) # try: mlmc_n_collected, nn_mlmc_n_collected, n_ops, n_ops_predict, orig_moments_mean, predict_moments_mean, \ @@ -829,7 +834,7 @@ def analyze_statistics(config, get_model=True): mlmc_hdf_file=config['mlmc_hdf_path'], stats=True, learning_time=learning_time, - dataset_config=data_dict.get("dataset_config", {})) + dataset_config=dataset_config) # except: # continue @@ -1130,6 +1135,7 @@ def analyze_statistics(config, get_model=True): # print("max train RMSE ", np.max(train_RMSE)) # print("max train MAE ", np.max(train_MAE)) + print("learning time ", learning_times) print("mean learning time ", np.mean(learning_times)) print("max learning time ", np.max(learning_times)) @@ -1423,7 +1429,7 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): # min_key = np.min(list(states.keys())) # gnn = states[min_key] - train_targets, train_predictions = gnn.predict(loader_tr) + train_targets, train_predictions = gnn.predict(MixedLoader(data_tr, batch_size=batch_size, epochs=1)) train_predictions = np.squeeze(train_predictions) val_targets, val_predictions = gnn.predict(loader_va) @@ -1502,6 +1508,8 @@ def predict_level_zero(nn, output_dir, hdf_path, mesh, conv_layer, batch_size=10 sample_time = 0 if corr_field_config: sample_time = corr_field_sample_time(mesh, corr_field_config) + else: + raise Exception("No corr field config passed") data = FlowDataset(output_dir=output_dir, log=log, config=config)#, mesh=mesh, corr_field_config=corr_field_config) #data = data # [:10000] diff --git a/mlmc/metamodel/postprocessing.py b/mlmc/metamodel/postprocessing.py index 4728392c..ede6804e 100644 --- a/mlmc/metamodel/postprocessing.py +++ b/mlmc/metamodel/postprocessing.py @@ -57,13 +57,124 @@ def cut_original_test(mlmc_hdf_file): sample_storage_for_estimated = cut_samples(data_mlmc, sample_storage_for_estimated, n_estimated_orig, bootstrap=True) +def analyze_output(targets, mult_coef=1,dataset_config=None): + + if dataset_config.get('output_scale', False): + print("targets[:10] ", targets[:10]) + fig, ax = plt.subplots(1, 1, figsize=(15, 10)) + ax.hist(targets, bins=50, alpha=0.5, label='target', density=True) + # fig.colorbar(cont) + plt.title("training targets") + plt.show() + + mean_targets = dataset_config.get('mean_output', False) + var_targets = dataset_config.get('var_output', False) + + # mean_targets = np.mean(targets) + print("mean targets ", mean_targets) + # print("mean targets axis=0 " , np.mean(targets, axis=0)) + # var_targets = np.var(targets) + + targets = var_targets * targets + mean_targets + + targets_orig = np.exp(targets) + fig, ax = plt.subplots(1, 1, figsize=(15, 10)) + + ax.hist(targets_orig, bins=50, alpha=0.5, label='targets orig', density=True) + # fig.colorbar(cont) + plt.title("targets orig") + plt.show() + + new_mult_coef = 1 / np.mean(targets_orig) + print("np.log(targets_orig * new_mult_coef)[:10] ", np.log(targets_orig * new_mult_coef)[:10]) + fig, ax = plt.subplots(1, 1, figsize=(15, 10)) + ax.hist(np.log(targets_orig * new_mult_coef), bins=50, alpha=0.5, label='targets orig', density=True) + # fig.colorbar(cont) + plt.title("log(targets orig * mult coef)") + plt.show() + + new_mult_coef = 1 / np.mean(np.log(targets_orig)) + fig, ax = plt.subplots(1, 1, figsize=(15, 10)) + ax.hist(np.log(targets_orig) * new_mult_coef, bins=50, alpha=0.5, label='targets orig', density=True) + # fig.colorbar(cont) + plt.title("log(targets orig) * mult coef") + plt.show() + + print("mean log targets orig ", np.mean(np.log(targets_orig))) + + log_scaled_targets_orig = (np.log(targets_orig) - np.mean(np.log(targets_orig))) / np.var(np.log(targets_orig)) + print("log scaled targets orig[:10] ", log_scaled_targets_orig[:10]) + fig, ax = plt.subplots(1, 1, figsize=(15, 10)) + ax.hist(log_scaled_targets_orig, bins=50, alpha=0.5, label='targets orig', density=True) + # fig.colorbar(cont) + plt.title("log scaled target orig") + plt.show() + + fig, ax = plt.subplots(1, 1, figsize=(15, 10)) + ax.hist(np.log(targets_orig), bins=50, alpha=0.5, label='target orig log', density=True) + # fig.colorbar(cont) + plt.title("log targets orig") + plt.show() + + + else: + fig, ax = plt.subplots(1, 1, figsize=(15, 10)) + ax.hist(targets, bins=50, alpha=0.5, label='target', density=True) + #fig.colorbar(cont) + plt.title("training targets") + plt.show() + + targets_orig = np.exp(targets) / mult_coef + fig, ax = plt.subplots(1, 1, figsize=(15, 10)) + + ax.hist(targets_orig, bins=50, alpha=0.5, label='targets orig', density=True) + # fig.colorbar(cont) + plt.title("targets orig") + plt.show() + + new_mult_coef = 1/np.mean(targets_orig) + fig, ax = plt.subplots(1, 1, figsize=(15, 10)) + ax.hist(np.log(targets_orig*new_mult_coef), bins=50, alpha=0.5, label='targets orig', density=True) + # fig.colorbar(cont) + plt.title("log(targets orig * mult coef)") + plt.show() + + new_mult_coef = 1 / np.mean(np.log(targets_orig)) + fig, ax = plt.subplots(1, 1, figsize=(15, 10)) + ax.hist(np.log(targets_orig) * new_mult_coef, bins=50, alpha=0.5, label='targets orig', density=True) + # fig.colorbar(cont) + plt.title("log(targets orig) * mult coef") + plt.show() + + log_scaled_targets_orig = (np.log(targets_orig) - np.mean(np.log(targets_orig))) / np.var(np.log(targets_orig)) + fig, ax = plt.subplots(1, 1, figsize=(15, 10)) + ax.hist(log_scaled_targets_orig, bins=50, alpha=0.5, label='targets orig', density=True) + # fig.colorbar(cont) + plt.title("log scaled target orig") + plt.show() + + + + fig, ax = plt.subplots(1, 1, figsize=(15, 10)) + ax.hist(np.log(targets_orig), bins=50, alpha=0.5, label='target orig log', density=True) + # fig.colorbar(cont) + plt.title("log targets orig") + plt.show() + + exit() def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predictions, train_targets, train_predictions, val_targets, l_0_targets=None, l_0_predictions=None, l1_sample_time=None, l0_sample_time=None, nn_level=0, replace_level=False, stats=False, mlmc_hdf_file=None, learning_time=0, dataset_config={}): + """ + :param l1_sample_time: preprocess_time / len(data) + learning_time / len(data), + preprocess_time includes graph creation time and FlowDataset creation time + """ # level_zero = False cut_est = True + all_samples = False + n0, nL = 2000, 100 domain_largest = False # If False than domain=None - domain is determined given the simulation samples distr_domain_largest = False @@ -71,58 +182,98 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic # exit() output_mult_factor = dataset_config.get('output_mult_factor', 1) + #print("output mult factor ", output_mult_factor) + + analyze_output(targets, output_mult_factor, dataset_config=dataset_config) + + if dataset_config.get('output_normalization', False): + min_out = dataset_config.get('min_output') + max_out = dataset_config.get('max_output') + + targets = targets * (max_out - min_out) + min_out + predictions = predictions * (max_out - min_out) + min_out + l_0_targets = l_0_targets * (max_out - min_out) + min_out + l_0_predictions = l_0_predictions * (max_out - min_out) + min_out + + if dataset_config.get('output_scale', False): + # mean_targets = np.mean(targets) + # var_targets = np.var(targets) + + mean_targets = dataset_config.get('mean_output', False) + var_targets = dataset_config.get('var_output', False) + + targets = var_targets * targets + mean_targets + predictions = var_targets * predictions + mean_targets + + # mean_l_0_targets = mean_targets + # var_l_0_targets = var_targets + + l_0_targets = var_targets * l_0_targets + mean_targets + l_0_predictions = var_targets * l_0_predictions + mean_targets + if dataset_config.get('output_log', False): targets = np.exp(targets) predictions = np.exp(predictions) l_0_predictions = np.exp(l_0_predictions) l_0_targets = np.exp(l_0_targets) + if dataset_config.get('first_log_output', False): + targets = np.exp(targets) + predictions = np.exp(predictions) + l_0_predictions = np.exp(l_0_predictions) + l_0_targets = np.exp(l_0_targets) + if mlmc_hdf_file is None: mlmc_hdf_file = nn_hdf_file - if not stats: - # print("nn_level ", nn_level) - # print("replace level ", replace_level) - - # targets = np.exp(targets) - # predictions = np.exp(predictions) - # l_0_predictions = np.exp(l_0_predictions) - # l_0_targets = np.exp(l_0_targets) - - # print("targets ", targets) - # print("predictions ", predictions) - print("targets ", targets) - plt.hist(targets, bins=50, alpha=0.5, label='target', density=True) - #plt.hist(predictions, bins=50, alpha=0.5, label='predictions', density=True) - - # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) - plt.legend(loc='upper right') - # plt.xlim(-0.5, 1000) - plt.yscale('log') - plt.show() + print("targets ", targets) + print("len targets ", len(targets)) - plt.hist(l_0_targets, bins=50, alpha=0.5, label='l_0_target', density=True) - plt.hist(l_0_predictions, bins=50, alpha=0.5, label='l_0_predictions', density=True) + #if not stats: + # print("nn_level ", nn_level) + # print("replace level ", replace_level) - # print("lo targets ", l_0_targets) - # print("l0 predictions ", l_0_predictions) - # exit() + # targets = np.exp(targets) + # predictions = np.exp(predictions) + # l_0_predictions = np.exp(l_0_predictions) + # l_0_targets = np.exp(l_0_targets) + + # print("targets ", targets) + # print("predictions ", predictions) + print("targets ", targets) + print("predictions ", predictions) + plt.hist(targets, bins=50, alpha=0.5, label='target', density=True) + plt.hist(predictions, bins=50, alpha=0.5, label='predictions', density=True) + + # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) + plt.legend(loc='upper right') + # plt.xlim(-0.5, 1000) + plt.yscale('log') + plt.show() + + plt.hist(l_0_targets, bins=50, alpha=0.5, label='l_0_target', density=True) + plt.hist(l_0_predictions, bins=50, alpha=0.5, label='l_0_predictions', density=True) + + # print("lo targets ", l_0_targets) + # print("l0 predictions ", l_0_predictions) + # exit() + + # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) + plt.legend(loc='upper right') + # plt.xlim(-0.5, 1000) + plt.yscale('log') + plt.show() - # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) - plt.legend(loc='upper right') - # plt.xlim(-0.5, 1000) - plt.yscale('log') - plt.show() - # targets = np.exp(targets) - # predictions = np.exp(predictions) + # targets = np.exp(targets) + # predictions = np.exp(predictions) - # # Creating plot - # plt.boxplot(targets) - # plt.boxplot(predictions) - # # show plot - # plt.show() - #exit() + # # Creating plot + # plt.boxplot(targets) + # plt.boxplot(predictions) + # # show plot + # plt.show() + #exit() ####### ### Create storage fromm original MLMC data @@ -139,7 +290,6 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic sample_storage_nn = SampleStorageHDF(file_path=nn_hdf_file) original_moments_nn, estimator_nn, original_true_domain_nn, _ = estimate_moments(sample_storage_nn) - print("nn moments.mean ", original_moments_nn.mean) print("nn moments.var ", original_moments_nn.var) @@ -167,10 +317,15 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic print("original level params", sample_storage.get_level_parameters()) sample_storage = create_quantity_mlmc(data_mlmc, level_parameters=sample_storage.get_level_parameters()) if cut_est: - n0 = 2000 - nL = 100 + # n0 = 2000 + # nL = 100 n_levels = sample_storage.get_n_levels() - n_samples = np.round(np.exp2(np.linspace(np.log2(n0), np.log2(nL), n_levels))).astype(int) + + if all_samples: + n_samples = sample_storage.get_n_collected() + else: + n_samples = np.round(np.exp2(np.linspace(np.log2(n0), np.log2(nL), n_levels))).astype(int) + print('n samples ', n_samples) sample_storage_for_estimated = cut_samples(data_mlmc, sample_storage, n_samples) # [2300]) @@ -195,7 +350,7 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic print("l1_sample_time ", l1_sample_time) ############################ - ### calculate level 1 n ops + ### Calculate First level (follows meta-level) n_ops ############################ len_data = 20000 # 50000#80000 len_train_data = 2000 @@ -203,10 +358,20 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic preprocess_time = l1_sample_time * len_data - learning_time preprocess_time_per_sample = preprocess_time / len_data n_ops_train = preprocess_time_per_sample + (learning_time / len_train_data) + l0_predict_time - # n_ops_test = (preprocess_time / len_test_data) + #### + # Notes: + # l1_sample_time = preprocess_time / len(data) + learning_time / len(data) + # + # n_ops_train includes: + # preprocess time per sample + # learning time per training sample + # l0_prediction_time - the time necessary to predict neural network outcome for given already preprocessed data + l1_sample_time = n_ops_train + # New l1_sample_time corresponds to the time necessary for Level 1 samples (fine + coarse) + # + ################################# print("L1 sample time ", l1_sample_time) - l1_sample_time = n_ops_train print("learning time ", learning_time) print("preprocess time ", preprocess_time) @@ -235,6 +400,9 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic # level_samples = np.ones((1, len(l_0_predictions), 1)) ft_index = nn_level + if output_mult_factor != 1: + level_samples /= output_mult_factor + if nn_level > 0: ft_index = nn_level - 1 n_ops_predict.append(l0_sample_time) # + field_times[ft_index] / 2) @@ -246,6 +414,7 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic if replace_level: level_id = l_id level_samples = estimator_nn.get_level_samples(level_id=level_id, n_samples=nn_n_collected[level_id]) + n_ops_predict.append(n_ops[level_id]) else: if l_id < mlmc_nn_diff_level: @@ -269,6 +438,9 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic level_samples = np.concatenate((fine_level_samples, coarse_level_samples), axis=2) print("nn level ", nn_level) + if output_mult_factor != 1: + level_samples /= output_mult_factor + print("level sampels ", level_samples) print("fine - coarse ", fine_level_samples - coarse_level_samples) print("fine - coarse ", np.var(fine_level_samples - coarse_level_samples)) @@ -293,8 +465,7 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic # if output_mult_factor != 1: # level_samples /= output_mult_factor - if output_mult_factor != 1: - level_samples /= output_mult_factor + print("level samples rescaled ", level_samples) @@ -316,10 +487,17 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic print("level params ", level_params) sample_storage_predict = create_quantity_mlmc(data_nn, level_parameters=level_params) if cut_est: - n0 = 2000 - nL = 100 + # n0 = 2000 + # nL = 100 n_levels = sample_storage_predict.get_n_levels() - n_samples = np.round(np.exp2(np.linspace(np.log2(n0), np.log2(nL), n_levels))).astype(int) + + #n_samples = np.round(np.exp2(np.linspace(np.log2(n0), np.log2(nL), n_levels))).astype(int) + + if all_samples: + n_samples = sample_storage.get_n_collected() + else: + n_samples = np.round(np.exp2(np.linspace(np.log2(n0), np.log2(nL), n_levels))).astype(int) + print('n samples predict', n_samples) sample_storage_predict_for_estimate = cut_samples(data_nn, sample_storage_predict, n_samples) # [4500, 1500]) # print("n ops predict ", n_ops_predict) @@ -372,7 +550,7 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic print("n estimated orig ", n_estimated_orig) print("n ops est ", n_ops_est) - print("sample storage for estimated n collected ", sample_storage_for_estimated.get_n_collected()) + #print("sample storage for estimated n collected ", sample_storage_for_estimated.get_n_collected()) ###### ## initial N geuss @@ -442,6 +620,10 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic # predict_q_estimator.quantity = predict_q_estimator.quantity.subsample(sample_vec=n_estimated_nn) + + ############### + ## Recalculate first level (level of metamodel and simulation difference) n ops + ############### n_ops_test = (preprocess_time_per_sample + l0_predict_time) # * (n_estimated_nn[1] - len_train_data) if n_estimated_nn[1] > len_train_data: orig_n_ops = n_ops_predict[1] - l1_sample_time @@ -457,6 +639,8 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic print("cost_tr ", cost_tr) print("cost te ", cost_te) + n_ops_predict_orig = n_ops_predict + n_estimated_nn, l_var_nn, n_samples_nn = get_n_estimated(sample_storage_predict, predict_q_estimator, n_ops=n_ops_predict) print("n ops ", n_ops) diff --git a/mlmc/metamodel/random_field_time.py b/mlmc/metamodel/random_field_time.py index fc29f9c6..a353dadc 100644 --- a/mlmc/metamodel/random_field_time.py +++ b/mlmc/metamodel/random_field_time.py @@ -13,10 +13,9 @@ def conc_rnd_sample_time(mesh_file, corr_field_config): start_time = time.process_time() mesh_data = FlowSim.extract_mesh(mesh_file) - fields = conc_create_corr_fields(dim=2, log=corr_field_config["log"], mode_no=10000) - n_samples = 200 for i in range(n_samples): + fields = conc_create_corr_fields(dim=2, log=corr_field_config["log"], mode_no=10000) fields.set_points(mesh_data['points'], mesh_data['point_region_ids'], mesh_data['region_map']) @@ -114,6 +113,48 @@ def corr_field_sample_time(mesh_file=None, corr_length_config=None): # ax.set_aspect("equal") + +def conc_corr_field(mesh_file, corr_field_config): + start_time = time.process_time() + mesh_data = FlowSim.extract_mesh(mesh_file) + + fields = conc_create_corr_fields(dim=2, log=corr_field_config["log"], mode_no=10000) + + n_samples = 200 + for i in range(n_samples): + fields.set_points(mesh_data['points'], mesh_data['point_region_ids'], + mesh_data['region_map']) + + fine_input_sample, coarse_input_sample = FlowSimProcConc.generate_random_sample(fields, coarse_step=0, + n_fine_elements=len( + mesh_data['points'])) + # print("fine input sample ", fine_input_sample) + + + # Xfinal, Yfinal = fields.fields[0].correlated_field.points[:, 0], fields.fields[0].correlated_field.points[:, 1] + + # cont = ax.tricontourf(Xfinal, + # Yfinal, + # fine_input_sample['conductivity'].ravel())#, locator=ticker.LogLocator()) + + # fig.colorbar(cont) + # fig.savefig("cl_{}_var_{}.pdf".format(cl, s ** 2)) + # plt.show() + + # print("fields ", fields) + # model = gs.Exponential(dim=2, len_scale=cl) + # srf = gs.SRF(model, mesh_type="unstructed", seed=20170519, mode_no=1000, generator='RandMeth') + # print("model.var ", model.var) + # field = srf( + # (fields.fields[0].correlated_field.points[:, 0], fields.fields[0].correlated_field.points[:, 1])) + # srf.vtk_export("field") + # ax = srf.plot() + # ax.set_aspect("equal") + + rnd_time = time.process_time() - start_time + print("rnd_time / n_samples ", rnd_time / n_samples) + + if __name__ == "__main__": import cProfile import pstats @@ -121,14 +162,23 @@ def corr_field_sample_time(mesh_file=None, corr_length_config=None): pr.enable() corr_file_config = {"02_conc": True, 'log': True} - mesh_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_por/l_step_1.0_common_files/repo.msh" + #mesh_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_conr/l_step_1.0_common_files/repo.msh" + mesh_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/l_step_0.1414213562373095_common_files/repo.msh" # corr_file_config = {"02_conc": False, 'log': True, 'corr_length':0.1, 'sigma':1} # mesh_file = "/home/martin/Documents/metamodels/data/1000_ele/l_step_0.055_common_files/mesh.msh" - my_result = corr_field_sample_time(mesh_file, corr_file_config) + # my_result = corr_field_sample_time(mesh_file, corr_file_config) + # + # pr.disable() + # ps = pstats.Stats(pr).sort_stats('cumtime') + # ps.print_stats() + # + + + + ############################### + ### generate conc random sample - pr.disable() - ps = pstats.Stats(pr).sort_stats('cumtime') - ps.print_stats() + conc_corr_field(mesh_file, corr_file_config) From f9bab57fc1fe1161dbfa0d1b05687fd18a93ef01 Mon Sep 17 00:00:00 2001 From: Martin Spetlik Date: Thu, 10 Feb 2022 15:02:17 +0100 Subject: [PATCH 50/67] learning rate schedule experiments --- mlmc/metamodel/analyze_nn.py | 14 +++++++++----- mlmc/metamodel/flow_task_GNN_2.py | 12 ++++++++++-- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/mlmc/metamodel/analyze_nn.py b/mlmc/metamodel/analyze_nn.py index 093372e7..9187357d 100644 --- a/mlmc/metamodel/analyze_nn.py +++ b/mlmc/metamodel/analyze_nn.py @@ -349,7 +349,7 @@ def predict_level_zero_SVR(nn, output_dir, hdf_path, mesh, batch_size=1000, log= def statistics(config): - n_subsamples = 1 + n_subsamples = 25 model_title, mch_l_model, log = config['machine_learning_model'] model_data = {} @@ -380,12 +380,16 @@ def statistics(config): config['iter_dir'] = iter_dir - model, targets, predictions, learning_time, train_targets, train_predictions, \ + gnn, targets, predictions, learning_time, train_targets, train_predictions, \ val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, total_steps = \ mch_l_model(config, stats=True, train=config.get('train_model', True), log=log, seed=i) if config['save_model']: - model_data["model"] = model + model_data["model"] = gnn._model + model_data["train_loss"] = gnn._train_loss + model_data["val_loss"] = gnn._val_loss + model_data["test_loss"] = gnn._test_loss + model_data["learning_rates"] = gnn._learning_rates model_data["test_targets"] = targets model_data["test_predictions"] = predictions model_data["train_targets"] = train_targets @@ -412,7 +416,7 @@ def statistics(config): # print("##################################################") - analyze_statistics(config) + #analyze_statistics(config) # plot_loss(train_losses, val_losses) # analyze_results(np.mean(all_test_outputs, axis=0), np.mean(all_predictions, axis=0)) @@ -1500,7 +1504,7 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): # l_0_predictions, l1_sample_time, l0_sample_time, nn_level=level, replace_level=replace_level, # stats=stats) - return gnn._model, targets, predictions, learning_time, train_targets, train_predictions,\ + return gnn, targets, predictions, learning_time, train_targets, train_predictions,\ val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, total_steps save_times(config['save_path'], False, (preprocess_time, len(data)), learning_time, (predict_l_0_time, len(l_0_targets))) diff --git a/mlmc/metamodel/flow_task_GNN_2.py b/mlmc/metamodel/flow_task_GNN_2.py index 0db7da9f..6bbca16d 100644 --- a/mlmc/metamodel/flow_task_GNN_2.py +++ b/mlmc/metamodel/flow_task_GNN_2.py @@ -14,6 +14,7 @@ from spektral.utils.sparse import sp_matrix_to_sp_tensor from tensorflow.keras.layers.experimental import preprocessing from mlmc.metamodel.custom_methods import abs_activation, var_loss_function +import keras.backend as K from mlmc.metamodel.graph_models import Net1 @@ -42,6 +43,7 @@ def __init__(self, **kwargs): self._train_loss = [] self._val_loss = [] self._test_loss = [] + self._learning_rates = [] self.val_targets = [] self._states = {} @@ -54,6 +56,7 @@ def __init__(self, **kwargs): else: model = kwargs.get('model') + if model is None: self._model = Net1(conv_layer=self._conv_layer, hidden_activation=self._hidden_activation, output_activation=self._output_activation, @@ -67,6 +70,8 @@ def __init__(self, **kwargs): # normalizer=self._normalizer) #self._model = model(n_labels=1, output_activation="relu") + self._model.optimizer = self._optimizer + def fit(self, loader_tr, loader_va, loader_te): """ Training procedure @@ -120,6 +125,9 @@ def fit(self, loader_tr, loader_va, loader_te): print("Early stopping") break + lr = K.eval(self._optimizer._decayed_lr(tf.float32)) + self._learning_rates.append(lr) + # Print results results_tr = np.array(results_tr) results_tr = np.average(results_tr[:, :-1], 0, weights=results_tr[:, -1]) @@ -127,8 +135,8 @@ def fit(self, loader_tr, loader_va, loader_te): print( "Train loss: {:.12f}, acc: {:.12f} | " "Valid loss: {:.12f}, acc: {:.12f} | " - "Test loss: {:.12f}, acc: {:.12f}".format( - *results_tr, *results_va, *results_te + "Test loss: {:.12f}, acc: {:.12f} | LR: {:.12f}".format( + *results_tr, *results_va, *results_te, lr ) ) From 1f03019c062c4d5c44a6423aac0787e7bb5024c1 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Thu, 10 Feb 2022 15:04:32 +0100 Subject: [PATCH 51/67] data augmentation --- mlmc/metamodel/analyze_nn.py | 118 +++++++++++++++++++++++++++------ mlmc/metamodel/flow_dataset.py | 118 +++++++++++++++++++++++++++++---- 2 files changed, 202 insertions(+), 34 deletions(-) diff --git a/mlmc/metamodel/analyze_nn.py b/mlmc/metamodel/analyze_nn.py index 093372e7..18558278 100644 --- a/mlmc/metamodel/analyze_nn.py +++ b/mlmc/metamodel/analyze_nn.py @@ -19,10 +19,10 @@ from scipy.stats import ks_2samp import sklearn.model_selection from mlmc.metamodel.custom_methods import abs_activation, MSE_moments -from mlmc.metamodel.postprocessing import analyze_results, plot_loss, estimate_density, process_mlmc, plot_progress +from mlmc.metamodel.postprocessing import analyze_results, plot_loss, estimate_density, process_mlmc, plot_progress, plot_learning_rate from mlmc.metamodel.flow_task_NN import DNN from mlmc.metamodel.flow_task_CNN import CNN - +import keras.backend as K from mlmc.metamodel.flow_task_GNN_2 import GNN from tensorflow.keras.losses import MeanSquaredError from spektral.data import MixedLoader @@ -355,6 +355,7 @@ def statistics(config): model_data = {} model_data["log"] = log + # seeds = [] # for i in range(n_subsamples): # seeds.append(i * 125) @@ -380,12 +381,16 @@ def statistics(config): config['iter_dir'] = iter_dir - model, targets, predictions, learning_time, train_targets, train_predictions, \ + gnn, targets, predictions, learning_time, train_targets, train_predictions, \ val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, total_steps = \ mch_l_model(config, stats=True, train=config.get('train_model', True), log=log, seed=i) if config['save_model']: - model_data["model"] = model + model_data["model"] = gnn._model + model_data["train_loss"] = gnn._train_loss + model_data["val_loss"] = gnn._val_loss + model_data["test_loss"] = gnn._test_loss + model_data["learning_rates"] = gnn._learning_rates model_data["test_targets"] = targets model_data["test_predictions"] = predictions model_data["train_targets"] = train_targets @@ -412,7 +417,7 @@ def statistics(config): # print("##################################################") - analyze_statistics(config) + return analyze_statistics(config) # plot_loss(train_losses, val_losses) # analyze_results(np.mean(all_test_outputs, axis=0), np.mean(all_predictions, axis=0)) @@ -433,6 +438,10 @@ def save_statistics(save_dir_path, model_data): def load_statistics(dir_path): models_data = {} models_data["model"] = [] + models_data["train_loss"] = [] + models_data["val_loss"] = [] + models_data["test_loss"] = [] + models_data["learning_rates"] = [] models_data["test_targets"] = [] models_data["test_predictions"] = [] models_data["train_targets"] = [] @@ -448,6 +457,7 @@ def load_statistics(dir_path): models_data["log"] = [] models_data["dataset_config"] = [] + #dirs = (os.path.split(dir_path)[-1]).split("_") n_iters = 25 for i in range(n_iters): @@ -509,7 +519,10 @@ def check_loss(config, model, log=True, dataset_config={}): config['dataset_config'] = dataset_config - data = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log, config=config) + print("config ", config) + print("dataset config ", config["dataset_config"]) + + data = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log, config=config, index=None) data = data # [:10000] data.a = config['conv_layer'].preprocess(data.a) @@ -533,15 +546,27 @@ def check_loss(config, model, log=True, dataset_config={}): test_targets, test_predictions = model_predict(model, loader_te) test_predictions = np.squeeze(test_predictions) + #print("(train_predictions - train_targets) ", (train_predictions - train_targets)) + train_MSE = np.mean((train_predictions - train_targets) ** 2) - test_MSE = np.mean((test_predictions - test_targets) ** 2) + train_bias = np.mean((train_targets - np.mean(train_predictions))**2) + train_variance = np.mean((train_predictions - np.mean(train_predictions))**2) + train_variance_2 = np.var(train_predictions) - print("test targets ", np.sort(test_targets)[:10]) - print("test predictions ", test_predictions) + test_MSE = np.mean((test_predictions - test_targets) ** 2) + test_bias = np.mean((test_targets - np.mean(test_predictions)) ** 2) + test_variance = np.mean((test_predictions - np.mean(test_predictions)) ** 2) + test_variance_2 = np.var(test_predictions) + # print("test targets ", np.sort(test_targets)[:10]) + # print("test predictions ", test_predictions) print("train MSE: {}, test MSE: {}".format(train_MSE, test_MSE)) + print("train MSE: {}, bias: {}, variance: {}, var2: {}".format(train_MSE, train_bias, train_variance, train_variance_2)) + print("test MSE: {}, bias: {}, variance: {}, var2: {}".format(test_MSE, test_bias, test_variance, test_variance_2)) + + exit() conv_layers = {} dense_layers = {} @@ -569,7 +594,8 @@ def predict_data(config, model, mesh_file, log=True): if model is None: return batch_size = config['batch_size'] - data = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log) + data = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log, config=config, index=0, + predict=True) data = data # [:10000] data.a = config['conv_layer'].preprocess(data.a) @@ -759,6 +785,13 @@ def analyze_statistics(config, get_model=True): orth_mlmc_means_mse = [] orth_nn_means_mse = [] + train_MSE_list = [] + train_bias = [] + train_variance = [] + test_MSE_list = [] + test_bias = [] + test_variance = [] + limit = 1e10 # 0.008#0.01#0.0009 #limit = 0.37 @@ -767,6 +800,9 @@ def analyze_statistics(config, get_model=True): # if i == 4: # break + # if i == 1: + # continue + #print("index ", i) # if i not in [0,2]: @@ -797,29 +833,67 @@ def analyze_statistics(config, get_model=True): try: model = data_dict["model"][i] + model_train_loss = data_dict["train_loss"][i] + model_val_loss = data_dict["val_loss"][i] + model_test_loss = data_dict["test_loss"][i] + model_learning_rates = data_dict["learning_rates"][i] except: model = None - #check_loss(config, model, dataset_config=data_dict.get("dataset_config", {})) - #predict_data(config, model, mesh_file=config["mesh"]) + plot_loss(model_train_loss, model_val_loss) + plot_learning_rate(model_learning_rates) + print("model learning rates ", model_learning_rates) + + print("model ", model) + print("dir(model.optimizer) ", dir(model.optimizer)) + #print("model weights ", model.weights) + print("model.optimizer", model.optimizer) + # print("model.optimizer", K.eval(model.optimizer.lr)) + # exit() iter_test_MSE = np.mean((predictions - targets) ** 2) + iter_test_bias = np.sqrt(np.mean((targets - np.mean(predictions)) ** 2)) + iter_test_variance = np.mean((predictions - np.mean(predictions)) ** 2) + iter_train_MSE = np.mean((train_predictions - train_targets) ** 2) + iter_train_bias = np.sqrt(np.mean((train_targets - np.mean(train_predictions)) ** 2)) + iter_train_variance = np.mean((train_predictions - np.mean(train_predictions)) ** 2) + + train_MSE_list.append(iter_train_MSE) + train_bias.append(iter_train_bias) + train_variance.append(iter_train_variance) + test_MSE_list.append(iter_test_MSE) + test_bias.append(iter_test_bias) + test_variance.append(iter_test_variance) if iter_test_MSE > limit: continue - print("iter test MSE ", iter_test_MSE) - print("iter train MSE ", iter_train_MSE) + print("iter test MSE: {}, bias: {}, variance:{} ".format(iter_test_MSE, iter_test_bias, iter_test_variance)) + print("iter train MSE: {}, bias: {}, variance:{} ".format(iter_train_MSE, iter_train_bias, iter_train_variance)) + # if "current_patience" in data_dict: # current_patience = data_dict["current_patience"][i] # print("current patience ", current_patience) - if 'dataset_config' in data_dict: + dataset_config = {} + if 'dataset_config' in data_dict and len(data_dict.get("dataset_config")) > 0: dataset_config = data_dict.get("dataset_config")[i] else: - dataset_config = {} + if os.path.exists(os.path.join(config['save_path'], "dataset_config.pkl")): + # Save config to Pickle + import pickle + # create a binary pickle file + with open(os.path.join(config['save_path'], "dataset_config.pkl"), "rb") as reader: + dataset_config = pickle.load(reader) + + config['dataset_config'] = dataset_config + + #check_loss(config, model, dataset_config=config["dataset_config"]) + + #predict_data(config, model, mesh_file=config["mesh"]) + #exit() print("total steps ", total_steps) # try: @@ -842,7 +916,6 @@ def analyze_statistics(config, get_model=True): # except: # continue - mlmc_n_collected_all.append(mlmc_n_collected) nn_n_collected_all.append(nn_mlmc_n_collected) n_ops_all.append(n_ops) @@ -1162,6 +1235,9 @@ def analyze_statistics(config, get_model=True): print("test MSE ", np.mean(test_MSE)) print("test MSE ", test_MSE) print("stats.sem(test_MSE) ", stats.sem(test_MSE)) + print("train MSE: {}, bias: {}, variance: {}".format(np.mean(train_MSE_list), np.mean(train_bias), np.mean(train_variance))) + print("test MSE: {}, bias: {}, variance: {}".format(np.mean(test_MSE_list), np.mean(test_bias), + np.mean(test_variance))) # print("test MSE std", np.sqrt(np.var(test_MSE))) print("train RSE ", np.mean(all_train_RSE)) print("test RSE ", np.mean(all_test_RSE)) @@ -1176,7 +1252,8 @@ def analyze_statistics(config, get_model=True): print("max learning time ", np.max(learning_times)) print("######################################") - return train_MSE, test_MSE, np.mean(all_train_RSE), np.mean(all_test_RSE) + return train_MSE, test_MSE, all_train_RSE, all_test_RSE, nn_total_time, mlmc_total_time, kl_mlmc_all, kl_nn_all,\ + learning_times def plot_sse(data_nn, data_mlmc, x_label="ith moment", y_label="MSE", title=""): @@ -1340,6 +1417,7 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): graph_creation_time = config['graph_creation_time'] if graph_creation_time == 0: graph_creator_preproces_time = time.process_time() + graph_creator(config['output_dir'], config['hdf_path'], config['mesh'], level=config['level'], feature_names=config.get('feature_names', [['conductivity']])) graph_creation_time = time.process_time() - graph_creator_preproces_time @@ -1377,7 +1455,7 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): data_tr = data data_te = data else: - data_tr = data[seed*train_data_len: seed*train_data_len + train_data_len] + data_tr = data.get_train_data(seed, train_data_len) print("data tr ", data_tr) data_te = data.get_test_data(seed, train_data_len) #data_tr, data_te = data[:train_data_len], data[train_data_len:] @@ -1500,7 +1578,7 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): # l_0_predictions, l1_sample_time, l0_sample_time, nn_level=level, replace_level=replace_level, # stats=stats) - return gnn._model, targets, predictions, learning_time, train_targets, train_predictions,\ + return gnn, targets, predictions, learning_time, train_targets, train_predictions,\ val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, total_steps save_times(config['save_path'], False, (preprocess_time, len(data)), learning_time, (predict_l_0_time, len(l_0_targets))) diff --git a/mlmc/metamodel/flow_dataset.py b/mlmc/metamodel/flow_dataset.py index 5aba0c4b..ef934b80 100644 --- a/mlmc/metamodel/flow_dataset.py +++ b/mlmc/metamodel/flow_dataset.py @@ -21,8 +21,8 @@ class FlowDataset(Dataset): GRAPHS_FILE = "graphs" DATA_FILE = "data" - def __init__(self, output_dir=None, level=0, log=False, mesh=None, corr_field_config=None, config={}, - index=None, predict=False, **kwargs): + def __init__(self, output_dir=None, level=0, log=False, mesh=None, corr_field_config=None, config={}, index=None, + predict=False, **kwargs): self._output_dir = output_dir # if self._output_dir is None: # self._output_dir = OUTPUT_DIR @@ -36,6 +36,7 @@ def __init__(self, output_dir=None, level=0, log=False, mesh=None, corr_field_co self._index = index self._predict = predict self._dataset_config = config.get('dataset_config', {}) + self._augment_data = config.get('augment_data', False) if predict: self._min_features = self._dataset_config.get('min_features', None) @@ -58,16 +59,31 @@ def __init__(self, output_dir=None, level=0, log=False, mesh=None, corr_field_co self._var_output = None self._output_mult_factor = 1 + self._aug_data = [] + self._columns = None + super().__init__(**kwargs) #self.a = self.adjacency_matrix self.dataset = pd.DataFrame(self.data) + self._df_for_augmentation = pd.DataFrame(self._aug_data, columns=self._columns) + + #self._data_augmentation() + + def get_train_data(self, index, length): + new_dataset = self.dataset[index * length: index * length + length] + if self._augment_data: + new_dataset = self._data_augmentation(self._df_for_augmentation[index * length: index * length + length]) + + new_obj = copy.deepcopy(self) + new_obj.dataset = new_dataset + return new_obj + def get_test_data(self, index, length): new_dataset = self.dataset[0:index * length] + self.dataset[index * length + length:] new_obj = copy.deepcopy(self) new_obj.dataset = new_dataset - return new_obj def shuffle(self, seed=None): @@ -77,6 +93,51 @@ def shuffle(self, seed=None): random.shuffle(self.data) self.dataset = pd.DataFrame(self.data) + def _data_augmentation(self, df_slice): + import smogn + import matplotlib.pyplot as plt + import seaborn + + # df_slice = self._df_for_augmentation[self._index * self._config['n_train_samples']: + # self._index * self._config['n_train_samples'] + self._config['n_train_samples']] + + # print("index ", self._index) + # print("df slice shape ", df_slice.shape) + if 'augmentation_config' in self._config: + print("len df slice 0", len(df_slice)) + dataset_modified = smogn.smoter(data=df_slice, y="y", **self._config["augmentation_config"]) + else: + dataset_modified = smogn.smoter(data=df_slice, y="y", k=9, samp_method="extreme") + + # print("dataset modified shape", dataset_modified.shape) + # print("dataset modified ", dataset_modified) + # + # print("df stats ", smogn.box_plot_stats(df_slice['y'])['stats']) + # print("modified stats ", smogn.box_plot_stats(dataset_modified['y'])['stats']) + # + # fig, ax = plt.subplots(1, 1, figsize=(15, 10)) + # ax.hist(df_slice['y'], bins=50, alpha=0.5, label='target', density=True) + # plt.title("original") + # plt.show() + # + # fig, ax = plt.subplots(1, 1, figsize=(15, 10)) + # ax.hist(dataset_modified['y'], bins=50, alpha=0.5, label='target', density=True) + # plt.title("modified") + # plt.show() + # + # appended_dataset = df_slice.append(dataset_modified) + # + # fig, ax = plt.subplots(1, 1, figsize=(15, 10)) + # ax.hist(appended_dataset['y'], bins=50, alpha=0.5, label='target', density=True) + # plt.title("appended") + # plt.show() + + return dataset_modified + + # seaborn.kdeplot(df_slice['y'], label="Original") + # seaborn.kdeplot(dataset_modified['y'], label="Modified") + + # def generate_data(self): # n_samples = 10**5 # graphs = [] @@ -147,16 +208,19 @@ def read(self): # max_output = np.max(all_outputs) if not self._predict: - train_outputs = all_outputs[self._index * self._config['n_train_samples']: - self._index * self._config['n_train_samples'] + self._config['n_train_samples']] - train_features = all_features[self._index * self._config['n_train_samples']: - self._index * self._config['n_train_samples'] + self._config['n_train_samples']] + if self._index is not None: + train_outputs = all_outputs[self._index * self._config['n_train_samples']: + self._index * self._config['n_train_samples'] + self._config['n_train_samples']] + train_features = all_features[self._index * self._config['n_train_samples']: + self._index * self._config['n_train_samples'] + self._config['n_train_samples']] + else: + train_outputs = all_outputs + train_features = all_features if self._dataset_config.get("calc_output_mult_factor", False) is True: self._output_mult_factor = 1/np.mean(train_outputs) self._dataset_config["output_mult_factor"] = self._output_mult_factor self._save_data_config() - print("output mult factor ", self._dataset_config["output_mult_factor"]) if self._dataset_config.get("features_normalization", False): self._min_features = np.min(train_features, axis=0) @@ -219,7 +283,6 @@ def read(self): if self._log and self._dataset_config.get("features_log", False) is False and self._dataset_config.get("output_log", False) is False: features = np.log(features) output = np.log(output) - print("self._log SET") if self._dataset_config.get("last_log_features", False): features = np.log(features) @@ -232,6 +295,32 @@ def read(self): # Save data for pandas dataframe creation, not used with Graph neural network self.data.append({'x': features, 'y': output}) + if self._augment_data: + if self._columns is None: + d = [pd.DataFrame(features.reshape((features.shape[1], features.shape[0])).tolist()).add_prefix("x_")] + new_df = pd.concat(d, axis=1) + #new_df.insert(loc=0, column="y", value=output) + self._columns = [] + for col in new_df.columns: + self._columns.append(col) + self._columns.append("y") + + squeezed_features = list(np.squeeze(features)) + squeezed_features.append(output) + self._aug_data.append(squeezed_features) + + + #new_df["y"] = output + # print("new df y ", new_df["y"]) + # print("new df .shape ", new_df.shape) + #print("new df ", new_df) + + # if self._df_for_augmentation is not None: + # self._df_for_augmentation = self._df_for_augmentation.append(new_df) + # else: + # self._df_for_augmentation = new_df + + self.a = self.adjacency_matrix return graphs @@ -240,12 +329,13 @@ def _save_data_config(self): import pickle import shutil - if os.path.exists(os.path.join(self._config['iter_dir'], "dataset_config.pkl")): - os.remove(os.path.join(self._config['iter_dir'], "dataset_config.pkl")) + if "iter_dir" in self._config: + if os.path.exists(os.path.join(self._config['iter_dir'], "dataset_config.pkl")): + os.remove(os.path.join(self._config['iter_dir'], "dataset_config.pkl")) - # create a binary pickle file - with open(os.path.join(self._config['iter_dir'], "dataset_config.pkl"), "wb") as writer: - pickle.dump(self._dataset_config, writer) + # create a binary pickle file + with open(os.path.join(self._config['iter_dir'], "dataset_config.pkl"), "wb") as writer: + pickle.dump(self._dataset_config, writer) @staticmethod def pickle_data(data, output_dir, file_path): From 81507f463a15b37ea017ad378c5e526647ca6519 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Thu, 10 Feb 2022 15:06:59 +0100 Subject: [PATCH 52/67] predict on different mesh --- test/metamodels/predict_on_different_mesh.py | 321 ++++++++++++++----- 1 file changed, 246 insertions(+), 75 deletions(-) diff --git a/test/metamodels/predict_on_different_mesh.py b/test/metamodels/predict_on_different_mesh.py index 2fd75905..84d5852a 100644 --- a/test/metamodels/predict_on_different_mesh.py +++ b/test/metamodels/predict_on_different_mesh.py @@ -22,6 +22,7 @@ from mlmc.plot import plots from tensorflow.keras.layers.experimental import preprocessing from mlmc.metamodel.analyze_nn import load_statistics, process_data +from scipy import stats warnings.filterwarnings("ignore", category=DeprecationWarning) @@ -254,6 +255,113 @@ def get_config(data_dir, case=0): graph_creation_time = 285 + elif case == "case_2": # mesh size comparison + data_dir = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/case_2" + # cl = "cl_0_1_s_1" + level = "3" + nn_level = 0 + replace_level = False + # mesh = os.path.join(data_dir, "l_step_1.0_common_files/repo.msh") # L1, 7s + # mesh = os.path.join(data_dir, "l_step_0.3760603093086394_common_files/repo.msh") #L2 10.5 s + mesh = os.path.join(data_dir, "l_step_0.1414213562373095_common_files/repo.msh") #L3 12s + # mesh = os.path.join(data_dir, "l_step_0.053182958969449884_common_files/repo.msh") # L4 + output_dir = os.path.join(data_dir, "L1_{}/test/02_conc/output/".format(level)) + hdf_path = os.path.join(data_dir, "L1_{}/mlmc_1.hdf5".format(level)) + mlmc_hdf_path = None # os.path.join(data_dir, "mlmc_hdf/L1_{}/mlmc_1.hdf5".format(level)) + save_path = data_dir + l_0_output_dir = os.path.join(data_dir, "L0_MC/L1_{}/test/02_conc/output/".format(level)) + l_0_hdf_path = os.path.join(data_dir, "L0_MC/L1_{}/mlmc_1.hdf5".format(level)) + sampling_info_path = os.path.join(data_dir, "sampling_info") + + # ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_3/mlmc_1.hdf5" + # ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_1/mlmc_1.hdf5" + ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_benchmark/mlmc_1.hdf5" + #ref_mlmc_file = os.path.join(data_dir, "L1_benchmark/mlmc_1.hdf5") + + feature_names = [['conductivity_top', 'conductivity_bot', 'conductivity_repo']] + + graph_creation_time = 670 # case_2 + + elif case == "case_3": # mesh size comparison + data_dir = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/case_3" + # cl = "cl_0_1_s_1" + level = "3" + nn_level = 0 + replace_level = False + # mesh = os.path.join(data_dir, "l_step_1.0_common_files/repo.msh") # L1, 7s + # mesh = os.path.join(data_dir, "l_step_0.3760603093086394_common_files/repo.msh") #L2 10.5 s + mesh = os.path.join(data_dir, "l_step_0.1414213562373095_common_files/repo.msh") #L3 12s + # mesh = os.path.join(data_dir, "l_step_0.053182958969449884_common_files/repo.msh") # L4 + output_dir = os.path.join(data_dir, "L1_{}/test/02_conc/output/".format(level)) + hdf_path = os.path.join(data_dir, "L1_{}/mlmc_1.hdf5".format(level)) + mlmc_hdf_path = None # os.path.join(data_dir, "mlmc_hdf/L1_{}/mlmc_1.hdf5".format(level)) + save_path = data_dir + l_0_output_dir = os.path.join(data_dir, "L0_MC/L1_{}/test/02_conc/output/".format(level)) + l_0_hdf_path = os.path.join(data_dir, "L0_MC/L1_{}/mlmc_1.hdf5".format(level)) + sampling_info_path = os.path.join(data_dir, "sampling_info") + + # ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_3/mlmc_1.hdf5" + # ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_1/mlmc_1.hdf5" + #ref_mlmc_file = os.path.join(data_dir, "L1_benchmark/mlmc_1.hdf5") + ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_benchmark/mlmc_1.hdf5" + + feature_names = [['conductivity_top', 'conductivity_bot', 'conductivity_repo']] + + graph_creation_time = 601 # case_3 + + elif case == "case_4": # mesh size comparison + data_dir = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/case_4" + # cl = "cl_0_1_s_1" + level = "3" + nn_level = 0 + replace_level = False + # mesh = os.path.join(data_dir, "l_step_1.0_common_files/repo.msh") # L1, 7s + # mesh = os.path.join(data_dir, "l_step_0.3760603093086394_common_files/repo.msh") #L2 10.5 s + mesh = os.path.join(data_dir, "l_step_0.1414213562373095_common_files/repo.msh") #L3 12s + # mesh = os.path.join(data_dir, "l_step_0.053182958969449884_common_files/repo.msh") # L4 + output_dir = os.path.join(data_dir, "L1_{}/test/02_conc/output/".format(level)) + hdf_path = os.path.join(data_dir, "L1_{}/mlmc_1.hdf5".format(level)) + mlmc_hdf_path = os.path.join(data_dir, "mlmc_hdf/L1_{}/mlmc_1.hdf5".format(level)) + save_path = data_dir + l_0_output_dir = os.path.join(data_dir, "L0_MC/L1_{}/test/02_conc/output/".format(level)) + l_0_hdf_path = os.path.join(data_dir, "L0_MC/L1_{}/mlmc_1.hdf5".format(level)) + sampling_info_path = os.path.join(data_dir, "sampling_info") + + # ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_3/mlmc_1.hdf5" + # ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_1/mlmc_1.hdf5" + #ref_mlmc_file = os.path.join(data_dir, "L1_benchmark/mlmc_1.hdf5") + ref_mlmc_file = mlmc_hdf_path #"/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_benchmark/mlmc_1.hdf5" + + feature_names = [['conductivity_top', 'conductivity_bot', 'conductivity_repo']] + + graph_creation_time = 608 # case_4 + + elif case == "case_5": # mesh size comparison + data_dir = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/case_5" + # cl = "cl_0_1_s_1" + level = "3" + nn_level = 0 + replace_level = False + # mesh = os.path.join(data_dir, "l_step_1.0_common_files/repo.msh") # L1, 7s + # mesh = os.path.join(data_dir, "l_step_0.3760603093086394_common_files/repo.msh") #L2 10.5 s + mesh = os.path.join(data_dir, "l_step_0.1414213562373095_common_files/repo.msh") #L3 12s + # mesh = os.path.join(data_dir, "l_step_0.053182958969449884_common_files/repo.msh") # L4 + output_dir = os.path.join(data_dir, "L1_{}/test/02_conc/output/".format(level)) + hdf_path = os.path.join(data_dir, "L1_{}/mlmc_1.hdf5".format(level)) + mlmc_hdf_path = None # os.path.join(data_dir, "mlmc_hdf/L1_{}/mlmc_1.hdf5".format(level)) + save_path = data_dir + l_0_output_dir = os.path.join(data_dir, "L0_MC/L1_{}/test/02_conc/output/".format(level)) + l_0_hdf_path = os.path.join(data_dir, "L0_MC/L1_{}/mlmc_1.hdf5".format(level)) + sampling_info_path = os.path.join(data_dir, "sampling_info") + + # ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_3/mlmc_1.hdf5" + # ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_1/mlmc_1.hdf5" + #ref_mlmc_file = os.path.join(data_dir, "L1_benchmark/mlmc_1.hdf5") + ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_benchmark/mlmc_1.hdf5" + + feature_names = [['conductivity_top', 'conductivity_bot', 'conductivity_repo']] + graph_creation_time = 587 # case_5 + return output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file,\ replace_level, nn_level, mlmc_hdf_path, feature_names, graph_creation_time @@ -297,93 +405,156 @@ def get_arguments(arguments): ####################### # Load trained model # ####################### - machine_learning_model = ("L1_3_02_conc_cond_log_output_mult_T19_case_1", run_GNN, False) + #machine_learning_model = ("L1_3_02_conc_cond_log_output_mult_T19_case_1", run_GNN, False) + machine_learning_model = ("L1_3_T19_case_1_out_log_scale", run_GNN, False) save_path = os.path.join("/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/", machine_learning_model[0]) data_dict = load_statistics(save_path) #data_dict = process_data(data_dict) #for i in range(len(data_dict["test_targets"])): - model = data_dict["model"][0] - # newInput = Input(batch_shape=(None, 108, 1)) - # newOutputs = model(newInput) - # newModel = Model(newInput, newOutputs) - # - # if type(model._conv_layers[0]).__name__ == 'OwnChebConv': - # conv_layer = OwnChebConv - # - # print("conv layer ", conv_layer) + list_train_MSE, list_test_MSE, list_all_train_RSE, list_all_test_RSE, list_nn_total_time, list_mlmc_total_time,\ + list_kl_mlmc_all, list_kl_nn_all, list_learning_times = [], [], [], [], [], [], [], [], [] + + for index, model in enumerate(data_dict["model"][:2]): + # newInput = Input(batch_shape=(None, 108, 1)) + # newOutputs = model(newInput) + # newModel = Model(newInput, newOutputs) + # + # if type(model._conv_layers[0]).__name__ == 'OwnChebConv': + # conv_layer = OwnChebConv + # + # print("conv layer ", conv_layer) + ##################### + ## New case config ## + ##################### + args = get_arguments(sys.argv[1:]) + data_dir = args.data_dir + work_dir = args.work_dir + # case = "L2" + # case = 521 + case = "case_3" + #data_dir = "/home/martin/Documents/metamodels/data/1000_ele/" + output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file,\ + replace_level, nn_level, mlmc_hdf_path, feature_names, graph_creation_time = get_config(data_dir, case) - ##################### - ## New case config ## - ##################### - args = get_arguments(sys.argv[1:]) - data_dir = args.data_dir - work_dir = args.work_dir - case = "L2" - case = 521 - #data_dir = "/home/martin/Documents/metamodels/data/1000_ele/" - output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file,\ - replace_level, nn_level, mlmc_hdf_path, feature_names, graph_creation_time = get_config(data_dir, case) + machine_learning_model = ("L1_2_02_conc_cond_log_output_mult_T19_case_1_trained_{}_{}".format(case, index), run_GNN, False) + save_path = os.path.join(save_path, machine_learning_model[0]) - machine_learning_model = ("L1_2_02_conc_cond_log_output_mult_T19_case_1_trained_L1_3", run_GNN, False) - save_path = os.path.join(save_path, machine_learning_model[0]) + if os.path.exists(save_path): + shutil.rmtree(save_path) - if os.path.exists(save_path): - shutil.rmtree(save_path) + #corr_field_config = {'02_conc': True, 'corr_length': 0.1, 'sigma': 1, 'log': True} - corr_field_config = {'02_conc': True, 'corr_length': 0.1, 'sigma': 1, 'log': True} + gnn, conv_layer, corr_field_config, model_config = get_gnn() - gnn, conv_layer, corr_field_config, model_config = get_gnn() - - #gnn_au = gnn(**model_config) - # - # print("gnn au model ", gnn_au._model) - # print("model ", model) - - #set_weights(gnn_au._model, model) - - dataset_config = {"features_normalization": False, - "calc_output_mult_factor": True, - "output_mult_factor": 1, - "features_mult_factor": 1, - "features_log": False, - "output_log": True - } - - config = {'machine_learning_model': machine_learning_model, - 'save_path': save_path, - 'output_dir': output_dir, - 'hdf_path': hdf_path, - 'mlmc_hdf_path': mlmc_hdf_path, - 'mesh': mesh, - 'l_0_output_dir': l_0_output_dir, - 'l_0_hdf_path': l_0_hdf_path, - 'sampling_info_path': sampling_info_path, - 'ref_mlmc_file': ref_mlmc_file, - 'level': nn_level, - 'conv_layer': conv_layer, - 'gnn': gnn, - 'model_config': model_config, - 'replace_level': replace_level, - 'corr_field_config': corr_field_config, - 'n_train_samples': 2000, - 'val_samples_ratio': 0.2, - 'batch_size': 200, - 'epochs': 1, - 'learning_rate': 0.001, - 'graph_creation_time': graph_creation_time, - 'save_model': True, - 'feature_names': feature_names, - "train_model": False, - "set_model": model, - "dataset_config": dataset_config - } - - model_title, mch_l_model, log = config['machine_learning_model'] - - statistics(config) + #gnn_au = gnn(**model_config) + # + # print("gnn au model ", gnn_au._model) + # print("model ", model) + + #set_weights(gnn_au._model, model) + + # dataset_config = {"features_normalization": False, + # "calc_output_mult_factor": True, + # "output_mult_factor": 1, + # "features_mult_factor": 1, + # "features_log": False, + # "output_log": True + # } + + dataset_config = {"first_log_features": False, + "first_log_output": True, + "features_normalization": False, + "output_normalization": False, + "calc_output_mult_factor": False, + "output_mult_factor": 1, + "features_mult_factor": 1, + "features_scale": False, + "output_scale": True, + } + + config = {'machine_learning_model': machine_learning_model, + 'save_path': save_path, + 'output_dir': output_dir, + 'hdf_path': hdf_path, + 'mlmc_hdf_path': mlmc_hdf_path, + 'mesh': mesh, + 'l_0_output_dir': l_0_output_dir, + 'l_0_hdf_path': l_0_hdf_path, + 'sampling_info_path': sampling_info_path, + 'ref_mlmc_file': ref_mlmc_file, + 'level': nn_level, + 'conv_layer': conv_layer, + 'gnn': gnn, + 'model_config': model_config, + 'replace_level': replace_level, + 'corr_field_config': corr_field_config, + 'n_train_samples': 2000, + 'val_samples_ratio': 0.2, + 'batch_size': 200, + 'epochs': 1, + 'learning_rate': 0.001, + 'graph_creation_time': graph_creation_time, + 'save_model': True, + 'feature_names': feature_names, + "train_model": False, + "set_model": model, + "dataset_config": dataset_config + } + + model_title, mch_l_model, log = config['machine_learning_model'] + + train_MSE, test_MSE, all_train_RSE, all_test_RSE, nn_total_time, mlmc_total_time, kl_mlmc_all, kl_nn_all, \ + learning_times = statistics(config) + + list_train_MSE.append(train_MSE) + list_test_MSE.append(test_MSE) + list_all_train_RSE.append(all_train_RSE) + list_all_test_RSE.append(all_test_RSE) + list_nn_total_time.append(nn_total_time) + list_mlmc_total_time.append(mlmc_total_time) + list_kl_mlmc_all.append(kl_mlmc_all) + list_kl_nn_all.append(kl_nn_all) + list_learning_times.append(learning_times) + + print("learning time ", list_learning_times) + print("mean learning time ", np.mean(list_learning_times)) + print("max learning time ", np.max(list_learning_times)) + + + print("############# OUTPUT ################") + print("len(train MSE) ", len(list_train_MSE)) + print("train MSE ", np.mean(list_all_train_RSE)) + # print("train MSE sqrt var", np.sqrt(np.var(train_MSE))) + # print("train MSE std", np.std(train_MSE)) + + # output_mult_factor = 1437603411 + # print("orig train MSE ", train_MSE) + # train_MSE = np.array(train_MSE) * output_mult_factor + # print("train MSE ", train_MSE) + # test_MSE = np.array(test_MSE) * output_mult_factor + + print("train MSE ", list_train_MSE) + print("stats.sem(train_MSE) ", stats.sem(list_train_MSE)) + print("test MSE ", np.mean(list_test_MSE)) + print("test MSE ", list_test_MSE) + print("stats.sem(test_MSE) ", stats.sem(list_test_MSE)) + # print("test MSE std", np.sqrt(np.var(test_MSE))) + print("train RSE ", np.mean(list_all_train_RSE)) + print("test RSE ", np.mean(list_all_test_RSE)) + + print("nn total time ", list_nn_total_time) + print("mlmc total time ", list_mlmc_total_time) + + print("KL mlmc ", np.mean(list_kl_mlmc_all)) + print("KL nn ", np.mean(list_kl_nn_all)) + + print("mean learning time ", np.mean(list_learning_times)) + print("max learning time ", np.max(list_learning_times)) + + print("######################################") #analyze_statistics(config) From 2e5f94ed5fe37af401e39a08af30f1b0668a912b Mon Sep 17 00:00:00 2001 From: Martin Spetlik Date: Wed, 23 Feb 2022 18:45:12 +0100 Subject: [PATCH 53/67] flow dataset --- mlmc/metamodel/flow_dataset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mlmc/metamodel/flow_dataset.py b/mlmc/metamodel/flow_dataset.py index ef934b80..097a3987 100644 --- a/mlmc/metamodel/flow_dataset.py +++ b/mlmc/metamodel/flow_dataset.py @@ -32,6 +32,7 @@ def __init__(self, output_dir=None, level=0, log=False, mesh=None, corr_field_co self._corr_field_config = corr_field_config self.adjacency_matrix = np.load(os.path.join(self._output_dir, "adjacency_matrix.npy"), allow_pickle=True) # adjacency matrix self.data = [] + self._aug_data = [] self._config = config self._index = index self._predict = predict @@ -59,7 +60,6 @@ def __init__(self, output_dir=None, level=0, log=False, mesh=None, corr_field_co self._var_output = None self._output_mult_factor = 1 - self._aug_data = [] self._columns = None super().__init__(**kwargs) @@ -96,15 +96,15 @@ def shuffle(self, seed=None): def _data_augmentation(self, df_slice): import smogn import matplotlib.pyplot as plt - import seaborn + #import seaborn # df_slice = self._df_for_augmentation[self._index * self._config['n_train_samples']: # self._index * self._config['n_train_samples'] + self._config['n_train_samples']] # print("index ", self._index) # print("df slice shape ", df_slice.shape) + df_slice = df_slice.reset_index(drop=True) if 'augmentation_config' in self._config: - print("len df slice 0", len(df_slice)) dataset_modified = smogn.smoter(data=df_slice, y="y", **self._config["augmentation_config"]) else: dataset_modified = smogn.smoter(data=df_slice, y="y", k=9, samp_method="extreme") From a1f754ec0a3efa3f1087b3594097e8751d2be2d5 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Wed, 23 Feb 2022 19:36:14 +0100 Subject: [PATCH 54/67] dataset split --- mlmc/metamodel/analyze_nn.py | 33 ++++++++++++--------- mlmc/metamodel/flow_dataset.py | 52 +++++++++++++++++++++++++--------- 2 files changed, 59 insertions(+), 26 deletions(-) diff --git a/mlmc/metamodel/analyze_nn.py b/mlmc/metamodel/analyze_nn.py index 2a0402ce..1e9a3ce8 100644 --- a/mlmc/metamodel/analyze_nn.py +++ b/mlmc/metamodel/analyze_nn.py @@ -839,16 +839,17 @@ def analyze_statistics(config, get_model=True): except: model = None - plot_loss(model_train_loss, model_val_loss) - plot_learning_rate(model_learning_rates) - print("model learning rates ", model_learning_rates) - - print("model ", model) - print("dir(model.optimizer) ", dir(model.optimizer)) - #print("model weights ", model.weights) - print("model.optimizer", model.optimizer) - # print("model.optimizer", K.eval(model.optimizer.lr)) - # exit() + if model is not None: + plot_loss(model_train_loss, model_val_loss) + plot_learning_rate(model_learning_rates) + print("model learning rates ", model_learning_rates) + + print("model ", model) + print("dir(model.optimizer) ", dir(model.optimizer)) + #print("model weights ", model.weights) + print("model.optimizer", model.optimizer) + # print("model.optimizer", K.eval(model.optimizer.lr)) + # exit() iter_test_MSE = np.mean((predictions - targets) ** 2) iter_test_bias = np.sqrt(np.mean((targets - np.mean(predictions)) ** 2)) @@ -1454,9 +1455,11 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): data_tr = data data_te = data else: + #data_tr = data[seed*train_data_len: seed*train_data_len + train_data_len] data_tr = data.get_train_data(seed, train_data_len) - print("data tr ", data_tr) + #print("data tr ", data_tr) data_te = data.get_test_data(seed, train_data_len) + print("data te ", data_te) #data_tr, data_te = data[:train_data_len], data[train_data_len:] gnn = config['gnn'](**config['model_config']) @@ -1470,9 +1473,13 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): # #accuracy_func = MSE_moments(moments_fn=moments_fn) # gnn._loss = MSE_moments(moments_fn=moments_fn) - np.random.shuffle(data_tr) + #np.random.shuffle(data_tr) val_data_len = int(len(data_tr) * config['val_samples_ratio']) - data_tr, data_va = data_tr[:-val_data_len], data_tr[-val_data_len:] + data_tr, data_va = data_tr.split_val_train(val_data_len) + #data_tr, data_va = data_tr[:-val_data_len], data_tr[-val_data_len:] + + print("data tr ", data_tr) + print("data va ", data_va) # print("data_tr len ", len(data_tr)) # print("data_va len ", len(data_va)) diff --git a/mlmc/metamodel/flow_dataset.py b/mlmc/metamodel/flow_dataset.py index ef934b80..4bed6c4f 100644 --- a/mlmc/metamodel/flow_dataset.py +++ b/mlmc/metamodel/flow_dataset.py @@ -73,17 +73,42 @@ def __init__(self, output_dir=None, level=0, log=False, mesh=None, corr_field_co def get_train_data(self, index, length): new_dataset = self.dataset[index * length: index * length + length] + new_graphs = self.graphs[index * length: index * length + length] # self.graphs is read() method output + if self._augment_data: - new_dataset = self._data_augmentation(self._df_for_augmentation[index * length: index * length + length]) + new_dataset, new_graphs = self._data_augmentation(self._df_for_augmentation[index * length: index * length + length], new_graphs) new_obj = copy.deepcopy(self) new_obj.dataset = new_dataset + new_obj.graphs = new_graphs + return new_obj + def split_val_train(self, len_val_data): + tr_dataset = self.dataset[:-len_val_data] + va_dataset = self.dataset[-len_val_data:] + + tr_graphs = self.graphs[:-len_val_data] + va_graphs = self.graphs[-len_val_data:] + #new_graphs = self.graphs[index * length: index * length + length] # self.graphs is read() method output + + tr_obj = copy.deepcopy(self) + va_obj = copy.deepcopy(self) + + tr_obj.dataset = tr_dataset + va_obj.dataset = va_dataset + + tr_obj.graphs = tr_graphs + va_obj.graphs = va_graphs + + return tr_obj, va_obj + def get_test_data(self, index, length): new_dataset = self.dataset[0:index * length] + self.dataset[index * length + length:] + new_graphs = self.graphs[0:index * length] + self.graphs[index * length + length:] new_obj = copy.deepcopy(self) new_obj.dataset = new_dataset + new_obj.graphs = new_graphs return new_obj def shuffle(self, seed=None): @@ -93,7 +118,7 @@ def shuffle(self, seed=None): random.shuffle(self.data) self.dataset = pd.DataFrame(self.data) - def _data_augmentation(self, df_slice): + def _data_augmentation(self, df_slice, new_graphs): import smogn import matplotlib.pyplot as plt import seaborn @@ -101,10 +126,9 @@ def _data_augmentation(self, df_slice): # df_slice = self._df_for_augmentation[self._index * self._config['n_train_samples']: # self._index * self._config['n_train_samples'] + self._config['n_train_samples']] - # print("index ", self._index) - # print("df slice shape ", df_slice.shape) + df_slice = df_slice.reset_index(drop=True) + if 'augmentation_config' in self._config: - print("len df slice 0", len(df_slice)) dataset_modified = smogn.smoter(data=df_slice, y="y", **self._config["augmentation_config"]) else: dataset_modified = smogn.smoter(data=df_slice, y="y", k=9, samp_method="extreme") @@ -124,15 +148,17 @@ def _data_augmentation(self, df_slice): # ax.hist(dataset_modified['y'], bins=50, alpha=0.5, label='target', density=True) # plt.title("modified") # plt.show() - # - # appended_dataset = df_slice.append(dataset_modified) - # - # fig, ax = plt.subplots(1, 1, figsize=(15, 10)) - # ax.hist(appended_dataset['y'], bins=50, alpha=0.5, label='target', density=True) - # plt.title("appended") - # plt.show() - return dataset_modified + appended_dataset = df_slice.append(dataset_modified) + + numpy_frame = dataset_modified.to_numpy() + + for i in range(numpy_frame.shape[0]): + features = numpy_frame[i][:-1] + features = features.reshape((len(features), 1)) + new_graphs.append(Graph(x=features, y=numpy_frame[i][-1])) + + return appended_dataset, copy.deepcopy(new_graphs) # seaborn.kdeplot(df_slice['y'], label="Original") # seaborn.kdeplot(dataset_modified['y'], label="Modified") From 61fd680b0583c927900d466c81a564ca30510ec7 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Thu, 24 Feb 2022 17:00:29 +0100 Subject: [PATCH 55/67] train val split --- mlmc/metamodel/analyze_nn.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mlmc/metamodel/analyze_nn.py b/mlmc/metamodel/analyze_nn.py index 1e9a3ce8..1ca975ad 100644 --- a/mlmc/metamodel/analyze_nn.py +++ b/mlmc/metamodel/analyze_nn.py @@ -1475,8 +1475,9 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): #np.random.shuffle(data_tr) val_data_len = int(len(data_tr) * config['val_samples_ratio']) - data_tr, data_va = data_tr.split_val_train(val_data_len) - #data_tr, data_va = data_tr[:-val_data_len], data_tr[-val_data_len:] + print("val data len ", val_data_len) + #data_tr, data_va = data_tr.split_val_train(val_data_len) + data_tr, data_va = data_tr[:-val_data_len], data_tr[-val_data_len:] print("data tr ", data_tr) print("data va ", data_va) From 5dbed89472dfa92a8bb797a67b0347572130e381 Mon Sep 17 00:00:00 2001 From: Martin Spetlik Date: Tue, 8 Mar 2022 15:34:13 +0100 Subject: [PATCH 56/67] save train accuracy --- mlmc/metamodel/analyze_nn.py | 3 +++ mlmc/metamodel/flow_task_GNN_2.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/mlmc/metamodel/analyze_nn.py b/mlmc/metamodel/analyze_nn.py index 1ca975ad..514540a4 100644 --- a/mlmc/metamodel/analyze_nn.py +++ b/mlmc/metamodel/analyze_nn.py @@ -388,6 +388,7 @@ def statistics(config): if config['save_model']: model_data["model"] = gnn._model model_data["train_loss"] = gnn._train_loss + model_data["train_acc"] = gnn._train_acc model_data["val_loss"] = gnn._val_loss model_data["test_loss"] = gnn._test_loss model_data["learning_rates"] = gnn._learning_rates @@ -438,6 +439,7 @@ def load_statistics(dir_path): models_data = {} models_data["model"] = [] models_data["train_loss"] = [] + models_data["train_acc"] = [] models_data["val_loss"] = [] models_data["test_loss"] = [] models_data["learning_rates"] = [] @@ -833,6 +835,7 @@ def analyze_statistics(config, get_model=True): try: model = data_dict["model"][i] model_train_loss = data_dict["train_loss"][i] + model_train_acc = data_dict["train_acc"][i] model_val_loss = data_dict["val_loss"][i] model_test_loss = data_dict["test_loss"][i] model_learning_rates = data_dict["learning_rates"][i] diff --git a/mlmc/metamodel/flow_task_GNN_2.py b/mlmc/metamodel/flow_task_GNN_2.py index 6bbca16d..527e0359 100644 --- a/mlmc/metamodel/flow_task_GNN_2.py +++ b/mlmc/metamodel/flow_task_GNN_2.py @@ -43,6 +43,7 @@ def __init__(self, **kwargs): self._train_loss = [] self._val_loss = [] self._test_loss = [] + self._train_acc = [] self._learning_rates = [] self.val_targets = [] @@ -99,6 +100,7 @@ def fit(self, loader_tr, loader_va, loader_te): loss, acc = self.train_on_batch(inputs, target) self._train_loss.append(loss) + self._train_acc.append(acc) results_tr.append((loss, acc, len(target))) results_va = self.evaluate(loader_va) From 2c26d12eb66cf14ee38d8c5f880294acff943e12 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Wed, 9 Mar 2022 16:27:54 +0100 Subject: [PATCH 57/67] evaluate batch fn function --- mlmc/metamodel/flow_task_GNN_2.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/mlmc/metamodel/flow_task_GNN_2.py b/mlmc/metamodel/flow_task_GNN_2.py index 527e0359..184deaaa 100644 --- a/mlmc/metamodel/flow_task_GNN_2.py +++ b/mlmc/metamodel/flow_task_GNN_2.py @@ -154,7 +154,6 @@ def train_on_batch(self, inputs, target): with tf.GradientTape() as tape: predictions = self._model(inputs, training=True) loss = self._loss(target, predictions) + sum(self._model.losses) #+ 5 * var_loss_function(target, predictions) - #loss = 100 * var_loss_function(target, predictions) acc = tf.reduce_mean(self._accuracy_func(target, predictions)) gradients = tape.gradient(loss, self._model.trainable_variables) @@ -178,17 +177,26 @@ def evaluate(self, loader): if val_targets: self.val_targets.extend(target) - predictions = self._model(inputs, training=False) + loss, acc = self.evaluate_batch(inputs, target) - loss = self._loss(target, predictions) - #print("target ", target) - #print("loss ", np.mean((target - predictions)**2)) - acc = tf.reduce_mean(self._accuracy_func(target, predictions)) + #predictions = self._model(inputs, training=False) + #loss = self._loss(target, predictions) + #acc = tf.reduce_mean(self._accuracy_func(target, predictions)) results.append((loss, acc, len(target))) # Keep track of batch size if step == loader.steps_per_epoch: results = np.array(results) return np.average(results[:, :-1], axis=0, weights=results[:, -1]) + @tf.function + def evaluate_batch(self, inputs, target): + predictions = self._model(inputs, training=False) + + loss = self._loss(target, predictions) + acc = tf.reduce_mean(self._accuracy_func(target, predictions)) + + return loss, acc + + def predict(self, loader): targets = [] predictions = [] From d0347cd36d8d55cfe6a31b17e269af5822bf6d5d Mon Sep 17 00:00:00 2001 From: Martin Spetlik Date: Thu, 24 Mar 2022 10:41:48 +0100 Subject: [PATCH 58/67] val loss best state --- mlmc/metamodel/analyze_nn.py | 8 ++++---- mlmc/metamodel/flow_task_GNN_2.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mlmc/metamodel/analyze_nn.py b/mlmc/metamodel/analyze_nn.py index 514540a4..b60d39bb 100644 --- a/mlmc/metamodel/analyze_nn.py +++ b/mlmc/metamodel/analyze_nn.py @@ -1519,10 +1519,10 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): learning_time = time.process_time() - learning_time_start - # states = gnn._states - # if len(states) > 0: - # min_key = np.min(list(states.keys())) - # gnn = states[min_key] + states = gnn._states + if len(states) > 0: + min_key = np.min(list(states.keys())) + gnn = states[min_key] train_targets, train_predictions = gnn.predict(MixedLoader(data_tr, batch_size=batch_size, epochs=1)) train_predictions = np.squeeze(train_predictions) diff --git a/mlmc/metamodel/flow_task_GNN_2.py b/mlmc/metamodel/flow_task_GNN_2.py index 184deaaa..bb0c5f2e 100644 --- a/mlmc/metamodel/flow_task_GNN_2.py +++ b/mlmc/metamodel/flow_task_GNN_2.py @@ -121,8 +121,8 @@ def fit(self, loader_tr, loader_va, loader_te): else: current_patience -= 1 #results_tr_0 = np.array(results_tr) - loss_tr = results_va[0] - self._states[loss_tr] = self + loss_va = results_va[0] + self._states[loss_va] = self if current_patience == 0: print("Early stopping") break From 1f0f8c85da2d893456f6b3d1761a775c92398165 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Thu, 24 Mar 2022 13:06:54 +0100 Subject: [PATCH 59/67] flow dataset train test split --- mlmc/metamodel/analyze_nn.py | 57 ++++++++++++++++++++-------------- mlmc/metamodel/flow_dataset.py | 50 +++++++++++++++++++++-------- 2 files changed, 70 insertions(+), 37 deletions(-) diff --git a/mlmc/metamodel/analyze_nn.py b/mlmc/metamodel/analyze_nn.py index 514540a4..7dc63aa9 100644 --- a/mlmc/metamodel/analyze_nn.py +++ b/mlmc/metamodel/analyze_nn.py @@ -797,9 +797,9 @@ def analyze_statistics(config, get_model=True): #limit = 0.37 for i in range(len(data_dict["test_targets"])): - print("index i ", i) + # print("index i ", i) # if i == 4: - # break + # continue # if i == 1: # continue @@ -835,7 +835,10 @@ def analyze_statistics(config, get_model=True): try: model = data_dict["model"][i] model_train_loss = data_dict["train_loss"][i] - model_train_acc = data_dict["train_acc"][i] + if "train_acc" in data_dict and len(data_dict["train_acc"]) > 0: + model_train_acc = data_dict["train_acc"][i] + else: + model_train_acc = None model_val_loss = data_dict["val_loss"][i] model_test_loss = data_dict["test_loss"][i] model_learning_rates = data_dict["learning_rates"][i] @@ -843,7 +846,7 @@ def analyze_statistics(config, get_model=True): model = None if model is not None: - plot_loss(model_train_loss, model_val_loss) + plot_loss(model_train_loss, model_val_loss, model_train_acc) plot_learning_rate(model_learning_rates) print("model learning rates ", model_learning_rates) @@ -899,25 +902,25 @@ def analyze_statistics(config, get_model=True): #exit() print("total steps ", total_steps) - # try: - mlmc_n_collected, nn_mlmc_n_collected, n_ops, n_ops_predict, orig_moments_mean, predict_moments_mean, \ - ref_moments_mean, orig_level_params, nn_level_params, kl_mlmc, kl_nn, target_variance, \ - orig_orth_moments, predict_orth_moments, ref_orth_moments = process_mlmc(config['hdf_path'], - config['sampling_info_path'], - config['ref_mlmc_file'], targets, - predictions, train_targets, - train_predictions, - val_targets, l_0_targets, - l_0_predictions, l1_sample_time, - l0_sample_time, - nn_level=config['level'], - replace_level=config['replace_level'], - mlmc_hdf_file=config['mlmc_hdf_path'], - stats=True, - learning_time=learning_time, - dataset_config=dataset_config) - # except: - # continue + try: + mlmc_n_collected, nn_mlmc_n_collected, n_ops, n_ops_predict, orig_moments_mean, predict_moments_mean, \ + ref_moments_mean, orig_level_params, nn_level_params, kl_mlmc, kl_nn, target_variance, \ + orig_orth_moments, predict_orth_moments, ref_orth_moments = process_mlmc(config['hdf_path'], + config['sampling_info_path'], + config['ref_mlmc_file'], targets, + predictions, train_targets, + train_predictions, + val_targets, l_0_targets, + l_0_predictions, l1_sample_time, + l0_sample_time, + nn_level=config['level'], + replace_level=config['replace_level'], + mlmc_hdf_file=config['mlmc_hdf_path'], + stats=True, + learning_time=learning_time, + dataset_config=dataset_config) + except: + continue mlmc_n_collected_all.append(mlmc_n_collected) nn_n_collected_all.append(nn_mlmc_n_collected) @@ -1072,6 +1075,8 @@ def analyze_statistics(config, get_model=True): data_dict["test_predictions"] = np.array(data_dict["test_predictions"]) data_dict["train_targets"] = np.array(data_dict["train_targets"]) data_dict["train_predictions"] = np.array(data_dict["train_predictions"]) + data_dict["val_targets"] = np.array(data_dict["val_targets"]) + data_dict["val_predictions"] = np.array(data_dict["val_predictions"]) print("data dict train predictions ", data_dict["train_predictions"]) @@ -1107,10 +1112,14 @@ def analyze_statistics(config, get_model=True): # print("test targets shape ", data_dict["test_targets"].shape) test_MSE = np.mean((data_dict["test_predictions"] - data_dict["test_targets"]) ** 2, axis=1) + print("val predictions ", data_dict["val_predictions"]) + print("val targets ", data_dict["val_targets"]) + #val_MSE = np.mean((data_dict["val_predictions"] - data_dict["val_targets"]) ** 2, axis=1) test_RMSE = np.sqrt(test_MSE) test_MAE = np.mean(np.abs(data_dict["test_predictions"] - data_dict["test_targets"]), axis=1) + #print("val MSE ", val_MSE) print("test MSE ", test_MSE) all_test_RSE = [] @@ -1453,6 +1462,7 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): #train_data_len = int(len(data) * 0.8) train_data_len = config['n_train_samples'] # Train/valid/test split + print("train data len ", train_data_len) if not train: data_tr = data @@ -1462,7 +1472,6 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): data_tr = data.get_train_data(seed, train_data_len) #print("data tr ", data_tr) data_te = data.get_test_data(seed, train_data_len) - print("data te ", data_te) #data_tr, data_te = data[:train_data_len], data[train_data_len:] gnn = config['gnn'](**config['model_config']) diff --git a/mlmc/metamodel/flow_dataset.py b/mlmc/metamodel/flow_dataset.py index 45019c7b..1fc65f2b 100644 --- a/mlmc/metamodel/flow_dataset.py +++ b/mlmc/metamodel/flow_dataset.py @@ -21,7 +21,7 @@ class FlowDataset(Dataset): GRAPHS_FILE = "graphs" DATA_FILE = "data" - def __init__(self, output_dir=None, level=0, log=False, mesh=None, corr_field_config=None, config={}, index=None, + def __init__(self, output_dir=None, level=0, log=False, mesh=None, corr_field_config=None, config={}, index=None, adj_matrix=None, dataset=None, graphs=None, predict=False, **kwargs): self._output_dir = output_dir # if self._output_dir is None: @@ -30,7 +30,9 @@ def __init__(self, output_dir=None, level=0, log=False, mesh=None, corr_field_co self.level = level self._mesh = mesh self._corr_field_config = corr_field_config - self.adjacency_matrix = np.load(os.path.join(self._output_dir, "adjacency_matrix.npy"), allow_pickle=True) # adjacency matrix + self.adjacency_matrix = adj_matrix + if self.adjacency_matrix is None and self._output_dir is not None and os.path.exists(os.path.join(self._output_dir, "adjacency_matrix.npy")): + self.adjacency_matrix = np.load(os.path.join(self._output_dir, "adjacency_matrix.npy"), allow_pickle=True) # adjacency matrix self.data = [] self._aug_data = [] self._config = config @@ -62,9 +64,14 @@ def __init__(self, output_dir=None, level=0, log=False, mesh=None, corr_field_co self._columns = None - super().__init__(**kwargs) + self.dataset = dataset + self.graphs = graphs + + if self.dataset is None or self.graphs is None: + super().__init__(**kwargs) + else: + self.a = self.adjacency_matrix - #self.a = self.adjacency_matrix self.dataset = pd.DataFrame(self.data) self._df_for_augmentation = pd.DataFrame(self._aug_data, columns=self._columns) @@ -78,9 +85,22 @@ def get_train_data(self, index, length): if self._augment_data: new_dataset, new_graphs = self._data_augmentation(self._df_for_augmentation[index * length: index * length + length], new_graphs) - new_obj = copy.deepcopy(self) - new_obj.dataset = new_dataset - new_obj.graphs = new_graphs + new_obj = FlowDataset(output_dir=self._output_dir, level=self.level, log=self._log, mesh=self._mesh, + corr_field_config=self._corr_field_config, config=self._config, index=self._index, + adj_matrix=self.adjacency_matrix, dataset=new_dataset, graphs=new_graphs) + + # self_dict = self.__dict__ + # self_dict["dataset"] = new_dataset + # self_dict["graphs"] = new_graphs + # + #new_obj.__dict__.update(self_dict) + # + print("new_obj.dataset ", len(new_obj.dataset)) + print("self dataset len ", len(self.dataset)) + + #new_obj = copy.deepcopy(self) + #new_obj.dataset = new_dataset + #new_obj.graphs = new_graphs return new_obj @@ -106,9 +126,15 @@ def split_val_train(self, len_val_data): def get_test_data(self, index, length): new_dataset = self.dataset[0:index * length] + self.dataset[index * length + length:] new_graphs = self.graphs[0:index * length] + self.graphs[index * length + length:] - new_obj = copy.deepcopy(self) - new_obj.dataset = new_dataset - new_obj.graphs = new_graphs + + new_obj = FlowDataset(output_dir=self._output_dir, level=self.level, log=self._log, mesh=self._mesh, + corr_field_config=self._corr_field_config, config=self._config, index=self._index, + adj_matrix=self.adjacency_matrix, dataset=new_dataset, graphs=new_graphs) + + + # new_obj = copy.deepcopy(self) + # new_obj.dataset = new_dataset + # new_obj.graphs = new_graphs return new_obj def shuffle(self, seed=None): @@ -201,12 +227,11 @@ def _data_augmentation(self, df_slice, new_graphs): # # self.a = self.adjacency_matrix # return graphs - def read(self): all_outputs = [] all_features = [] - for s_dir in os.listdir(self._output_dir): + for idx, s_dir in enumerate(os.listdir(self._output_dir)): try: l = re.findall(r'L(\d+)_S', s_dir)[0] if int(l) != self.level: @@ -345,7 +370,6 @@ def read(self): # else: # self._df_for_augmentation = new_df - self.a = self.adjacency_matrix return graphs From ffd2e88c0db788ec1b4d1cfb9bfc1658c345e671 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Fri, 25 Mar 2022 16:59:27 +0100 Subject: [PATCH 60/67] gcn model state --- mlmc/metamodel/analyze_nn.py | 25 ++++++++++++++++++++----- mlmc/metamodel/flow_dataset.py | 13 ++----------- mlmc/metamodel/flow_task_GNN_2.py | 14 +++++++++----- 3 files changed, 31 insertions(+), 21 deletions(-) diff --git a/mlmc/metamodel/analyze_nn.py b/mlmc/metamodel/analyze_nn.py index 95aa756a..f7121293 100644 --- a/mlmc/metamodel/analyze_nn.py +++ b/mlmc/metamodel/analyze_nn.py @@ -1,4 +1,9 @@ +import warnings import os +import logging +logging.getLogger('tensorflow').disabled = True +logging.getLogger('absl').disabled = True + import numpy as np import time import glob @@ -15,6 +20,7 @@ from scipy import stats # np.set_printoptions(precision=9, suppress=True) import tensorflow as tf + from tensorflow import keras from scipy.stats import ks_2samp import sklearn.model_selection @@ -28,6 +34,8 @@ from spektral.data import MixedLoader from spektral.utils.sparse import sp_matrix_to_sp_tensor +tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR) + print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU'))) epochs = 100 @@ -1462,7 +1470,7 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): #train_data_len = int(len(data) * 0.8) train_data_len = config['n_train_samples'] # Train/valid/test split - print("train data len ", train_data_len) + #print("train data len ", train_data_len) if not train: data_tr = data @@ -1487,12 +1495,12 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): #np.random.shuffle(data_tr) val_data_len = int(len(data_tr) * config['val_samples_ratio']) - print("val data len ", val_data_len) + #print("val data len ", val_data_len) #data_tr, data_va = data_tr.split_val_train(val_data_len) data_tr, data_va = data_tr[:-val_data_len], data_tr[-val_data_len:] - print("data tr ", data_tr) - print("data va ", data_va) + # print("data tr ", data_tr) + # print("data va ", data_va) # print("data_tr len ", len(data_tr)) # print("data_va len ", len(data_va)) @@ -1529,10 +1537,17 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): learning_time = time.process_time() - learning_time_start states = gnn._states + # print("states ", states) + # for state in states.values(): + # print("state._model", state._model) + if len(states) > 0: min_key = np.min(list(states.keys())) gnn = states[min_key] + # print("gnn._model.layers[min].get_weights() ", states[np.min(list(states.keys()))]._model.layers[0].get_weights()) + # print("gnn._model.layers[max].get_weights() ", states[np.max(list(states.keys()))]._model.layers[0].get_weights()) + train_targets, train_predictions = gnn.predict(MixedLoader(data_tr, batch_size=batch_size, epochs=1)) train_predictions = np.squeeze(train_predictions) @@ -1625,7 +1640,7 @@ def predict_level_zero(nn, output_dir, hdf_path, mesh, conv_layer, batch_size=10 predict_time_start = time.process_time() data.a = conv_layer.preprocess(data.a) - data.a = sp_matrix_to_sp_tensor(data.a) + #data.a = sp_matrix_to_sp_tensor(data.a) loader_te = MixedLoader(data, batch_size=batch_size) diff --git a/mlmc/metamodel/flow_dataset.py b/mlmc/metamodel/flow_dataset.py index 1fc65f2b..9065310f 100644 --- a/mlmc/metamodel/flow_dataset.py +++ b/mlmc/metamodel/flow_dataset.py @@ -69,13 +69,11 @@ def __init__(self, output_dir=None, level=0, log=False, mesh=None, corr_field_co if self.dataset is None or self.graphs is None: super().__init__(**kwargs) + self.dataset = pd.DataFrame(self.data) + self._df_for_augmentation = pd.DataFrame(self._aug_data, columns=self._columns) else: self.a = self.adjacency_matrix - self.dataset = pd.DataFrame(self.data) - - self._df_for_augmentation = pd.DataFrame(self._aug_data, columns=self._columns) - #self._data_augmentation() def get_train_data(self, index, length): @@ -94,9 +92,6 @@ def get_train_data(self, index, length): # self_dict["graphs"] = new_graphs # #new_obj.__dict__.update(self_dict) - # - print("new_obj.dataset ", len(new_obj.dataset)) - print("self dataset len ", len(self.dataset)) #new_obj = copy.deepcopy(self) #new_obj.dataset = new_dataset @@ -131,10 +126,6 @@ def get_test_data(self, index, length): corr_field_config=self._corr_field_config, config=self._config, index=self._index, adj_matrix=self.adjacency_matrix, dataset=new_dataset, graphs=new_graphs) - - # new_obj = copy.deepcopy(self) - # new_obj.dataset = new_dataset - # new_obj.graphs = new_graphs return new_obj def shuffle(self, seed=None): diff --git a/mlmc/metamodel/flow_task_GNN_2.py b/mlmc/metamodel/flow_task_GNN_2.py index bb0c5f2e..a4931eaf 100644 --- a/mlmc/metamodel/flow_task_GNN_2.py +++ b/mlmc/metamodel/flow_task_GNN_2.py @@ -1,4 +1,9 @@ import os +import warnings +import logging +logging.getLogger('tensorflow').disabled = True +logging.getLogger('absl').disabled = True +warnings.simplefilter("ignore") import numpy as np #os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only import tensorflow as tf @@ -13,10 +18,12 @@ from spektral.layers import GCNConv, GlobalSumPool, ChebConv, GraphSageConv, ARMAConv, GATConv, APPNPConv, GINConv from spektral.utils.sparse import sp_matrix_to_sp_tensor from tensorflow.keras.layers.experimental import preprocessing +import copy from mlmc.metamodel.custom_methods import abs_activation, var_loss_function import keras.backend as K from mlmc.metamodel.graph_models import Net1 +tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR) class GNN: @@ -57,7 +64,6 @@ def __init__(self, **kwargs): else: model = kwargs.get('model') - if model is None: self._model = Net1(conv_layer=self._conv_layer, hidden_activation=self._hidden_activation, output_activation=self._output_activation, @@ -115,14 +121,13 @@ def fit(self, loader_tr, loader_va, loader_te): if results_va[0] < best_val_loss: best_val_loss = results_va[0] current_patience = self._patience - self._states = {} + #self._states = {} + self._states[results_va[0]] = copy.deepcopy(self) results_te = self.evaluate(loader_te) self._test_loss.append(results_te[0]) else: current_patience -= 1 #results_tr_0 = np.array(results_tr) - loss_va = results_va[0] - self._states[loss_va] = self if current_patience == 0: print("Early stopping") break @@ -196,7 +201,6 @@ def evaluate_batch(self, inputs, target): return loss, acc - def predict(self, loader): targets = [] predictions = [] From 8a5d0b9c39c8ef3f07854825358043e4185d9ab0 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Sat, 26 Mar 2022 13:56:45 +0100 Subject: [PATCH 61/67] flow dataset refactoring --- mlmc/metamodel/analyze_nn.py | 117 ++++++++++++++++++++------------- mlmc/metamodel/flow_dataset.py | 75 +++++++++++++++++---- 2 files changed, 134 insertions(+), 58 deletions(-) diff --git a/mlmc/metamodel/analyze_nn.py b/mlmc/metamodel/analyze_nn.py index f7121293..dc0e6225 100644 --- a/mlmc/metamodel/analyze_nn.py +++ b/mlmc/metamodel/analyze_nn.py @@ -1445,59 +1445,83 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): exit() preprocess_start_time = time.process_time() - # Load data - data = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log, config=config, index=seed) - data = data#[:10000] - # Dataset preprocess config - config['dataset_config'] = data._dataset_config + if train: + data_tr = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log, config=config, index=seed, train_samples=True) + print("data tr") + data_te = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log, config=config, index=seed, predict=True, test_samples=True) + else: + data = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log, config=config, index=seed) + data_tr = data + data_te = data - # print("n node features ", data.graphs[0].n_node_features) - # print("graph x", data.graphs[0].x) - # print("graphs[0] ", repr(data.graphs[0])) - # exit() + # Dataset preprocess config + config['dataset_config'] = data_tr._dataset_config - #print("len data ", len(data)) - #data.shuffle(seed=seed) preprocess_time = time.process_time() - preprocess_start_time - #print("preproces time ", preprocess_time) preprocess_time = preprocess_time + graph_creation_time - #print("total preprocess time ", preprocess_time) + print("preprocess time ", preprocess_time) learning_time_start = time.process_time() - data.a = config['conv_layer'].preprocess(data.a) - data.a = sp_matrix_to_sp_tensor(data.a) - #train_data_len = int(len(data) * 0.8) - train_data_len = config['n_train_samples'] - # Train/valid/test split - #print("train data len ", train_data_len) + data_tr.a = sp_matrix_to_sp_tensor(config['conv_layer'].preprocess(data_tr.a)) + data_te.a = data_tr.a #sp_matrix_to_sp_tensor(config['conv_layer'].preprocess(data_te.a)) - if not train: - data_tr = data - data_te = data - else: - #data_tr = data[seed*train_data_len: seed*train_data_len + train_data_len] - data_tr = data.get_train_data(seed, train_data_len) - #print("data tr ", data_tr) - data_te = data.get_test_data(seed, train_data_len) - #data_tr, data_te = data[:train_data_len], data[train_data_len:] + val_data_len = int(len(data_tr) * config['val_samples_ratio']) + # print("val data len ", val_data_len) + # data_tr, data_va = data_tr.split_val_train(val_data_len) + data_tr, data_va = data_tr[:-val_data_len], data_tr[-val_data_len:] - gnn = config['gnn'](**config['model_config']) + print("len data tr ", len(data_tr)) + print("len data va ", len(data_va)) + print("len data te ", len(data_te)) - # if hasattr(gnn._loss,'__name__') and gnn._loss.__name__ == "MSE_moments": - # tr_output = [g.y for g in data_tr] - # n_moments = 3 - # quantile = 0.001 - # domain = np.percentile(tr_output, [100 * quantile, 100 * (1 - quantile)]) - # moments_fn = Legendre_tf(n_moments, domain) - # #accuracy_func = MSE_moments(moments_fn=moments_fn) - # gnn._loss = MSE_moments(moments_fn=moments_fn) + # ############################# + # #### OLD version + # ############################# + # # Load data + # data = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log, config=config, index=seed) + # data = data#[:10000] + # + # # Dataset preprocess config + # config['dataset_config'] = data._dataset_config + # + # #print("len data ", len(data)) + # #data.shuffle(seed=seed) + # preprocess_time = time.process_time() - preprocess_start_time + # #print("preproces time ", preprocess_time) + # preprocess_time = preprocess_time + graph_creation_time + # #print("total preprocess time ", preprocess_time) + # + # learning_time_start = time.process_time() + # data.a = config['conv_layer'].preprocess(data.a) + # data.a = sp_matrix_to_sp_tensor(data.a) + # #train_data_len = int(len(data) * 0.8) + # train_data_len = config['n_train_samples'] + # # Train/valid/test split + # #print("train data len ", train_data_len) + # + # if not train: + # data_tr = data + # data_te = data + # else: + # #data_tr = data[seed*train_data_len: seed*train_data_len + train_data_len] + # data_tr = data.get_train_data(seed, train_data_len) + # #print("data tr ", data_tr) + # data_te = data.get_test_data(seed, train_data_len) + # #data_tr, data_te = data[:train_data_len], data[train_data_len:] + # + # #np.random.shuffle(data_tr) + # val_data_len = int(len(data_tr) * config['val_samples_ratio']) + # #print("val data len ", val_data_len) + # #data_tr, data_va = data_tr.split_val_train(val_data_len) + # data_tr, data_va = data_tr[:-val_data_len], data_tr[-val_data_len:] + + + ########################################### + ########################################### + ########################################### + ########################################### - #np.random.shuffle(data_tr) - val_data_len = int(len(data_tr) * config['val_samples_ratio']) - #print("val data len ", val_data_len) - #data_tr, data_va = data_tr.split_val_train(val_data_len) - data_tr, data_va = data_tr[:-val_data_len], data_tr[-val_data_len:] # print("data tr ", data_tr) # print("data va ", data_va) @@ -1506,6 +1530,9 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): # print("data_va len ", len(data_va)) # print("data_te len ", len(data_te)) + + gnn = config['gnn'](**config['model_config']) + # We use a MixedLoader since the dataset is in mixed mode loader_tr = MixedLoader(data_tr, batch_size=batch_size, epochs=epochs) loader_va = MixedLoader(data_va, batch_size=batch_size) @@ -1600,7 +1627,7 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): #predict_l_0_time = time.process_time() - predict_l_0_start_time if stats: - l1_sample_time = preprocess_time / len(data) + learning_time / len(data) + l1_sample_time = preprocess_time / (len(data_tr) + len(data_te)) + learning_time / (len(data_tr) + len(data_te)) l0_sample_time = predict_l_0_time / len(l_0_targets) # print("targets ", targets) @@ -1615,7 +1642,7 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): return gnn, targets, predictions, learning_time, train_targets, train_predictions,\ val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, total_steps - save_times(config['save_path'], False, (preprocess_time, len(data)), learning_time, (predict_l_0_time, len(l_0_targets))) + save_times(config['save_path'], False, (preprocess_time, (len(data_tr) + len(data_te))), learning_time, (predict_l_0_time, len(l_0_targets))) save_load_data(config['save_path'], False, targets, predictions, train_targets, train_predictions, val_targets, l_0_targets, l_0_predictions) @@ -1640,7 +1667,7 @@ def predict_level_zero(nn, output_dir, hdf_path, mesh, conv_layer, batch_size=10 predict_time_start = time.process_time() data.a = conv_layer.preprocess(data.a) - #data.a = sp_matrix_to_sp_tensor(data.a) + data.a = sp_matrix_to_sp_tensor(data.a) loader_te = MixedLoader(data, batch_size=batch_size) diff --git a/mlmc/metamodel/flow_dataset.py b/mlmc/metamodel/flow_dataset.py index 9065310f..20781ba0 100644 --- a/mlmc/metamodel/flow_dataset.py +++ b/mlmc/metamodel/flow_dataset.py @@ -22,7 +22,7 @@ class FlowDataset(Dataset): DATA_FILE = "data" def __init__(self, output_dir=None, level=0, log=False, mesh=None, corr_field_config=None, config={}, index=None, adj_matrix=None, dataset=None, graphs=None, - predict=False, **kwargs): + predict=False, train_samples=False, test_samples=False, n_test_samples=20000, **kwargs): self._output_dir = output_dir # if self._output_dir is None: # self._output_dir = OUTPUT_DIR @@ -38,6 +38,9 @@ def __init__(self, output_dir=None, level=0, log=False, mesh=None, corr_field_co self._config = config self._index = index self._predict = predict + self._train_samples = train_samples + self._test_samples = test_samples + self._n_test_samples = n_test_samples self._dataset_config = config.get('dataset_config', {}) self._augment_data = config.get('augment_data', False) @@ -83,9 +86,11 @@ def get_train_data(self, index, length): if self._augment_data: new_dataset, new_graphs = self._data_augmentation(self._df_for_augmentation[index * length: index * length + length], new_graphs) - new_obj = FlowDataset(output_dir=self._output_dir, level=self.level, log=self._log, mesh=self._mesh, - corr_field_config=self._corr_field_config, config=self._config, index=self._index, - adj_matrix=self.adjacency_matrix, dataset=new_dataset, graphs=new_graphs) + new_obj = FlowDataset(output_dir=copy.deepcopy(self._output_dir), level=copy.deepcopy(self.level), + log=copy.deepcopy(self._log), mesh=copy.deepcopy(self._mesh), + corr_field_config=copy.deepcopy(self._corr_field_config), + config=copy.deepcopy(self._config), index=copy.deepcopy(self._index), + adj_matrix=copy.deepcopy(self.adjacency_matrix), dataset=new_dataset, graphs=new_graphs) # self_dict = self.__dict__ # self_dict["dataset"] = new_dataset @@ -107,14 +112,26 @@ def split_val_train(self, len_val_data): va_graphs = self.graphs[-len_val_data:] #new_graphs = self.graphs[index * length: index * length + length] # self.graphs is read() method output - tr_obj = copy.deepcopy(self) - va_obj = copy.deepcopy(self) + tr_obj = FlowDataset(output_dir=copy.deepcopy(self._output_dir), level=copy.deepcopy(self.level), + log=copy.deepcopy(self._log), mesh=copy.deepcopy(self._mesh), + corr_field_config=copy.deepcopy(self._corr_field_config), + config=copy.deepcopy(self._config), index=copy.deepcopy(self._index), + adj_matrix=copy.deepcopy(self.adjacency_matrix), dataset=tr_dataset, graphs=tr_graphs) - tr_obj.dataset = tr_dataset - va_obj.dataset = va_dataset + va_obj = FlowDataset(output_dir=copy.deepcopy(self._output_dir), level=copy.deepcopy(self.level), + log=copy.deepcopy(self._log), mesh=copy.deepcopy(self._mesh), + corr_field_config=copy.deepcopy(self._corr_field_config), + config=copy.deepcopy(self._config), index=copy.deepcopy(self._index), + adj_matrix=copy.deepcopy(self.adjacency_matrix), dataset=va_dataset, graphs=va_graphs) - tr_obj.graphs = tr_graphs - va_obj.graphs = va_graphs + # tr_obj = copy.deepcopy(self) + # va_obj = copy.deepcopy(self) + # + # tr_obj.dataset = tr_dataset + # va_obj.dataset = va_dataset + # + # tr_obj.graphs = tr_graphs + # va_obj.graphs = va_graphs return tr_obj, va_obj @@ -122,9 +139,9 @@ def get_test_data(self, index, length): new_dataset = self.dataset[0:index * length] + self.dataset[index * length + length:] new_graphs = self.graphs[0:index * length] + self.graphs[index * length + length:] - new_obj = FlowDataset(output_dir=self._output_dir, level=self.level, log=self._log, mesh=self._mesh, - corr_field_config=self._corr_field_config, config=self._config, index=self._index, - adj_matrix=self.adjacency_matrix, dataset=new_dataset, graphs=new_graphs) + new_obj = FlowDataset(output_dir=copy.deepcopy(self._output_dir), level=copy.deepcopy(self.level), log=copy.deepcopy(self._log), mesh=copy.deepcopy(self._mesh), + corr_field_config=copy.deepcopy(self._corr_field_config), config=copy.deepcopy(self._config), index=copy.deepcopy(self._index), + adj_matrix=copy.deepcopy(self.adjacency_matrix), dataset=new_dataset, graphs=new_graphs) return new_obj @@ -218,6 +235,7 @@ def _data_augmentation(self, df_slice, new_graphs): # # self.a = self.adjacency_matrix # return graphs + def read(self): all_outputs = [] all_features = [] @@ -233,6 +251,13 @@ def read(self): sample_dir = os.path.join(self._output_dir, s_dir) if os.path.exists(os.path.join(sample_dir, "nodes_features.npy")): features = np.load(os.path.join(sample_dir, "nodes_features.npy")) + # print("type features ", type(features)) + # print("features shape ", features.shape) + # features = np.expand_dims(features, axis=0) + # if all_features is None: + # all_features = features + # else: + # all_features = np.vstack((all_features, features)) all_features.append(features) output = np.load(os.path.join(sample_dir, "output.npy")) @@ -289,6 +314,30 @@ def read(self): self._save_data_config() + if self._train_samples: + all_features = train_features + all_outputs = train_outputs + + elif self._test_samples: + if isinstance(all_outputs, list): + all_outputs = all_outputs[0:self._index * self._config['n_train_samples']] + all_outputs[self._index * self._config['n_train_samples'] + + self._config['n_train_samples']:] + else: + all_outputs = np.concatenate([all_outputs[0:self._index * self._config['n_train_samples']], all_outputs[self._index * self._config['n_train_samples'] + self._config['n_train_samples']:]]) + + if isinstance(all_features, list): + all_features = all_features[0:self._index * self._config['n_train_samples']] + all_features[ + self._index * self._config['n_train_samples'] + self._config[ + 'n_train_samples']:] + else: + all_features = np.concatenate([all_features[0:self._index * self._config['n_train_samples']], + all_features[ + self._index * self._config['n_train_samples'] + self._config[ + 'n_train_samples']:]]) + + all_outputs = all_outputs[:self._n_test_samples] + all_features = all_features[:self._n_test_samples] + graphs = [] for features, output in zip(all_features, all_outputs): if self._dataset_config.get("features_normalization", False): From a57fa341d011615e57e33ade8bfa55697b1b691d Mon Sep 17 00:00:00 2001 From: Martin Spetlik Date: Wed, 13 Apr 2022 16:52:07 +0200 Subject: [PATCH 62/67] restore states --- mlmc/metamodel/flow_task_GNN_2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlmc/metamodel/flow_task_GNN_2.py b/mlmc/metamodel/flow_task_GNN_2.py index a4931eaf..d4762083 100644 --- a/mlmc/metamodel/flow_task_GNN_2.py +++ b/mlmc/metamodel/flow_task_GNN_2.py @@ -121,7 +121,7 @@ def fit(self, loader_tr, loader_va, loader_te): if results_va[0] < best_val_loss: best_val_loss = results_va[0] current_patience = self._patience - #self._states = {} + self._states = {} self._states[results_va[0]] = copy.deepcopy(self) results_te = self.evaluate(loader_te) self._test_loss.append(results_te[0]) From e9aac3140745cf9561ca96c3d09d752104b9ce32 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Wed, 13 Apr 2022 17:22:12 +0200 Subject: [PATCH 63/67] independent samples --- mlmc/metamodel/analyze_nn.py | 217 ++++++++++++++++++++++++++++----- mlmc/metamodel/flow_dataset.py | 54 +++++--- 2 files changed, 224 insertions(+), 47 deletions(-) diff --git a/mlmc/metamodel/analyze_nn.py b/mlmc/metamodel/analyze_nn.py index dc0e6225..a913debf 100644 --- a/mlmc/metamodel/analyze_nn.py +++ b/mlmc/metamodel/analyze_nn.py @@ -33,6 +33,7 @@ from tensorflow.keras.losses import MeanSquaredError from spektral.data import MixedLoader from spektral.utils.sparse import sp_matrix_to_sp_tensor +from sklearn.metrics import r2_score tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR) @@ -391,7 +392,7 @@ def statistics(config): gnn, targets, predictions, learning_time, train_targets, train_predictions, \ val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, total_steps = \ - mch_l_model(config, stats=True, train=config.get('train_model', True), log=log, seed=i) + mch_l_model(config, stats=True, train=config.get('train_model', True), log=log, index=i) if config['save_model']: model_data["model"] = gnn._model @@ -751,6 +752,8 @@ def analyze_statistics(config, get_model=True): data_dict = process_data(data_dict) + rescale_data = True + # print("train predictions type ", type(data_dict["train_predictions"])) # print("train predictions type ", type(data_dict["train_predictions"][0])) # print("train predictions shape ", np.array(data_dict["train_predictions"]).shape) @@ -788,6 +791,16 @@ def analyze_statistics(config, get_model=True): nn_vars_mse = [] mlmc_means_mse = [] nn_means_mse = [] + mlmc_means_diff = [] + nn_means_diff = [] + + mlmc_vars_mse_2 = [] + nn_vars_mse_2 = [] + mlmc_means_mse_2 = [] + nn_means_mse_2 = [] + mlmc_means_diff_2 = [] + nn_means_diff_2 = [] + kl_mlmc_all = [] kl_nn_all = [] @@ -800,8 +813,16 @@ def analyze_statistics(config, get_model=True): test_MSE_list = [] test_bias = [] test_variance = [] + train_RSE_list = [] + train_RMSE_list = [] + train_MAE_list = [] + + test_RSE_list = [] + test_RMSE_list = [] + test_MAE_list = [] - limit = 1e10 # 0.008#0.01#0.0009 + + limit = 5 # 0.008#0.01#0.0009 #limit = 0.37 for i in range(len(data_dict["test_targets"])): @@ -855,7 +876,7 @@ def analyze_statistics(config, get_model=True): if model is not None: plot_loss(model_train_loss, model_val_loss, model_train_acc) - plot_learning_rate(model_learning_rates) + #plot_learning_rate(model_learning_rates) print("model learning rates ", model_learning_rates) print("model ", model) @@ -865,18 +886,79 @@ def analyze_statistics(config, get_model=True): # print("model.optimizer", K.eval(model.optimizer.lr)) # exit() + if rescale_data: + if "dataset_config" in data_dict: + dataset_config = data_dict["dataset_config"][i] + + if dataset_config.get('output_normalization', False): + min_out = dataset_config.get('min_output') + max_out = dataset_config.get('max_output') + + targets = targets * (max_out - min_out) + min_out + predictions = predictions * (max_out - min_out) + min_out + train_targets = train_targets * (max_out - min_out) + min_out + train_predictions = train_predictions * (max_out - min_out) + min_out + + if dataset_config.get('output_scale', False): + # mean_targets = np.mean(targets) + # var_targets = np.var(targets) + + mean_targets = dataset_config.get('mean_output', False) + var_targets = dataset_config.get('var_output', False) + + targets = var_targets * targets + mean_targets + predictions = var_targets * predictions + mean_targets + + # mean_l_0_targets = mean_targets + # var_l_0_targets = var_targets + + train_targets = var_targets * train_targets + mean_targets + train_predictions = var_targets * train_predictions + mean_targets + + if dataset_config.get('output_log', False): + targets = np.exp(targets) + predictions = np.exp(predictions) + train_predictions = np.exp(train_predictions) + train_targets = np.exp(train_targets) + + if dataset_config.get('first_log_output', False): + targets = np.exp(targets) + predictions = np.exp(predictions) + train_predictions = np.exp(train_predictions) + train_targets = np.exp(train_targets) + iter_test_MSE = np.mean((predictions - targets) ** 2) + iter_test_bias = np.sqrt(np.mean((targets - np.mean(predictions)) ** 2)) iter_test_variance = np.mean((predictions - np.mean(predictions)) ** 2) iter_train_MSE = np.mean((train_predictions - train_targets) ** 2) + iter_train_bias = np.sqrt(np.mean((train_targets - np.mean(train_predictions)) ** 2)) iter_train_variance = np.mean((train_predictions - np.mean(train_predictions)) ** 2) + mean_t = np.mean(targets) + iter_test_RSE = np.sum((predictions - targets) ** 2) / np.sum((targets - mean_t) ** 2) + + mean_tr = np.mean(train_targets) + iter_train_RSE = np.sum((train_predictions - train_targets) ** 2) / np.sum((train_targets - mean_tr) ** 2) + + iter_test_MAE = np.abs((predictions - targets)) + iter_train_MAE = np.abs((train_predictions - train_targets)) + train_MSE_list.append(iter_train_MSE) + train_RSE_list.append(iter_train_RSE) + train_RMSE_list.append(np.sqrt(iter_train_MSE)) + train_MAE_list.append(iter_train_MAE) + train_bias.append(iter_train_bias) train_variance.append(iter_train_variance) + test_MSE_list.append(iter_test_MSE) + test_RSE_list.append(iter_test_RSE) + test_RMSE_list.append(np.sqrt(iter_test_MSE)) + test_MAE_list.append(iter_test_MAE) + test_bias.append(iter_test_bias) test_variance.append(iter_test_variance) @@ -913,13 +995,18 @@ def analyze_statistics(config, get_model=True): try: mlmc_n_collected, nn_mlmc_n_collected, n_ops, n_ops_predict, orig_moments_mean, predict_moments_mean, \ ref_moments_mean, orig_level_params, nn_level_params, kl_mlmc, kl_nn, target_variance, \ - orig_orth_moments, predict_orth_moments, ref_orth_moments = process_mlmc(config['hdf_path'], + orig_orth_moments, predict_orth_moments, ref_orth_moments,\ + ref_orig_moments, ref_predict_moments = process_mlmc(config['hdf_path'], config['sampling_info_path'], - config['ref_mlmc_file'], targets, - predictions, train_targets, - train_predictions, - val_targets, l_0_targets, - l_0_predictions, l1_sample_time, + config['ref_mlmc_file'], + data_dict["test_targets"][i], + data_dict["test_predictions"][i], + data_dict["train_targets"][i], + data_dict["train_predictions"][i], + data_dict["val_targets"][i], + data_dict["l_0_targets"][i], + data_dict["l_0_predictions"][i], + l1_sample_time, l0_sample_time, nn_level=config['level'], replace_level=config['replace_level'], @@ -952,7 +1039,24 @@ def analyze_statistics(config, get_model=True): nn_vars_mse.append((ref_moments_mean.var - predict_moments_mean.var) ** 2) mlmc_means_mse.append((ref_moments_mean.mean - orig_moments_mean.mean) ** 2) - nn_means_mse.append((ref_moments_mean.mean) ** 2 - predict_moments_mean.mean) + nn_means_mse.append((ref_moments_mean.mean - predict_moments_mean.mean) ** 2) + + mlmc_means_diff.append(np.abs(ref_moments_mean.mean - orig_moments_mean.mean)) + nn_means_diff.append(np.abs(ref_moments_mean.mean - predict_moments_mean.mean)) + + + ####################################### + # Moments same domain for comparison # + ####################################### + + mlmc_vars_mse_2.append((ref_orig_moments[0].var - ref_orig_moments[1].var) ** 2) + nn_vars_mse_2.append((ref_predict_moments[0].var - ref_predict_moments[1].var) ** 2) + + mlmc_means_mse_2.append((ref_orig_moments[0].mean - ref_orig_moments[1].mean) ** 2) + nn_means_mse_2.append((ref_predict_moments[0].mean - ref_predict_moments[1].mean) ** 2) + + mlmc_means_diff_2.append(np.abs(ref_orig_moments[0].mean - ref_orig_moments[1].mean)) + nn_means_diff_2.append(np.abs(ref_predict_moments[0].mean - ref_predict_moments[1].mean)) # print("np.min(len(ref_orth_moments.mean), len(orig_orth_moments.mean)) ", np.min(len(ref_orth_moments.mean), len(orig_orth_moments.mean))) # print("ref_orth_moments.mean[:np.min(len(ref_orth_moments.mean), len(orig_orth_moments.mean))] ", ref_orth_moments.mean[:np.min(len(ref_orth_moments.mean), len(orig_orth_moments.mean))]) @@ -1004,6 +1108,9 @@ def analyze_statistics(config, get_model=True): print("mlmc means MSE ", np.mean(mlmc_means_mse, axis=0)) print("nn means MSE ", np.mean(nn_means_mse, axis=0)) + print("mlmc means diff ", np.mean(mlmc_means_diff, axis=0)) + print("nn means diff ", np.mean(nn_means_diff, axis=0)) + print("mlmc times ", mlmc_times) print("nn times ", nn_times) @@ -1041,10 +1148,10 @@ def analyze_statistics(config, get_model=True): mlmc_n_collected = np.mean(mlmc_n_collected_all, axis=0) nn_n_collected = np.mean(nn_n_collected_all, axis=0) - # print("mlmc n collected ", mlmc_n_collected_all) - # print("nn n collected all ", nn_n_collected_all) - # print("mlmc n collected ", mlmc_n_collected) - # print("nn n collected ", nn_n_collected) + print("mlmc n collected ", mlmc_n_collected_all) + print("nn n collected all ", nn_n_collected_all) + print("mlmc n collected ", mlmc_n_collected) + print("nn n collected ", nn_n_collected) plt_var = plots.VarianceNN() plt_var.set_n_ops(np.mean(n_ops_predict_all, axis=0)) @@ -1074,6 +1181,11 @@ def analyze_statistics(config, get_model=True): plot_sse(nn_means_mse, mlmc_means_mse, title="moments_mean") plot_sse(mlmc_means_mse, mlmc_means_mse, title="mlmc moments_mean") + plot_sse(nn_vars_mse_2, mlmc_vars_mse_2, title="moments_var same domain") + plot_sse(nn_means_mse_2, mlmc_means_mse_2, title="moments_mean same domain") + plot_sse(mlmc_means_mse_2, mlmc_means_mse_2, title="mlmc moments_mean same domain") + + # if ref_orth_moments is not None: # print("orth nn means mse ", orth_nn_means_mse) # print("orth mlmc means mse ", orth_mlmc_means_mse) @@ -1167,6 +1279,11 @@ def analyze_statistics(config, get_model=True): print("test RAE ", test_RAE) print("test_MSE ", test_MSE) + + for pred, target in zip(data_dict["test_predictions"], data_dict["test_targets"]): + R_squared = r2_score(target, pred) + print("R squared ", R_squared) + t_mse_sum = [] for t_mse in test_MSE: # Note: je mozne odstranit vetsi hodnoty MSE pro L4, protoze by slo dosahnout mensich hodnot pokud by se navysil pocet iteraci nebo by se vysledek pro nejlepsi train + val MSE a ne posledni vysledek @@ -1212,9 +1329,9 @@ def analyze_statistics(config, get_model=True): print("NN moments MSE sum ", np.sum(np.mean(nn_means_mse, axis=0))) print("mean test MSE ", np.mean(test_MSE)) - # print("mean test RSE ", np.mean(test_RSE)) - # print("mean test RMSE ", np.mean(test_RMSE)) - # print("mean test MAE ", np.mean(test_MAE)) + print("mean test RSE ", np.mean(test_RSE)) + print("mean test RMSE ", np.mean(test_RMSE)) + print("mean test MAE ", np.mean(test_MAE)) print("max test MSE ", np.max(test_MSE)) # print("max test RMSE ", np.max(test_RMSE)) # print("max test MAE ", np.max(test_MAE)) @@ -1225,9 +1342,9 @@ def analyze_statistics(config, get_model=True): print("test RSE ", np.mean(all_test_RSE)) print("test RSE ", np.mean(all_train_RSE)) - # print("mean train RSE ", np.mean(train_RSE)) - # print("mean train RMSE ", np.mean(train_RMSE)) - # print("mean train MAE ", np.mean(train_MAE)) + print("mean train RSE ", np.mean(train_RSE)) + print("mean train RMSE ", np.mean(train_RMSE)) + print("mean train MAE ", np.mean(train_MAE)) print("max train MSE ", np.max(train_MSE)) # print("max train RMSE ", np.max(train_RMSE)) # print("max train MAE ", np.max(train_MAE)) @@ -1258,9 +1375,33 @@ def analyze_statistics(config, get_model=True): print("train MSE: {}, bias: {}, variance: {}".format(np.mean(train_MSE_list), np.mean(train_bias), np.mean(train_variance))) print("test MSE: {}, bias: {}, variance: {}".format(np.mean(test_MSE_list), np.mean(test_bias), np.mean(test_variance))) + print("SEM train MSE: {}".format(stats.sem(train_MSE_list))) + print("SEM test MSE: {}".format(stats.sem(test_MSE_list))) # print("test MSE std", np.sqrt(np.var(test_MSE))) - print("train RSE ", np.mean(all_train_RSE)) - print("test RSE ", np.mean(all_test_RSE)) + print("RMSE train: {}, test: {}".format(np.mean(train_RMSE_list), np.mean(test_RMSE_list))) + print("RSE train: {}, test: {}".format(np.mean(train_RSE_list), np.mean(test_RSE_list))) + print("MAE train: {}, test: {}".format(np.mean(train_MAE_list), np.mean(test_MAE_list))) + + # print("train RSE ", np.mean(train_RSE)) + # print("test RSE ", np.mean(test_RMSE)) + # + # print("train RMSE ", np.mean(train_RMSE)) + # print("test RMSE ", np.mean(test_RMSE)) + + print("MC mom mean MSE: total: {:0.5g}, to 10: {:0.5g}, above: {:0.5g}".format(np.sum(np.mean(mlmc_means_mse, axis=0)), np.sum(np.mean(mlmc_means_mse, axis=0)[:10]), + np.sum(np.mean(mlmc_means_mse, axis=0)[10:]))) + print("NN mom mean MSE: total: {:0.5g}, to 10: {:0.5g}, above: {:0.5g}".format(np.sum(np.mean(nn_means_mse, axis=0)), np.sum(np.mean(nn_means_mse, axis=0)[:10]), + np.sum(np.mean(nn_means_mse, axis=0)[10:]))) + + print("compare moments each couple has same domain (common domain)") + print("MC mom mean MSE: total: {:0.5g}, to 10: {:0.5g}, above: {:0.5g}".format( + np.sum(np.mean(mlmc_means_mse_2, axis=0)), np.sum(np.mean(mlmc_means_mse_2, axis=0)[:10]), + np.sum(np.mean(mlmc_means_mse_2, axis=0)[10:]))) + print("NN mom mean MSE: total: {:0.5g}, to 10: {:0.5g}, above: {:0.5g}".format(np.sum(np.mean(nn_means_mse_2, axis=0)), + np.sum(np.mean(nn_means_mse_2, axis=0)[ + :10]), + np.sum(np.mean(nn_means_mse_2, axis=0)[ + 10:]))) print("nn total time ", nn_total_time) print("mlmc total time ", mlmc_total_time) @@ -1420,8 +1561,8 @@ def set_model_layers(new_model, old_model): # exit() -def run_GNN(config, stats=True, train=True, log=False, seed=0): - print("seed ", seed) +def run_GNN(config, stats=True, train=True, log=False, index=0): + print("seed ", index) loss = MeanSquaredError() # var_loss_function# accuracy_func = MSE_moments @@ -1447,14 +1588,30 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): preprocess_start_time = time.process_time() if train: - data_tr = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log, config=config, index=seed, train_samples=True) - print("data tr") - data_te = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log, config=config, index=seed, predict=True, test_samples=True) + data_tr = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log, config=config, index=index, train_samples=True) + data_te = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log, config=config, index=index, predict=True, test_samples=True) else: - data = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log, config=config, index=seed) + data = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log, config=config, index=index) data_tr = data data_te = data + independent_samples = config.get("independent_samples", False) + + if independent_samples and train: + len_all_samples = len(data_tr) + len(data_te) + + last_train_sample = index * config['n_train_samples'] + config['n_train_samples'] + last_test_sample = len_all_samples - (index * config['n_train_samples'] + config['n_train_samples']) + + if last_train_sample > last_test_sample: + return + + data_tr = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log, config=config, + index=index, train_samples=True, independent_sample=True) + + data_te = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log, config=config, + index=index, predict=True, test_samples=True, independent_samples=True) + # Dataset preprocess config config['dataset_config'] = data_tr._dataset_config @@ -1522,7 +1679,6 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): ########################################### ########################################### - # print("data tr ", data_tr) # print("data va ", data_va) @@ -1530,7 +1686,6 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): # print("data_va len ", len(data_va)) # print("data_te len ", len(data_te)) - gnn = config['gnn'](**config['model_config']) # We use a MixedLoader since the dataset is in mixed mode @@ -1620,7 +1775,7 @@ def run_GNN(config, stats=True, train=True, log=False, seed=0): config['conv_layer'], batch_size, log, stats=stats, corr_field_config=config['corr_field_config'], - seed=seed, + seed=index, feature_names=config.get('feature_names', [['conductivity']]), config=config ) diff --git a/mlmc/metamodel/flow_dataset.py b/mlmc/metamodel/flow_dataset.py index 20781ba0..60d01b7b 100644 --- a/mlmc/metamodel/flow_dataset.py +++ b/mlmc/metamodel/flow_dataset.py @@ -22,7 +22,7 @@ class FlowDataset(Dataset): DATA_FILE = "data" def __init__(self, output_dir=None, level=0, log=False, mesh=None, corr_field_config=None, config={}, index=None, adj_matrix=None, dataset=None, graphs=None, - predict=False, train_samples=False, test_samples=False, n_test_samples=20000, **kwargs): + predict=False, train_samples=False, test_samples=False, n_test_samples=20000, independent_samples=False, **kwargs): self._output_dir = output_dir # if self._output_dir is None: # self._output_dir = OUTPUT_DIR @@ -40,6 +40,7 @@ def __init__(self, output_dir=None, level=0, log=False, mesh=None, corr_field_co self._predict = predict self._train_samples = train_samples self._test_samples = test_samples + self._independent_samples = independent_samples self._n_test_samples = n_test_samples self._dataset_config = config.get('dataset_config', {}) self._augment_data = config.get('augment_data', False) @@ -136,8 +137,17 @@ def split_val_train(self, len_val_data): return tr_obj, va_obj def get_test_data(self, index, length): - new_dataset = self.dataset[0:index * length] + self.dataset[index * length + length:] - new_graphs = self.graphs[0:index * length] + self.graphs[index * length + length:] + if self._independent_samples: + if index > 0: + new_dataset =self.dataset[-index * length - length:-index * length] + new_graphs = self.graphs[-index * length - length:-index * length] + else: + new_dataset = self.dataset[-index * length - length:] + new_graphs = self.graphs[-index * length - length:] + + else: + new_dataset = self.dataset[0:index * length] + self.dataset[index * length + length:] + new_graphs = self.graphs[0:index * length] + self.graphs[index * length + length:] new_obj = FlowDataset(output_dir=copy.deepcopy(self._output_dir), level=copy.deepcopy(self.level), log=copy.deepcopy(self._log), mesh=copy.deepcopy(self._mesh), corr_field_config=copy.deepcopy(self._corr_field_config), config=copy.deepcopy(self._config), index=copy.deepcopy(self._index), @@ -319,21 +329,33 @@ def read(self): all_outputs = train_outputs elif self._test_samples: - if isinstance(all_outputs, list): - all_outputs = all_outputs[0:self._index * self._config['n_train_samples']] + all_outputs[self._index * self._config['n_train_samples'] + - self._config['n_train_samples']:] - else: - all_outputs = np.concatenate([all_outputs[0:self._index * self._config['n_train_samples']], all_outputs[self._index * self._config['n_train_samples'] + self._config['n_train_samples']:]]) + if self._independent_samples: + if self._index > 0: + all_outputs = all_outputs[-self._index * self._config['n_train_samples']- self._config['n_train_samples']:-self._index * self._config['n_train_samples']] + all_features = all_features[-self._index * self._config['n_train_samples']- self._config['n_train_samples']:-self._index *self._config['n_train_samples']] + else: + all_outputs = all_outputs[-self._index * self._config['n_train_samples'] - self._config[ + 'n_train_samples']:] + all_features = all_features[-self._index * self._config['n_train_samples'] - self._config[ + 'n_train_samples']:] - if isinstance(all_features, list): - all_features = all_features[0:self._index * self._config['n_train_samples']] + all_features[ - self._index * self._config['n_train_samples'] + self._config[ - 'n_train_samples']:] else: - all_features = np.concatenate([all_features[0:self._index * self._config['n_train_samples']], - all_features[ - self._index * self._config['n_train_samples'] + self._config[ - 'n_train_samples']:]]) + + if isinstance(all_outputs, list): + all_outputs = all_outputs[0:self._index * self._config['n_train_samples']] + all_outputs[self._index * self._config['n_train_samples'] + + self._config['n_train_samples']:] + else: + all_outputs = np.concatenate([all_outputs[0:self._index * self._config['n_train_samples']], all_outputs[self._index * self._config['n_train_samples'] + self._config['n_train_samples']:]]) + + if isinstance(all_features, list): + all_features = all_features[0:self._index * self._config['n_train_samples']] + all_features[ + self._index * self._config['n_train_samples'] + self._config[ + 'n_train_samples']:] + else: + all_features = np.concatenate([all_features[0:self._index * self._config['n_train_samples']], + all_features[ + self._index * self._config['n_train_samples'] + self._config[ + 'n_train_samples']:]]) all_outputs = all_outputs[:self._n_test_samples] all_features = all_features[:self._n_test_samples] From 7125462de77da51f5b114d81f32a2d338ecb2349 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Thu, 14 Apr 2022 13:29:11 +0200 Subject: [PATCH 64/67] fix test samples size --- mlmc/metamodel/analyze_nn.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/mlmc/metamodel/analyze_nn.py b/mlmc/metamodel/analyze_nn.py index a913debf..a01a4807 100644 --- a/mlmc/metamodel/analyze_nn.py +++ b/mlmc/metamodel/analyze_nn.py @@ -1587,18 +1587,12 @@ def run_GNN(config, stats=True, train=True, log=False, index=0): preprocess_start_time = time.process_time() - if train: - data_tr = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log, config=config, index=index, train_samples=True) - data_te = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log, config=config, index=index, predict=True, test_samples=True) - else: - data = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log, config=config, index=index) - data_tr = data - data_te = data - independent_samples = config.get("independent_samples", False) if independent_samples and train: - len_all_samples = len(data_tr) + len(data_te) + data = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log, config=config, + index=index, n_test_samples=100000) + len_all_samples = len(data) last_train_sample = index * config['n_train_samples'] + config['n_train_samples'] last_test_sample = len_all_samples - (index * config['n_train_samples'] + config['n_train_samples']) @@ -1612,6 +1606,19 @@ def run_GNN(config, stats=True, train=True, log=False, index=0): data_te = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log, config=config, index=index, predict=True, test_samples=True, independent_samples=True) + else: + if train: + data_tr = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log, config=config, + index=index, train_samples=True) + data_te = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log, config=config, + index=index, predict=True, test_samples=True) + else: + data = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log, config=config, + index=index) + data_tr = data + data_te = data + + # Dataset preprocess config config['dataset_config'] = data_tr._dataset_config From 6a6ca9419319aea59cdb4702b0846b4abf48e55e Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Fri, 22 Apr 2022 16:18:57 +0200 Subject: [PATCH 65/67] data for estimates --- mlmc/metamodel/analyze_nn.py | 154 ++++++++++++++++++++++++++------- mlmc/metamodel/flow_dataset.py | 2 +- 2 files changed, 125 insertions(+), 31 deletions(-) diff --git a/mlmc/metamodel/analyze_nn.py b/mlmc/metamodel/analyze_nn.py index a01a4807..53866788 100644 --- a/mlmc/metamodel/analyze_nn.py +++ b/mlmc/metamodel/analyze_nn.py @@ -391,7 +391,8 @@ def statistics(config): config['iter_dir'] = iter_dir gnn, targets, predictions, learning_time, train_targets, train_predictions, \ - val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, total_steps = \ + val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time,\ + total_steps, targets_to_est, predictions_to_est = \ mch_l_model(config, stats=True, train=config.get('train_model', True), log=log, index=i) if config['save_model']: @@ -413,6 +414,8 @@ def statistics(config): model_data["l0_sample_time"] = l0_sample_time model_data["total_steps"] = total_steps model_data["learning_times"] = learning_time + model_data["targets_to_est"] = targets_to_est + model_data["predictions_to_est"] = predictions_to_est save_statistics(iter_dir, model_data) @@ -466,7 +469,8 @@ def load_statistics(dir_path): models_data["learning_times"] = [] models_data["log"] = [] models_data["dataset_config"] = [] - + models_data["targets_to_est"] = [] + models_data["predictions_to_est"] = [] #dirs = (os.path.split(dir_path)[-1]).split("_") n_iters = 25 @@ -796,10 +800,18 @@ def analyze_statistics(config, get_model=True): mlmc_vars_mse_2 = [] nn_vars_mse_2 = [] + mlmc_nn_vars_mse_2 = [] mlmc_means_mse_2 = [] nn_means_mse_2 = [] + mlmc_nn_means_mse_2 = [] mlmc_means_diff_2 = [] nn_means_diff_2 = [] + mlmc_nn_means_diff_2 = [] + + mlmc_moments_mean = [] + mlmc_nn_moments_mean = [] + mlmc_moments_var = [] + mlmc_nn_moments_var = [] kl_mlmc_all = [] kl_nn_all = [] @@ -816,6 +828,12 @@ def analyze_statistics(config, get_model=True): train_RSE_list = [] train_RMSE_list = [] train_MAE_list = [] + train_relRMSE_list = [] + test_relRMSE_list = [] + + + all_train_samples = [] + all_test_samples = [] test_RSE_list = [] test_RMSE_list = [] @@ -835,7 +853,7 @@ def analyze_statistics(config, get_model=True): #print("index ", i) - # if i not in [0,2]: + # if i not in [0]: # continue # if i in [2, 11, 12]: @@ -877,7 +895,7 @@ def analyze_statistics(config, get_model=True): if model is not None: plot_loss(model_train_loss, model_val_loss, model_train_acc) #plot_learning_rate(model_learning_rates) - print("model learning rates ", model_learning_rates) + #print("model learning rates ", model_learning_rates) print("model ", model) print("dir(model.optimizer) ", dir(model.optimizer)) @@ -937,6 +955,9 @@ def analyze_statistics(config, get_model=True): iter_train_bias = np.sqrt(np.mean((train_targets - np.mean(train_predictions)) ** 2)) iter_train_variance = np.mean((train_predictions - np.mean(train_predictions)) ** 2) + all_test_samples.append(targets) + all_train_samples.append(train_targets) + mean_t = np.mean(targets) iter_test_RSE = np.sum((predictions - targets) ** 2) / np.sum((targets - mean_t) ** 2) @@ -949,6 +970,7 @@ def analyze_statistics(config, get_model=True): train_MSE_list.append(iter_train_MSE) train_RSE_list.append(iter_train_RSE) train_RMSE_list.append(np.sqrt(iter_train_MSE)) + train_relRMSE_list.append(np.sqrt(iter_train_MSE)/np.mean(train_targets)) train_MAE_list.append(iter_train_MAE) train_bias.append(iter_train_bias) @@ -957,6 +979,7 @@ def analyze_statistics(config, get_model=True): test_MSE_list.append(iter_test_MSE) test_RSE_list.append(iter_test_RSE) test_RMSE_list.append(np.sqrt(iter_test_MSE)) + test_relRMSE_list.append(np.sqrt(iter_train_MSE) / np.mean(targets)) test_MAE_list.append(iter_test_MAE) test_bias.append(iter_test_bias) @@ -992,30 +1015,32 @@ def analyze_statistics(config, get_model=True): #exit() print("total steps ", total_steps) - try: - mlmc_n_collected, nn_mlmc_n_collected, n_ops, n_ops_predict, orig_moments_mean, predict_moments_mean, \ - ref_moments_mean, orig_level_params, nn_level_params, kl_mlmc, kl_nn, target_variance, \ - orig_orth_moments, predict_orth_moments, ref_orth_moments,\ - ref_orig_moments, ref_predict_moments = process_mlmc(config['hdf_path'], - config['sampling_info_path'], - config['ref_mlmc_file'], - data_dict["test_targets"][i], - data_dict["test_predictions"][i], - data_dict["train_targets"][i], - data_dict["train_predictions"][i], - data_dict["val_targets"][i], - data_dict["l_0_targets"][i], - data_dict["l_0_predictions"][i], - l1_sample_time, - l0_sample_time, - nn_level=config['level'], - replace_level=config['replace_level'], - mlmc_hdf_file=config['mlmc_hdf_path'], - stats=True, - learning_time=learning_time, - dataset_config=dataset_config) - except: - continue + #try: + mlmc_n_collected, nn_mlmc_n_collected, n_ops, n_ops_predict, orig_moments_mean, predict_moments_mean, \ + ref_moments_mean, orig_level_params, nn_level_params, kl_mlmc, kl_nn, target_variance, \ + orig_orth_moments, predict_orth_moments, ref_orth_moments,\ + ref_orig_moments, ref_predict_moments, mlmc_predict_moments = process_mlmc(config['hdf_path'], + config['sampling_info_path'], + config['ref_mlmc_file'], + data_dict["test_targets"][i], + data_dict["test_predictions"][i], + data_dict["train_targets"][i], + data_dict["train_predictions"][i], + data_dict["val_targets"][i], + data_dict["l_0_targets"][i], + data_dict["l_0_predictions"][i], + l1_sample_time, + l0_sample_time, + nn_level=config['level'], + replace_level=config['replace_level'], + mlmc_hdf_file=config['mlmc_hdf_path'], + stats=True, + learning_time=learning_time, + dataset_config=dataset_config, + targets_to_est=data_dict["targets_to_est"][i], + predictions_to_est=data_dict["predictions_to_est"][i]) + # except: + # continue mlmc_n_collected_all.append(mlmc_n_collected) nn_n_collected_all.append(nn_mlmc_n_collected) @@ -1058,6 +1083,16 @@ def analyze_statistics(config, get_model=True): mlmc_means_diff_2.append(np.abs(ref_orig_moments[0].mean - ref_orig_moments[1].mean)) nn_means_diff_2.append(np.abs(ref_predict_moments[0].mean - ref_predict_moments[1].mean)) + mlmc_nn_vars_mse_2.append((mlmc_predict_moments[0].var - mlmc_predict_moments[1].var) ** 2) + mlmc_nn_means_mse_2.append((mlmc_predict_moments[0].mean - mlmc_predict_moments[1].mean) ** 2) + mlmc_nn_means_diff_2.append(np.abs(mlmc_predict_moments[0].mean - mlmc_predict_moments[1].mean)) + + + mlmc_moments_mean.append(mlmc_predict_moments[0].mean) + mlmc_moments_var.append(mlmc_predict_moments[0].var) + mlmc_nn_moments_mean.append(mlmc_predict_moments[1].mean) + mlmc_nn_moments_var.append(mlmc_predict_moments[1].var) + # print("np.min(len(ref_orth_moments.mean), len(orig_orth_moments.mean)) ", np.min(len(ref_orth_moments.mean), len(orig_orth_moments.mean))) # print("ref_orth_moments.mean[:np.min(len(ref_orth_moments.mean), len(orig_orth_moments.mean))] ", ref_orth_moments.mean[:np.min(len(ref_orth_moments.mean), len(orig_orth_moments.mean))]) if ref_orth_moments is not None: @@ -1091,6 +1126,14 @@ def analyze_statistics(config, get_model=True): orig_moments_mean.var), label="orig moments") moments_plot.show(None) + moments_plot_2 = plots.MomentsPlots(log_var_y=True, title="Moments MLMC domain") + moments_plot_2.add_moments((np.mean(mlmc_moments_mean, axis=0), + np.mean(mlmc_moments_var, axis=0)), label="mlmc moments") + moments_plot_2.add_moments((np.mean(mlmc_nn_moments_mean, axis=0), + np.mean(mlmc_nn_moments_var, axis=0)), label="mlmc nn moments") + + moments_plot_2.show(None) + display_vars(mlmc_vars, nn_vars, target_variance=target_variance) @@ -1171,7 +1214,14 @@ def analyze_statistics(config, get_model=True): l_vars = np.mean(nn_l_vars, axis=0) # print("nn l vars ", l_vars) # print("nn level parsm ", nn_level_params) - level_params = np.squeeze(nn_level_params) + if len(nn_level_params) > 1: + level_params = np.squeeze(nn_level_params) + else: + level_params = nn_level_params[0] + + print("level params ", level_params) + + level_params[0] *= 2 plt_var.add_level_variances_nn(level_params, l_vars) plt_var.show("nn_vars") @@ -1378,7 +1428,13 @@ def analyze_statistics(config, get_model=True): print("SEM train MSE: {}".format(stats.sem(train_MSE_list))) print("SEM test MSE: {}".format(stats.sem(test_MSE_list))) # print("test MSE std", np.sqrt(np.var(test_MSE))) + print("mean train sample: {}".format(np.mean(all_train_samples))) + print("mean test sample: {}".format(np.mean(all_test_samples))) + #print("mean sample : {}".format(np.mean(np.array(all_train_samples).flatten() + np.array(all_test_samples).flatten()))) print("RMSE train: {}, test: {}".format(np.mean(train_RMSE_list), np.mean(test_RMSE_list))) + print("relative RMSE train: {}, test: {}".format(np.mean(train_RMSE_list)/np.mean(all_train_samples), np.mean(test_RMSE_list)/np.mean(all_test_samples))) + print("iter relative RMSE train: {}, test: {}".format(np.mean(train_relRMSE_list), + np.mean(test_relRMSE_list))) print("RSE train: {}, test: {}".format(np.mean(train_RSE_list), np.mean(test_RSE_list))) print("MAE train: {}, test: {}".format(np.mean(train_MAE_list), np.mean(test_MAE_list))) @@ -1403,6 +1459,14 @@ def analyze_statistics(config, get_model=True): np.sum(np.mean(nn_means_mse_2, axis=0)[ 10:]))) + print("MLMC vs NN mom mean MSE: total: {:0.5g}, to 10: {:0.5g}, above: {:0.5g}".format( + np.sum(np.mean(mlmc_nn_means_mse_2, axis=0)), np.sum(np.mean(mlmc_nn_means_mse_2, axis=0)[:10]), + np.sum(np.mean(mlmc_nn_means_mse_2, axis=0)[10:]))) + print("MLMC vs NN mom var MSE: total: {:0.5g}, to 10: {:0.5g}, above: {:0.5g}".format( + np.sum(np.mean(mlmc_nn_vars_mse_2, axis=0)), + np.sum(np.mean(mlmc_nn_vars_mse_2, axis=0)[:10]), + np.sum(np.mean(mlmc_nn_vars_mse_2, axis=0)[10:]))) + print("nn total time ", nn_total_time) print("mlmc total time ", mlmc_total_time) @@ -1597,15 +1661,23 @@ def run_GNN(config, stats=True, train=True, log=False, index=0): last_train_sample = index * config['n_train_samples'] + config['n_train_samples'] last_test_sample = len_all_samples - (index * config['n_train_samples'] + config['n_train_samples']) + print("last train sample ", last_train_sample) + print("last test sample ", last_test_sample) + if last_train_sample > last_test_sample: return data_tr = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log, config=config, index=index, train_samples=True, independent_sample=True) + print("len data tr ", len(data_tr)) + data_te = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log, config=config, index=index, predict=True, test_samples=True, independent_samples=True) + print("len data te ", len(data_te)) + + else: if train: data_tr = FlowDataset(output_dir=config['output_dir'], level=config['level'], log=log, config=config, @@ -1626,6 +1698,17 @@ def run_GNN(config, stats=True, train=True, log=False, index=0): preprocess_time = preprocess_time + graph_creation_time print("preprocess time ", preprocess_time) + if config["predict_dir"] is not None: + # data_te_predict = FlowDataset(output_dir=config['predict_dir'], level=config['level'], log=log, config=config, + # index=index, n_test_samples=50000) + + data_te_predict = FlowDataset(output_dir=config['predict_dir'], config=config, predict=True) + + data_te_predict.a = config['conv_layer'].preprocess(data_te_predict.a) + data_te_predict.a = sp_matrix_to_sp_tensor(data_te_predict.a) + + + learning_time_start = time.process_time() data_tr.a = sp_matrix_to_sp_tensor(config['conv_layer'].preprocess(data_tr.a)) data_te.a = data_tr.a #sp_matrix_to_sp_tensor(config['conv_layer'].preprocess(data_te.a)) @@ -1700,6 +1783,8 @@ def run_GNN(config, stats=True, train=True, log=False, index=0): loader_va = MixedLoader(data_va, batch_size=batch_size) loader_te = MixedLoader(data_te, batch_size=batch_size) + loader_te_predict = MixedLoader(data_te_predict, batch_size=batch_size) + if not train: gnn.fit(MixedLoader(data_tr[:10], batch_size=batch_size, epochs=epochs), MixedLoader(data_tr[10:20], batch_size=batch_size), MixedLoader(data_tr[20:30], batch_size=batch_size)) @@ -1725,6 +1810,8 @@ def run_GNN(config, stats=True, train=True, log=False, index=0): learning_time = time.process_time() - learning_time_start + print("learning time ", learning_time) + states = gnn._states # print("states ", states) # for state in states.values(): @@ -1749,6 +1836,11 @@ def run_GNN(config, stats=True, train=True, log=False, index=0): targets, predictions = gnn.predict(loader_te) predictions = np.squeeze(predictions) + targets_to_est, predictions_to_est = gnn.predict(loader_te_predict) + predictions_to_est = np.squeeze(predictions_to_est) + targets_to_est = np.array(targets_to_est) + predictions_to_est = np.array(predictions_to_est) + #print("learning time ", learning_time) targets = np.array(targets) @@ -1759,6 +1851,8 @@ def run_GNN(config, stats=True, train=True, log=False, index=0): if log: targets = np.exp(targets) predictions = np.exp(predictions) + target_to_est = np.exp(targets_to_est) + predictions_to_est = np.exp(predictions_to_est) if not stats: analyze_results(targets, predictions) @@ -1802,7 +1896,7 @@ def run_GNN(config, stats=True, train=True, log=False, index=0): # stats=stats) return gnn, targets, predictions, learning_time, train_targets, train_predictions,\ - val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, total_steps + val_targets, val_predictions, l_0_targets, l_0_predictions, l1_sample_time, l0_sample_time, total_steps, targets_to_est, predictions_to_est save_times(config['save_path'], False, (preprocess_time, (len(data_tr) + len(data_te))), learning_time, (predict_l_0_time, len(l_0_targets))) save_load_data(config['save_path'], False, targets, predictions, train_targets, train_predictions, val_targets, l_0_targets, diff --git a/mlmc/metamodel/flow_dataset.py b/mlmc/metamodel/flow_dataset.py index 60d01b7b..6616ded9 100644 --- a/mlmc/metamodel/flow_dataset.py +++ b/mlmc/metamodel/flow_dataset.py @@ -22,7 +22,7 @@ class FlowDataset(Dataset): DATA_FILE = "data" def __init__(self, output_dir=None, level=0, log=False, mesh=None, corr_field_config=None, config={}, index=None, adj_matrix=None, dataset=None, graphs=None, - predict=False, train_samples=False, test_samples=False, n_test_samples=20000, independent_samples=False, **kwargs): + predict=False, train_samples=False, test_samples=False, n_test_samples=50000, independent_samples=False, **kwargs): self._output_dir = output_dir # if self._output_dir is None: # self._output_dir = OUTPUT_DIR From 0d67640287380a62a8675b5c0dde80876005b604 Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Mon, 8 Aug 2022 15:35:19 +0200 Subject: [PATCH 66/67] kurtosis check --- mlmc/estimator.py | 84 +++++++++++++++++++++++++++++- mlmc/plot/diagnostic_plots.py | 0 mlmc/quantity/quantity_estimate.py | 53 +++++++++++++++++-- mlmc/tool/hdf5.py | 2 +- 4 files changed, 133 insertions(+), 6 deletions(-) create mode 100644 mlmc/plot/diagnostic_plots.py diff --git a/mlmc/estimator.py b/mlmc/estimator.py index af4f6e03..9e9fad64 100644 --- a/mlmc/estimator.py +++ b/mlmc/estimator.py @@ -3,6 +3,7 @@ import scipy.integrate as integrate import mlmc.quantity.quantity_estimate as qe import mlmc.tool.simple_distribution +from mlmc.quantity.quantity_estimate import mask_nan_samples from mlmc.quantity.quantity_types import ScalarType from mlmc.plot import plots from mlmc.quantity.quantity_spec import ChunkSpec @@ -16,6 +17,7 @@ def __init__(self, quantity, sample_storage, moments_fn=None): self._quantity = quantity self._sample_storage = sample_storage self._moments_fn = moments_fn + self._moments_mean = None @property def quantity(self): @@ -29,6 +31,16 @@ def quantity(self, quantity): def n_moments(self): return self._moments_fn.size + @property + def moments_mean_obj(self): + return self._moments_mean + + @moments_mean_obj.setter + def moments_mean_obj(self, moments_mean): + if not isinstance(moments_mean, mlmc.quantity.quantity.QuantityMean): + raise TypeError + self._moments_mean = moments_mean + def estimate_moments(self, moments_fn=None): """ Use collected samples to estimate moments and variance of this estimate. @@ -39,6 +51,7 @@ def estimate_moments(self, moments_fn=None): moments_fn = self._moments_fn moments_mean = qe.estimate_mean(qe.moments(self._quantity, moments_fn)) + self.moments_mean_obj = moments_mean return moments_mean.mean, moments_mean.var def estimate_covariance(self, moments_fn=None): @@ -340,6 +353,51 @@ def get_level_samples(self, level_id, n_samples=None): chunk_spec = next(self._sample_storage.chunks(level_id=level_id, n_samples=n_samples)) return self._quantity.samples(chunk_spec=chunk_spec) + def kurtosis_check(self, quantity=None): + if quantity is None: + quantity = self._quantity + moments_mean_quantity = qe.estimate_mean(quantity) + kurtosis = qe.level_kurtosis(quantity, moments_mean_quantity) + return kurtosis + + +def consistency_check(quantity, sample_storage=None): + + fine_samples = {} + coarse_samples = {} + for chunk_spec in quantity.get_quantity_storage().chunks(): + samples = quantity.samples(chunk_spec) + chunk, n_mask_samples = mask_nan_samples(samples) + + # No samples in chunk + if chunk.shape[1] == 0: + continue + + fine_samples.setdefault(chunk_spec.level_id, []).extend(chunk[:, :, 0]) + if chunk_spec.level_id > 0: + coarse_samples.setdefault(chunk_spec.level_id, []).extend(chunk[:, :, 1]) + + cons_check_val = {} + for level_id in range(sample_storage.get_n_levels()): + if level_id > 0: + fine_mean = np.mean(fine_samples[level_id]) + coarse_mean = np.mean(coarse_samples[level_id]) + diff_mean = np.mean(np.array(fine_samples[level_id]) - np.array(coarse_samples[level_id])) + + fine_var = np.var(fine_samples[level_id]) + coarse_var = np.var(fine_samples[level_id]) + diff_var = np.var(np.array(fine_samples[level_id]) - np.array(coarse_samples[level_id])) + + val = np.abs(coarse_mean - fine_mean + diff_mean) / ( + 3 * (np.sqrt(coarse_var) + np.sqrt(fine_var) + np.sqrt(diff_var))) + + assert np.isclose(coarse_mean - fine_mean + diff_mean, 0) + assert val < 0.9 + + cons_check_val[level_id] = val + + return cons_check_val + def estimate_domain(quantity, sample_storage, quantile=None): """ @@ -363,7 +421,23 @@ def estimate_domain(quantity, sample_storage, quantile=None): return np.min(ranges[:, 0]), np.max(ranges[:, 1]) -def estimate_n_samples_for_target_variance(target_variance, prescribe_vars, n_ops, n_levels): +def coping_with_high_kurtosis(vars, costs, kurtosis, kurtosis_threshold=100): + """ + Coping with high kurtosis is recommended by prof. M. Giles in http://people.maths.ox.ac.uk/~gilesm/talks/MCQMC_22_b.pdf + :param vars: vars[L, M] for all levels L and moments_fn M safe the (zeroth) constant moment with zero variance. + :param costs: cost of level's sample + :param kurtosis: each level's sample kurtosis + :param kurtosis_threshold: Kurtosis is considered to be too high if it is above this threshold. + Original variances are underestimated and therefore modified in this metod + :return: vars + """ + for l_id in range(2, vars.shape[0]): + if kurtosis[l_id] > kurtosis_threshold: + vars[l_id] = np.maximum(vars[l_id], 0.5 * vars[l_id - 1] * costs[l_id - 1] / costs[l_id]) + return vars + + +def estimate_n_samples_for_target_variance(target_variance, prescribe_vars, n_ops, n_levels, theta=0, kurtosis=None): """ Estimate optimal number of samples for individual levels that should provide a target variance of resulting moment estimate. @@ -372,12 +446,20 @@ def estimate_n_samples_for_target_variance(target_variance, prescribe_vars, n_op :param prescribe_vars: vars[ L, M] for all levels L and moments_fn M safe the (zeroth) constant moment with zero variance. :param n_ops: number of operations at each level :param n_levels: number of levels + :param theta: number of samples N_l control parameter, suitable values: 0.25 ... 0.5 + :param kurtosis: levels' kurtosis :return: np.array with number of optimal samples for individual levels and moments_fn, array (LxR) """ vars = prescribe_vars + + if kurtosis is not None and len(vars) == len(kurtosis): + vars = coping_with_high_kurtosis(vars, n_ops, kurtosis) + sqrt_var_n = np.sqrt(vars.T * n_ops) # moments_fn in rows, levels in cols total = np.sum(sqrt_var_n, axis=1) # sum over levels n_samples_estimate = np.round((sqrt_var_n / n_ops).T * total / target_variance).astype(int) # moments_fn in cols + n_samples_estimate = 1/(1-theta) * n_samples_estimate + # Limit maximal number of samples per level n_samples_estimate_safe = np.maximum( np.minimum(n_samples_estimate, vars * n_levels / target_variance), 2) diff --git a/mlmc/plot/diagnostic_plots.py b/mlmc/plot/diagnostic_plots.py new file mode 100644 index 00000000..e69de29b diff --git a/mlmc/quantity/quantity_estimate.py b/mlmc/quantity/quantity_estimate.py index 2436d7b9..a1f63ddd 100644 --- a/mlmc/quantity/quantity_estimate.py +++ b/mlmc/quantity/quantity_estimate.py @@ -19,13 +19,17 @@ def cache_clear(): mlmc.quantity.quantity.QuantityConst.samples.cache_clear() -def estimate_mean(quantity): +def estimate_mean(quantity, form="diff", operation_func=None, **kwargs): """ MLMC mean estimator. The MLMC method is used to compute the mean estimate to the Quantity dependent on the collected samples. The squared error of the estimate (the estimator variance) is estimated using the central limit theorem. Data is processed by chunks, so that it also supports big data processing :param quantity: Quantity + :param form: if "diff" estimates based on difference between fine and coarse data = MLMC approach + "fine" estimates based on level's fine data + "coarse" estimates based on level's coarse data + :param operation_func: function to process level data, e.g. kurtosis estimation :return: QuantityMean which holds both mean and variance """ cache_clear() @@ -56,10 +60,26 @@ def estimate_mean(quantity): sums = [np.zeros(chunk.shape[0]) for _ in range(n_levels)] sums_of_squares = [np.zeros(chunk.shape[0]) for _ in range(n_levels)] - if chunk_spec.level_id == 0: - chunk_diff = chunk[:, :, 0] + # Estimates of level's fine data + if form == "fine": + if chunk_spec.level_id == 0: + chunk_diff = chunk[:, :, 0] + else: + chunk_diff = chunk[:, :, 0] + # Estimate of level's coarse data + elif form == "coarse": + if chunk_spec.level_id == 0: + chunk_diff = np.zeros(chunk[:, :, 0].shape) + else: + chunk_diff = chunk[:, :, 1] else: - chunk_diff = chunk[:, :, 0] - chunk[:, :, 1] + if chunk_spec.level_id == 0: + chunk_diff = chunk[:, :, 0] + else: + chunk_diff = chunk[:, :, 0] - chunk[:, :, 1] + + if operation_func is not None: + chunk_diff = operation_func(chunk_diff, chunk_spec, **kwargs) sums[chunk_spec.level_id] += np.sum(chunk_diff, axis=1) sums_of_squares[chunk_spec.level_id] += np.sum(chunk_diff**2, axis=1) @@ -154,3 +174,28 @@ def eval_cov(x): else: moments_qtype = qt.ArrayType(shape=(moments_fn.size, moments_fn.size, ), qtype=quantity.qtype) return mlmc.quantity.quantity.Quantity(quantity_type=moments_qtype, input_quantities=[quantity], operation=eval_cov) + + +def kurtosis_numerator(chunk_diff, chunk_spec, l_means): + """ + Estimate sample kurtosis nominator: + E[(Y_l - E[Y_l])^4] + :param chunk_diff: np.ndarray, [quantity shape, number of samples] + :param chunk_spec: quantity_spec.ChunkSpec + :return: np.ndarray, unchanged shape + """ + chunk_diff = (chunk_diff - l_means[chunk_spec.level_id]) ** 4 + return chunk_diff + + +def level_kurtosis(quantity, means_obj): + """ + Estimate sample kurtosis at each level as: + E[(Y_l - E[Y_l])^4] / (Var[Y_l])^2, where Y_l = fine_l - coarse_l + :param quantity: Quantity + :param means_obj: Quantity.QuantityMean + :return: np.ndarray, kurtosis per level + """ + numerator_means_obj = estimate_mean(quantity, operation_func=kurtosis_numerator, l_means=means_obj.l_means) + kurtosis = numerator_means_obj.l_means / (means_obj.l_vars)**2 + return kurtosis diff --git a/mlmc/tool/hdf5.py b/mlmc/tool/hdf5.py index f6f3219c..a83f6a8a 100644 --- a/mlmc/tool/hdf5.py +++ b/mlmc/tool/hdf5.py @@ -357,7 +357,7 @@ def chunks(self, n_samples=None): dataset = hdf_file["/".join([self.level_group_path, "collected_values"])] if n_samples is not None: - yield ChunkSpec(chunk_id=0, chunk_slice=slice(0, n_samples, 1), level_id=int(self.level_id)) + yield ChunkSpec(chunk_id=0, chunk_slice=slice(0, n_samples, ...), level_id=int(self.level_id)) else: for chunk_id, chunk in enumerate(dataset.iter_chunks()): yield ChunkSpec(chunk_id=chunk_id, chunk_slice=chunk[0], level_id=int(self.level_id)) # slice, level_id From 6a089cc1c08eb17c4fa785cfdcfba822feaf5efb Mon Sep 17 00:00:00 2001 From: martinspetlik Date: Mon, 8 Aug 2022 15:37:39 +0200 Subject: [PATCH 67/67] diffpool experiments --- mlmc/metamodel/own_diffpool.py | 0 mlmc/metamodel/own_src.py | 0 mlmc/metamodel/postprocessing.py | 955 +++++++++++++++---- mlmc/plot/plots.py | 205 +++- test/metamodels/predict_on_different_mesh.py | 116 ++- 5 files changed, 1068 insertions(+), 208 deletions(-) create mode 100644 mlmc/metamodel/own_diffpool.py create mode 100644 mlmc/metamodel/own_src.py diff --git a/mlmc/metamodel/own_diffpool.py b/mlmc/metamodel/own_diffpool.py new file mode 100644 index 00000000..e69de29b diff --git a/mlmc/metamodel/own_src.py b/mlmc/metamodel/own_src.py new file mode 100644 index 00000000..e69de29b diff --git a/mlmc/metamodel/postprocessing.py b/mlmc/metamodel/postprocessing.py index df3fdb53..262656ff 100644 --- a/mlmc/metamodel/postprocessing.py +++ b/mlmc/metamodel/postprocessing.py @@ -16,11 +16,91 @@ QUANTILE = 1e-6 -N_MOMENTS = 25 +# QUANTILE = 0.001 +N_MOMENTS = 20 TARGET_VAR = 1e-5 -def cut_original_test(mlmc_hdf_file): +def use_levels_from_mlmc(sample_storage, use_levels): + print("mlmc sample storage get N collected ", sample_storage.get_n_collected()) + n_levels = len(sample_storage.get_level_ids()) + original_moments, estimator, original_true_domain, _ = estimate_moments(sample_storage) + data_mlmc = [] + new_coarse_data = [] + level_parameters = sample_storage.get_level_parameters() + new_level_parameters = [] + + mlmc_n_collected = estimator._sample_storage.get_n_collected() + for l_id, l_n_collected in zip(range(n_levels), mlmc_n_collected): + print("l id ", l_id) + if l_id == np.max(use_levels): + new_coarse_data = estimator.get_level_samples(level_id=1, n_samples=mlmc_n_collected[1])[:, :, 1] + + if l_id in use_levels: + level_samples = estimator.get_level_samples(level_id=l_id, n_samples=l_n_collected) + #fine_samples = level_samples[:, :, 0] + #coarse_samples = level_samples[:, :, 1] + print("level samples shape ", level_samples.shape) + else: + continue + + print("level samples shape ", level_samples.shape) + + + # if len(data_mlmc) == 0: + # level_samples[:, :, 1] = 0 + if len(new_coarse_data) > 0: + print("new coarse data len ", len(new_coarse_data[0])) + print("new_coarse_data[:, :level_samples.shape[1]] ", new_coarse_data[:, :level_samples.shape[1]]) + level_samples[:, :, 1] = new_coarse_data[:, :level_samples.shape[1]] + + print("level samples ", level_samples) + new_level_parameters.append(level_parameters[l_id]) + data_mlmc.append(level_samples) + + exit() + + final_sample_storage = create_quantity_mlmc(data_mlmc, level_parameters=np.array(new_level_parameters)) + + return final_sample_storage + + +def remove_level(mlmc_hdf_file, rm_level_id=0, use_levels=[]): + sample_storage = SampleStorageHDF(file_path=mlmc_hdf_file) + + print("mlmc sample storage get N collected ", sample_storage.get_n_collected()) + n_levels = len(sample_storage.get_level_ids()) + original_moments, estimator, original_true_domain, _ = estimate_moments(sample_storage) + + n_ops, field_times, coarse_flow, fine_flow = get_sample_times_mlmc(mlmc_hdf_file) + n_ops_est = n_ops + + if len(use_levels) > 0: + final_sample_storage = use_levels_from_mlmc(sample_storage, use_levels) + else: + data_mlmc = [] + mlmc_n_collected = estimator._sample_storage.get_n_collected() + for l_id, l_n_collected in zip(range(n_levels), mlmc_n_collected): + if l_id <= rm_level_id: + continue + print("l id ", l_id) + if rm_level_id + 1 == l_id: + level_samples = estimator.get_level_samples(level_id=l_id, n_samples=l_n_collected) + level_samples[:, :, 1] = 0 + print("level sampels shape ", level_samples.shape) + + else: + level_samples = estimator.get_level_samples(level_id=l_id, n_samples=l_n_collected) + print("level samples ", level_samples) + data_mlmc.append(level_samples) + + final_sample_storage = create_quantity_mlmc(data_mlmc, level_parameters=sample_storage.get_level_parameters()[rm_level_id+1:]) + + return final_sample_storage + + +def cut_original_test(mlmc_hdf_file, n_levels=None): + print("mlmc hdf file ", mlmc_hdf_file) sample_storage = SampleStorageHDF(file_path=mlmc_hdf_file) print("mlmc sample storage get N collected ", sample_storage.get_n_collected()) @@ -163,10 +243,120 @@ def analyze_output(targets, mult_coef=1,dataset_config=None): exit() + +def _variance_cost_analysis(l_var_nn, n_ops_predict): + + print("np.max(l_var_nn, axis=0) ", np.max(l_var_nn, axis=1)) + vl_cl = np.max(l_var_nn, axis=1) * n_ops_predict + + print("VlCl ", vl_cl) + + beta = {} + gamma = {} + beta_1 = {} + gamma_1 = {} + moments_m = [1, 5, 10, 15, 20] + #n_collected = estimator._sample_storage.get_n_collected() + # cost_levels = n_collected * np.array(new_n_ops) + cost_levels = np.array(n_ops_predict) + + max_level_var = True + beta_levels = [] + gamma_levels = [] + beta_levels_2 = [] + gamma_levels_2 = [] + + level_vars = np.max(l_var_nn, axis=1) + + level_costs = n_ops_predict + + print("cost levels ", cost_levels) + for l_id, l_vars in enumerate(l_var_nn, start=1): + # print("l_vars ", l_vars) + # print("l id ", l_id) + if max_level_var is True: + if l_id < len(level_vars): + beta_levels.append(-1 * np.log(level_vars[l_id]/level_vars[l_id-1])) + gamma_levels.append(np.log2(level_costs[l_id] / level_costs[l_id - 1])) + + if l_id == 1: + beta_levels_2.append(-1 * (np.log(level_vars[l_id - 1]))) + gamma_levels_2.append(np.log2(level_costs[l_id-1])) + else: + beta_levels_2.append(-1 * (np.log(level_vars[l_id-1])/(l_id-1))) + gamma_levels_2.append(np.log2(level_costs[l_id - 1])/(l_id-1)) + else: + for moment in moments_m: + if moment not in beta: + beta[moment] = {} + beta_1[moment] = {} + + if l_id not in beta[moment]: + beta[moment][l_id] = [] + beta_1[moment][l_id] = [] + + if l_id not in gamma: + gamma[l_id] = [] + gamma_1[l_id] = [] + + # print("l_id ", l_id) + # print("len l vars ", len(l_vars)) + # print("moment ", moment) + # print("beta ", beta) + beta[moment][l_id].append(-1 * np.log2(l_vars[moment]) / l_id) + + # print("level l vars ", l_vars) + # print("moments_mean.l_vars[l_id-2] ", moments_mean.l_vars[l_id-1]) + if l_id < len(cost_levels): + beta_1[moment][l_id].append(-1 * np.log2(l_var_nn[l_id][moment] / l_vars[moment])) + + gamma[l_id].append(np.log2(cost_levels[l_id - 1]) / l_id) + + if l_id < len(cost_levels): + # print("cost levles ", cost_levels) + # print("l id ", l_id) + gamma_1[l_id].append(np.log2(cost_levels[l_id] / cost_levels[l_id - 1])) + + print("beta levels ", beta_levels) + print("gamma levels ", gamma_levels) + + print("beta levels 2", beta_levels_2) + print("gamma levels 2", gamma_levels_2) + + beta_curve = [] + gamma_curve = [] + + fig, ax = plt.subplots(1, 1, figsize=(15, 10)) + ax.plot(level_vars, label="vars") + ax.plot(level_costs, label="costs") + + for b, g in zip(beta_levels, gamma_levels): + beta_curve = [] + gamma_curve = [] + for l in range(len(level_vars)): + beta_curve.append(2**(-1*b*l)) + gamma_curve.append(2 ** (g * l)) + + ax.plot(beta_curve, label="beta: {}".format(b)) + ax.plot(gamma_curve, label="gamma: {}".format(g)) + + # fig.colorbar(cont) + plt.title("levels - var, cost") + plt.yscale("log") + plt.legend() + plt.show() + + + print("beta: {}, beta_1:{}".format(beta, beta_1)) + print("gamma: {}, gamma_1:{}".format(gamma, gamma_1)) + + exit() + + def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predictions, train_targets, train_predictions, val_targets, l_0_targets=None, l_0_predictions=None, l1_sample_time=None, l0_sample_time=None, nn_level=0, replace_level=False, stats=False, mlmc_hdf_file=None, - learning_time=0, dataset_config={}): + learning_time=0, dataset_config={}, targets_to_est=None, predictions_to_est=None): """ :param l1_sample_time: preprocess_time / len(data) + learning_time / len(data), preprocess_time includes graph creation time and FlowDataset creation time @@ -175,16 +365,91 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic cut_est = True all_samples = False n0, nL = 2000, 100 - domain_largest = False # If False than domain=None - domain is determined given the simulation samples - distr_domain_largest = False + domain_largest = True # If False than domain=None - domain is determined given the simulation samples + distr_domain_largest = False#False + replace_level = False#False - # cut_original_test(nn_hdf_file) + rm_level = None#0 + rm_level_nn = None#None + use_levels = []#[0, 2] + + + #cut_original_test(nn_hdf_file) # exit() - output_mult_factor = dataset_config.get('output_mult_factor', 1) - #print("output mult factor ", output_mult_factor) + # output_mult_factor = dataset_config.get('output_mult_factor', 1) + # #print("output mult factor ", output_mult_factor) + # #analyze_output(targets, output_mult_factor, dataset_config=dataset_config) + # plt.hist(targets, bins=50, alpha=0.5, label='target', density=True) + # plt.hist(predictions, bins=50, alpha=0.5, label='predictions', density=True) + # + # # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) + # plt.legend(loc='upper right') + # # plt.xlim(-0.5, 1000) + # plt.yscale('log') + # plt.show() + # + # plt.hist(l_0_targets, bins=50, alpha=0.5, label='l_0_target', density=True) + # plt.hist(l_0_predictions, bins=50, alpha=0.5, label='l_0_predictions', density=True) + # + # # print("lo targets ", l_0_targets) + # # print("l0 predictions ", l_0_predictions) + # # exit() + # + # # # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) + # plt.legend(loc='upper right') + # # plt.xlim(-0.5, 1000) + # plt.yscale('log') + # plt.show() + # + # + # plt.hist(targets, bins=50, alpha=0.5, label='target', density=True) + # plt.hist(predictions, bins=50, alpha=0.5, label='predictions', density=True) + # + # plt.legend(loc='upper right') + # # plt.xlim(-0.5, 1000) + # plt.yscale('log') + # + # plt.show() + # # + # plt.hist(targets_to_est, bins=50, alpha=0.5, label='target to est', density=True) + # plt.hist(predictions_to_est, bins=50, alpha=0.5, label='predictions to est', density=True) + # + # plt.legend(loc='upper right') + # # plt.xlim(-0.5, 1000) + # plt.yscale('log') + # plt.show() + + if targets_to_est is not None and predictions_to_est is not None: + print("len targets_to_est ", len(targets_to_est)) + targets = np.concatenate((targets, targets_to_est), axis=0) + predictions = np.concatenate((predictions, predictions_to_est), axis=0) + # targets = np.concatenate((targets, targets_to_est), axis=0) + # predictions = np.concatenate((predictions, predictions_to_est), axis=0) + + # l_0_targets = np.concatenate((l_0_targets, targets_to_est), axis=0) + # l_0_predictions = np.concatenate((l_0_predictions, predictions_to_est), axis=0) + + # targets = np.concatenate((targets, l_0_targets), axis=0) + # predictions = np.concatenate((predictions, l_0_predictions), axis=0) + + # targets = np.concatenate((targets, targets_to_est[:5000]), axis=0) + # predictions = np.concatenate((predictions, predictions_to_est[:5000]), axis=0) + + # l_0_targets = np.concatenate((l_0_targets, targets_to_est), axis=0) + # l_0_predictions = np.concatenate((l_0_predictions, predictions_to_est), axis=0) + + # targets = np.concatenate((targets, l_0_targets[-1000:]), axis=0) + # predictions = np.concatenate((predictions, l_0_predictions[-1000:]), axis=0) + + # print("len targets to est", len(targets_to_est)) + # print("len predictions to est ", len(predictions_to_est)) + # exit() + + + print("len(targets ", len(targets)) + print("len l0 targets ", len(l_0_targets)) - #analyze_output(targets, output_mult_factor, dataset_config=dataset_config) if dataset_config.get('output_normalization', False): min_out = dataset_config.get('min_output') @@ -223,13 +488,10 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic l_0_predictions = np.exp(l_0_predictions) l_0_targets = np.exp(l_0_targets) - if mlmc_hdf_file is None: - mlmc_hdf_file = nn_hdf_file - print("targets ", targets) print("len targets ", len(targets)) - #if not stats: + # if not stats: # print("nn_level ", nn_level) # print("replace level ", replace_level) @@ -240,30 +502,30 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic # print("targets ", targets) # print("predictions ", predictions) - print("targets ", targets) - print("predictions ", predictions) - plt.hist(targets, bins=50, alpha=0.5, label='target', density=True) - plt.hist(predictions, bins=50, alpha=0.5, label='predictions', density=True) - - # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) - plt.legend(loc='upper right') - # plt.xlim(-0.5, 1000) - plt.yscale('log') - plt.show() - - plt.hist(l_0_targets, bins=50, alpha=0.5, label='l_0_target', density=True) - plt.hist(l_0_predictions, bins=50, alpha=0.5, label='l_0_predictions', density=True) - - # print("lo targets ", l_0_targets) - # print("l0 predictions ", l_0_predictions) - # exit() - - # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) - plt.legend(loc='upper right') - # plt.xlim(-0.5, 1000) - plt.yscale('log') - plt.show() - + # print("targets ", targets) + # print("predictions ", predictions) + # plt.hist(targets, bins=50, alpha=0.5, label='target', density=True) + # plt.hist(predictions, bins=50, alpha=0.5, label='predictions', density=True) + # + # # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) + # plt.legend(loc='upper right') + # # plt.xlim(-0.5, 1000) + # plt.yscale('log') + # plt.show() + # + # plt.hist(l_0_targets, bins=50, alpha=0.5, label='l_0_target', density=True) + # plt.hist(l_0_predictions, bins=50, alpha=0.5, label='l_0_predictions', density=True) + # + # # print("lo targets ", l_0_targets) + # # print("l0 predictions ", l_0_predictions) + # # exit() + # + # # plt.hist(targets - predictions, bins=50, alpha=0.5, label='predictions', density=True) + # plt.legend(loc='upper right') + # # plt.xlim(-0.5, 1000) + # plt.yscale('log') + # plt.show() + # # targets = np.exp(targets) # predictions = np.exp(predictions) @@ -278,7 +540,13 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic ####### ### Create storage fromm original MLMC data ###### - sample_storage = SampleStorageHDF(file_path=mlmc_hdf_file) + if mlmc_hdf_file is None: + mlmc_hdf_file = nn_hdf_file + + if rm_level is not None: + sample_storage = remove_level(mlmc_hdf_file, rm_level_id=rm_level, use_levels=use_levels) + else: + sample_storage = SampleStorageHDF(file_path=mlmc_hdf_file) print("mlmc sample storage get N collected ", sample_storage.get_n_collected()) @@ -288,7 +556,12 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic print("moments.mean ", original_moments.mean) print("moments.var ", original_moments.var) - sample_storage_nn = SampleStorageHDF(file_path=nn_hdf_file) + if rm_level_nn is not None: + sample_storage_nn = remove_level(nn_hdf_file, rm_level_id=rm_level_nn, use_levels=use_levels) + else: + sample_storage_nn = SampleStorageHDF(file_path=nn_hdf_file) + + original_moments_nn, estimator_nn, original_true_domain_nn, _ = estimate_moments(sample_storage_nn) print("nn moments.mean ", original_moments_nn.mean) print("nn moments.var ", original_moments_nn.var) @@ -300,9 +573,15 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic ### Get n ops ###### n_ops, field_times, coarse_flow, fine_flow = get_sample_times_mlmc(mlmc_hdf_file) + #n_ops = [20.279451930908973, 87.91808330548963, 216.172210888505, 892.2780022583306, 2646.912581985272] # n_ops, _, _ = get_sample_times(sampling_info_path) print("n ops ", n_ops) + n_ops = n_ops[-sample_storage.get_n_levels():] + # field_times = field_times[-sample_storage.get_n_levels():] + # coarse_flow = coarse_flow[-sample_storage.get_n_levels():] + # fine_flow = fine_flow[-sample_storage.get_n_levels():] + if n_ops is None: n_ops = sample_storage.get_n_ops() field_times = np.zeros(len(n_ops)) @@ -326,7 +605,7 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic else: n_samples = np.round(np.exp2(np.linspace(np.log2(n0), np.log2(nL), n_levels))).astype(int) - print('n samples ', n_samples) + print('cut n samples for sample storgae estimate', n_samples) sample_storage_for_estimated = cut_samples(data_mlmc, sample_storage, n_samples) # [2300]) print("Original storage") @@ -352,12 +631,34 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic ############################ ### Calculate First level (follows meta-level) n_ops ############################ - len_data = 20000 # 50000#80000 + print("learning time ", learning_time) + print("l1_sample_time ", l1_sample_time) + + + #learning_time = learning_time / 4 + C_1_cost = 0 + C_2_cost = 0 + + #len_data = 50000 # 50000#80000 + len_data = 4000 # case with independent samples len_train_data = 2000 + len_data_preprocess = len_data + 50000 l0_predict_time = 1e-3 preprocess_time = l1_sample_time * len_data - learning_time - preprocess_time_per_sample = preprocess_time / len_data - n_ops_train = preprocess_time_per_sample + (learning_time / len_train_data) + l0_predict_time + preprocess_time_per_sample = preprocess_time / len_data_preprocess + + learning_time = 310 # cl_0_1_s_1: L1 - 180, L2 - 220, L3 - 310, L4 - 820, L5 - 3320.261721275 + learning_time = 2840 # 02 conc cond, L1 - 400, L2 - 400 , L3 - 450, L4 - 550 , L5 - 2840 + + #n_ops_train = preprocess_time_per_sample + (learning_time / len_train_data) + l0_predict_time + + C_1_cost = l0_sample_time # time for generating random field + C_2_cost = preprocess_time_per_sample + l0_predict_time + n_ops[0] + + + print("C1 cost: {}, C2 cost: {}".format(C_1_cost, C_2_cost)) + + #### # Notes: # l1_sample_time = preprocess_time / len(data) + learning_time / len(data) @@ -366,18 +667,19 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic # preprocess time per sample # learning time per training sample # l0_prediction_time - the time necessary to predict neural network outcome for given already preprocessed data - l1_sample_time = n_ops_train + #l1_sample_time = n_ops_train # New l1_sample_time corresponds to the time necessary for Level 1 samples (fine + coarse) # ################################# - print("L1 sample time ", l1_sample_time) + #print("L1 sample time ", l1_sample_time) print("learning time ", learning_time) print("preprocess time ", preprocess_time) print("preprocess time per sample ", preprocess_time_per_sample) - print("new l1 sample time ", l1_sample_time) + #print("new l1 sample time ", l1_sample_time) + # print("n ops train ", n_ops_train) # print("n ops test ", n_ops_test) @@ -400,13 +702,15 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic # level_samples = np.ones((1, len(l_0_predictions), 1)) ft_index = nn_level - if output_mult_factor != 1: - level_samples /= output_mult_factor + # if output_mult_factor != 1: + # level_samples /= output_mult_factor if nn_level > 0: ft_index = nn_level - 1 n_ops_predict.append(l0_sample_time) # + field_times[ft_index] / 2) + print("L0 n ops predict ", n_ops_predict) + # print("l0_sample_time ", l0_sample_time) # print("len l0 predictions ", len(l_0_predictions)) # exit() @@ -416,6 +720,7 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic level_samples = estimator_nn.get_level_samples(level_id=level_id, n_samples=nn_n_collected[level_id]) n_ops_predict.append(n_ops[level_id]) + print("replace level n ops ", n_ops_predict) else: if l_id < mlmc_nn_diff_level: continue @@ -438,8 +743,9 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic level_samples = np.concatenate((fine_level_samples, coarse_level_samples), axis=2) print("nn level ", nn_level) - if output_mult_factor != 1: - level_samples /= output_mult_factor + # if output_mult_factor != 1: + # level_samples /= output_mult_factor + print("level sampels corase and fine shape", fine_level_samples.shape) print("level sampels ", level_samples) print("fine - coarse ", fine_level_samples - coarse_level_samples) @@ -454,30 +760,31 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic level_id = l_id # - 1 else: level_id = l_id + nn_level - 1 - level_samples = estimator_nn.get_level_samples(level_id=level_id, - n_samples=nn_n_collected[level_id]) - print('n ops ', n_ops) - print("level id ", level_id) + if level_id >= len(nn_n_collected): + level_samples = [] + n_ops_predict[-1] = n_ops[level_id] + continue + else: + level_samples = estimator_nn.get_level_samples(level_id=level_id, + n_samples=nn_n_collected[level_id]) + print('n ops ', n_ops) + print("level id ", level_id) - n_ops_predict.append(n_ops[level_id]) + n_ops_predict.append(n_ops[level_id]) # print("n ops predict append", n_ops_predict) # if output_mult_factor != 1: # level_samples /= output_mult_factor - - print("level samples rescaled ", level_samples) #level_samples = np.log(level_samples) - #level_samples /= output_mult_factor print("leel samples exp ", level_samples) #print("level samples exp ", level_samples) - data_nn.append(level_samples) print("n ops predict ", n_ops_predict) @@ -498,8 +805,9 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic else: n_samples = np.round(np.exp2(np.linspace(np.log2(n0), np.log2(nL), n_levels))).astype(int) - print('n samples predict', n_samples) - sample_storage_predict_for_estimate = cut_samples(data_nn, sample_storage_predict, n_samples) # [4500, 1500]) + print('cut n samples for MC+NN estimate', n_samples) + sample_storage_predict_for_estimate = cut_samples(data_nn, sample_storage_predict, n_samples) + # print("n ops predict ", n_ops_predict) print("Storage predict info") predict_storage_n_collected, predict_storage_max_vars = get_storage_info(sample_storage_predict_for_estimate) @@ -521,6 +829,10 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic predict_q_estimator = get_quantity_estimator(sample_storage_predict, true_domain=domain) if cut_est: + if domain_largest: + domain = get_largest_domain([sample_storage_for_estimated, sample_storage_predict_for_estimate, ref_sample_storage]) + else: + domain = None original_q_estimator_est = get_quantity_estimator(sample_storage_for_estimated, true_domain=domain) predict_q_estimator_est = get_quantity_estimator(sample_storage_predict_for_estimate, true_domain=domain) # ref_estimator = get_quantity_estimator(ref_sample_storage, true_domain=domain) @@ -541,62 +853,55 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic # n_ops = n_ops[2:] # # # ##### + # print("n ops ", n_ops) + # print("n ops predict ", n_ops_predict) + # exit() + # n_ops = [30.1978988484516, 2321.096786450282] + # n_ops_predict = [0.33790084, 30.91978988484516, 2321.096786450282] + #### Original data n_ops_est = copy.deepcopy(n_ops) - # n_ops_est[0] = n_ops_est[0] / 1000 - print("sample_storage.get_n_collected() ", sample_storage.get_n_collected()) - n_estimated_orig, l_vars_orig, n_samples_orig = get_n_estimated(sample_storage, original_q_estimator, - n_ops=n_ops_est) + if not cut_est: + # n_ops_est[0] = n_ops_est[0] / 1000 + print("sample_storage.get_n_collected() ", sample_storage.get_n_collected()) + n_estimated_orig, l_vars_orig, n_samples_orig = get_n_estimated(sample_storage, original_q_estimator, + n_ops=n_ops_est) - print("n estimated orig ", n_estimated_orig) - print("n ops est ", n_ops_est) - #print("sample storage for estimated n collected ", sample_storage_for_estimated.get_n_collected()) + print("n estimated orig ", n_estimated_orig) + print("n ops est ", n_ops_est) + #print("sample storage for estimated n collected ", sample_storage_for_estimated.get_n_collected()) ###### - ## initial N geuss + ## initial N guess if cut_est: print("n ops est ", n_ops_est) n_estimated_orig, l_vars_orig, n_samples_orig = get_n_estimated(sample_storage_for_estimated, original_q_estimator_est, n_ops=n_ops_est) print("n estimated orig ", n_estimated_orig) - sample_storage_for_estimated = cut_samples(data_mlmc, sample_storage_for_estimated, n_estimated_orig, - bootstrap=True) + sample_storage_for_estimated = cut_samples(data_mlmc, sample_storage, n_estimated_orig, + bootstrap=False) - # print("new n estimated orig ", n_estimated_orig) - # print("l vars orig ", np.array(l_vars_orig) / np.array(sample_storage.get_n_collected())[:, np.newaxis]) + # Another estimate simulate adding samples algo + original_q_estimator_est_2 = get_quantity_estimator(sample_storage_for_estimated, true_domain=domain) + n_estimated_orig, l_vars_orig, n_samples_orig = get_n_estimated(sample_storage_for_estimated, + original_q_estimator_est_2, n_ops=n_ops_est) print("new n estimated orig ", n_estimated_orig) - sample_storage = cut_samples(data_mlmc, sample_storage, n_estimated_orig, bootstrap=True) - - # original_q_estimator.quantity = original_q_estimator.quantity.subsample(sample_vec=n_estimated_orig) + sample_storage = cut_samples(data_mlmc, sample_storage, n_estimated_orig, bootstrap=False) - # original_q_estimator = get_quantity_estimator(sample_storage, true_domain=None, quantity=original_q_estimator.quantity) - - n_ops_predict_orig = copy.deepcopy(n_ops_predict) - # n_ops_predict_orig[0] = n_ops_predict_orig[0] /5 - # n_ops_predict = np.array(n_ops_predict)**2 - # n_ops_predict[0] = n_ops_predict[0] / 1000 - # print("n ops predict for estimate ", n_ops_predict) - # n_samples = [10000, 2000, 500, 150, 40, 11] - - n_estimated_nn, l_var_nn, n_samples_nn = get_n_estimated(sample_storage_predict, predict_q_estimator, - n_ops=n_ops_predict) - ###### - ## initial N geuss - if cut_est: - n_estimated_nn, l_vars_nn, n_samples_nn = get_n_estimated(sample_storage_predict_for_estimate, - predict_q_estimator_est, n_ops=n_ops_predict) - sample_storage_predict_for_estimate = cut_samples(data_nn, sample_storage_predict, n_estimated_nn) # exit() - print("l var nn ", l_var_nn) - print("n ops predict ", n_ops_predict) - print("n estimated nn ", n_estimated_nn) + n_estimated_nn, l_var_nn, n_samples_nn = get_n_estimated(sample_storage_predict_for_estimate, predict_q_estimator_est, + n_ops=n_ops_predict) - print("n estimated orig ", n_estimated_orig) + # print("l var nn ", l_var_nn) + # print("n ops predict ", n_ops_predict) + # print("n estimated nn ", n_estimated_nn) + # + # print("n estimated orig ", n_estimated_orig) # new_n_estimated_nn = [] # for n_est in n_estimated_nn: @@ -620,43 +925,123 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic # predict_q_estimator.quantity = predict_q_estimator.quantity.subsample(sample_vec=n_estimated_nn) + print("n estiamted nn ", n_estimated_nn) + print("n ops predict ", n_ops_predict) ############### ## Recalculate first level (level of metamodel and simulation difference) n ops ############### - n_ops_test = (preprocess_time_per_sample + l0_predict_time) # * (n_estimated_nn[1] - len_train_data) - if n_estimated_nn[1] > len_train_data: - orig_n_ops = n_ops_predict[1] - l1_sample_time - cost_tr = l1_sample_time * (len_train_data) # / n_estimated_nn[1]) - cost_te = n_ops_test * (n_estimated_nn[1] - len_train_data) # / n_estimated_nn[1]) + # Use NN as new MC + if len(n_estimated_nn) == 1: + cost_tr = C_2_cost * len_train_data + if n_estimated_nn[0] > len_train_data: + cost_te = C_1_cost * (n_estimated_nn[0] - len_train_data) + else: + cost_te = 0 + + # n_ops_predict_C_2 = [C_2_cost] + # print("l1 sample time ", l1_sample_time) + # #orig_n_ops = n_ops_predict[0] - l1_sample_time + # cost_tr = (l1_sample_time + n_ops[0]) * (len_train_data) + # n_ops_test = (preprocess_time_per_sample + l0_predict_time) + # cost_te = n_ops_test * (n_estimated_nn[0] - len_train_data) + # + # print("cost tr ", cost_tr) + # print("cost te ", cost_te) - # note L1 sample time is n_ops_train + #n_ops_predict[1] = orig_n_ops + ((cost_tr + cost_te) / n_estimated_nn[1]) - n_ops_predict[1] = orig_n_ops + ((cost_tr + cost_te) / n_estimated_nn[1]) + n_ops_predict[0] = ((cost_tr + cost_te) / n_estimated_nn[0]) - print("preprocess time per sample", preprocess_time_per_sample) - print("orig n ops ", orig_n_ops) - print("cost_tr ", cost_tr) - print("cost te ", cost_te) + print("n ops predict final", n_ops_predict) + print("n ops ", n_ops) + else: + # n_ops_test = (preprocess_time_per_sample + l0_predict_time) # * (n_estimated_nn[1] - len_train_data) + # if n_estimated_nn[1] > len_train_data: + # orig_n_ops = n_ops_predict[1] - l1_sample_time + # cost_tr = l1_sample_time * (len_train_data) # / n_estimated_nn[1]) + # cost_te = n_ops_test * (n_estimated_nn[1] - len_train_data) # / n_estimated_nn[1]) + # + # # note L1 sample time is n_ops_train + # + # n_ops_predict[1] = orig_n_ops + ((cost_tr + cost_te) / n_estimated_nn[1]) + # + # print("preprocess time per sample", preprocess_time_per_sample) + # print("orig n ops ", orig_n_ops) + # print("cost_tr ", cost_tr) + # print("cost te ", cost_te) - n_ops_predict_orig = n_ops_predict + n_ops_predict[0] = C_1_cost + n_ops_predict[1] = C_2_cost + + #n_ops_predict = [0.33790084, 30.91978988484516, 2321.096786450282] + n_ops_predict_orig = n_ops_predict + + ###### + ## initial N guess + if cut_est: + n_estimated_nn, l_var_nn, n_samples_nn = get_n_estimated(sample_storage_predict_for_estimate, + predict_q_estimator_est, n_ops=n_ops_predict) + + print("NN FIRST n estimated nn ", n_estimated_nn) + all_n_estimated_nn = [] + for i in range(10): + + sample_storage_predict_for_est_2 = cut_samples(data_nn, sample_storage_predict, n_estimated_nn, bootstrap=True) + + predict_q_estimator_est_2 = get_quantity_estimator(sample_storage_predict_for_est_2, true_domain=domain) + + n_estimated_nn, l_var_nn, n_samples_nn = get_n_estimated(sample_storage_predict_for_est_2, + predict_q_estimator_est_2, + n_ops=n_ops_predict) + + all_n_estimated_nn.append(n_estimated_nn) + + print("all n estimated nn ", all_n_estimated_nn) + n_estimated_nn = np.mean(all_n_estimated_nn, axis=0, dtype=np.int32) + print("mean n estimated nn ", n_estimated_nn) + + + # n_estimated_nn, l_var_nn, n_samples_nn = get_n_estimated(sample_storage_predict_for_est_2, predict_q_estimator_est_2, + # n_ops=n_ops_predict) + + print("NN SECOND n collected before estimate ", sample_storage_predict_for_est_2.get_n_collected()) + print("NN SECOND n estimated nn ", n_estimated_nn) + + # n_estimated_nn, l_var_nn, n_samples_nn = get_n_estimated(sample_storage_predict, + # predict_q_estimator, + # n_ops=n_ops_predict) + + # n_estimated_nn_C1_C2, _, _ = get_n_estimated(sample_storage_predict, predict_q_estimator, + # n_ops=n_ops_predict) - n_estimated_nn, l_var_nn, n_samples_nn = get_n_estimated(sample_storage_predict, predict_q_estimator, - n_ops=n_ops_predict) print("n ops ", n_ops) print("new n ops predict ", n_ops_predict) print("new n estimated nn ", n_estimated_nn) - + #print("n ops predict C1 C2", n_ops_predict_C1_C2) + #print("new n estimated nn C1 C2 ", n_estimated_nn_C1_C2) # exit() sample_storage_predict = cut_samples(data_nn, sample_storage_predict, n_estimated_nn) + moms, _, _, _ = estimate_moments(sample_storage_predict) + print("moms.vars ", moms.var) + # exit() + + #sample_storage_predict = sample_storage_predict_for_estimate + # predict_q_estimator = get_quantity_estimator(sample_storage_predict_0, true_domain=domain) # predict_q_estimator.quantity = predict_q_estimator.quantity.subsample(sample_vec=n_estimated_nn) print("new n estimated nn ", n_estimated_nn) print("new l var nn ", l_var_nn) + ############################### + ### Variance cost relation ### + ############################### + #_variance_cost_analysis(l_var_nn, n_ops_predict) + + # predict_moments = compute_moments(sample_storage_predict) # print("predict moments var ", predict_moments.var) @@ -666,7 +1051,8 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic print("NN estimated ", n_estimated_nn) print("MLMC estimated ", n_estimated_orig) print("n ops predict_orig ", n_ops_predict_orig) - print("n estimated nn[1] ", n_estimated_nn[1]) + print("n ops ", n_ops) + #print("n estimated nn[1] ", n_estimated_nn[1]) # n_ops_predict_orig = n_ops_predict # n_ops_test = preprocess_time / (n_estimated_nn[1] - len_train_data) # if n_estimated_nn[1] > len_train_data: @@ -680,10 +1066,12 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic print("NN time levels ", NN_time_levels) print("MLMC time levels", n_collected_times) - nn_total_time = np.sum(NN_time_levels) + nn_total_time = np.sum(NN_time_levels) + learning_time print("NN total time ", nn_total_time) mlmc_total_time = np.sum(n_collected_times) print("MLMC total time ", mlmc_total_time) + # nn_total_time = np.sum(n_ops_predict_C1_C2 * np.array(n_estimated_nn_C1_C2)) + learning_time + # print("NN total time + learning time ", np.sum(n_ops_predict_C1_C2 * np.array(n_estimated_nn_C1_C2)) + learning_time) # original_moments, estimator, original_true_domain, _ = estimate_moments(sample_storage) @@ -730,20 +1118,27 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic # for l_id, (l_mom, l_mom_pred) in enumerate(zip(moments_nn.l_means, moments_predict.l_means)): # print("L id: {}, mom diff: {}".format(l_id, l_mom - l_mom_pred)) - if domain_largest: - domain = get_largest_domain([sample_storage, sample_storage_predict, ref_sample_storage]) - common_domain = get_largest_domain([sample_storage, sample_storage_predict, ref_sample_storage]) - else: - domain = None + # if domain_largest: + # domain = get_largest_domain([sample_storage, sample_storage_predict, ref_sample_storage]) + # common_domain = get_largest_domain([sample_storage, sample_storage_predict, ref_sample_storage]) + # else: + # domain = None #domain = get_largest_domain([sample_storage, sample_storage_predict, ref_sample_storage]) - original_q_estimator = get_quantity_estimator(sample_storage, true_domain=domain) predict_q_estimator = get_quantity_estimator(sample_storage_predict, true_domain=domain) - if cut_est: - original_q_estimator = get_quantity_estimator(sample_storage_for_estimated, true_domain=domain) - predict_q_estimator = get_quantity_estimator(sample_storage_predict_for_estimate, true_domain=domain) + final_sample_storage_moments, _, _, _ = estimate_moments(sample_storage) + print("final_sample_storage_moments.mean ", final_sample_storage_moments.mean) + print("final_sample_storage_moments.var ", final_sample_storage_moments.var) + + final_sample_storage_predict_moments, _, _, _ = estimate_moments(sample_storage_predict) + print("final_sample_storage_predict_moments.mean ", final_sample_storage_predict_moments.mean) + print("final_sample_storage_predict_moments.var ", final_sample_storage_predict_moments.var) + + # if cut_est: + # original_q_estimator = get_quantity_estimator(sample_storage_for_estimated, true_domain=domain) + # predict_q_estimator = get_quantity_estimator(sample_storage_predict_for_estimate, true_domain=domain) print("ref samples ", ref_sample_storage) print("domain ", domain) @@ -761,6 +1156,10 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic orig_moments_mean, predict_moments_mean, ref_moments_mean = compare_moments(original_q_estimator, predict_q_estimator, ref_estimator) + ref_orig_moments, ref_predict_moments, mlmc_predict_moments = compare_moments_2(sample_storage, sample_storage_predict, ref_sample_storage) + + level_kurtosis(original_q_estimator, predict_q_estimator) + kl_mlmc, kl_nn = -1, -1 orig_orth_moments, predict_orth_moments, ref_orth_moments = None, None, None kl_mlmc, kl_nn, orig_orth_moments, predict_orth_moments, ref_orth_moments = compare_densities(original_q_estimator, predict_q_estimator, ref_estimator, @@ -771,16 +1170,38 @@ def process_mlmc(nn_hdf_file, sampling_info_path, ref_mlmc_file, targets, predic return n_estimated_orig, n_estimated_nn, n_ops, n_ops_predict, orig_moments_mean, \ predict_moments_mean, ref_moments_mean, sample_storage.get_level_parameters(), \ sample_storage_predict.get_level_parameters(), kl_mlmc, kl_nn, TARGET_VAR, \ - orig_orth_moments, predict_orth_moments, ref_orth_moments + orig_orth_moments, predict_orth_moments, ref_orth_moments,\ + ref_orig_moments, ref_predict_moments, mlmc_predict_moments, learning_time plot_moments({"ref": ref_estimator, "orig": original_q_estimator, "nn": predict_q_estimator}) -def plot_loss(train_loss, val_loss): +def plot_loss(train_loss, val_loss, train_acc=None): plt.plot(train_loss, label='loss') + if train_acc is not None: + plt.plot(train_acc, label='train acc') plt.plot(val_loss, label='val_loss') - #plt.ylim([0, 10]) + + print("len train loss ", len(train_loss)) + print("len val loss ", len(val_loss)) + print("final train loss ", train_loss[-1]) + print("final val loss ", val_loss[-1]) + + + print("val loss min: {}, corresponding train loss: {}".format(np.min(val_loss), train_loss[np.argmin(val_loss)])) + print("min val loss position: {}".format(np.argmin(val_loss))) + print("train loss min: {}, corresponding val loss: {}".format(np.min(train_loss), val_loss[np.argmin(train_loss)])) + print("min train loss position: {}".format(np.argmin(train_loss))) + + print("min train after min val: {}".format(np.min(train_loss[np.argmin(val_loss):]))) + print("min train after min val position: {}".format(np.argmin(val_loss) + np.argmin(train_loss[np.argmin(val_loss):]))) + + #plt.ylim([1, 3]) plt.yscale("log") + #plt.axhline(np.min(train_loss)) + #plt.axhline(np.min(val_loss)) + plt.axvline(x=np.argmin(train_loss), color="blue") + plt.axvline(x=np.argmin(val_loss), color="green") plt.xlabel('Epoch') plt.ylabel('Error') plt.legend() @@ -1051,12 +1472,144 @@ def get_largest_domain(storages): true_domains = np.array(true_domains) + print("true domains ", true_domains) + + # print("true domains ", true_domains[0]) + #true_domain = true_domains[-1] # ref + + #true_domain = [np.min(true_domains[:, 0]), np.max(true_domains[:, 1])] - true_domain = [np.max(true_domains[:, 0]), np.min(true_domains[:, 1])] + #true_domain = [np.max(true_domains[:, 0]), np.min(true_domains[:, 1])] #true_domain = [np.mean(true_domains[:, 0]), np.mean(true_domains[:, 1])] + + #true_domain = true_domains[-1] # ref domain + true_domain = true_domains[0] # MC domain return true_domain +def compare_moments_2(sample_storage, sample_storage_predict, ref_sample_storage): + true_domains = [] + for storage in [sample_storage, sample_storage_predict, ref_sample_storage]: + result_format = storage.load_result_format() + root_quantity = make_root_quantity(storage, result_format) + + conductivity = root_quantity['conductivity'] + time = conductivity[1] # times: [1] + location = time['0'] # locations: ['0'] + q_value = location[0, 0] + + # @TODO: How to estimate true_domain? + quantile = QUANTILE + domain = mlmc.estimator.Estimate.estimate_domain(q_value, storage, quantile=quantile) + true_domains.append([domain[0], domain[1]]) + + mlmc_ref_domain = [np.max([true_domains[0][0], true_domains[-1][0]]), + np.min([true_domains[0][1], true_domains[-1][1]])] + + print("true domain ", true_domains) + + # just mlmc domain + mlmc_ref_domain = true_domains[-1] # ref domain + mlmc_ref_domain = true_domains[0] # mlmc domain + + + nn_ref_domain = [np.max([true_domains[1][0], true_domains[-1][0]]), + np.min([true_domains[1][1], true_domains[-1][1]])] + + # just nn domain + nn_ref_domain = true_domains[-1] # ref domain + nn_ref_domain = true_domains[1] # mlmc domain + + + print("mlmc ref domain ", mlmc_ref_domain) + print("nn ref domain ", nn_ref_domain) + + print("############################ COMPARE MOMENTS 2 ####################################") + #### + ## MLMC vs REF + ### + ref_estimator = get_quantity_estimator(ref_sample_storage, true_domain=mlmc_ref_domain) + mlmc_estimator = get_quantity_estimator(sample_storage, true_domain=mlmc_ref_domain) + + ref_estimator.estimate_moments() + ref_moments_mean = ref_estimator.moments_mean + + mlmc_estimator.estimate_moments() + orig_moments_mean = mlmc_estimator.moments_mean + + # print("ref moments mean ", ref_moments_mean.mean) + # print("orig moments mean ", orig_moments_mean.mean) + + ref_orig_moments = [ref_moments_mean, orig_moments_mean] + + print("ref orig mean SSE ", np.sum((ref_moments_mean.mean - orig_moments_mean.mean) ** 2)) + print("ref orig mean SE ", np.sum(np.abs((ref_moments_mean.mean - orig_moments_mean.mean)))) + + print("ref orig var SSE ", np.sum((ref_moments_mean.var - orig_moments_mean.var) ** 2)) + print("ref orig var SE ", np.sum(np.abs((ref_moments_mean.var - orig_moments_mean.var)))) + + ############################## + ############################## + ### NN PREDICT vs REF + ### + ref_estimator = get_quantity_estimator(ref_sample_storage, true_domain=nn_ref_domain) + predict_estimator = get_quantity_estimator(sample_storage_predict, true_domain=nn_ref_domain) + + ref_estimator.estimate_moments() + ref_moments_mean = ref_estimator.moments_mean + + predict_estimator.estimate_moments() + predict_moments_mean = predict_estimator.moments_mean + + ref_predict_moments = [ref_moments_mean, predict_moments_mean] + + print("ref predict mean SSE ", np.sum((ref_moments_mean.mean - predict_moments_mean.mean) ** 2)) + # print("predict moments mean ", predict_moments_mean.mean) + print("ref predict mean SE ", np.sum(np.abs((ref_moments_mean.mean - predict_moments_mean.mean)))) + + print("ref predict var SSE ", np.sum((ref_moments_mean.var - predict_moments_mean.var) ** 2)) + print("ref predict var SE ", np.sum(np.abs((ref_moments_mean.var - predict_moments_mean.var)))) + + + ############################## + ###############################place + ### MLMC vs MC +NN + mlmc_estimator = get_quantity_estimator(sample_storage, true_domain=true_domains[0]) + predict_estimator = get_quantity_estimator(sample_storage_predict, true_domain=true_domains[0]) + + mlmc_estimator.estimate_moments() + mlmc_moments_mean = mlmc_estimator.moments_mean + + predict_estimator.estimate_moments() + predict_moments_mean = predict_estimator.moments_mean + + mlmc_predict_moments = [mlmc_moments_mean, predict_moments_mean] + + + # print("ref moments var ", ref_moments_mean.var) + #print("orig moments var ", orig_moments_mean.var) + #print("predict moments var ", predict_moments_mean.var) + + # print("MAX orig moments var ", np.max(orig_moments_mean.l_vars, axis=1)) + # print("MAX predict moments var ", np.max(predict_moments_mean.l_vars, axis=1)) + + print("MC moments ", mlmc_moments_mean.mean) + print("MC + NN moments ", predict_moments_mean.mean) + + print("mlmc predict mean SSE ", np.sum((mlmc_moments_mean.mean - predict_moments_mean.mean) ** 2)) + # print("predict moments mean ", predict_moments_mean.mean) + print("mlmc predict mean SE ", np.sum(np.abs((mlmc_moments_mean.mean - predict_moments_mean.mean)))) + + print("mlmc predict var SSE ", np.sum((mlmc_moments_mean.var - predict_moments_mean.var) ** 2)) + print("mlmc predict var SE ", np.sum(np.abs((mlmc_moments_mean.var - predict_moments_mean.var)))) + + + + print("##############################################################") + + return ref_orig_moments, ref_predict_moments, mlmc_predict_moments + + def compare_moments(original_q_estimator, predict_q_estimator, ref_estimator=None): print("############################ COMPARE MOMENTS ####################################") original_q_estimator.estimate_moments() @@ -1070,7 +1623,7 @@ def compare_moments(original_q_estimator, predict_q_estimator, ref_estimator=Non ref_estimator.estimate_moments() ref_moments_mean = ref_estimator.moments_mean - #print("ref moments mean ", ref_moments_mean.mean) + print("ref moments mean ", ref_moments_mean.mean) print("orig moments mean ", orig_moments_mean.mean) print("predict moments mean ", predict_moments_mean.mean) @@ -1125,10 +1678,12 @@ def compare_moments(original_q_estimator, predict_q_estimator, ref_estimator=Non # print("fine l_1_samples.var ", np.var(np.squeeze(l_1_samples[..., 0]))) # print("fine l_2_samples.var ", np.var(np.squeeze(l_2_samples[..., 0]))) +def level_kurtosis(original_q_estimator, predict_q_estimator): + original_q_estimator.kurtosis_check() -def compare_densities(estimator_1, estimator_2, ref_estimator, label_1="", label_2=""): - distr_plot = plots.ArticleDistributionPDF(title="densities", log_density=True, set_x_lim=False) +def compare_densities(estimator_1, estimator_2, ref_estimator, label_1="", label_2=""): + distr_plot = plots.ArticleDistributionPDF(title="densities", log_density=True, set_x_lim=False, quantity_name="$c [kgm^{-3}]$") tol = 1e-7 reg_param = 0 @@ -1167,12 +1722,13 @@ def compare_densities(estimator_1, estimator_2, ref_estimator, label_1="", label #distr_plot.add_distribution(distr_obj_1, label=label_1 + ", KL(ref|orig):{:0.4g}".format(kl_div_ref_mlmc), color="blue") #distr_plot.add_distribution(distr_obj_2, label=label_2 + ", KL(ref|gnn):{:0.4g}".format(kl_div_ref_gnn), color="red", line_style="--") - distr_plot.add_distribution(distr_obj_1, label=r"$D_{MC}:$" + "{:0.4g}".format(kl_div_ref_mlmc), color="blue") - distr_plot.add_distribution(distr_obj_2, label=r"$D_{meta}:$" + "{:0.4g}".format(kl_div_ref_gnn), color="red", line_style="--") + distr_plot.add_distribution(distr_obj_1, label=r"$D_{3LMC}:$" + "{:0.4g}".format(kl_div_ref_mlmc), color="blue") + distr_plot.add_distribution(distr_obj_2, label=r"$D_{3LMC-M}:$" + "{:0.4g}".format(kl_div_ref_gnn), color="red", line_style="--") distr_plot.add_distribution(ref_distr_obj, label="MC ref", color="black", line_style=":") - distr_plot.show(file="densities.pdf") distr_plot.show(file=None) + distr_plot.show(file="densities.pdf") + return kl_div_ref_mlmc, kl_div_ref_gnn, orig_orth_moments, predict_orth_moments, ref_orth_moments @@ -1194,6 +1750,7 @@ def get_quantity_estimator(sample_storage, true_domain=None, quantity=None, n_mo print("true domain") moments_fn = Legendre(n_moments, true_domain) + #moments_fn = Monomial(n_moments, true_domain) return mlmc.estimator.Estimate(quantity=quantity, sample_storage=sample_storage, moments_fn=moments_fn) @@ -1205,15 +1762,15 @@ def get_n_estimated(sample_storage, estimator, n_ops=None): n_level_samples = sample_storage.get_n_collected() # New estimation according to already finished samples - print("n level samples ", n_level_samples) + #print("n level samples ", n_level_samples) variances, n_samples = estimator.estimate_diff_vars() - print("n samples ", n_samples) + #print("n samples ", n_samples) #variances, est_n_ops = estimator.estimate_diff_vars_regression(n_level_samples) if n_ops is None: n_ops = n_samples - print("get n estimated n ops ", n_ops) - print("variances ", variances) + # print("get n estimated n ops ", n_ops) + # print("variances ", variances) n_estimated = mlmc.estimator.estimate_n_samples_for_target_variance(target_var, variances, n_ops, n_levels=len(n_level_samples)) return n_estimated, variances, n_samples @@ -1227,7 +1784,6 @@ def get_storage_info(sample_storage): print("moments.l_vars max ", max_vars) - print('moments l vars ', moments.l_vars) return n_collected, max_vars @@ -1236,12 +1792,15 @@ def cut_samples(data, sample_storage, new_n_collected, new_l_0=0, bootstrap=Fals new_data = [] for l_id, (d, n_est) in enumerate(zip(data, new_n_collected)): # print("len d :", d.shape[1]) - # print("n est ", n_est) + #print("cut samples n est", n_est) if n_est > 0: if l_id == new_l_0: if bootstrap: sample_idx = np.random.choice(list(range(0, d.shape[1]-1)), size=n_est, replace=True) - fine_samples = d[:, sample_idx, 0].reshape(1, np.min([d.shape[1], len(sample_idx)]), 1) + if len(sample_idx) > d.shape[1]: + fine_samples = d[:, :np.min([d.shape[1], n_est]), 0].reshape(1, np.min([d.shape[1], n_est]), 1) + else: + fine_samples = d[:, sample_idx, 0].reshape(1, np.min([d.shape[1], len(sample_idx)]), 1) else: fine_samples = d[:, :np.min([d.shape[1], n_est]), 0].reshape(1, np.min([d.shape[1], n_est]), 1) @@ -1250,7 +1809,10 @@ def cut_samples(data, sample_storage, new_n_collected, new_l_0=0, bootstrap=Fals else: if bootstrap: sample_idx = np.random.choice(list(range(0, d.shape[1] - 1)), size=n_est, replace=True) - new_data.append(d[:, sample_idx, :]) + if len(sample_idx) > d.shape[1]: + new_data.append(d[:, sample_idx, :]) + else: + new_data.append(d[:, :np.min([d.shape[1], n_est]), :]) else: new_data.append(d[:, :np.min([d.shape[1], n_est]), :]) @@ -1279,24 +1841,33 @@ def plot_progress(conv_layers, dense_layers, output_flatten, mesh_file=None, n_s plt.show() # Note: weights have different shape than the mesh + print("inputs ", inputs) + print("weights ", weights) + print("outputs ", outputs) + for index, input in enumerate(inputs[:n_samples]): if mesh_file: - fig, ax = plt.subplots(1, 1, figsize=(15, 10)) - - cont = ax.tricontourf(X, Y, input.ravel(), levels=16) - fig.colorbar(cont) - plt.title("input") - plt.show() - - # for i in range(outputs[index].shape[1]): - # channel_output = outputs[index][:, i] - # print("channel output shape ", channel_output.shape) - # fig, ax = plt.subplots(1, 1, figsize=(15, 10)) - # cont = ax.tricontourf(X, Y, channel_output, levels=16) - # fig.colorbar(cont) - # plt.title("channel {}".format(i)) - # - # plt.show() + for i in range(inputs[index].shape[1]): + input_feature = inputs[index][:, i] + fig, ax = plt.subplots(1, 1, figsize=(15, 10)) + print("inputs.shape ", input_feature.shape) + print("X.shape ", X.shape) + print("Y.shape ", Y.shape) + print("input.ravel().shape ", input_feature.ravel().shape) + cont = ax.tricontourf(X, Y, input_feature.ravel(), levels=16) + fig.colorbar(cont) + plt.title("input feature {}".format(i)) + plt.show() + + for i in range(outputs[index].shape[1]): + channel_output = outputs[index][:, i] + print("channel output shape ", channel_output.shape) + fig, ax = plt.subplots(1, 1, figsize=(15, 10)) + cont = ax.tricontourf(X, Y, channel_output, levels=16) + fig.colorbar(cont) + plt.title("output channel {}".format(i)) + + plt.show() else: plt.matshow(input[0]) @@ -1337,7 +1908,7 @@ def plot_progress(conv_layers, dense_layers, output_flatten, mesh_file=None, n_s plt.show() - #exit() + exit() def plot_moments(mlmc_estimators): @@ -1395,15 +1966,69 @@ def analyze_mlmc_data(): assert np.allclose(original_moments.var, moments_2.var) -def get_sample_times_mlmc(mlmc_file): - sample_storage = SampleStorageHDF(file_path=mlmc_file) +def load_sim_data(sim_data_file): + """ + Load saved simulation data + :return: dict + """ + import json + + with open(sim_data_file, 'r') as file: + # First we load existing data into a dict. + data = json.load(file) + + generate_rnd = [] + extract_mesh = [] + make_fields = [] + coarse_flow = [] + fine_flow = [] + for level_data in data.values(): + generate_rnd_level = np.zeros(2) + extract_mesh_level = np.zeros(2) + make_fields_level = np.zeros(2) + coarse_flow_level = np.zeros(2) + fine_flow_level = np.zeros(2) + for d in level_data: + generate_rnd_level[0] += d["generate_rnd"] + generate_rnd_level[1] += 1 + + extract_mesh_level[0] += d["extract_mesh"] + extract_mesh_level[1] += 1 + + make_fields_level[0] += d["make_field"] + make_fields_level[1] += 1 + + coarse_flow_level[0] += d["coarse_flow"] + coarse_flow_level[1] += 1 + + fine_flow_level[0] += d["fine_flow"] + fine_flow_level[1] += 1 + + generate_rnd.append(generate_rnd_level) + extract_mesh.append(extract_mesh_level) + make_fields.append(make_fields_level) + coarse_flow.append(coarse_flow_level) + fine_flow.append(fine_flow_level) + + return generate_rnd, extract_mesh, make_fields, coarse_flow, fine_flow + + + +def get_sample_times_mlmc(mlmc_file, sample_storage=None): + if sample_storage is None: + sample_storage = SampleStorageHDF(file_path=mlmc_file) + + sim_data_file = os.path.join(os.path.dirname(mlmc_file), 'sim_data.json') n_ops = sample_storage.get_n_ops() - generate_rnd = sample_storage.get_generate_rnd_times() - extract_mesh = sample_storage.get_extract_mesh_times() - make_fields = sample_storage.get_make_field_times() - coarse_flow = sample_storage.get_coarse_flow_times() - fine_flow = sample_storage.get_fine_flow_times() + if os.path.exists(sim_data_file): + generate_rnd, extract_mesh, make_fields, coarse_flow, fine_flow = load_sim_data(sim_data_file) + else: + generate_rnd = sample_storage.get_generate_rnd_times() + extract_mesh = sample_storage.get_extract_mesh_times() + make_fields = sample_storage.get_make_field_times() + coarse_flow = sample_storage.get_coarse_flow_times() + fine_flow = sample_storage.get_fine_flow_times() def time_for_sample_func(data): new_n_ops = [] diff --git a/mlmc/plot/plots.py b/mlmc/plot/plots.py index 71e0ba06..f97cadaf 100644 --- a/mlmc/plot/plots.py +++ b/mlmc/plot/plots.py @@ -25,6 +25,7 @@ def create_color_bar(range, label, ax=None, colormap=None): except TypeError: min_r, max_r = 0, range normalize = plt.Normalize(vmin=min_r, vmax=max_r) + #colormap = (matplotlib.colors.ListedColormap(['red', 'green', 'blue', 'orange'])) scalar_mappable = plt.cm.ScalarMappable(norm=normalize, cmap=colormap) if type(max_r) is int: cb_values = np.arange(min_r, max_r) @@ -33,6 +34,11 @@ def create_color_bar(range, label, ax=None, colormap=None): cb_values = np.linspace(min_r, max_r, 100) #ticks = np.linspace(min_r, int(size / 10) * 10, 9) ticks = None + ticks = [5,10,15,20] + ticks = [2, 4, 6, 8, 10] + ticks = [2, 3, 4, 5] + + scalar_mappable.set_array(cb_values) clb = plt.colorbar(scalar_mappable, ticks=ticks, aspect=50, pad=0.01, ax=ax) clb.set_label(label) @@ -958,7 +964,7 @@ def show(self, file=""): label='r"$MLMC_{meta}:$"') # Line2D([], [], color='black', marker='|') handles.append(mlmc_marker_meta) - labels.append("MLMC meta") + labels.append("MLMC-M") legend._legend_box = None legend._init_legend_box(handles, labels) @@ -967,6 +973,198 @@ def show(self, file=""): _show_and_save(self.fig, file, self.title) + +class VarianceNN2: + """ + Plot level variances, i.e. Var X^l as a function of the mesh step. + Selected moments are plotted. + """ + def __init__(self, moments=None): + """ + :param moments: Size or type of moments subset, see moments_subset function. + """ + matplotlib.rcParams.update({'font.size': 26}) + matplotlib.rcParams.update({'lines.markersize': 7}) + self.fig = plt.figure(figsize=(15, 8)) + + matplotlib.rcParams.update({'font.size': 16}) + matplotlib.rcParams.update({'lines.markersize': 8}) + # fig, axes = plt.subplots(1, 1, figsize=(22, 10)) + self.fig = plt.figure(figsize=(8, 5)) + + self.title = ""#"Level variances" + self.fig.suptitle(self.title) + + self.ax = self.fig.add_subplot(1, 1, 1) + self.ax.set_xlabel("mesh elements") + #self.ax.set_ylabel("Var $X^h$") + self.ax.set_ylabel("$\hat{V}^r_l$") + self.ax.set_xscale('log') + self.ax.set_yscale('log') + + #self.ax.set_xlim([1e-3, 1e0]) + self.ax.set_xlim([5e1, 1e5]) + + self.n_moments = None + self.subset_type = moments + self.min_step = 1e300 + self.max_step = 0 + self.data = {} + + self.nn_min_step = 1e300 + self.nn_max_step = 0 + self.nn_data = {} + + self._colormap = plt.cm.tab20 + self._n_ops = None + + def set_n_ops(self, n_ops): + print("n ops ", n_ops) + self._n_ops = n_ops + + def add_level_variances(self, steps, variances): + """ + Add variances for single MLMC instance. + :param steps, variances : as returned by Estimate.estimate_level_vars + :param n_levels: + """ + n_levels, n_moments = variances.shape + if self.n_moments is None: + self.n_moments = n_moments + self.moments_subset = moments_subset(n_moments, self.subset_type) + else: + assert self.n_moments == n_moments + + variances = variances[:, self.moments_subset] + self.min_step = min(self.min_step, steps[-1]) + self.max_step = max(self.max_step, steps[0]) + for m, vars in enumerate(variances.T): + X, Y = self.data.get(m, ([], [])) + X.extend(steps.tolist()) + Y.extend(vars.tolist()) + self.data[m] = (X, Y) + + def add_level_variances_nn(self, steps, variances): + """ + Add variances for single MLMC instance. + :param steps, variances : as returned by Estimate.estimate_level_vars + :param n_levels: + """ + n_levels, n_moments = variances.shape + if self.n_moments is None: + self.n_moments = n_moments + self.moments_subset = moments_subset(n_moments, self.subset_type) + else: + assert self.n_moments == n_moments + + variances = variances[:, self.moments_subset] + self.nn_min_step = min(self.min_step, steps[-1]) + self.nn_max_step = max(self.max_step, steps[0]) + for m, vars in enumerate(variances.T): + X, Y = self.nn_data.get(m, ([], [])) + X.extend(steps.tolist()) + Y.extend(vars.tolist()) + self.nn_data[m] = (X, Y) + + def show(self, file=""): + self._colormap = create_color_bar(range=[1, self.n_moments], label=r'$r$', ax=self.ax, colormap=plt.cm.tab20) + res = 5 + step_range = self.max_step / self.min_step + log_scale = step_range ** 0.001 - 1 + #rv = st.lognorm(scale=1, s=log_scale) + for m, (X, Y) in self.data.items(): + + # if m == 5: + # break + print("m+1+m ", m+1+m) + col = self._colormap(m+1) + label = "M{}".format(self.moments_subset[m]) + label = "MLMC" + print("data m: {}, col: {}".format(m, col)) + # print("X ", X) + # print("len(X) ", len(X)) + # #print("rv.rvs(size=len(X)) ", rv.rvs(size=len(X))) + # print("Y ", Y) + XX = np.array(X) #* rv.rvs(size=len(X)) + # print("XX ", X) + self.ax.scatter(XX, Y, color=col) + XX, YY = make_monotone(X, Y) + + # step_range = self.nn_max_step / self.nn_min_step + # log_scale = step_range ** 0.01 - 1 + # rv = st.lognorm(scale=1, s=log_scale) + levels = {} + + print("nn data ", self.nn_data) + for m, (X, Y) in self.nn_data.items(): + #m += 1 + # if m == 5: + # break + #col = plt.cm.tab20(m) + col = self._colormap(m+1) + print("nn_data m: {}, col: {}".format(m, col)) + XX = np.array(X) * 0.84 # rv.rvs(size=len(X)) + print("X ", X) + X = np.array(X) + #XY = np.concatenate(([X[0] * 0.35], X[1:] * 1.16), axis=0) + + XY = np.concatenate(([X[0] * 0.6], [X[1] * 1.1], X[2:] * 0.95), axis=0) + + self.ax.scatter(XX, Y, color=col, marker='v') + # XX, YY = make_monotone(X, Y) + + for x, y in zip(X, Y): + if x not in levels: + levels[x] = [] + levels[x].append(y) + + print("XX ", XX) + print("np.max(Y) + np.max(Y)*0.3) ", np.max(Y) + np.max(Y) * 0.3) + print("self._n_ops ", self._n_ops) + self._n_ops = [0.338, 29.7, 223, 2320] + if self._n_ops is not None: + for index, n_ops in enumerate(self._n_ops): + if index == 0: + self.ax.annotate("{}".format(n_ops), (XY[index], np.max(levels[X[index]])*1.7)) + elif index == 1: + self.ax.annotate("{}".format(n_ops), (XY[index], np.max(levels[X[index]])*1.9)) + else: + #self.ax.annotate("{:0.3g}".format(n_ops), (XY[index], np.max(levels[X[index]]))) + self.ax.annotate("{}".format(n_ops), (XY[index], np.max(levels[X[index]])*1.7)) + + #self.fig.legend() + + legend = self.ax.legend() + ax = legend.axes + + from matplotlib.lines import Line2D + from matplotlib.patches import Rectangle, RegularPolygon, FancyBboxPatch + + handles, labels = ax.get_legend_handles_labels() + + mlmc_marker = Line2D([], [], color='black', marker='o', linestyle='None', + markersize=8, markeredgewidth=1.7, + label='MLMC') # Line2D([], [], color='black', marker='|') + + handles.append(mlmc_marker) + labels.append("3LMC") + + if self.nn_data: + mlmc_marker_meta = Line2D([], [], color='black', marker='v', linestyle='None', + markersize=8, markeredgewidth=1.7, + label='r"$MLMC_{meta}:$"') # Line2D([], [], color='black', marker='|') + + handles.append(mlmc_marker_meta) + labels.append("3LMC-M") + + legend._legend_box = None + legend._init_legend_box(handles, labels) + legend._set_loc(legend._loc) + legend.set_title(legend.get_title().get_text()) + + _show_and_save(self.fig, file, self.title) + + class MomentsPlots(Distribution): def __init__(self, title="", quantity_name="i-th moment", legend_title="", log_mean_y=False, log_var_y=False): """ @@ -1805,7 +2003,11 @@ def __init__(self, exact_distr=None, title="", quantity_name="Y", legend_title=" integrand of KL divergence: exact_pdf * log(exact_pdf / approx_pdf). Simple difference is used for CDF for both options. """ + plt.ticklabel_format(style='sci') matplotlib.rcParams.update({'font.size': 16}) + from matplotlib import ticker + formatter = ticker.ScalarFormatter(useMathText=True) + formatter.set_scientific(True) #matplotlib.rcParams.update({'lines.markersize': 8}) self._exact_distr = exact_distr self._log_density = log_density @@ -1838,6 +2040,7 @@ def __init__(self, exact_distr=None, title="", quantity_name="Y", legend_title=" self.ax_pdf.set_ylabel("PDF")#, color=self.pdf_color) #self.ax_pdf.set_ylabel("probability density") self.ax_pdf.set_xlabel(x_axis_label) + #self.ax_pdf.ticklabel_format(style='sci') #self.ax_pdf.tick_params(axis='y', labelcolor=self.pdf_color) if self._log_x: self.ax_pdf.set_xscale('log') diff --git a/test/metamodels/predict_on_different_mesh.py b/test/metamodels/predict_on_different_mesh.py index 84d5852a..7192c443 100644 --- a/test/metamodels/predict_on_different_mesh.py +++ b/test/metamodels/predict_on_different_mesh.py @@ -1,7 +1,7 @@ import os import numpy as np import warnings -os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only +#os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # Run on CPU only import sys import shutil import subprocess @@ -45,7 +45,7 @@ def get_gnn(): # loss = KLDivergence() # loss = total_loss_function optimizer = tf.optimizers.Adam(learning_rate=0.001) - patience = 150 + patience = 1000 hidden_regularization = None # l2(2e-10) net_model_config = { @@ -81,7 +81,7 @@ def __init__(self, conv_layer, hidden_activation, output_activation, kernel_regu conv_layer(8, K=4, activation=hidden_activation, kernel_regularizer=kernel_regularization)] self.flatten = GlobalSumPool() - self._dense_layers = [Dense(32, activation=hidden_activation), Dense(16, activation=hidden_activation), + self._dense_layers = [Dense(64, activation=hidden_activation), Dense(32, activation=hidden_activation), Dense(1)] # T34 @@ -111,31 +111,35 @@ def call(self, inputs): def get_config(data_dir, case=0): - if case == 12: # mesh size comparison data_dir = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/" # cl = "cl_0_1_s_1" - level = 3 + level = "4" nn_level = 0 replace_level = False mesh = os.path.join(data_dir, "l_step_1.0_common_files/repo.msh") # L1, 7s - #mesh = os.path.join(data_dir, "l_step_0.3760603093086394_common_files/repo.msh") #L2 10.5 s - mesh = os.path.join(data_dir, "l_step_0.1414213562373095_common_files/repo.msh") #L3 12s - #mesh = os.path.join(data_dir, "l_step_0.053182958969449884_common_files/repo.msh") # L4 - output_dir = os.path.join(data_dir, "L1_{}/test/02_conc/output/".format(level)) - hdf_path = os.path.join(data_dir, "L1_{}/mlmc_1.hdf5".format(level)) + # mesh = os.path.join(data_dir, "l_step_0.3760603093086394_common_files/repo.msh") #L2 10.5 s + # mesh = os.path.join(data_dir, "l_step_0.1414213562373095_common_files/repo.msh") #L3 12s + mesh = os.path.join(data_dir, "l_step_0.053182958969449884_common_files/repo.msh") # L4 # 4388 - for 50k + # mesh = os.path.join(data_dir, "l_step_0.027_common_files/repo.msh") # L5 - graph creation time: 2564.6843196170003 + output_dir = os.path.join(data_dir, "L1_{}_50k/test/02_conc/output/".format(level)) + predict_dir = os.path.join(data_dir, "L1_{}/test/02_conc/output/".format(level)) + predict_hdf = os.path.join(data_dir, "L1_{}/mlmc_1.hdf5".format(level)) + hdf_path = os.path.join(data_dir, "L1_{}_50k/mlmc_1.hdf5".format(level)) mlmc_hdf_path = os.path.join(data_dir, "mlmc_hdf/L1_{}/mlmc_1.hdf5".format(level)) save_path = data_dir - l_0_output_dir = os.path.join(data_dir, "L0_MC/L1_{}/test/02_conc/output/".format(level)) - l_0_hdf_path = os.path.join(data_dir, "L0_MC/L1_{}/mlmc_1.hdf5".format(level)) + l_0_output_dir = os.path.join(data_dir, "L0_MC/L1_{}_50k/test/02_conc/output/".format(level)) + l_0_hdf_path = os.path.join(data_dir, "L0_MC/L1_{}_50k/mlmc_1.hdf5".format(level)) sampling_info_path = os.path.join(data_dir, "sampling_info") - #ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_3/mlmc_1.hdf5" - #ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_1/mlmc_1.hdf5" + # ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_3/mlmc_1.hdf5" + # ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_1/mlmc_1.hdf5" ref_mlmc_file = os.path.join(data_dir, "L1_benchmark/mlmc_1.hdf5") feature_names = [['conductivity_top', 'conductivity_bot', 'conductivity_repo']] + graph_creation_time = 4400 + if case == "L2": # mesh size comparison data_dir = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/" level = 2 @@ -144,12 +148,12 @@ def get_config(data_dir, case=0): mesh = os.path.join(data_dir, "l_step_0.3760603093086394_common_files/repo.msh") graph_creation_time = 35 - output_dir = os.path.join(data_dir, "L1_{}/test/02_conc/output/".format(level)) - hdf_path = os.path.join(data_dir, "L1_{}/mlmc_1.hdf5".format(level)) + output_dir = os.path.join(data_dir, "L1_{}_50k/test/02_conc/output/".format(level)) + hdf_path = os.path.join(data_dir, "L1_{}_50k/mlmc_1.hdf5".format(level)) mlmc_hdf_path = os.path.join(data_dir, "mlmc_hdf/L1_{}/mlmc_1.hdf5".format(level)) save_path = data_dir - l_0_output_dir = os.path.join(data_dir, "L0_MC/L1_{}/test/02_conc/output/".format(level)) - l_0_hdf_path = os.path.join(data_dir, "L0_MC/L1_{}/mlmc_1.hdf5".format(level)) + l_0_output_dir = os.path.join(data_dir, "L0_MC/L1_{}_50k/test/02_conc/output/".format(level)) + l_0_hdf_path = os.path.join(data_dir, "L0_MC/L1_{}_50k/mlmc_1.hdf5".format(level)) sampling_info_path = os.path.join(data_dir, "sampling_info") #ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_3/mlmc_1.hdf5" @@ -165,12 +169,12 @@ def get_config(data_dir, case=0): replace_level = False mesh = os.path.join(data_dir, "l_step_1.0_common_files/repo.msh") graph_creation_time = 68 - output_dir = os.path.join(data_dir, "L1_{}/test/02_conc/output/".format(level)) - hdf_path = os.path.join(data_dir, "L1_{}/mlmc_1.hdf5".format(level)) + output_dir = os.path.join(data_dir, "L1_{}_50k/test/02_conc/output/".format(level)) + hdf_path = os.path.join(data_dir, "L1_{}_50k/mlmc_1.hdf5".format(level)) mlmc_hdf_path = os.path.join(data_dir, "mlmc_hdf/L1_{}/mlmc_1.hdf5".format(level)) save_path = data_dir - l_0_output_dir = os.path.join(data_dir, "L0_MC/L1_{}/test/02_conc/output/".format(level)) - l_0_hdf_path = os.path.join(data_dir, "L0_MC/L1_{}/mlmc_1.hdf5".format(level)) + l_0_output_dir = os.path.join(data_dir, "L0_MC/L1_{}_50k/test/02_conc/output/".format(level)) + l_0_hdf_path = os.path.join(data_dir, "L0_MC/L1_{}_50k/mlmc_1.hdf5".format(level)) sampling_info_path = os.path.join(data_dir, "sampling_info") #ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_3/mlmc_1.hdf5" @@ -185,13 +189,37 @@ def get_config(data_dir, case=0): nn_level = 0 replace_level = False mesh = os.path.join(data_dir, "l_step_0.1414213562373095_common_files/repo.msh") # L3 12s - graph_creation_time = 250 - output_dir = os.path.join(data_dir, "L1_{}/test/02_conc/output/".format(level)) - hdf_path = os.path.join(data_dir, "L1_{}/mlmc_1.hdf5".format(level)) + graph_creation_time = 500 + output_dir = os.path.join(data_dir, "L1_{}_50k/test/02_conc/output/".format(level)) + hdf_path = os.path.join(data_dir, "L1_{}_50k/mlmc_1.hdf5".format(level)) mlmc_hdf_path = os.path.join(data_dir, "mlmc_hdf/L1_{}/mlmc_1.hdf5".format(level)) save_path = data_dir - l_0_output_dir = os.path.join(data_dir, "L0_MC/L1_{}/test/02_conc/output/".format(level)) - l_0_hdf_path = os.path.join(data_dir, "L0_MC/L1_{}/mlmc_1.hdf5".format(level)) + l_0_output_dir = os.path.join(data_dir, "L0_MC/L1_{}_50k/test/02_conc/output/".format(level)) + l_0_hdf_path = os.path.join(data_dir, "L0_MC/L1_{}_50k/mlmc_1.hdf5".format(level)) + sampling_info_path = os.path.join(data_dir, "sampling_info") + + # output_dir = l_0_output_dir + # hdf_path = l_0_hdf_path + + #ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_3/mlmc_1.hdf5" + #ref_mlmc_file = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/L1_1/mlmc_1.hdf5" + ref_mlmc_file = os.path.join(data_dir, "L1_benchmark/mlmc_1.hdf5") + + feature_names = [['conductivity_top', 'conductivity_bot', 'conductivity_repo']] + + if case == 'L5': # mesh size comparison + data_dir = "/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/" + level = 5 + nn_level = 0 + replace_level = False + mesh = os.path.join(data_dir, "l_step_0.027_common_files/repo.msh") # L5 - graph creation time: 2564.6843196170003 + graph_creation_time = 5000 + output_dir = os.path.join(data_dir, "L1_{}_50k/test/02_conc/output/".format(level)) + hdf_path = os.path.join(data_dir, "L1_{}_50k/mlmc_1.hdf5".format(level)) + mlmc_hdf_path = os.path.join(data_dir, "mlmc_hdf/L1_{}/mlmc_1.hdf5".format(level)) + save_path = data_dir + l_0_output_dir = os.path.join(data_dir, "L0_MC/L1_{}_50k/test/02_conc/output/".format(level)) + l_0_hdf_path = os.path.join(data_dir, "L0_MC/L1_{}_50k/mlmc_1.hdf5".format(level)) sampling_info_path = os.path.join(data_dir, "sampling_info") # output_dir = l_0_output_dir @@ -406,7 +434,7 @@ def get_arguments(arguments): # Load trained model # ####################### #machine_learning_model = ("L1_3_02_conc_cond_log_output_mult_T19_case_1", run_GNN, False) - machine_learning_model = ("L1_3_T19_case_1_out_log_scale", run_GNN, False) + machine_learning_model = ("L1_4_T27_case_1_out_log_scale_is", run_GNN, False) save_path = os.path.join("/home/martin/Documents/metamodels/data/mesh_size/02_conc_cond/", machine_learning_model[0]) data_dict = load_statistics(save_path) #data_dict = process_data(data_dict) @@ -415,7 +443,7 @@ def get_arguments(arguments): list_train_MSE, list_test_MSE, list_all_train_RSE, list_all_test_RSE, list_nn_total_time, list_mlmc_total_time,\ list_kl_mlmc_all, list_kl_nn_all, list_learning_times = [], [], [], [], [], [], [], [], [] - for index, model in enumerate(data_dict["model"][:2]): + for index, model in enumerate(data_dict["model"][:12]): # newInput = Input(batch_shape=(None, 108, 1)) # newOutputs = model(newInput) @@ -434,12 +462,14 @@ def get_arguments(arguments): work_dir = args.work_dir # case = "L2" # case = 521 - case = "case_3" + case = 12#"case_3" + case = "L5" + case="L1" #data_dir = "/home/martin/Documents/metamodels/data/1000_ele/" output_dir, hdf_path, l_0_output_dir, l_0_hdf_path, save_path, mesh, sampling_info_path, ref_mlmc_file,\ replace_level, nn_level, mlmc_hdf_path, feature_names, graph_creation_time = get_config(data_dir, case) - machine_learning_model = ("L1_2_02_conc_cond_log_output_mult_T19_case_1_trained_{}_{}".format(case, index), run_GNN, False) + machine_learning_model = ("L1_5_02_conc_cond_log_output_mult_T27_case_1_trained_{}_{}".format(case, index), run_GNN, False) save_path = os.path.join(save_path, machine_learning_model[0]) if os.path.exists(save_path): @@ -464,14 +494,13 @@ def get_arguments(arguments): # "output_log": True # } - dataset_config = {"first_log_features": False, - "first_log_output": True, - "features_normalization": False, - "output_normalization": False, + # # 02_conc config + dataset_config = {"features_normalization": False, "calc_output_mult_factor": False, "output_mult_factor": 1, "features_mult_factor": 1, - "features_scale": False, + "first_log_features": False, + "first_log_output": True, "output_scale": True, } @@ -486,22 +515,25 @@ def get_arguments(arguments): 'sampling_info_path': sampling_info_path, 'ref_mlmc_file': ref_mlmc_file, 'level': nn_level, - 'conv_layer': conv_layer, - 'gnn': gnn, - 'model_config': model_config, + 'conv_layer': conv_layer, + 'gnn': gnn, + 'model_config': model_config, 'replace_level': replace_level, 'corr_field_config': corr_field_config, 'n_train_samples': 2000, 'val_samples_ratio': 0.2, 'batch_size': 200, - 'epochs': 1, + 'epochs': 2, 'learning_rate': 0.001, 'graph_creation_time': graph_creation_time, 'save_model': True, - 'feature_names': feature_names, "train_model": False, + 'feature_names': feature_names, "set_model": model, - "dataset_config": dataset_config + 'dataset_config': dataset_config, + 'independent_samples': False, + # 'predict_dir': predict_dir, + # 'predict_hdf ': predict_hdf } model_title, mch_l_model, log = config['machine_learning_model']