Conversation
| res = np.array(res) | ||
| fine_coarse_res = res[:, 1] | ||
|
|
||
| result_type = np.dtype((np.float, np.array(fine_coarse_res[0]).shape)) |
There was a problem hiding this comment.
Reason: float seems to be incompatible with new Numpy versions
| for i_level in range(len(self._hdf_object.level_parameters)): | ||
| self._level_groups.append(self._hdf_object.add_level_group(str(i_level))) | ||
|
|
||
| def _hdf_result_format(self, locations, times): |
There was a problem hiding this comment.
Simplified, moved into hdf5.py as it is implementation detail how the format is stored.
There was a problem hiding this comment.
Moereover the format saving was wrong since it same format has been assumed for all dict elements.
| Load result format | ||
| """ | ||
| results_format = self._hdf_object.load_result_format() | ||
| quantities = [] |
There was a problem hiding this comment.
Same thing, but using constructing function with named parameters.
| for level, samples in successful_samples.items(): | ||
| if len(samples) > 0: | ||
| self._level_groups[level].append_successful(np.array(samples)) | ||
| ids, sample_values = zip(*samples) |
There was a problem hiding this comment.
need to make pairs here and convert to numpy arrays just before storage.
| except Exception: | ||
| str_list = traceback.format_exception(*sys.exc_info()) | ||
| err_msg = "".join(str_list) | ||
| print(err_msg) |
There was a problem hiding this comment.
TODO: replace by suitable logging? Only in case of debugging turned on.
Problem is that the exceptions may not be propagated to the Sampler if pool somehow fail. So it would be great to log the exception within the sampling job process.
| if not os.path.isdir(sample_dir): | ||
| os.makedirs(sample_dir, mode=0o775, exist_ok=True) | ||
|
|
||
| # We have observed possible problems with directory creation on the |
There was a problem hiding this comment.
I guessed that there is some problem with NFS, however in fact there was probably problem with various threads writing to the same directory, randomly cleaning it after it was created.
I suggest to keep the code commented, it allows to better check that we are able to write into the directory. There could be problem with rights anyway.
| self._save_running_time(level_sim._level_id, running_time) | ||
|
|
||
| if not err_msg: | ||
| self._queues.setdefault(level_sim._level_id, queue.Queue()).put((sample_id, (result[0], result[1]))) |
There was a problem hiding this comment.
just to improve readibility
| Execute pbs script | ||
| :return: None | ||
| """ | ||
| if len(self._scheduled) > 0: |
There was a problem hiding this comment.
Decrease indention level
| pbs_process.write_pbs_id(pbs_id) | ||
|
|
||
| process = subprocess.run(['qsub', job_file], stderr=subprocess.PIPE, stdout=subprocess.PIPE) | ||
| try: |
There was a problem hiding this comment.
I quite didn't get reason for this try block. What is reason for PIPEs? I propose to just let subproces.run finish and take resulting stdout and stderr.
| import time | ||
| import logging | ||
|
|
||
| class FileSafe(h5py.File): |
There was a problem hiding this comment.
Context manager to open the file with some timeout.
2 participants
Various changes made during application to the Endorse software.