Merge pull request #811 from rhayes777/feature/database_output

Feature/database output

Author: Jammy2211

autofit/database/aggregator/scrape.py

@@ -233,7 +233,6 @@ def _add_pickles(fit: m.Fit, pickle_path: Path):
     try:
         filenames = os.listdir(pickle_path)
     except FileNotFoundError as e:
-        logger.info(e)
         filenames = []
 
     for filename in filenames:

autofit/non_linear/analysis/analysis.py

@@ -6,6 +6,7 @@
 
 from autofit.mapper.prior_model.abstract import AbstractPriorModel
 from autofit.non_linear.paths.abstract import AbstractPaths
+from autofit.non_linear.paths.database import DatabasePaths
 from autofit.non_linear.result import Result
 
 logger = logging.getLogger(__name__)
@@ -52,7 +53,9 @@ def should_visualize(
         3) Visualization can be forced to run via the `force_visualization_overwrite`, for example if a user
         wants to plot additional images that were not output on the original run.
 
-        4) If PyAutoFit test mode is on visualization is disabled, irrespective of the `force_visualization_overwite`
+        4) If the analysis is running a database session visualization is switched off.
+
+        5) If PyAutoFit test mode is on visualization is disabled, irrespective of the `force_visualization_overwite`
         config input.
 
         Parameters
@@ -70,6 +73,9 @@ def should_visualize(
         if os.environ.get("PYAUTOFIT_TEST_MODE") == "1":
             return False
 
+        if isinstance(paths, DatabasePaths):
+            return False
+
         if conf.instance["general"]["output"]["force_visualize_overwrite"]:
             return True
 

autofit/non_linear/grid/grid_search/__init__.py

@@ -253,6 +253,9 @@ def write_results():
             self.logger.debug(
                 "Writing results"
             )
+
+            os.makedirs(self.paths.output_path, exist_ok=True)
+
             with open(self.paths.output_path / "results.csv", "w+") as f:
                 writer = csv.writer(f)
                 writer.writerow([

autofit/non_linear/mock/mock_samples.py

@@ -74,9 +74,6 @@ def gaussian_priors_at_sigma(self, sigma=None):
 
         return self._gaussian_tuples
 
-    def write_table(self, filename):
-        pass
-
     @property
     def unconverged_sample_size(self):
         return self.samples_info["unconverged_sample_size"]

autofit/non_linear/parallel/sneaky.py

@@ -389,9 +389,12 @@ def prior_transform_cache(x):
     """
     Prior transform call
     """
+
     return FunctionCache.prior_transform(x, *FunctionCache.prior_transform_args,
                                          **FunctionCache.prior_transform_kwargs)
 
+
+
 class SneakierPool:
     def __init__(
             self,
@@ -417,11 +420,10 @@ def __init__(
         try:
             from mpi4py import MPI
             self.comm = MPI.COMM_WORLD
+            self._processes = self.comm.size
         except ModuleNotFoundError:
-            pass
+            self._processes = 1
 
-        self._processes = self.comm.size
-        
         init_args = (
             self.fitness_init, self.prior_transform_init,
             self.fitness_args, self.fitness_kwargs,

autofit/non_linear/paths/abstract.py

@@ -1,3 +1,4 @@
+import dill
 import json
 import logging
 import os
@@ -197,7 +198,10 @@ def search_internal_path(self) -> Path:
         """
         The path to the samples folder.
         """
-        return self.output_path / "search_internal"
+
+        os.makedirs(self._files_path / "search_internal", exist_ok=True)
+
+        return self._files_path / "search_internal"
 
     @property
     def image_path(self) -> Path:
@@ -237,7 +241,12 @@ def output_path(self) -> Path:
 
     @property
     def _files_path(self) -> Path:
-        raise NotImplementedError
+        """
+        This is private for a reason, use the save_json etc. methods to save and load json
+        """
+        files_path = self.output_path / "files"
+        os.makedirs(files_path, exist_ok=True)
+        return files_path
 
     def zip_remove(self):
         """
@@ -343,11 +352,11 @@ def _zip_path(self) -> str:
         return f"{self.output_path}.zip"
 
     @abstractmethod
-    def save_json(self, name, object_dict: dict):
+    def save_json(self, name, object_dict: dict, prefix : str = ""):
         pass
 
     @abstractmethod
-    def load_json(self, name) -> dict:
+    def load_json(self, name, prefix : str = "") -> dict:
         pass
 
     @abstractmethod
@@ -359,19 +368,19 @@ def load_array(self, name) -> np.ndarray:
         pass
 
     @abstractmethod
-    def save_fits(self, name: str, hdu):
+    def save_fits(self, name: str, hdu, prefix : str = ""):
         pass
 
     @abstractmethod
-    def load_fits(self, name: str):
+    def load_fits(self, name: str, prefix : str = ""):
         pass
 
     @abstractmethod
-    def save_object(self, name: str, obj: object):
+    def save_object(self, name: str, obj: object, prefix : str = ""):
         pass
 
     @abstractmethod
-    def load_object(self, name: str):
+    def load_object(self, name: str, prefix : str = ""):
         pass
 
     @abstractmethod
@@ -382,21 +391,11 @@ def remove_object(self, name: str):
     def is_object(self, name: str) -> bool:
         pass
 
-    @abstractmethod
-    def save_results_internal(self, obj: object):
-        pass
-
-    @abstractmethod
-    def load_results_internal(self):
-        pass
-
-    @abstractmethod
-    def save_results_internal_json(self, results_internal_dict: Dict):
-        pass
+    def save_search_internal(self, obj):
+        raise NotImplementedError
 
-    @abstractmethod
-    def load_results_internal_json(self) -> Dict:
-        pass
+    def load_search_internal(self):
+        raise NotImplementedError
 
     @property
     @abstractmethod

autofit/non_linear/paths/database.py

@@ -121,7 +121,7 @@ def __getstate__(self):
         del d["session"]
         return d
 
-    def save_json(self, name, object_dict: Union[dict, list]):
+    def save_json(self, name, object_dict: Union[dict, list], prefix : str = ""):
         """
         Save a dictionary as a json file in the database
 
@@ -134,7 +134,7 @@ def save_json(self, name, object_dict: Union[dict, list]):
         """
         self.fit.set_json(name, object_dict)
 
-    def load_json(self, name: str) -> Union[dict, list]:
+    def load_json(self, name: str, prefix : str = "") -> Union[dict, list]:
         """
         Load a json file from the database
 
@@ -177,7 +177,7 @@ def load_array(self, name: str) -> np.ndarray:
         """
         return self.fit.get_array(name)
 
-    def save_fits(self, name: str, hdu):
+    def save_fits(self, name: str, hdu, prefix : str = ""):
         """
         Save a fits file in the database
 
@@ -188,9 +188,9 @@ def save_fits(self, name: str, hdu):
         hdu
             The hdu to save
         """
-        self.fit.set_fits(name, hdu)
+        self.fit.set_hdu(name, hdu)
 
-    def load_fits(self, name: str):
+    def load_fits(self, name: str, prefix : str = ""):
         """
         Load a fits file from the database
 
@@ -205,30 +205,24 @@ def load_fits(self, name: str):
         """
         return self.fit.get_hdu(name)
 
-    def save_object(self, name: str, obj: object):
+    def save_object(self, name: str, obj: object, prefix : str = ""):
         self.fit[name] = obj
 
-    def load_object(self, name: str):
+    def load_object(self, name: str, prefix : str = ""):
         return self.fit[name]
 
-    def save_results_internal(self, obj: object):
-        pass
-
-    def load_results_internal(self):
-        pass
-
-    def save_results_internal_json(self, results_internal_dict: Dict):
-        pass
-
-    def load_results_internal_json(self) -> Dict:
-        pass
-
     def remove_object(self, name: str):
         del self.fit[name]
 
     def is_object(self, name: str) -> bool:
         return name in self.fit
 
+    def save_search_internal(self, obj):
+        pass
+
+    def load_search_internal(self):
+        pass
+
     @property
     def fit(self) -> Fit:
         if self._fit is None:
@@ -260,7 +254,6 @@ def completed(self):
     def save_summary(self, samples, log_likelihood_function_time):
         self.fit.instance = samples.max_log_likelihood()
         self.fit.max_log_likelihood = samples.max_log_likelihood_sample.log_likelihood
-        super().save_summary(samples, log_likelihood_function_time)
 
     def save_samples(self, samples):
         if not self.save_all_samples:
@@ -286,10 +279,12 @@ def load_samples_info(self):
         return self._load_samples().samples_info
 
     def save_all(self, info, *_, **kwargs):
-        self.save_identifier()
+
         self.fit.info = info
         self.fit.model = self.model
-
+        if info:
+            self.save_json("info", info)
         self.save_json("search", to_dict(self.search))
+        self.save_json("model", self.model.dict())
 
         self.session.commit()