Fixes in Integration and Unit Test to run with the new workspace structure

* In order to run cleaner tests, now the MLCLocal receives the scripts as a
  parameter when a new experiment is created. If no scripts are given, MLC
  will use the template scripts as it was working before #35

Author: Ezetowers

MLC/api/Experiment.py

@@ -56,7 +56,8 @@ def set_configuration(self, new_configuration):
         Config.get_instance().read(self._config_file)
 
     @staticmethod
-    def make(working_dir, experiment_name, experiment_configuration):
+    def make(working_dir, experiment_name, experiment_configuration,
+             evaluation_script, preevaluation_script):
         # Obtain experiment filenames
         experiment_dir = os.path.join(working_dir, experiment_name)
         experiment_cf, experiment_db = Experiment.get_experiment_files(experiment_dir, experiment_name)
@@ -79,17 +80,14 @@ def make(working_dir, experiment_name, experiment_configuration):
         simulation = Simulation(experiment_name)
 
         def folder_structure_creator(folder_to_create, file_to_copy):
-            templates_dir = path_solver.get_templates_path()
-            file_template = os.path.join(templates_dir, file_to_copy)
             folder_path = os.path.join(experiment_dir, folder_to_create)
-            file_copied = os.path.join(folder_path, file_to_copy)
             os.makedirs(folder_path)
-            shutil.copyfile(file_template, file_copied)
+            shutil.copy(file_to_copy, folder_path)
             init_file = os.path.join(folder_path, "__init__.py")
             open(init_file, "w").close()
 
-        folder_structure_creator("Evaluation", "toy_problem.py")
-        folder_structure_creator("Preevaluation", "default.py")
+        folder_structure_creator("Evaluation", evaluation_script)
+        folder_structure_creator("Preevaluation", preevaluation_script)
 
         # Load experiment
         try:

MLC/api/MLCLocal.py

@@ -14,6 +14,8 @@
 from MLC.api.mlc import ExperimentNotExistException
 from MLC.api.mlc import DuplicatedExperimentError
 from MLC.api.mlc import InvalidExperimentException
+from MLC.api.mlc import EvaluationScriptNotExistException
+from MLC.api.mlc import PreevaluationScriptNotExistException
 from MLC.Application import Application
 from MLC.config import get_templates_path
 from MLC.config import set_working_directory
@@ -27,6 +29,8 @@
 
 class MLCLocal(MLC):
     DEFAULT_EXPERIMENT_CONFIG = os.path.join(get_templates_path(), "configuration.ini")
+    DEFAULT_EVALUATION_SCRIPT = os.path.join(get_templates_path(), "toy_problem.py")
+    DEFAULT_PREEVALUATION_SCRIPT = os.path.join(get_templates_path(), "default.py")
 
     def __init__(self, working_dir):
         if not os.path.exists(working_dir):
@@ -85,14 +89,19 @@ def close_experiment(self, experiment_name):
         sys.path.remove(experiment_dir)
         del self._open_experiments[experiment_name]
 
-    def new_experiment(self, experiment_name, experiment_configuration=None):
+    def new_experiment(self, experiment_name,
+                       experiment_configuration=None,
+                       evaluation_script=None,
+                       preevaluation_script=None):
         self._create_experiment_dir(experiment_name)
 
         config = experiment_configuration
         if experiment_configuration is None:
             config = MLCLocal.DEFAULT_EXPERIMENT_CONFIG
 
-        self._load_new_experiment(experiment_name, config)
+        self._load_new_experiment(experiment_name, config,
+                                  evaluation_script,
+                                  preevaluation_script)
 
     def delete_experiment(self, experiment_name):
         if experiment_name not in self._experiments:
@@ -249,15 +258,30 @@ def _create_experiment_dir(self, experiment_name):
                          "Experiment already exists".format(experiment_name))
             raise DuplicatedExperimentError(experiment_name)
 
-    def _load_new_experiment(self, experiment_name, config_path):
+    def _load_new_experiment(self, experiment_name, config_path,
+                             evaluation_script, preevaluation_script):
         experiment_config = ConfigParser.ConfigParser()
         experiment_config.read(config_path)
         config = Config.to_dictionary(experiment_config)
 
+        if evaluation_script is None:
+            evaluation_script = MLCLocal.DEFAULT_EVALUATION_SCRIPT
+
+        if preevaluation_script is None:
+            preevaluation_script = MLCLocal.DEFAULT_PREEVALUATION_SCRIPT
+
+        if not os.path.exists(evaluation_script):
+            raise EvaluationScriptNotExistException(experiment_name, evaluation_script)
+
+        if not os.path.exists(preevaluation_script):
+            raise PreevaluationScriptNotExistException(experiment_name, evaluation_script)
+
         try:
             self._experiments[experiment_name] = Experiment.make(self._working_dir,
                                                                  experiment_name,
-                                                                 config)
+                                                                 config,
+                                                                 evaluation_script,
+                                                                 preevaluation_script)
         except Exception, err:
             logger.error("Cannot create a new experiment. Error message: %s " % err)
             raise

MLC/api/mlc.py

@@ -20,6 +20,22 @@ def __init__(self, experiment_name):
         MLCException.__init__(self, "Experiment '%s' already exists." % experiment_name)
 
 
+class EvaluationScriptNotExistException(MLCException):
+
+    def __init__(self, experiment_name, script_path):
+        MLCException.__init__(self, "Experiment {0} is trying to use a non "
+                                    "existent evaluation script. Script: {1}"
+                                    .format(experiment_name, script_path))
+
+
+class PreevaluationScriptNotExistException(MLCException):
+
+    def __init__(self, experiment_name, script_path):
+        MLCException.__init__(self, "Experiment {0} is trying to use a non "
+                                    "existent evaluation script. Script: {1}"
+                                    .format(experiment_name, script_path))
+
+
 class MLC:
 
     def open_experiment(self, experiment_name):

MLC/db/sqlite/sqlite_repository.py

@@ -12,7 +12,6 @@ class SQLiteRepository(MLCRepository):
     IN_MEMORY_DB = ":memory:"
 
     def __init__(self, database, init_db=False):
-        print database
         self._conn = sqlite3.connect(database)
         self._database = database
 

tests/integration_tests/integration_tests.py

@@ -1,10 +1,13 @@
-import sys, os
+import sys
+import os
 sys.path.append("../..")
 import matlab.engine
 import numpy as np
+import shutil
 import unittest
 import yaml
 
+from MLC.api.MLCLocal import MLCLocal
 from MLC.Application import Application
 from MLC.Common.RandomManager import RandomManager
 from MLC.db.mlc_repository import MLCRepository
@@ -20,6 +23,8 @@
 class MLCIntegrationTest(unittest.TestCase):
     TEST_DIRECTORY = None
     GENERATIONS = None
+    WORKSPACE_DIR = os.path.abspath("/tmp/mlc_workspace/")
+    EXPERIMENT_NAME = "integration_tests"
 
     @classmethod
     def _populate_indiv_dict(cls):
@@ -69,6 +74,8 @@ def _populate_pop_dict(cls):
     @classmethod
     def setUpClass(cls):
         config_file = os.path.join(MLCIntegrationTest.TEST_DIRECTORY, './configuration.ini')
+        ev_script = os.path.join(MLCIntegrationTest.TEST_DIRECTORY, './default_evaluation_script.py')
+        preev_script = os.path.join(MLCIntegrationTest.TEST_DIRECTORY, './default_preevaluation_script.py')
 
         # Load the config
         config = Config.get_instance()
@@ -79,46 +86,47 @@ def setUpClass(cls):
         RandomManager.clear_random_values()
         RandomManager.load_random_values(random_file)
 
-        # clear state
-        if Config.get_instance().getboolean("BEHAVIOUR", "save"):
-            try:
-                os.remove(Config.get_instance().get("BEHAVIOUR", "savedir"))
-            except OSError:
-                pass
+        # Create the workspace directory
+        try:
+            os.makedirs(MLCIntegrationTest.WORKSPACE_DIR)
+        except OSError:
+            shutil.rmtree(MLCIntegrationTest.WORKSPACE_DIR)
+            os.makedirs(MLCIntegrationTest.WORKSPACE_DIR)
+
+        # Create a new experiment, which will be used to
+        cls._mlc = MLCLocal(working_dir=MLCIntegrationTest.WORKSPACE_DIR)
+        cls._mlc.new_experiment(experiment_name=MLCIntegrationTest.EXPERIMENT_NAME,
+                                experiment_configuration=config_file,
+                                evaluation_script=ev_script,
+                                preevaluation_script=preev_script)
+        cls._mlc.open_experiment(experiment_name=MLCIntegrationTest.EXPERIMENT_NAME)
 
         for generation_params in MLCIntegrationTest.GENERATIONS:
-            # clear static values
-            if Config.get_instance().getboolean("BEHAVIOUR", "save"):
-                MLCRepository._instance = None
-            simulation = Simulation("integration_test")
-            cls._app = Application(simulation)
-
             if isinstance(generation_params, int):
-                cls._app.go(to_generation=generation_params)
+                cls._mlc.go(experiment_name=MLCIntegrationTest.EXPERIMENT_NAME,
+                            to_generation=generation_params)
 
             elif isinstance(generation_params, list):
-                cls._app.go(from_generation=generation_params[0],
+                cls._mlc.go(experiment_name=MLCIntegrationTest.EXPERIMENT_NAME,
+                            from_generation=generation_params[0],
                             to_generation=generation_params[1])
 
             else:
                 raise Exception("Integration test, bad value for generations param")
 
-        if Config.get_instance().getboolean("BEHAVIOUR", "save"):
-            MLCRepository._instance._conn.close()
-            MLCRepository._instance = None
-            cls._app = Application(Simulation("integration_test"))
-
-        a = cls._app.get_simulation().number_of_generations()
-        print "Number of populations: " + str(a)
-
         # List with individuals data
         cls._indivs = []
         cls._populate_indiv_dict()
 
         cls._pops = []
         cls ._populate_pop_dict()
 
-    ## FIXME: Test methods should be cerated dynamically in the setUp method.
+    @classmethod
+    def tearDownClass(cls):
+        cls._mlc.close_experiment(experiment_name=MLCIntegrationTest.EXPERIMENT_NAME)
+        shutil.rmtree(MLCIntegrationTest.WORKSPACE_DIR)
+
+    # FIXME: Test methods should be crated dynamically in the setUp method.
     def test_generation_1(self):
         if max(MLCIntegrationTest.GENERATIONS) >= 1:
             self._run_x_generation(1)
@@ -160,7 +168,7 @@ def _run_x_generation(self, gen_number):
         self._check_indiv_values(gen_number)
 
         print "Check Indvidual properties..."
-        pop = self._app.get_simulation().get_generation(gen_number)
+        pop = self._mlc.get_generation(MLCIntegrationTest.EXPERIMENT_NAME, gen_number)
         self._check_indiv_property(gen_number, pop.get_individuals(), 'index', 'int')
         self._check_indiv_property(gen_number, pop.get_costs(), 'cost', 'float')
 
@@ -169,7 +177,7 @@ def _run_x_generation(self, gen_number):
 
     def _check_indiv_values(self, gen_number):
         i = 1
-        indexes = self._app.get_simulation().get_generation(gen_number).get_individuals()
+        indexes = self._mlc.get_generation(MLCIntegrationTest.EXPERIMENT_NAME, gen_number).get_individuals()
         print "Check %s indviduals from generation %s" % (len(indexes), gen_number)
         for index in indexes:
             indiv = MLCRepository.get_instance().get_individual(index)
@@ -187,10 +195,6 @@ def _check_indiv_property(self, gen_number, values, map_property, type=None):
         # List of dictionaries with the values of every individual
         pop = self._pops[gen_number - 1]
         for i in range(len(pop)):
-        #     if str(values[i]) != str(pop[i][map_property]):
-        #         print "Indiv N#{0} - Value obtained: {1}".format(i + 1, values[i])
-        #         print "Indiv N#{0} - Value expected: {1}".format(i + 1, str(pop[i][map_property]))
-
             if type == 'int':
                 self.assertEqual(int(values[i]), int(pop[i][map_property]))
             elif type == 'float':
@@ -203,6 +207,7 @@ def _check_indiv_property(self, gen_number, values, map_property, type=None):
             else:
                 self.assertEqual(values[i], pop[i][map_property])
 
+
 def get_test_class(test_dir, integration_test):
     generations = integration_test['generations']
     if isinstance(generations, int):
@@ -211,10 +216,12 @@ def get_test_class(test_dir, integration_test):
     class IntegrationTest(MLCIntegrationTest):
         MLCIntegrationTest.TEST_DIRECTORY = test_dir
         MLCIntegrationTest.GENERATIONS = generations
+        MLCIntegrationTest.EXPERIMENT_NAME = test_dir
 
     return IntegrationTest
 
-def execute_integration_test(test_name, integration_test):
+
+def execute_integration_test(test_dir, integration_test):
     print "Running '%s' with %s generations: %s" % (integration_test['name'],
                                                     integration_test['generations'],
                                                     integration_test['description'])

tests/integration_tests/test_basic/configuration.ini

@@ -48,11 +48,8 @@ cascade = 1,1
 
 [EVALUATOR]
 #  Evaluator
-# evaluation_method = standalone_function
-# evaluation_method = standalone_files
 evaluation_method = mfile_standalone
-evaluation_function = toy_problem
-# evaluation_function = toy_problem_python_ev
+evaluation_function = default_evaluation_script
 indfile = ind.dat
 Jfile = J.dat
 # exchangedir = fullfile(pwd,evaluator0)
@@ -67,7 +64,7 @@ badvalues_elim = first
 %badvalues_elim = none
 %badvalues_elim = all
 preevaluation = false
-preev_function = default
+preev_function = default_preevaluation_script
 problem_variables.gamma = 0.1
 
 [BEHAVIOUR]

tests/integration_tests/test_basic/default_evaluation_script.py

@@ -0,0 +1,74 @@
+import numpy as np
+import MLC.Log.log as lg
+import matplotlib.pyplot as plt
+import sys
+
+from MLC.matlab_engine import MatlabEngine
+from MLC.mlc_parameters.mlc_parameters import Config
+
+
+def individual_data(indiv):
+    x = np.linspace(-10.0, 10.0, num=201)
+    y = np.tanh(x**3 - x**2 - 1)
+
+    eng = MatlabEngine.engine()
+    config = Config.get_instance()
+    # artificial_noise = self._config.getint('EVALUATOR', 'artificialnoise')
+
+    # In this test we have no noise by config file. But, if this were the
+    # case, we would a have problem because the random of MATLAB is not
+    # the random of Python :(
+    # WORKAROUND: Generate the noise in matlab and process it in python
+
+    # MAKE SOME NOISE!!!
+    # noise = eng.eval('rand(length(zeros(1, ' + str(len(x)) + ')))-0.5')
+    # np_noise = np.array([s for s in noise[0]])
+    # y2 = y + np_noise * 500 * artificial_noise
+
+    y2 = y
+
+    if isinstance(indiv.get_formal(), str):
+        formal = indiv.get_formal().replace('S0', 'x')
+    else:
+        # toy problem support for multiple controls
+        formal = indiv.get_formal()[0].replace('S0', 'x')
+
+    eng.workspace['x'] = eng.eval('-10:0.1:10')
+    eng.workspace['formal'] = formal
+
+    # Calculate J like the sum of the square difference of the
+    # functions in every point
+
+    lg.logger_.debug('[POP][TOY_PROBLEM] Individual Formal: ' + formal)
+    eng.workspace['y3'] = eng.eval('zeros(1, ' + str(len(x)) + ')')
+    eng.eval('eval([formal])')
+    y3 = eng.eval('eval([formal])')
+
+    try:
+        np_y3 = np.array([s for s in y3[0]])
+    except TypeError:
+        np_y3 = np.repeat(y3, len(x))
+
+    return x, y, y2, np_y3
+
+
+def cost(indiv):
+    x, y, y2, np_y3 = individual_data(indiv)
+    cost_np_y3 = float(np.sum((np_y3 - y2)**2))
+    return cost_np_y3
+
+
+def show_best(index, indiv, cost, block=True):
+    x, y, y2, np_y3 = individual_data(indiv)
+    # FIXME: Absolute only makes sense if we're working with complex numbers. It's not the case...
+    y4 = np.sqrt((y - np_y3)**2 / (1 + np.absolute(x**2)))
+
+    plt.clf()
+    plt.suptitle("Individual N#{0} - Cost: {1} \n Formal: {2}".format(index, cost, indiv.get_formal()))
+    plt.subplot(2, 1, 1)
+    plt.plot(x, y, x, y2, '*', x, np_y3)
+
+    plt.subplot(2, 1, 2)
+    plt.plot(x, y4, '*r')
+    plt.yscale('log')
+    plt.show(block=block)

tests/integration_tests/test_basic/default_preevaluation_script.py

@@ -0,0 +1,8 @@
+import MLC.Log.log as lg
+from MLC.mlc_parameters.mlc_parameters import Config
+from MLC.individual.Individual import Individual
+
+
+def preev(indiv):
+    # TODO: Do something
+    return True