#44 remove_population_from/remove_population_to implemented in the sqlite_repository. cut_population(N) can me implemented by executing remove_population_from(N+1), remove_population_from(N-1), and removing the unused individuals

Author: raulopez0

MLC/db/mlc_repository.py

@@ -61,9 +61,6 @@ def add_population(self, population):
     def update_population(self, generation, population):
         raise NotImplementedError("This method must be implemented")
 
-    def remove_population(self, generation):
-        raise NotImplementedError("This method must be implemented")
-
     def get_population(self, generation):
         raise NotImplementedError("This method must be implemented")
 
@@ -74,9 +71,6 @@ def count_population(self):
     def remove_population_from(self, from_generation):
         raise NotImplementedError("This method must be implemented")
 
-    def remove_last_population(self):
-        raise NotImplementedError("This method must be implemented")
-
     # operations over individuals
     def add_individual(self, individual):
         raise NotImplementedError("This method must be implemented")

MLC/db/sqlite/sql_statements.py

@@ -26,6 +26,9 @@ def stmt_delete_from_generations(from_generation):
     return """DELETE FROM population
               WHERE gen >= %s""" % (from_generation,)
 
+def stmt_delete_to_generations(to_generation):
+    return """DELETE FROM population
+              WHERE gen <= %s""" % (to_generation,)
 
 def stmt_delete_unused_individuals():
     return '''DELETE FROM individual

MLC/db/sqlite/sqlite_repository.py

@@ -24,7 +24,9 @@ def __init__(self, database, init_db=False):
         self.__execute(stmt_enable_foreign_key())
 
         # cache for population
-        self.__generations = self._how_many_generations()
+        gen_numbers = self._get_generations()
+        self.__generations = len(gen_numbers)
+        self.__base_gen = gen_numbers[0] if gen_numbers else 1
 
         # all individuals: {individual_id: (Individual, generated(bool))
         self.__individuals = self.__load_individuals()
@@ -77,13 +79,14 @@ def add_population(self, population):
         cursor = conn.cursor()
 
         try:
+            next_gen_id = self.__base_gen+self.__generations
             for i in range(len(population._individuals)):
                 individual_id = population._individuals[i]
                 individual_cost = population._costs[i]
                 evaluation_time = population._ev_time[i]
                 individual_gen_method = population._gen_method[i]
                 individual_parents = ','.join(str(elem) for elem in population._parents[i])
-                cursor.execute(stmt_insert_individual_in_population(self.__generations + 1,
+                cursor.execute(stmt_insert_individual_in_population(next_gen_id,
                                                                     individual_id,
                                                                     individual_cost,
                                                                     evaluation_time,
@@ -97,25 +100,30 @@ def add_population(self, population):
 
         self.__generations += 1
 
-    def remove_population(self, generation):
-        self.__execute(stmt_delete_generation(generation))
-        self.__generations -= 1
-
     def get_population(self, generation):
-        pop = self.__load_population(generation)
+        gen_id = self.__base_gen + generation - 1
+        pop = self.__load_population(gen_id)
         return pop
 
     def count_population(self):
         return self.__generations
 
     # special methods
     def remove_population_from(self, from_generation):
-        self.__execute(stmt_delete_from_generations(from_generation))
+        gen_id = self.__base_gen+from_generation-1
+        self.__execute(stmt_delete_from_generations(gen_id))
         self.__generations = from_generation - 1
-
-    def remove_last_population(self):
-        if self.__generations > 0:
-            self.remove_population_from(self.__generations)
+        if from_generation == 1:
+            self.__base_gen = 1
+
+    def remove_population_to(self, to_generation):
+        gen_id = self.__base_gen + to_generation - 1
+        self.__execute(stmt_delete_to_generations(gen_id))
+        self.__generations = self.__generations-to_generation
+        if self.__generations == 0:
+            self.__base_gen = 1
+        else:
+            self.__base_gen = to_generation + 1
 
     def remove_unused_individuals(self):
         to_delete = []
@@ -232,15 +240,15 @@ def __execute(self, statement):
         conn.commit()
         return cursor.lastrowid
 
-    def _how_many_generations(self):
+    def _get_generations(self):
         generations = []
         conn = self.__get_db_connection()
         cursor = conn.execute(stmt_get_generations())
         for row in cursor:
             generations.append(int(row[0]))
         cursor.close()
         conn.commit()
-        return len(sorted(generations))
+        return sorted(generations)
 
     def __load_population(self, generation):
         i = 0

tests/mlc/db/test_mlc_repository.py

@@ -315,13 +315,14 @@ def test_reload_individuals_from_file(self):
             self.assertEqual(mlc_repo.count_individual(), 2)
             self.assertEqual(mlc_repo.count_population(), 1)
 
-    def test_remove_last_population(self):
+    def test_remove_from_population(self):
         mlc_repo = self.__get_new_repo()
 
         # add individuals
         mlc_repo.add_individual(Individual("1+1"))
         mlc_repo.add_individual(Individual("2+2"))
         mlc_repo.add_individual(Individual("3+3"))
+        mlc_repo.add_individual(Individual("4+4"))
 
         # add first population
         p = Population(3, 0, Config.get_instance(), mlc_repo)
@@ -333,8 +334,13 @@ def test_remove_last_population(self):
         p._individuals = [1, 2, 3]
         mlc_repo.add_population(p)
 
+        # add third population
+        p = Population(3, 0, Config.get_instance(), mlc_repo)
+        p._individuals = [1, 2, 4]
+        mlc_repo.add_population(p)
+
         # remove last population
-        mlc_repo.remove_last_population()
+        mlc_repo.remove_population_from(2)
 
         # last generation must be removed
         self.assertEqual(mlc_repo.count_population(), 1)
@@ -344,16 +350,11 @@ def test_remove_last_population(self):
         self.assertEqual(p._individuals, [1, 2, 1])
 
         # all individuals exists and the third individual do not appear in any generation
-        self.assertEqual(mlc_repo.count_individual(), 3)
-
-        data = mlc_repo.get_individual_data(3)
-        self.assertEqual(data.get_value(), "3+3")
-        self.assertEqual(data.get_appearances(), 0)
-        self.assertEqual(data.get_cost_history(), {})
+        self.assertEqual(mlc_repo.count_individual(), 4)
 
         # remove unused individuals
         deleted = mlc_repo.remove_unused_individuals()
-        self.assertEqual(deleted, 1)
+        self.assertEqual(deleted, 2)
         self.assertEqual(mlc_repo.count_individual(), 2)
 
         individual = mlc_repo.get_individual(1)
@@ -368,56 +369,174 @@ def test_remove_last_population(self):
         except KeyError:
             self.assertTrue(True)
 
-    def test_remove_from_population(self):
+    def test_remove_population_to(self):
         mlc_repo = self.__get_new_repo()
 
         # add individuals
         mlc_repo.add_individual(Individual("1+1"))
         mlc_repo.add_individual(Individual("2+2"))
         mlc_repo.add_individual(Individual("3+3"))
         mlc_repo.add_individual(Individual("4+4"))
+        mlc_repo.add_individual(Individual("5+5"))
 
-        # add first population
+        # add  population
         p = Population(3, 0, Config.get_instance(), mlc_repo)
-        p._individuals = [1, 2, 1]
+        p._individuals = [1, 1, 1]
         mlc_repo.add_population(p)
 
-        # add second population
+        # add population
         p = Population(3, 0, Config.get_instance(), mlc_repo)
-        p._individuals = [1, 2, 3]
+        p._individuals = [2, 2, 2]
         mlc_repo.add_population(p)
 
-        # add third population
+        # add population
         p = Population(3, 0, Config.get_instance(), mlc_repo)
-        p._individuals = [1, 2, 4]
+        p._individuals = [3, 3, 3]
         mlc_repo.add_population(p)
 
-        # remove last population
-        mlc_repo.remove_population_from(2)
+        # add population
+        p = Population(3, 0, Config.get_instance(), mlc_repo)
+        p._individuals = [4, 4, 4]
+        mlc_repo.add_population(p)
 
-        # last generation must be removed
-        self.assertEqual(mlc_repo.count_population(), 1)
+        self.assertEqual(mlc_repo.count_population(), 4)
+
+        # remove generations 1 to 2
+        mlc_repo.remove_population_to(2)
+        self.assertEqual(mlc_repo.count_population(), 2)
 
-        # first generation exists
         p = mlc_repo.get_population(1)
-        self.assertEqual(p._individuals, [1, 2, 1])
+        self.assertEqual(p._individuals, [3, 3, 3])
 
-        # all individuals exists and the third individual do not appear in any generation
-        self.assertEqual(mlc_repo.count_individual(), 4)
+        p = mlc_repo.get_population(2)
+        self.assertEqual(p._individuals, [4, 4, 4])
+
+        # New generation must be number 3
+        p = Population(3, 0, Config.get_instance(), mlc_repo)
+        p._individuals = [5, 5, 5]
+        mlc_repo.add_population(p)
+
+        self.assertEqual(mlc_repo.count_population(), 3)
+
+        p = mlc_repo.get_population(1)
+        self.assertEqual(p._individuals, [3, 3, 3])
+
+        p = mlc_repo.get_population(2)
+        self.assertEqual(p._individuals, [4, 4, 4])
+
+        p = mlc_repo.get_population(3)
+        self.assertEqual(p._individuals, [5, 5, 5])
+
+    def test_remove_population_to_clear_generations(self):
+        mlc_repo = self.__get_new_repo()
+
+        # add individuals
+        mlc_repo.add_individual(Individual("1+1"))
+        mlc_repo.add_individual(Individual("2+2"))
+        mlc_repo.add_individual(Individual("3+3"))
+        mlc_repo.add_individual(Individual("4+4"))
+        mlc_repo.add_individual(Individual("5+5"))
+
+        # add  population
+        p = Population(3, 0, Config.get_instance(), mlc_repo)
+        p._individuals = [1, 1, 1]
+        mlc_repo.add_population(p)
+
+        # add population
+        p = Population(3, 0, Config.get_instance(), mlc_repo)
+        p._individuals = [2, 2, 2]
+        mlc_repo.add_population(p)
+
+        # add population
+        p = Population(3, 0, Config.get_instance(), mlc_repo)
+        p._individuals = [3, 3, 3]
+        mlc_repo.add_population(p)
+
+        self.assertEqual(mlc_repo.count_population(), 3)
+
+        # Remove all generations (1 to 3)
+        mlc_repo.remove_population_to(3)
+        self.assertEqual(mlc_repo.count_population(), 0)
+
+        # Insert populations again
+        p = Population(3, 0, Config.get_instance(), mlc_repo)
+        p._individuals = [3, 3, 3]
+        mlc_repo.add_population(p)
+
+        # add population
+        p = Population(3, 0, Config.get_instance(), mlc_repo)
+        p._individuals = [4, 4, 4]
+        mlc_repo.add_population(p)
+
+        # add population
+        p = Population(3, 0, Config.get_instance(), mlc_repo)
+        p._individuals = [5, 5, 5]
+        mlc_repo.add_population(p)
+
+        self.assertEqual(mlc_repo.count_population(), 3)
+
+        p = mlc_repo.get_population(1)
+        self.assertEqual(p._individuals, [3, 3, 3])
+
+        p = mlc_repo.get_population(2)
+        self.assertEqual(p._individuals, [4, 4, 4])
+
+        p = mlc_repo.get_population(3)
+        self.assertEqual(p._individuals, [5, 5, 5])
+
+    def test_cut_generation(self):
+        """
+        Cut a generation using remove_population_from/remove_population_last
+        :return:
+        """
+        mlc_repo = self.__get_new_repo()
+
+        # add individuals
+        mlc_repo.add_individual(Individual("1+1"))
+        mlc_repo.add_individual(Individual("2+2"))
+        mlc_repo.add_individual(Individual("3+3"))
+        mlc_repo.add_individual(Individual("4+4"))
+        mlc_repo.add_individual(Individual("5+5"))
+
+        # add  population
+        p = Population(3, 0, Config.get_instance(), mlc_repo)
+        p._individuals = [1, 1, 1]
+        mlc_repo.add_population(p)
+
+        # add population
+        p = Population(3, 0, Config.get_instance(), mlc_repo)
+        p._individuals = [2, 2, 2]
+        mlc_repo.add_population(p)
+
+        # add population
+        p = Population(3, 0, Config.get_instance(), mlc_repo)
+        p._individuals = [3, 3, 3]
+        mlc_repo.add_population(p)
+
+        # add population
+        p = Population(3, 0, Config.get_instance(), mlc_repo)
+        p._individuals = [4, 4, 4]
+        mlc_repo.add_population(p)
+
+        # add population
+        p = Population(3, 0, Config.get_instance(), mlc_repo)
+        p._individuals = [5, 5, 5]
+        mlc_repo.add_population(p)
+
+        self.assertEqual(mlc_repo.count_population(), 5)
+
+        # Cut population 4
+        mlc_repo.remove_population_from(4+1)
+        mlc_repo.remove_population_to(4-1)
 
         # remove unused individuals
-        deleted = mlc_repo.remove_unused_individuals()
-        self.assertEqual(deleted, 2)
-        self.assertEqual(mlc_repo.count_individual(), 2)
+        mlc_repo.remove_unused_individuals()
 
-        individual = mlc_repo.get_individual(1)
-        self.assertEqual(individual.get_value(), "1+1")
+        self.assertEqual(mlc_repo.count_population(), 1)
+        self.assertEqual(mlc_repo.count_individual(), 1)
 
-        individual = mlc_repo.get_individual(2)
-        self.assertEqual(individual.get_value(), "2+2")
+        p = mlc_repo.get_population(1)
+        self.assertEqual(p._individuals, [4, 4, 4])
 
-        try:
-            individual = mlc_repo.get_individual(3)
-            self.assertTrue(False)
-        except KeyError:
-            self.assertTrue(True)
+        individual = mlc_repo.get_individual(4)
+        self.assertEqual(individual.get_value(), "4+4")