From 42c71d9ca2a135fcd4272ac79a7ec8f18205e886 Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Thu, 11 May 2023 14:07:31 +0100
Subject: [PATCH 01/40] creates a multiagent model

---
 Macodiac.ML/environment.py           |   2 +
 Macodiac.ML/main.py                  |   3 +-
 Macodiac.ML/multiagent_main.py       | 189 +++++++++++++++++++++++++++
 Macodiac.ML/multiagentenvironment.py |  76 +++++++++++
 4 files changed, 269 insertions(+), 1 deletion(-)
 create mode 100644 Macodiac.ML/multiagent_main.py
 create mode 100644 Macodiac.ML/multiagentenvironment.py

diff --git a/Macodiac.ML/environment.py b/Macodiac.ML/environment.py
index fa1d065..3303f83 100644
--- a/Macodiac.ML/environment.py
+++ b/Macodiac.ML/environment.py
@@ -41,6 +41,8 @@ def step(self, action):
         self.state += action - 1
         self.environment_timesteps -=1
 
+        
+
 
         if self.state > 0:
             reward = 1
diff --git a/Macodiac.ML/main.py b/Macodiac.ML/main.py
index 829146b..bd62e8c 100644
--- a/Macodiac.ML/main.py
+++ b/Macodiac.ML/main.py
@@ -2,6 +2,7 @@
 import gymnasium as gym
 from gymnasium import Env
 from environment import MacodiacEnvironment
+from multiagentenvironment import MultiAgentMacodiacEnvironment
 from stable_baselines3.common.evaluation import evaluate_policy
 import numpy as np
 
@@ -22,7 +23,7 @@ def __init__(self):
         self.save_path =  os.path.join(filePath,'saved_models', 'model')
         self.save_path_intermittent =  os.path.join(filePath,'saved_models', 'intermittent_saved_models')
         self.env = MacodiacEnvironment(envTimesteps=100)
-        self.numTrainingIterations = 10_000_000
+        self.numTrainingIterations = 10_000
         self.numEpisodes = 10
 
 
diff --git a/Macodiac.ML/multiagent_main.py b/Macodiac.ML/multiagent_main.py
new file mode 100644
index 0000000..b31aefd
--- /dev/null
+++ b/Macodiac.ML/multiagent_main.py
@@ -0,0 +1,189 @@
+import os
+import gymnasium as gym
+from gymnasium import Env
+from environment import MacodiacEnvironment
+from multiagentenvironment import MultiAgentMacodiacEnvironment
+from stable_baselines3.common.evaluation import evaluate_policy
+import numpy as np
+
+
+
+from stable_baselines3 import PPO
+
+
+class MultiagentMain():
+    isRunning = False
+
+    def __init__(self):
+        """
+        init the class
+        """
+        filePath = os.path.join('Macodiac.ML', 'training_multiagent','results')
+        self.log_path =  os.path.join(filePath,'Logs')
+        self.save_path =  os.path.join(filePath,'saved_models', 'model')
+        self.save_path_intermittent =  os.path.join(filePath,'saved_models', 'intermittent_saved_models')
+        self.env = MultiAgentMacodiacEnvironment(envTimesteps=100)
+        self.numTrainingIterations = 100
+        self.numEpisodes = 10
+        self.numAgents = 2
+
+
+        # set to true if you want to load an existing model
+        # model loading happens first, then training
+        # NOTES: 
+        #   if loadmodel is set to false, and trainmodel is set to true, 
+        #   the currently saved model is overwritten
+        self.__MODE_LOADMODEL__ = False
+
+        # set to true if you want to train and then save the model
+        self.__MODE_TRAINMODEL__ = False
+
+        # set to true to use the randomsample mode for testing, 
+        # rather than the model version
+        self.__MODE_RANDOMSAMPLE__ = True
+
+
+    def Run(self):
+        """
+        Runs the project
+        """
+        model = self.create_model(self.env, self.log_path)
+
+        if self.__MODE_LOADMODEL__:
+            model = self.load_model(self.env, model, self.save_path)
+
+        if self.__MODE_TRAINMODEL__: 
+            model = self.train_model(model,
+                                     self.numTrainingIterations, self.save_path_intermittent)
+            self.save_model(model, self.save_path)
+
+        if self.__MODE_RANDOMSAMPLE__:
+            self.run_project_with_rand_test(self.env, self.numEpisodes)
+        else:
+            self.run_project(self.env, self.numEpisodes, model)
+            self.policy_evaluation(model, self.env, self.numEpisodes)
+
+
+    def run_multiagent_project_with_rand_test(self, env:MultiAgentMacodiacEnvironment, numEpisodes: int):
+        """
+        Runs the project with random sampling, using the multiagent env
+        """
+
+        for episode in range(numEpisodes):
+            obs = env.reset()
+            action = env.action_space.sample()
+            obs, rewards, isTerminal, infos = env.step(action)
+
+
+    def run_project_with_rand_test(self, env:MultiAgentMacodiacEnvironment, numEpisodes:int):
+        """
+        Runs the project with random sampling, instead
+        of a model
+
+        @param env: The environment to run this project with
+        @param numEpisodes: the count of episodes to run the environment for
+        """
+        for episode in range(numEpisodes):
+            obs = env.reset()
+            done = False
+            score = 0
+            while not done:
+                #env.render()
+                action = env.action_space.sample()
+                obs, reward, isTerminal, info = env.step(action)
+                score += reward
+                done = isTerminal
+            print(f'Episode:{episode}  | Score:{score}')
+        env.close()
+
+
+    def run_project(self, env:MultiAgentMacodiacEnvironment, numEpisodes: int, model):
+        """
+        Runs the project with an actual model, instead of random sampling
+        of a model
+
+        @param env: The environment to run this project with
+        @param numEpisodes: the count of episodes to run the environment for
+        """
+        scores = []
+        for episode in range(numEpisodes):
+            obs = env.reset()
+            done = False
+            score = 0
+            while not done:
+                #env.render()
+                action, _discard = model.predict(obs)
+                obs, reward, isTerminal, info = env.step(action)
+                score += reward
+                done = isTerminal
+            scores.append(score)
+
+            runningAvg = np.mean(scores)
+
+            print(f'Episode:{episode} \t| Score:{score} \t\t| RunningAvg: {round(runningAvg, 2)}')
+        env.close()
+
+
+    def create_model(self, env: MultiAgentMacodiacEnvironment, log_path: str):
+        model = PPO('MlpPolicy', env, verbose=1, tensorboard_log=log_path)
+        return model
+
+
+    def train_model(self, model, numTimesteps: int, savePath:str):
+        """
+        Trains a model with the number of iterations in 
+        numtimesteps. Creates a n intermediate save every 1m iterations
+
+        @param model: The model to train. The model must have been instantiated
+        @param numTimesteps: the number of training iterations
+        """
+        
+        saveEveryNSteps = 1_000_000
+        
+        if numTimesteps < saveEveryNSteps:
+            model.learn(total_timesteps=numTimesteps)
+
+        else:
+            rangeUpper = int(numTimesteps / saveEveryNSteps)
+            for i in range(1,rangeUpper+1):
+                model.learn(total_timesteps=saveEveryNSteps)
+                model.save(os.path.join(savePath, f'interim-{i}'))
+
+        return model
+
+    def policy_evaluation(self, model, env: MultiAgentMacodiacEnvironment, numEpisodes:int=50):
+        """
+        Prints a policy evaluation, including the mean episode reward
+        and the standard deviation
+
+        @param model:       The model to be evaluated
+        @param env:         The environment to evaluate the model against
+        @param numEpisodes: The count of episodes to evaluate against        
+        """
+        print('\nevalResult:(mean episode reward, standard deviation)')
+        print(f'evalResult:{evaluate_policy(model, env, n_eval_episodes=numEpisodes)}\n')
+
+
+
+    def save_model(self, model, modelPath):
+        """
+        Saves a model to a given path
+
+        @param model:       The model to save
+        @param modelPath:   The path to save to
+        """
+        model.save(modelPath)    
+    
+    
+    def load_model(self, env: MultiAgentMacodiacEnvironment, model, modelPath: str):
+        """
+        Saves a model to a given path
+
+        @param model:       The model to save
+        @param modelPath:   The modelPath to save to
+        """
+        model = PPO.load(modelPath, env=env)
+        return model
+
+main = MultiagentMain()
+main.Run()
\ No newline at end of file
diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
new file mode 100644
index 0000000..52ddd6d
--- /dev/null
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -0,0 +1,76 @@
+import gymnasium as gym
+from gymnasium import Env
+# use `gym.spaces` here, even though we're using `gymnasium`
+# https://stackoverflow.com/questions/75108957/assertionerror-the-algorithm-only-supports-class-gym-spaces-box-box-as-acti
+from gym import spaces
+import numpy as np
+import random
+
+
+class MultiAgentMacodiacEnvironment(Env):
+    """
+    Builds a profit maximising agent environment, supporting
+    up to n_agent agents
+    """
+    state = 0
+    environment_timesteps = 1000
+
+    def __init__(self, envTimesteps:int, n_agents: int):
+        """
+        Initialises the class
+        """
+        self.environment_timesteps = envTimesteps
+        self.action_space = spaces.Discrete(3)    
+        self.observation_space = spaces.Box(low=np.array([0]), high=np.array([100]))
+        self.reset()
+
+
+        
+        print('-- ENV SETTINGS --')
+        print(self.observation_space)
+        print(self.observation_space.sample())
+        print(self.action_space)
+        print(self.action_space.sample())
+        print(self.environment_timesteps)
+        print('-- ENV SETTINGS --')
+
+
+    def step(self, action):
+        """
+        Processes an action for an agent
+        """
+        self.state += action - 1
+        self.environment_timesteps -=1
+
+
+        if self.state > 0:
+            reward = 1
+        elif self.state == 0:
+            reward = 0
+        else:
+            reward = -1
+
+        if self.environment_timesteps <= 0:
+            done = True
+        else:
+            done = False
+
+        info = {}
+        return self.state, reward, done, info
+
+    def render(self) -> None:
+        """
+        Does nothing, the environment is fully headless
+        """
+        pass
+
+    def reset(self) -> float:
+        """
+        Sets the application to its initial conditions
+
+        Sets state to a random float between negative 100 to  positive 100
+        """
+        self.state = np.array([0 + random.randint(-100,100)]).astype(float)
+        self.environment_timesteps = 1000
+        return self.state
+        
\ No newline at end of file

From fbada718c68a1bc6c08ba174e903f20accc40af3 Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Thu, 11 May 2023 15:00:46 +0100
Subject: [PATCH 02/40] begins to add an array'd environment

---
 Macodiac.ML/multiagent_main.py       | 16 ++++++++--
 Macodiac.ML/multiagentenvironment.py | 45 +++++++++++++++++++---------
 2 files changed, 44 insertions(+), 17 deletions(-)

diff --git a/Macodiac.ML/multiagent_main.py b/Macodiac.ML/multiagent_main.py
index b31aefd..06ab4a6 100644
--- a/Macodiac.ML/multiagent_main.py
+++ b/Macodiac.ML/multiagent_main.py
@@ -22,11 +22,13 @@ def __init__(self):
         self.log_path =  os.path.join(filePath,'Logs')
         self.save_path =  os.path.join(filePath,'saved_models', 'model')
         self.save_path_intermittent =  os.path.join(filePath,'saved_models', 'intermittent_saved_models')
-        self.env = MultiAgentMacodiacEnvironment(envTimesteps=100)
         self.numTrainingIterations = 100
         self.numEpisodes = 10
         self.numAgents = 2
 
+        self.env = MultiAgentMacodiacEnvironment(envTimesteps=100, numAgents=self.numAgents)
+
+
 
         # set to true if you want to load an existing model
         # model loading happens first, then training
@@ -71,8 +73,16 @@ def run_multiagent_project_with_rand_test(self, env:MultiAgentMacodiacEnvironmen
 
         for episode in range(numEpisodes):
             obs = env.reset()
-            action = env.action_space.sample()
-            obs, rewards, isTerminal, infos = env.step(action)
+            done = False
+            score = 0
+            while not done:
+                #env.render()
+                action = env.action_space.sample()
+                obs_arr, reward_arr, done_arr, isTerminal, info_arr = env.step(action)
+                score += reward
+                done = isTerminal
+            print(f'Episode:{episode}  | Score:{score}')
+        env.close()
 
 
     def run_project_with_rand_test(self, env:MultiAgentMacodiacEnvironment, numEpisodes:int):
diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index 52ddd6d..47b7fdf 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -15,13 +15,16 @@ class MultiAgentMacodiacEnvironment(Env):
     state = 0
     environment_timesteps = 1000
 
-    def __init__(self, envTimesteps:int, n_agents: int):
+    def __init__(self, envTimesteps:int, numAgents: int):
         """
         Initialises the class
         """
         self.environment_timesteps = envTimesteps
         self.action_space = spaces.Discrete(3)    
         self.observation_space = spaces.Box(low=np.array([0]), high=np.array([100]))
+        
+        self.policy_agents = [numAgents]
+        
         self.reset()
 
 
@@ -35,28 +38,42 @@ def __init__(self, envTimesteps:int, n_agents: int):
         print('-- ENV SETTINGS --')
 
 
-    def step(self, action):
+    def step(self, action_arr):
         """
-        Processes an action for an agent
+        Processes an action for an agent.
+
+        Loops through each agent and sets its action. 
+        Then calls world.step to progress the entire world's actions
+
+        Builds up arrays of results, and returns them in a tuple of arrays
+        
         """
-        self.state += action - 1
         self.environment_timesteps -=1
 
+        obs_arr = []
+        reward_arr = []
+        done_arr = []
+        info_arr = [{'n': []}]
+        
+        agent_arr = self.policy_agents
 
-        if self.state > 0:
-            reward = 1
-        elif self.state == 0:
-            reward = 0
-        else:
-            reward = -1
+        for i, agent in enumerate(agent_arr):
+            self.set_action(action_arr[i], agent, self.action_space[i])
+
+        self.world.step()
+
+        for agent in self.policy_agents:
+            obs_arr.append(self._get_obs(agent))
+            reward_arr.append(self._get_reward(agent))
+            done_arr.append(self._get_done(agent))
+            info_arr.append(self._get_info(agent))
 
         if self.environment_timesteps <= 0:
-            done = True
+            isTerminal = True
         else:
-            done = False
+            isTerminal = False
 
-        info = {}
-        return self.state, reward, done, info
+        return obs_arr, reward_arr, done_arr, isTerminal, info_arr
 
     def render(self) -> None:
         """

From cf85b4372671cb8ff39e408a5b2c92c7c0562a24 Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Thu, 11 May 2023 15:48:41 +0100
Subject: [PATCH 03/40] updates random sample `run` to use env arrays

---
 Macodiac.ML/multiagent_main.py       | 24 ++++++++++++++++-----
 Macodiac.ML/multiagentenvironment.py | 32 ++++++++++++++++++++--------
 2 files changed, 42 insertions(+), 14 deletions(-)

diff --git a/Macodiac.ML/multiagent_main.py b/Macodiac.ML/multiagent_main.py
index 06ab4a6..95ecd7a 100644
--- a/Macodiac.ML/multiagent_main.py
+++ b/Macodiac.ML/multiagent_main.py
@@ -24,7 +24,7 @@ def __init__(self):
         self.save_path_intermittent =  os.path.join(filePath,'saved_models', 'intermittent_saved_models')
         self.numTrainingIterations = 100
         self.numEpisodes = 10
-        self.numAgents = 2
+        self.numAgents = 5
 
         self.env = MultiAgentMacodiacEnvironment(envTimesteps=100, numAgents=self.numAgents)
 
@@ -49,6 +49,10 @@ def Run(self):
         """
         Runs the project
         """
+
+        self.run_multiagent_project_with_rand_test(self.env, 5)
+        return
+
         model = self.create_model(self.env, self.log_path)
 
         if self.__MODE_LOADMODEL__:
@@ -75,13 +79,22 @@ def run_multiagent_project_with_rand_test(self, env:MultiAgentMacodiacEnvironmen
             obs = env.reset()
             done = False
             score = 0
+            agent_scores = [len(env.policy_agents)]
             while not done:
                 #env.render()
-                action = env.action_space.sample()
-                obs_arr, reward_arr, done_arr, isTerminal, info_arr = env.step(action)
-                score += reward
+                action_arr = [len(env.policy_agents)]
+                for i in range(len(env.policy_agents)):
+                    action_arr.append(env.action_space[i].sample())
+                obs_arr, reward_arr, done_arr, isTerminal, info_arr = env.step(action_arr)
+                
+                for reward, i in reward_arr:
+                    agent_scores[i] += reward
+
+                if any(done_arr):
+                    isTerminal = True
+
                 done = isTerminal
-            print(f'Episode:{episode}  | Score:{score}')
+            print(f'Episode:{episode}  | Score:{agent_scores}')
         env.close()
 
 
@@ -195,5 +208,6 @@ def load_model(self, env: MultiAgentMacodiacEnvironment, model, modelPath: str):
         model = PPO.load(modelPath, env=env)
         return model
 
+
 main = MultiagentMain()
 main.Run()
\ No newline at end of file
diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index 47b7fdf..de5c07c 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -6,6 +6,9 @@
 import numpy as np
 import random
 
+class AgentObject:
+    def __init__(self):
+        self.state = []
 
 class MultiAgentMacodiacEnvironment(Env):
     """
@@ -20,10 +23,17 @@ def __init__(self, envTimesteps:int, numAgents: int):
         Initialises the class
         """
         self.environment_timesteps = envTimesteps
-        self.action_space = spaces.Discrete(3)    
-        self.observation_space = spaces.Box(low=np.array([0]), high=np.array([100]))
-        
-        self.policy_agents = [numAgents]
+        self.policy_agents = []
+        for i in range(numAgents):
+            self.policy_agents.append(AgentObject())
+
+        self.action_space = []
+        self.observation_space = []
+        self.agents = [numAgents]
+
+        for agent in self.policy_agents:
+            self.action_space.append(spaces.Discrete(3))
+            self.observation_space.append(spaces.Box(low=np.array([0]), high=np.array([100])))
         
         self.reset()
 
@@ -31,9 +41,9 @@ def __init__(self, envTimesteps:int, numAgents: int):
         
         print('-- ENV SETTINGS --')
         print(self.observation_space)
-        print(self.observation_space.sample())
+        #print(self.observation_space.sample())
         print(self.action_space)
-        print(self.action_space.sample())
+        #print(self.action_space.sample())
         print(self.environment_timesteps)
         print('-- ENV SETTINGS --')
 
@@ -46,7 +56,7 @@ def step(self, action_arr):
         Then calls world.step to progress the entire world's actions
 
         Builds up arrays of results, and returns them in a tuple of arrays
-        
+
         """
         self.environment_timesteps -=1
 
@@ -62,6 +72,8 @@ def step(self, action_arr):
 
         self.world.step()
 
+
+
         for agent in self.policy_agents:
             obs_arr.append(self._get_obs(agent))
             reward_arr.append(self._get_reward(agent))
@@ -87,7 +99,9 @@ def reset(self) -> float:
 
         Sets state to a random float between negative 100 to  positive 100
         """
-        self.state = np.array([0 + random.randint(-100,100)]).astype(float)
+        for i in range(len(self.policy_agents)):
+            self.policy_agents[i].state = np.array([0 + random.randint(-100,100)]).astype(float)
+
         self.environment_timesteps = 1000
-        return self.state
+        return self.environment_timesteps
         
\ No newline at end of file

From 5333a107edabddf6f87f2eb1d48ea14204240f11 Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Fri, 12 May 2023 13:11:20 +0100
Subject: [PATCH 04/40] agents pass actions and rewards back and forwards
 correctly

---
 Macodiac.ML/environment.py           |  3 --
 Macodiac.ML/main.py                  |  6 +--
 Macodiac.ML/multiagent_main.py       | 27 ++++++++++----
 Macodiac.ML/multiagentenvironment.py | 55 +++++++++++++++++++++++++---
 4 files changed, 72 insertions(+), 19 deletions(-)

diff --git a/Macodiac.ML/environment.py b/Macodiac.ML/environment.py
index 3303f83..498513f 100644
--- a/Macodiac.ML/environment.py
+++ b/Macodiac.ML/environment.py
@@ -41,9 +41,6 @@ def step(self, action):
         self.state += action - 1
         self.environment_timesteps -=1
 
-        
-
-
         if self.state > 0:
             reward = 1
         elif self.state == 0:
diff --git a/Macodiac.ML/main.py b/Macodiac.ML/main.py
index bd62e8c..01b7a50 100644
--- a/Macodiac.ML/main.py
+++ b/Macodiac.ML/main.py
@@ -32,14 +32,14 @@ def __init__(self):
         # NOTES: 
         #   if loadmodel is set to false, and trainmodel is set to true, 
         #   the currently saved model is overwritten
-        self.__MODE_LOADMODEL__ = True
+        self.__MODE_LOADMODEL__ =False
 
         # set to true if you want to train and then save the model
-        self.__MODE_TRAINMODEL__ = True
+        self.__MODE_TRAINMODEL__ = False
 
         # set to true to use the randomsample mode for testing, 
         # rather than the model version
-        self.__MODE_RANDOMSAMPLE__ = False
+        self.__MODE_RANDOMSAMPLE__ = True
 
 
     def Run(self):
diff --git a/Macodiac.ML/multiagent_main.py b/Macodiac.ML/multiagent_main.py
index 95ecd7a..6a85e01 100644
--- a/Macodiac.ML/multiagent_main.py
+++ b/Macodiac.ML/multiagent_main.py
@@ -24,9 +24,9 @@ def __init__(self):
         self.save_path_intermittent =  os.path.join(filePath,'saved_models', 'intermittent_saved_models')
         self.numTrainingIterations = 100
         self.numEpisodes = 10
-        self.numAgents = 5
+        self.numAgents = 25
 
-        self.env = MultiAgentMacodiacEnvironment(envTimesteps=100, numAgents=self.numAgents)
+        self.env = MultiAgentMacodiacEnvironment(envTimesteps=15, numAgents=self.numAgents)
 
 
 
@@ -79,22 +79,33 @@ def run_multiagent_project_with_rand_test(self, env:MultiAgentMacodiacEnvironmen
             obs = env.reset()
             done = False
             score = 0
-            agent_scores = [len(env.policy_agents)]
+            agent_scores = []
+            iterator = 0
             while not done:
                 #env.render()
-                action_arr = [len(env.policy_agents)]
+                iterator+=1
+                print(f'iterator:{iterator}')
+                action_arr = []
                 for i in range(len(env.policy_agents)):
-                    action_arr.append(env.action_space[i].sample())
+                    actionForAgent = env.action_space[i].sample()
+                    action_arr.append(actionForAgent)
+                
+                print(f'action for agents:\t{action_arr}')
+                
                 obs_arr, reward_arr, done_arr, isTerminal, info_arr = env.step(action_arr)
                 
-                for reward, i in reward_arr:
-                    agent_scores[i] += reward
+                # for i, reward in enumerate(reward_arr):
+                #     print(f'reward is {reward}')
+
+                agent_scores.append(sum(reward_arr))
+
+                print(f'rewards for agents:\t{reward_arr}')
 
                 if any(done_arr):
                     isTerminal = True
 
                 done = isTerminal
-            print(f'Episode:{episode}  | Score:{agent_scores}')
+            print(f'Episode:{episode} | Aggregate agent scores:(Sum:{sum(agent_scores)})')
         env.close()
 
 
diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index de5c07c..426ec82 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -16,7 +16,7 @@ class MultiAgentMacodiacEnvironment(Env):
     up to n_agent agents
     """
     state = 0
-    environment_timesteps = 1000
+    environment_timesteps = 15
 
     def __init__(self, envTimesteps:int, numAgents: int):
         """
@@ -48,6 +48,21 @@ def __init__(self, envTimesteps:int, numAgents: int):
         print('-- ENV SETTINGS --')
 
 
+    def set_agent_action(self, action, agent, actionSpace):
+        agent.state = action
+
+    def step_agent(self, agent):
+        if agent.state > 0:
+            reward = 1
+        elif agent.state == 0:
+            reward = 0
+        else:
+            reward = -1
+
+        info = {}
+        return agent.state, reward, False, info
+
+
     def step(self, action_arr):
         """
         Processes an action for an agent.
@@ -68,11 +83,10 @@ def step(self, action_arr):
         agent_arr = self.policy_agents
 
         for i, agent in enumerate(agent_arr):
-            self.set_action(action_arr[i], agent, self.action_space[i])
-
-        self.world.step()
-
+            self.set_agent_action(action_arr[i], agent, self.action_space[i])
 
+        for i, agent in enumerate(agent_arr):
+            agent.state, agent.reward, agent.done, agent.info = self.step_agent(agent)
 
         for agent in self.policy_agents:
             obs_arr.append(self._get_obs(agent))
@@ -82,11 +96,39 @@ def step(self, action_arr):
 
         if self.environment_timesteps <= 0:
             isTerminal = True
+        elif any(done_arr):
+             isTerminal = True
         else:
             isTerminal = False
 
         return obs_arr, reward_arr, done_arr, isTerminal, info_arr
 
+
+    def _get_obs(self, agent):
+        """
+            accepts an Agent, and returns its observation/state
+        """
+        return agent.state
+
+    def _get_reward(self, agent):
+        """
+            accepts an Agent, and returns its reward
+        """
+        return agent.reward
+
+    def _get_done(self, agent):
+        """
+            accepts an Agent, and returns its done/terminal property
+        """
+        return agent.done
+
+    def _get_info(self, agent):
+        """
+            accepts an Agent, and returns its info object
+        """
+        return agent.info
+
+
     def render(self) -> None:
         """
         Does nothing, the environment is fully headless
@@ -101,6 +143,9 @@ def reset(self) -> float:
         """
         for i in range(len(self.policy_agents)):
             self.policy_agents[i].state = np.array([0 + random.randint(-100,100)]).astype(float)
+            self.policy_agents[i].reward = 0
+            self.policy_agents[i].info = {}
+            self.policy_agents[i].done = False
 
         self.environment_timesteps = 1000
         return self.environment_timesteps

From 49c495bd08b879e32727f103974331750154f737 Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Fri, 12 May 2023 14:20:46 +0100
Subject: [PATCH 05/40] updates the config to run learn algo

---
 Macodiac.ML/multiagent_main.py       | 12 +++++-------
 Macodiac.ML/multiagentenvironment.py | 10 ++++++----
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/Macodiac.ML/multiagent_main.py b/Macodiac.ML/multiagent_main.py
index 6a85e01..4ff5273 100644
--- a/Macodiac.ML/multiagent_main.py
+++ b/Macodiac.ML/multiagent_main.py
@@ -23,8 +23,8 @@ def __init__(self):
         self.save_path =  os.path.join(filePath,'saved_models', 'model')
         self.save_path_intermittent =  os.path.join(filePath,'saved_models', 'intermittent_saved_models')
         self.numTrainingIterations = 100
-        self.numEpisodes = 10
-        self.numAgents = 25
+        self.numEpisodes = 100
+        self.numAgents = 2
 
         self.env = MultiAgentMacodiacEnvironment(envTimesteps=15, numAgents=self.numAgents)
 
@@ -38,11 +38,11 @@ def __init__(self):
         self.__MODE_LOADMODEL__ = False
 
         # set to true if you want to train and then save the model
-        self.__MODE_TRAINMODEL__ = False
+        self.__MODE_TRAINMODEL__ = True
 
         # set to true to use the randomsample mode for testing, 
         # rather than the model version
-        self.__MODE_RANDOMSAMPLE__ = True
+        self.__MODE_RANDOMSAMPLE__ = False
 
 
     def Run(self):
@@ -50,8 +50,6 @@ def Run(self):
         Runs the project
         """
 
-        self.run_multiagent_project_with_rand_test(self.env, 5)
-        return
 
         model = self.create_model(self.env, self.log_path)
 
@@ -64,7 +62,7 @@ def Run(self):
             self.save_model(model, self.save_path)
 
         if self.__MODE_RANDOMSAMPLE__:
-            self.run_project_with_rand_test(self.env, self.numEpisodes)
+            self.run_multiagent_project_with_rand_test(self.env, 5)
         else:
             self.run_project(self.env, self.numEpisodes, model)
             self.policy_evaluation(model, self.env, self.numEpisodes)
diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index 426ec82..1eac567 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -27,14 +27,16 @@ def __init__(self, envTimesteps:int, numAgents: int):
         for i in range(numAgents):
             self.policy_agents.append(AgentObject())
 
-        self.action_space = []
-        self.observation_space = []
+        self_action_space = []
+        self_observation_space = []
         self.agents = [numAgents]
 
         for agent in self.policy_agents:
-            self.action_space.append(spaces.Discrete(3))
-            self.observation_space.append(spaces.Box(low=np.array([0]), high=np.array([100])))
+            self_action_space.append(spaces.Discrete(3))
+            self_observation_space.append(spaces.Box(low=np.array([0]), high=np.array([100])))
         
+        self.action_space = np.array(self_action_space)
+        self.observation_space = np.array(self_observation_space)
         self.reset()
 
 

From 7918e20ac7c308303e92561313c607fb853799c9 Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Fri, 12 May 2023 16:35:49 +0100
Subject: [PATCH 06/40] creates a multidiscrete version

---
 Macodiac.ML/md_multiagent_main.py       | 207 ++++++++++++++++++++++++
 Macodiac.ML/md_multiagentenvironment.py | 166 +++++++++++++++++++
 2 files changed, 373 insertions(+)
 create mode 100644 Macodiac.ML/md_multiagent_main.py
 create mode 100644 Macodiac.ML/md_multiagentenvironment.py

diff --git a/Macodiac.ML/md_multiagent_main.py b/Macodiac.ML/md_multiagent_main.py
new file mode 100644
index 0000000..c1f77ec
--- /dev/null
+++ b/Macodiac.ML/md_multiagent_main.py
@@ -0,0 +1,207 @@
+import os
+import gymnasium as gym
+from gymnasium import Env
+from environment import MacodiacEnvironment
+from multiagentenvironment import MultiAgentMacodiacEnvironment
+from stable_baselines3.common.evaluation import evaluate_policy
+from stable_baselines3.common.env_checker import check_env
+import numpy as np
+import random
+
+
+
+from stable_baselines3 import PPO
+
+
+class MultiagentMain():
+    isRunning = False
+
+    def __init__(self):
+        """
+        init the class
+        """
+        filePath = os.path.join('Macodiac.ML', 'training_multiagent','results')
+        self.log_path =  os.path.join(filePath,'Logs')
+        self.save_path =  os.path.join(filePath,'saved_models', 'model')
+        self.save_path_intermittent =  os.path.join(filePath,'saved_models', 'intermittent_saved_models')
+        self.numTrainingIterations = 10
+        self.numEpisodes = 10
+        self.numAgents = 3
+
+        self.env = MultiAgentMacodiacEnvironment(envTimesteps=15, numAgents=self.numAgents)
+        check_env(self.env)
+
+
+        # set to true if you want to load an existing model
+        # model loading happens first, then training
+        # NOTES: 
+        #   if loadmodel is set to false, and trainmodel is set to true, 
+        #   the currently saved model is overwritten
+        self.__MODE_LOADMODEL__ = False
+
+        # set to true if you want to train and then save the model
+        self.__MODE_TRAINMODEL__ = True
+
+        # set to true to use the randomsample mode for testing, 
+        # rather than the model version
+        self.__MODE_RANDOMSAMPLE__ = False
+
+
+    def Run(self):
+        """
+        Runs the project
+        """
+
+        if self.__MODE_RANDOMSAMPLE__:
+            self.run_multiagent_project_with_rand_test(self.env, 5)
+            
+        model = self.create_model(self.env, self.log_path)
+
+        if self.__MODE_LOADMODEL__:
+            model = self.load_model(self.env, model, self.save_path)
+
+        if self.__MODE_TRAINMODEL__: 
+            model = self.train_model(model,
+                                     self.numTrainingIterations, self.save_path_intermittent)
+            self.save_model(model, self.save_path)
+
+        
+        else:
+            self.run_project(self.env, self.numEpisodes, model)
+            self.policy_evaluation(model, self.env, self.numEpisodes)
+
+
+    def run_multiagent_project_with_rand_test(self, env:MultiAgentMacodiacEnvironment, numEpisodes: int):
+        """
+        Runs the project with random sampling, using the multiagent env
+        """
+
+        for episode in range(numEpisodes):
+            obs = env.reset()
+            done = False
+            score = 0
+            agent_scores = []
+            iterator = 0
+            while not done:
+                #env.render()
+                iterator+=1
+                print(f'iterator:{iterator}')
+                action_arr = []
+                for i in range(len(env.policy_agents)):
+                    action_arr.append(random.randint(0,2))
+                    # agentActionSpace =  env.action_space[i]
+                    # actionForAgent = agentActionSpace.sample()
+                    # action_arr.append(actionForAgent)
+                
+                print(f'action for agents:\t{action_arr}')
+                
+                obs_arr, reward_arr, done_arr, isTerminal, info_arr = env.step(action_arr)
+                
+                # for i, reward in enumerate(reward_arr):
+                #     print(f'reward is {reward}')
+
+                agent_scores.append(sum(reward_arr))
+
+                print(f'rewards for agents:\t{reward_arr}')
+
+                if any(done_arr):
+                    isTerminal = True
+
+                done = isTerminal
+            print(f'Episode:{episode} | Aggregate agent scores:(Sum:{sum(agent_scores)})')
+        env.close()
+
+
+
+    def run_project(self, env:MultiAgentMacodiacEnvironment, numEpisodes: int, model):
+        """
+        Runs the project with an actual model, instead of random sampling
+        of a model
+
+        @param env: The environment to run this project with
+        @param numEpisodes: the count of episodes to run the environment for
+        """
+        scores = []
+        for episode in range(numEpisodes):
+            obs = env.reset()
+            done = False
+            score = 0
+            while not done:
+                #env.render()
+                action, _discard = model.predict(obs)
+                obs, reward, isTerminal, info = env.step(action)
+                score += reward
+                done = isTerminal
+            scores.append(score)
+
+            runningAvg = np.mean(scores)
+
+            print(f'Episode:{episode} \t| Score:{score} \t\t| RunningAvg: {round(runningAvg, 2)}')
+        env.close()
+
+
+    def create_model(self, env: MultiAgentMacodiacEnvironment, log_path: str):
+        env.reset()
+        model = PPO('MlpPolicy', env, verbose=1, tensorboard_log=log_path)
+        return model
+
+
+    def train_model(self, model, numTimesteps: int, savePath:str):
+        """
+        Trains a model with the number of iterations in 
+        numtimesteps. Creates a n intermediate save every 1m iterations
+
+        @param model: The model to train. The model must have been instantiated
+        @param numTimesteps: the number of training iterations
+        """
+        
+        saveEveryNSteps = 1_000_000
+        
+        if numTimesteps < saveEveryNSteps:
+            model.learn(total_timesteps=numTimesteps)
+
+        else:
+            rangeUpper = int(numTimesteps / saveEveryNSteps)
+            for i in range(1,rangeUpper+1):
+                model.learn(total_timesteps=saveEveryNSteps)
+                model.save(os.path.join(savePath, f'interim-{i}'))
+
+        return model
+
+    def policy_evaluation(self, model, env: MultiAgentMacodiacEnvironment, numEpisodes:int=50):
+        """
+        Prints a policy evaluation, including the mean episode reward
+        and the standard deviation
+
+        @param model:       The model to be evaluated
+        @param env:         The environment to evaluate the model against
+        @param numEpisodes: The count of episodes to evaluate against        
+        """
+        print('\nevalResult:(mean episode reward, standard deviation)')
+        print(f'evalResult:{evaluate_policy(model, env, n_eval_episodes=numEpisodes)}\n')
+
+
+
+    def save_model(self, model, modelPath):
+        """
+        Saves a model to a given path
+
+        @param model:       The model to save
+        @param modelPath:   The path to save to
+        """
+        model.save(modelPath)    
+    
+    
+    def load_model(self, env: MultiAgentMacodiacEnvironment, model, modelPath: str):
+        """
+        Saves a model to a given path
+
+        @param model:       The model to save
+        @param modelPath:   The modelPath to save to
+        """
+        model = PPO.load(modelPath, env=env)
+        return model
+
+
+main = MultiagentMain()
+main.Run()
\ No newline at end of file
diff --git a/Macodiac.ML/md_multiagentenvironment.py b/Macodiac.ML/md_multiagentenvironment.py
new file mode 100644
index 0000000..f5ac0fa
--- /dev/null
+++ b/Macodiac.ML/md_multiagentenvironment.py
@@ -0,0 +1,166 @@
+import gymnasium as gym
+from gym import Env
+# use `gym.spaces` here, even though we're using `gymnasium`
+# https://stackoverflow.com/questions/75108957/assertionerror-the-algorithm-only-supports-class-gym-spaces-box-box-as-acti
+from gym import spaces
+import numpy as np
+import random
+
+class AgentObject:
+    def __init__(self):
+        self.state = []
+
+class MultiAgentMacodiacEnvironment(Env):
+    """
+    Builds a profit maximising agent environment, supporting
+    up to n_agent agents
+    """
+    state = 0
+    environment_timesteps = 15
+
+    def __init__(self, envTimesteps:int, numAgents: int):
+        """
+        Initialises the class
+        """
+        self.environment_timesteps = envTimesteps
+        self.policy_agents = []
+        for i in range(numAgents):
+            self.policy_agents.append(AgentObject())
+
+        #self_action_space = []
+        #self.observation_space = []
+        self.agents = [numAgents]
+        mdActionSpace_arr = []
+        mdObservationSpace_arr = []
+
+        for agent in self.policy_agents:
+            mdActionSpace_arr.append(3)
+            mdObservationSpace_arr.append(10)
+            #md_action_space = spaces.MultiDiscrete(np.array([3,3]), seed=42)
+            #self_action_space.append(spaces.Discrete(3))
+            #self_observation_space.append(spaces.Box(low=np.array([0]), high=np.array([100])))
+            
+            #self.observation_space.append(spaces.Box(low=np.array([0]), high=np.array([100])))
+        
+        md_action_space = spaces.MultiDiscrete(np.array(mdActionSpace_arr))#, seed=42)
+        md_observation_space = spaces.MultiDiscrete(np.array(mdObservationSpace_arr))#, seed=42)
+        self.action_space = md_action_space
+        self.observation_space = md_observation_space
+        # self.action_space = np.array(self_action_space)
+        # self.observation_space = self_observation_space
+        self.reset()
+
+
+        
+        print('-- ENV SETTINGS --')
+        print(self.observation_space)
+        #print(self.observation_space.sample())
+        print(self.action_space)
+        #print(self.action_space.sample())
+        print(self.environment_timesteps)
+        print('-- ENV SETTINGS --')
+
+
+    def set_agent_action(self, action, agent, actionSpace):
+        agent.state = action
+
+    def step_agent(self, agent):
+        myState = agent.state - 1
+        if myState > 0:
+            reward = 1
+        elif myState == 0:
+            reward = 0
+        else:
+            reward = -1
+
+        info = {}
+        return agent.state, reward, False, info
+
+
+    def step(self, action_arr):
+        """
+        Processes an action for an agent.
+
+        Loops through each agent and sets its action. 
+        Then calls world.step to progress the entire world's actions
+
+        Builds up arrays of results, and returns them in a tuple of arrays
+
+        """
+        self.environment_timesteps -=1
+
+        obs_arr = []
+        reward_arr = []
+        done_arr = []
+        info_arr = [{'n': []}]
+        
+        agent_arr = self.policy_agents
+
+        for i, agent in enumerate(agent_arr):
+            self.set_agent_action(action_arr[i], agent, self.action_space[i])
+
+        for i, agent in enumerate(agent_arr):
+            agent.state, agent.reward, agent.done, agent.info = self.step_agent(agent)
+
+        for agent in self.policy_agents:
+            obs_arr.append(self._get_obs(agent))
+            reward_arr.append(self._get_reward(agent))
+            done_arr.append(self._get_done(agent))
+            info_arr.append(self._get_info(agent))
+
+        if self.environment_timesteps <= 0:
+            isTerminal = True
+        elif any(done_arr):
+             isTerminal = True
+        else:
+            isTerminal = False
+
+        return obs_arr, reward_arr, done_arr, isTerminal, info_arr
+
+
+    def _get_obs(self, agent):
+        """
+            accepts an Agent, and returns its observation/state
+        """
+        return agent.state
+
+    def _get_reward(self, agent):
+        """
+            accepts an Agent, and returns its reward
+        """
+        return agent.reward
+
+    def _get_done(self, agent):
+        """
+            accepts an Agent, and returns its done/terminal property
+        """
+        return agent.done
+
+    def _get_info(self, agent):
+        """
+            accepts an Agent, and returns its info object
+        """
+        return agent.info
+
+
+    def render(self) -> None:
+        """
+        Does nothing, the environment is fully headless
+        """
+        pass
+
+    def reset(self) -> float:
+        """
+        Sets the application to its initial conditions
+
+        Sets state to a random float between negative 100 to  positive 100
+        """
+        for i in range(len(self.policy_agents)):
+            self.policy_agents[i].state = np.array([0 + random.randint(0,10)]).astype(float)
+            self.policy_agents[i].reward = 0
+            self.policy_agents[i].info = {}
+            self.policy_agents[i].done = False
+
+        self.environment_timesteps = 10
+        return self.environment_timesteps
+        
\ No newline at end of file

From 5ee2d829915dc3c50a1393a22f3996bc4e5f6568 Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Fri, 12 May 2023 17:46:27 +0100
Subject: [PATCH 07/40] creates a multidiscrete version of the application

---
 Macodiac.ML/.gitignore                  |  4 +++
 Macodiac.ML/md_multiagent_main.py       | 14 ++++----
 Macodiac.ML/md_multiagentenvironment.py |  2 +-
 Macodiac.ML/multiagent_main.py          |  7 ++--
 Macodiac.ML/multiagentenvironment.py    | 44 +++++++++++++++----------
 5 files changed, 43 insertions(+), 28 deletions(-)

diff --git a/Macodiac.ML/.gitignore b/Macodiac.ML/.gitignore
index 197028e..ea7e1d9 100644
--- a/Macodiac.ML/.gitignore
+++ b/Macodiac.ML/.gitignore
@@ -1,7 +1,11 @@
 training/results/logs/*
+training_multiagent/results/logs/*
+!training_multiagent/results/logs/.keep
 !training/results/logs/.keep
 training/results/saved_models/*
 !training/results/saved_models/.keep
+training_multiagent/results/saved_models/*
+!training_multiagent/results/saved_models/.keep
 
 # Byte-compiled / optimized / DLL files
 __pycache__/
diff --git a/Macodiac.ML/md_multiagent_main.py b/Macodiac.ML/md_multiagent_main.py
index c1f77ec..9c307c9 100644
--- a/Macodiac.ML/md_multiagent_main.py
+++ b/Macodiac.ML/md_multiagent_main.py
@@ -2,7 +2,7 @@
 import gymnasium as gym
 from gymnasium import Env
 from environment import MacodiacEnvironment
-from multiagentenvironment import MultiAgentMacodiacEnvironment
+from md_multiagentenvironment import MdMultiAgentMacodiacEnvironment
 from stable_baselines3.common.evaluation import evaluate_policy
 from stable_baselines3.common.env_checker import check_env
 import numpy as np
@@ -28,7 +28,7 @@ def __init__(self):
         self.numEpisodes = 10
         self.numAgents = 3
 
-        self.env = MultiAgentMacodiacEnvironment(envTimesteps=15, numAgents=self.numAgents)
+        self.env = MdMultiAgentMacodiacEnvironment(envTimesteps=15, numAgents=self.numAgents)
         check_env(self.env)
 
 
@@ -71,7 +71,7 @@ def Run(self):
             self.policy_evaluation(model, self.env, self.numEpisodes)
 
 
-    def run_multiagent_project_with_rand_test(self, env:MultiAgentMacodiacEnvironment, numEpisodes: int):
+    def run_multiagent_project_with_rand_test(self, env:MdMultiAgentMacodiacEnvironment, numEpisodes: int):
         """
         Runs the project with random sampling, using the multiagent env
         """
@@ -113,7 +113,7 @@ def run_multiagent_project_with_rand_test(self, env:MultiAgentMacodiacEnvironmen
 
 
 
-    def run_project(self, env:MultiAgentMacodiacEnvironment, numEpisodes: int, model):
+    def run_project(self, env:MdMultiAgentMacodiacEnvironment, numEpisodes: int, model):
         """
         Runs the project with an actual model, instead of random sampling
         of a model
@@ -140,7 +140,7 @@ def run_project(self, env:MultiAgentMacodiacEnvironment, numEpisodes: int, model
         env.close()
 
 
-    def create_model(self, env: MultiAgentMacodiacEnvironment, log_path: str):
+    def create_model(self, env: MdMultiAgentMacodiacEnvironment, log_path: str):
         env.reset()
         model = PPO('MlpPolicy', env, verbose=1, tensorboard_log=log_path)
         return model
@@ -168,7 +168,7 @@ def train_model(self, model, numTimesteps: int, savePath:str):
 
         return model
 
-    def policy_evaluation(self, model, env: MultiAgentMacodiacEnvironment, numEpisodes:int=50):
+    def policy_evaluation(self, model, env: MdMultiAgentMacodiacEnvironment, numEpisodes:int=50):
         """
         Prints a policy evaluation, including the mean episode reward
         and the standard deviation
@@ -192,7 +192,7 @@ def save_model(self, model, modelPath):
         model.save(modelPath)    
     
     
-    def load_model(self, env: MultiAgentMacodiacEnvironment, model, modelPath: str):
+    def load_model(self, env: MdMultiAgentMacodiacEnvironment, model, modelPath: str):
         """
         Saves a model to a given path
 
diff --git a/Macodiac.ML/md_multiagentenvironment.py b/Macodiac.ML/md_multiagentenvironment.py
index f5ac0fa..6c23e8a 100644
--- a/Macodiac.ML/md_multiagentenvironment.py
+++ b/Macodiac.ML/md_multiagentenvironment.py
@@ -10,7 +10,7 @@ class AgentObject:
     def __init__(self):
         self.state = []
 
-class MultiAgentMacodiacEnvironment(Env):
+class MdMultiAgentMacodiacEnvironment(Env):
     """
     Builds a profit maximising agent environment, supporting
     up to n_agent agents
diff --git a/Macodiac.ML/multiagent_main.py b/Macodiac.ML/multiagent_main.py
index 4ff5273..16163db 100644
--- a/Macodiac.ML/multiagent_main.py
+++ b/Macodiac.ML/multiagent_main.py
@@ -4,6 +4,8 @@
 from environment import MacodiacEnvironment
 from multiagentenvironment import MultiAgentMacodiacEnvironment
 from stable_baselines3.common.evaluation import evaluate_policy
+from stable_baselines3.common.env_checker import check_env
+
 import numpy as np
 
 
@@ -22,11 +24,12 @@ def __init__(self):
         self.log_path =  os.path.join(filePath,'Logs')
         self.save_path =  os.path.join(filePath,'saved_models', 'model')
         self.save_path_intermittent =  os.path.join(filePath,'saved_models', 'intermittent_saved_models')
-        self.numTrainingIterations = 100
-        self.numEpisodes = 100
+        self.numTrainingIterations = 5
+        self.numEpisodes = 5
         self.numAgents = 2
 
         self.env = MultiAgentMacodiacEnvironment(envTimesteps=15, numAgents=self.numAgents)
+        check_env(self.env)
 
 
 
diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index 1eac567..2bccfa7 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -1,5 +1,5 @@
 import gymnasium as gym
-from gymnasium import Env
+from gym import Env
 # use `gym.spaces` here, even though we're using `gymnasium`
 # https://stackoverflow.com/questions/75108957/assertionerror-the-algorithm-only-supports-class-gym-spaces-box-box-as-acti
 from gym import spaces
@@ -16,36 +16,42 @@ class MultiAgentMacodiacEnvironment(Env):
     up to n_agent agents
     """
     state = 0
-    environment_timesteps = 15
+    environment_timesteps = 0
+    environment_starter_timesteps = 15
 
     def __init__(self, envTimesteps:int, numAgents: int):
         """
         Initialises the class
         """
-        self.environment_timesteps = envTimesteps
+        self.environment_starter_timesteps = envTimesteps
+
         self.policy_agents = []
         for i in range(numAgents):
             self.policy_agents.append(AgentObject())
-
-        self_action_space = []
-        self_observation_space = []
+        self.observation_space = []
         self.agents = [numAgents]
 
-        for agent in self.policy_agents:
-            self_action_space.append(spaces.Discrete(3))
-            self_observation_space.append(spaces.Box(low=np.array([0]), high=np.array([100])))
+        self.action_space = spaces.MultiDiscrete([3, 3])
+        self.observation_space = spaces.MultiDiscrete([3, 3])
+
+        # for agent in self.policy_agents:
+            #self_action_space.append(spaces.Discrete(3))
+            #self_observation_space.append(spaces.Box(low=np.array([0]), high=np.array([100])))
+            # self.observation_space.append(spaces.Box(low=-np.inf, high=+np.inf, shape=(5,), dtype=np.float32))
         
-        self.action_space = np.array(self_action_space)
-        self.observation_space = np.array(self_observation_space)
+        #self.action_space = np.array(self_action_space)
+        # self.observation_space = spaces.Box(low=np.array([0, 0, 0]), high=np.array([100, 100, 100]), 
+        #                                     shape=(3,3,3),
+        #                                     dtype=np.float64)
         self.reset()
 
 
         
         print('-- ENV SETTINGS --')
         print(self.observation_space)
-        #print(self.observation_space.sample())
+        print(self.observation_space[0].sample())
         print(self.action_space)
-        #print(self.action_space.sample())
+        print(self.action_space.sample())
         print(self.environment_timesteps)
         print('-- ENV SETTINGS --')
 
@@ -80,7 +86,7 @@ def step(self, action_arr):
         obs_arr = []
         reward_arr = []
         done_arr = []
-        info_arr = [{'n': []}]
+        info_arr = {'n': []}
         
         agent_arr = self.policy_agents
 
@@ -94,7 +100,7 @@ def step(self, action_arr):
             obs_arr.append(self._get_obs(agent))
             reward_arr.append(self._get_reward(agent))
             done_arr.append(self._get_done(agent))
-            info_arr.append(self._get_info(agent))
+            info_arr['n'].append(self._get_info(agent))
 
         if self.environment_timesteps <= 0:
             isTerminal = True
@@ -103,7 +109,7 @@ def step(self, action_arr):
         else:
             isTerminal = False
 
-        return obs_arr, reward_arr, done_arr, isTerminal, info_arr
+        return  np.array(obs_arr), sum(reward_arr), isTerminal,  info_arr
 
 
     def _get_obs(self, agent):
@@ -143,12 +149,14 @@ def reset(self) -> float:
 
         Sets state to a random float between negative 100 to  positive 100
         """
+        obs_arr =[]
         for i in range(len(self.policy_agents)):
             self.policy_agents[i].state = np.array([0 + random.randint(-100,100)]).astype(float)
+            obs_arr.append(self.policy_agents[i].state)
             self.policy_agents[i].reward = 0
             self.policy_agents[i].info = {}
             self.policy_agents[i].done = False
 
-        self.environment_timesteps = 1000
-        return self.environment_timesteps
+        self.environment_timesteps = self.environment_starter_timesteps
+        return np.zeros(len(self.policy_agents)).astype(np.int64)
         
\ No newline at end of file

From 7a7b7f0ec3062f7bbe56879974b72413f78df11d Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Sat, 13 May 2023 12:24:30 +0100
Subject: [PATCH 08/40] adjusts action and observation spaces to pass env
 validations

---
 Macodiac.ML/multiagent_main.py       | 23 +++++-----
 Macodiac.ML/multiagentenvironment.py | 63 ++++++++++++++++++----------
 2 files changed, 51 insertions(+), 35 deletions(-)

diff --git a/Macodiac.ML/multiagent_main.py b/Macodiac.ML/multiagent_main.py
index 16163db..69f2c56 100644
--- a/Macodiac.ML/multiagent_main.py
+++ b/Macodiac.ML/multiagent_main.py
@@ -26,7 +26,7 @@ def __init__(self):
         self.save_path_intermittent =  os.path.join(filePath,'saved_models', 'intermittent_saved_models')
         self.numTrainingIterations = 5
         self.numEpisodes = 5
-        self.numAgents = 2
+        self.numAgents = 3
 
         self.env = MultiAgentMacodiacEnvironment(envTimesteps=15, numAgents=self.numAgents)
         check_env(self.env)
@@ -41,11 +41,11 @@ def __init__(self):
         self.__MODE_LOADMODEL__ = False
 
         # set to true if you want to train and then save the model
-        self.__MODE_TRAINMODEL__ = True
+        self.__MODE_TRAINMODEL__ = False
 
         # set to true to use the randomsample mode for testing, 
         # rather than the model version
-        self.__MODE_RANDOMSAMPLE__ = False
+        self.__MODE_RANDOMSAMPLE__ = True
 
 
     def Run(self):
@@ -87,25 +87,22 @@ def run_multiagent_project_with_rand_test(self, env:MultiAgentMacodiacEnvironmen
                 iterator+=1
                 print(f'iterator:{iterator}')
                 action_arr = []
-                for i in range(len(env.policy_agents)):
-                    actionForAgent = env.action_space[i].sample()
-                    action_arr.append(actionForAgent)
+                # for i in range(len(env.policy_agents)):
+                #     actionForAgent = env.action_space.sample()
+                #     action_arr.append(actionForAgent)
                 
                 print(f'action for agents:\t{action_arr}')
                 
-                obs_arr, reward_arr, done_arr, isTerminal, info_arr = env.step(action_arr)
+                obs_arr, reward, isDone, info_arr = env.step(env.action_space.sample())
                 
                 # for i, reward in enumerate(reward_arr):
                 #     print(f'reward is {reward}')
 
-                agent_scores.append(sum(reward_arr))
+                agent_scores.append(reward)
 
-                print(f'rewards for agents:\t{reward_arr}')
+                print(f'rewards for agents:\t{reward}')
 
-                if any(done_arr):
-                    isTerminal = True
-
-                done = isTerminal
+                done = isDone
             print(f'Episode:{episode} | Aggregate agent scores:(Sum:{sum(agent_scores)})')
         env.close()
 
diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index 2bccfa7..c6d61d4 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -9,6 +9,7 @@
 class AgentObject:
     def __init__(self):
         self.state = []
+        self.vendingPrice = 0
 
 class MultiAgentMacodiacEnvironment(Env):
     """
@@ -17,7 +18,9 @@ class MultiAgentMacodiacEnvironment(Env):
     """
     state = 0
     environment_timesteps = 0
-    environment_starter_timesteps = 15
+    environment_starter_timesteps = 150
+    env_wholesale_price = 60        # the price agents pay to purchase goods
+    env_agent_marginal_cost = 5     # the marginal operating cost
 
     def __init__(self, envTimesteps:int, numAgents: int):
         """
@@ -30,26 +33,24 @@ def __init__(self, envTimesteps:int, numAgents: int):
             self.policy_agents.append(AgentObject())
         self.observation_space = []
         self.agents = [numAgents]
+        numActions = 3
 
-        self.action_space = spaces.MultiDiscrete([3, 3])
-        self.observation_space = spaces.MultiDiscrete([3, 3])
 
-        # for agent in self.policy_agents:
-            #self_action_space.append(spaces.Discrete(3))
-            #self_observation_space.append(spaces.Box(low=np.array([0]), high=np.array([100])))
-            # self.observation_space.append(spaces.Box(low=-np.inf, high=+np.inf, shape=(5,), dtype=np.float32))
-        
-        #self.action_space = np.array(self_action_space)
-        # self.observation_space = spaces.Box(low=np.array([0, 0, 0]), high=np.array([100, 100, 100]), 
-        #                                     shape=(3,3,3),
-        #                                     dtype=np.float64)
+        self.action_space = spaces.MultiDiscrete([200, 200, 200])
+
+        # the observation space is a nAgents by nActions array of float32 numbers
+        # between 0-100
+        self.observation_space = spaces.Box(low=-100,high=100, shape=(3, 3), dtype=np.float32)
+
+        print(f'obs_space.sample: {self.observation_space.sample()}')
+
         self.reset()
 
 
         
         print('-- ENV SETTINGS --')
-        print(self.observation_space)
-        print(self.observation_space[0].sample())
+        print(f'obs:{self.observation_space}')
+        print(f'sample:{self.observation_space.sample()}')
         print(self.action_space)
         print(self.action_space.sample())
         print(self.environment_timesteps)
@@ -57,7 +58,13 @@ def __init__(self, envTimesteps:int, numAgents: int):
 
 
     def set_agent_action(self, action, agent, actionSpace):
-        agent.state = action
+        # agent.state is the percentage price diff from the 
+        # wholesale price
+        agent.state = action - 100
+        agentBaseVendingPrice = self.env_wholesale_price * (agent.state / 100)
+        agentMarginalCostAddedVendingPrice = agentBaseVendingPrice + self.env_agent_marginal_cost
+        agent.vendingPrice = agentMarginalCostAddedVendingPrice
+        print(f'agent vending price was {agent.vendingPrice}')
 
     def step_agent(self, agent):
         if agent.state > 0:
@@ -88,12 +95,10 @@ def step(self, action_arr):
         done_arr = []
         info_arr = {'n': []}
         
-        agent_arr = self.policy_agents
-
-        for i, agent in enumerate(agent_arr):
+        for i, agent in enumerate(self.policy_agents):
             self.set_agent_action(action_arr[i], agent, self.action_space[i])
 
-        for i, agent in enumerate(agent_arr):
+        for i, agent in enumerate(self.policy_agents):
             agent.state, agent.reward, agent.done, agent.info = self.step_agent(agent)
 
         for agent in self.policy_agents:
@@ -109,7 +114,11 @@ def step(self, action_arr):
         else:
             isTerminal = False
 
-        return  np.array(obs_arr), sum(reward_arr), isTerminal,  info_arr
+        fakeObsArray = np.array([   [0, 0, 0],
+                            [0, 0, 0],
+                            [0, 0, 0]]).astype(np.float32) 
+
+        return  fakeObsArray, sum(reward_arr), isTerminal,  info_arr
 
 
     def _get_obs(self, agent):
@@ -151,12 +160,22 @@ def reset(self) -> float:
         """
         obs_arr =[]
         for i in range(len(self.policy_agents)):
-            self.policy_agents[i].state = np.array([0 + random.randint(-100,100)]).astype(float)
+            self.policy_agents[i].state = np.array(
+                                    [0.0, 60.0, 5.0], 
+                                     dtype=np.float32)
             obs_arr.append(self.policy_agents[i].state)
             self.policy_agents[i].reward = 0
             self.policy_agents[i].info = {}
             self.policy_agents[i].done = False
+            self.policy_agents[i].vendingPrice = 0
 
         self.environment_timesteps = self.environment_starter_timesteps
-        return np.zeros(len(self.policy_agents)).astype(np.int64)
+        
+        print(f'obsArr: {obs_arr}')
+        return np.array(obs_arr).astype(np.float32)
+        
+        return np.array([   [0, 0, 0],
+                            [0, 0, 0],
+                            [0, 0, 0]]).astype(np.float32) 
+
         
\ No newline at end of file

From 18aba5efee34ac2d635aa544f0dd03cebe86e3c0 Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Sat, 13 May 2023 12:27:54 +0100
Subject: [PATCH 09/40] configures the model to run training

---
 Macodiac.ML/multiagent_main.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Macodiac.ML/multiagent_main.py b/Macodiac.ML/multiagent_main.py
index 69f2c56..d6e7dbd 100644
--- a/Macodiac.ML/multiagent_main.py
+++ b/Macodiac.ML/multiagent_main.py
@@ -41,11 +41,11 @@ def __init__(self):
         self.__MODE_LOADMODEL__ = False
 
         # set to true if you want to train and then save the model
-        self.__MODE_TRAINMODEL__ = False
+        self.__MODE_TRAINMODEL__ = True
 
         # set to true to use the randomsample mode for testing, 
         # rather than the model version
-        self.__MODE_RANDOMSAMPLE__ = True
+        self.__MODE_RANDOMSAMPLE__ = False
 
 
     def Run(self):

From 349e8577db4834ba138aa08036b28dbfaab0dec5 Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Sat, 13 May 2023 14:34:44 +0100
Subject: [PATCH 10/40] model learns behaviour

---
 Macodiac.ML/main.py                  |  4 ++--
 Macodiac.ML/multiagent_main.py       |  6 +++---
 Macodiac.ML/multiagentenvironment.py | 25 +++++++++++++++++--------
 3 files changed, 22 insertions(+), 13 deletions(-)

diff --git a/Macodiac.ML/main.py b/Macodiac.ML/main.py
index 01b7a50..1619c2f 100644
--- a/Macodiac.ML/main.py
+++ b/Macodiac.ML/main.py
@@ -35,11 +35,11 @@ def __init__(self):
         self.__MODE_LOADMODEL__ =False
 
         # set to true if you want to train and then save the model
-        self.__MODE_TRAINMODEL__ = False
+        self.__MODE_TRAINMODEL__ = True
 
         # set to true to use the randomsample mode for testing, 
         # rather than the model version
-        self.__MODE_RANDOMSAMPLE__ = True
+        self.__MODE_RANDOMSAMPLE__ = False
 
 
     def Run(self):
diff --git a/Macodiac.ML/multiagent_main.py b/Macodiac.ML/multiagent_main.py
index d6e7dbd..e45986e 100644
--- a/Macodiac.ML/multiagent_main.py
+++ b/Macodiac.ML/multiagent_main.py
@@ -24,7 +24,7 @@ def __init__(self):
         self.log_path =  os.path.join(filePath,'Logs')
         self.save_path =  os.path.join(filePath,'saved_models', 'model')
         self.save_path_intermittent =  os.path.join(filePath,'saved_models', 'intermittent_saved_models')
-        self.numTrainingIterations = 5
+        self.numTrainingIterations = 1_000_000
         self.numEpisodes = 5
         self.numAgents = 3
 
@@ -41,11 +41,11 @@ def __init__(self):
         self.__MODE_LOADMODEL__ = False
 
         # set to true if you want to train and then save the model
-        self.__MODE_TRAINMODEL__ = True
+        self.__MODE_TRAINMODEL__ = False
 
         # set to true to use the randomsample mode for testing, 
         # rather than the model version
-        self.__MODE_RANDOMSAMPLE__ = False
+        self.__MODE_RANDOMSAMPLE__ = True
 
 
     def Run(self):
diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index c6d61d4..0b51b80 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -64,7 +64,7 @@ def set_agent_action(self, action, agent, actionSpace):
         agentBaseVendingPrice = self.env_wholesale_price * (agent.state / 100)
         agentMarginalCostAddedVendingPrice = agentBaseVendingPrice + self.env_agent_marginal_cost
         agent.vendingPrice = agentMarginalCostAddedVendingPrice
-        print(f'agent vending price was {agent.vendingPrice}')
+        # print(f'agent vending price was {agent.vendingPrice}')
 
     def step_agent(self, agent):
         if agent.state > 0:
@@ -114,11 +114,19 @@ def step(self, action_arr):
         else:
             isTerminal = False
 
-        fakeObsArray = np.array([   [0, 0, 0],
-                            [0, 0, 0],
-                            [0, 0, 0]]).astype(np.float32) 
+        # fakeObsArray = np.array([     [0, 0, 0],
+        #                               [0, 0, 0],
+        #                               [0, 0, 0]]).astype(np.float32) 
+        tmpFakeObsArray = []
+        for i, agent in enumerate(self.policy_agents):
+            partialResult = self.get_agent_default_observation_array()
+            partialResult[0] = self._get_obs(agent)
+            tmpFakeObsArray.append(partialResult)
+
+        fakeObsArray = np.array(tmpFakeObsArray).astype(np.float32) 
+
 
-        return  fakeObsArray, sum(reward_arr), isTerminal,  info_arr
+        return fakeObsArray, sum(reward_arr), isTerminal,  info_arr
 
 
     def _get_obs(self, agent):
@@ -161,7 +169,7 @@ def reset(self) -> float:
         obs_arr =[]
         for i in range(len(self.policy_agents)):
             self.policy_agents[i].state = np.array(
-                                    [0.0, 60.0, 5.0], 
+                                    self.get_agent_default_observation_array(), 
                                      dtype=np.float32)
             obs_arr.append(self.policy_agents[i].state)
             self.policy_agents[i].reward = 0
@@ -171,11 +179,12 @@ def reset(self) -> float:
 
         self.environment_timesteps = self.environment_starter_timesteps
         
-        print(f'obsArr: {obs_arr}')
+        #print(f'obsArr: {obs_arr}')
         return np.array(obs_arr).astype(np.float32)
         
         return np.array([   [0, 0, 0],
                             [0, 0, 0],
                             [0, 0, 0]]).astype(np.float32) 
 
-        
\ No newline at end of file
+    def get_agent_default_observation_array(self):
+        return [0.0, 60.0, 5.0]
\ No newline at end of file

From f16dcf07013d63eee805477e9261ae4ac49d3af5 Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Sat, 13 May 2023 14:42:40 +0100
Subject: [PATCH 11/40] refactors some var names

---
 Macodiac.ML/multiagentenvironment.py | 19 ++++++-------------
 1 file changed, 6 insertions(+), 13 deletions(-)

diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index 0b51b80..0eb3804 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -114,19 +114,16 @@ def step(self, action_arr):
         else:
             isTerminal = False
 
-        # fakeObsArray = np.array([     [0, 0, 0],
-        #                               [0, 0, 0],
-        #                               [0, 0, 0]]).astype(np.float32) 
-        tmpFakeObsArray = []
+        tmpObsArray = []
         for i, agent in enumerate(self.policy_agents):
             partialResult = self.get_agent_default_observation_array()
-            partialResult[0] = self._get_obs(agent)
-            tmpFakeObsArray.append(partialResult)
+            partialResult[0] = self._get_obs(agent) #The agent's result is present in the 0th element of its result
+            tmpObsArray.append(partialResult)
 
-        fakeObsArray = np.array(tmpFakeObsArray).astype(np.float32) 
+        concatObsArray = np.array(tmpObsArray).astype(np.float32) 
 
 
-        return fakeObsArray, sum(reward_arr), isTerminal,  info_arr
+        return concatObsArray, sum(reward_arr), isTerminal,  info_arr
 
 
     def _get_obs(self, agent):
@@ -179,12 +176,8 @@ def reset(self) -> float:
 
         self.environment_timesteps = self.environment_starter_timesteps
         
-        #print(f'obsArr: {obs_arr}')
         return np.array(obs_arr).astype(np.float32)
-        
-        return np.array([   [0, 0, 0],
-                            [0, 0, 0],
-                            [0, 0, 0]]).astype(np.float32) 
+
 
     def get_agent_default_observation_array(self):
         return [0.0, 60.0, 5.0]
\ No newline at end of file

From edd425d2c935708bc06e67c1dc48ca73b7d160a9 Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Sat, 13 May 2023 15:30:37 +0100
Subject: [PATCH 12/40] WIP, setting up consumers

---
 Macodiac.ML/multiagent_main.py       | 33 ++++++++----------
 Macodiac.ML/multiagentenvironment.py | 51 +++++++++++++++++++++-------
 2 files changed, 52 insertions(+), 32 deletions(-)

diff --git a/Macodiac.ML/multiagent_main.py b/Macodiac.ML/multiagent_main.py
index e45986e..aff7afc 100644
--- a/Macodiac.ML/multiagent_main.py
+++ b/Macodiac.ML/multiagent_main.py
@@ -24,11 +24,12 @@ def __init__(self):
         self.log_path =  os.path.join(filePath,'Logs')
         self.save_path =  os.path.join(filePath,'saved_models', 'model')
         self.save_path_intermittent =  os.path.join(filePath,'saved_models', 'intermittent_saved_models')
-        self.numTrainingIterations = 1_000_000
-        self.numEpisodes = 5
-        self.numAgents = 3
+        self.numTrainingIterations = 1_000
+        self.numEpisodes = 15
+        self.envTimesteps = 15
+        self.numAgents = 10
 
-        self.env = MultiAgentMacodiacEnvironment(envTimesteps=15, numAgents=self.numAgents)
+        self.env = MultiAgentMacodiacEnvironment(envTimesteps=self.envTimesteps, numAgents=self.numAgents)
         check_env(self.env)
 
 
@@ -38,21 +39,22 @@ def __init__(self):
         # NOTES: 
         #   if loadmodel is set to false, and trainmodel is set to true, 
         #   the currently saved model is overwritten
-        self.__MODE_LOADMODEL__ = False
+        self.__MODE_LOADMODEL__ = True
 
         # set to true if you want to train and then save the model
-        self.__MODE_TRAINMODEL__ = False
+        self.__MODE_TRAINMODEL__ = True
 
         # set to true to use the randomsample mode for testing, 
         # rather than the model version
-        self.__MODE_RANDOMSAMPLE__ = True
+        self.__MODE_RANDOMSAMPLE__ = False
 
 
     def Run(self):
         """
         Runs the project
         """
-
+        if self.__MODE_RANDOMSAMPLE__:
+            self.run_multiagent_project_with_rand_test(self.env, 5)
 
         model = self.create_model(self.env, self.log_path)
 
@@ -64,9 +66,7 @@ def Run(self):
                                      self.numTrainingIterations, self.save_path_intermittent)
             self.save_model(model, self.save_path)
 
-        if self.__MODE_RANDOMSAMPLE__:
-            self.run_multiagent_project_with_rand_test(self.env, 5)
-        else:
+        if not self.__MODE_RANDOMSAMPLE__:
             self.run_project(self.env, self.numEpisodes, model)
             self.policy_evaluation(model, self.env, self.numEpisodes)
 
@@ -86,21 +86,16 @@ def run_multiagent_project_with_rand_test(self, env:MultiAgentMacodiacEnvironmen
                 #env.render()
                 iterator+=1
                 print(f'iterator:{iterator}')
-                action_arr = []
-                # for i in range(len(env.policy_agents)):
-                #     actionForAgent = env.action_space.sample()
-                #     action_arr.append(actionForAgent)
+                action_arr = env.action_space.sample()
                 
                 print(f'action for agents:\t{action_arr}')
                 
-                obs_arr, reward, isDone, info_arr = env.step(env.action_space.sample())
+                obs_arr, reward, isDone, info_arr = env.step(action_arr)
                 
-                # for i, reward in enumerate(reward_arr):
-                #     print(f'reward is {reward}')
-
                 agent_scores.append(reward)
 
                 print(f'rewards for agents:\t{reward}')
+                print(f'obs for agents:\t{obs_arr}')
 
                 done = isDone
             print(f'Episode:{episode} | Aggregate agent scores:(Sum:{sum(agent_scores)})')
diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index 0eb3804..aaa601e 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -11,6 +11,13 @@ def __init__(self):
         self.state = []
         self.vendingPrice = 0
 
+class ConsumerObject:
+    def __init__(self):
+        self.demand = 1
+        self.utility = 0
+        self.money = 100
+        self.daily_income = 20
+
 class MultiAgentMacodiacEnvironment(Env):
     """
     Builds a profit maximising agent environment, supporting
@@ -20,27 +27,34 @@ class MultiAgentMacodiacEnvironment(Env):
     environment_timesteps = 0
     environment_starter_timesteps = 150
     env_wholesale_price = 60        # the price agents pay to purchase goods
-    env_agent_marginal_cost = 5     # the marginal operating cost
+    env_agent_marginal_cost = 5     # the marginal cost of vending
+    num_consumers = 5
+    consumers_arr = []
 
     def __init__(self, envTimesteps:int, numAgents: int):
         """
         Initialises the class
         """
         self.environment_starter_timesteps = envTimesteps
-
+        
         self.policy_agents = []
         for i in range(numAgents):
             self.policy_agents.append(AgentObject())
+        
+        for i in range(self.num_consumers):
+            self.consumers_arr.append(ConsumerObject())
+
         self.observation_space = []
         self.agents = [numAgents]
-        numActions = 3
+
+        arr = np.full(numAgents, 200) # creates an array full of 200's shaped [200,200,200], with numAgents number of element
 
 
-        self.action_space = spaces.MultiDiscrete([200, 200, 200])
+        self.action_space = spaces.MultiDiscrete(arr)
 
-        # the observation space is a nAgents by nActions array of float32 numbers
-        # between 0-100
-        self.observation_space = spaces.Box(low=-100,high=100, shape=(3, 3), dtype=np.float32)
+        # the observation space is a nAgents by nActions array of float32 numbers between 0-100
+        # also contains the static value for marginal cost and wholesale price
+        self.observation_space = spaces.Box(low=-100,high=100, shape=(numAgents, 4), dtype=np.float32)
 
         print(f'obs_space.sample: {self.observation_space.sample()}')
 
@@ -101,6 +115,9 @@ def step(self, action_arr):
         for i, agent in enumerate(self.policy_agents):
             agent.state, agent.reward, agent.done, agent.info = self.step_agent(agent)
 
+        # for i, consumer in enumerate(self.consumers_arr):
+        #     self.step_consumer(self.policy_agents)
+
         for agent in self.policy_agents:
             obs_arr.append(self._get_obs(agent))
             reward_arr.append(self._get_reward(agent))
@@ -116,16 +133,21 @@ def step(self, action_arr):
 
         tmpObsArray = []
         for i, agent in enumerate(self.policy_agents):
-            partialResult = self.get_agent_default_observation_array()
-            partialResult[0] = self._get_obs(agent) #The agent's result is present in the 0th element of its result
-            tmpObsArray.append(partialResult)
+            partialObservationResult = self.get_agent_default_observation_array()
+            partialObservationResult[0] = self._get_obs(agent) #The agent's result is present in the 0th element of its result
+            partialObservationResult[1] = self._get_final_vend_price(agent) #The agent's result is present in the 0th element of its result
+            tmpObsArray.append(partialObservationResult)
 
         concatObsArray = np.array(tmpObsArray).astype(np.float32) 
-
-
         return concatObsArray, sum(reward_arr), isTerminal,  info_arr
 
 
+    def _get_final_vend_price(self, agent):
+        """
+            accepts an agent and returns its final vending
+        """
+        return agent.vendingPrice
+    
     def _get_obs(self, agent):
         """
             accepts an Agent, and returns its observation/state
@@ -173,6 +195,9 @@ def reset(self) -> float:
             self.policy_agents[i].info = {}
             self.policy_agents[i].done = False
             self.policy_agents[i].vendingPrice = 0
+        
+        for i in range(len(self.consumers_arr)):
+            self.consumers_arr[i].money += self.consumers_arr[i].daily_income
 
         self.environment_timesteps = self.environment_starter_timesteps
         
@@ -180,4 +205,4 @@ def reset(self) -> float:
 
 
     def get_agent_default_observation_array(self):
-        return [0.0, 60.0, 5.0]
\ No newline at end of file
+        return [0.0, 0.0, 60.0, 5.0]
\ No newline at end of file

From 5e56200869f2a1d9dc1f85d4d1913fb525ae1ed5 Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Sun, 14 May 2023 11:18:43 +0100
Subject: [PATCH 13/40] adds some vending based on price points

---
 Macodiac.ML/multiagent_main.py       |  6 +-
 Macodiac.ML/multiagentenvironment.py | 92 +++++++++++++++++++++++-----
 2 files changed, 79 insertions(+), 19 deletions(-)

diff --git a/Macodiac.ML/multiagent_main.py b/Macodiac.ML/multiagent_main.py
index aff7afc..a3329d4 100644
--- a/Macodiac.ML/multiagent_main.py
+++ b/Macodiac.ML/multiagent_main.py
@@ -39,14 +39,14 @@ def __init__(self):
         # NOTES: 
         #   if loadmodel is set to false, and trainmodel is set to true, 
         #   the currently saved model is overwritten
-        self.__MODE_LOADMODEL__ = True
+        self.__MODE_LOADMODEL__ = False
 
         # set to true if you want to train and then save the model
-        self.__MODE_TRAINMODEL__ = True
+        self.__MODE_TRAINMODEL__ = False
 
         # set to true to use the randomsample mode for testing, 
         # rather than the model version
-        self.__MODE_RANDOMSAMPLE__ = False
+        self.__MODE_RANDOMSAMPLE__ = True
 
 
     def Run(self):
diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index aaa601e..7a6f73c 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -10,13 +10,13 @@ class AgentObject:
     def __init__(self):
         self.state = []
         self.vendingPrice = 0
+        self.reward = 0
 
 class ConsumerObject:
     def __init__(self):
         self.demand = 1
         self.utility = 0
-        self.money = 100
-        self.daily_income = 20
+        self.money = 0
 
 class MultiAgentMacodiacEnvironment(Env):
     """
@@ -26,9 +26,10 @@ class MultiAgentMacodiacEnvironment(Env):
     state = 0
     environment_timesteps = 0
     environment_starter_timesteps = 150
-    env_wholesale_price = 60        # the price agents pay to purchase goods
-    env_agent_marginal_cost = 5     # the marginal cost of vending
-    num_consumers = 5
+    env_wholesale_price = 50        # the price agents pay to purchase goods
+    env_agent_marginal_cost = 0     # the marginal cost of vending
+    num_consumers = 2
+    consumer_total_money_per_turn = 25000
     consumers_arr = []
 
     def __init__(self, envTimesteps:int, numAgents: int):
@@ -81,13 +82,13 @@ def set_agent_action(self, action, agent, actionSpace):
         # print(f'agent vending price was {agent.vendingPrice}')
 
     def step_agent(self, agent):
-        if agent.state > 0:
-            reward = 1
-        elif agent.state == 0:
-            reward = 0
-        else:
-            reward = -1
-
+        # if agent.state > 0:
+        #     reward = 1
+        # elif agent.state == 0:
+        #     reward = 0
+        # else:
+        #     reward = -1
+        reward = agent.reward
         info = {}
         return agent.state, reward, False, info
 
@@ -112,12 +113,12 @@ def step(self, action_arr):
         for i, agent in enumerate(self.policy_agents):
             self.set_agent_action(action_arr[i], agent, self.action_space[i])
 
+        for i, consumer in enumerate(self.consumers_arr):
+            self.set_consumer_purchases(self.policy_agents, consumer)
+
         for i, agent in enumerate(self.policy_agents):
             agent.state, agent.reward, agent.done, agent.info = self.step_agent(agent)
 
-        # for i, consumer in enumerate(self.consumers_arr):
-        #     self.step_consumer(self.policy_agents)
-
         for agent in self.policy_agents:
             obs_arr.append(self._get_obs(agent))
             reward_arr.append(self._get_reward(agent))
@@ -142,6 +143,30 @@ def step(self, action_arr):
         return concatObsArray, sum(reward_arr), isTerminal,  info_arr
 
 
+    def set_consumer_purchases(self, agents_arr, consumer):
+        """
+        So long as the consumer has money, loops through the agents, and selects the lowest
+        price agent. 
+
+        Purchases as many items from the agent as possible
+        """
+        lowestPriceAgnetIndex = 0
+            
+        for i, agent in agents_arr:
+            if agent.vendingPrice < agents_arr[lowestPriceAgnetIndex]:
+                lowestPriceAgnetIndex = i
+
+        while consumer.money > 0:            
+            if lowestPriceAgnetIndex.vendingPrice < consumer.money:
+                agents_arr[lowestPriceAgnetIndex].reward += agents_arr[lowestPriceAgnetIndex].vendingPrice
+                consumer.money -= agents_arr[lowestPriceAgnetIndex].vendingPrice
+            else:
+                # set the consumer's money to 0 if the vend price
+                # is less than the remaining money (stops infinite loop)
+                consumer.money = 0
+
+
+
     def _get_final_vend_price(self, agent):
         """
             accepts an agent and returns its final vending
@@ -196,13 +221,48 @@ def reset(self) -> float:
             self.policy_agents[i].done = False
             self.policy_agents[i].vendingPrice = 0
         
+        consumerMoneyEach = self.consumer_total_money_per_turn / self.num_consumers
         for i in range(len(self.consumers_arr)):
-            self.consumers_arr[i].money += self.consumers_arr[i].daily_income
+            self.consumers_arr[i].money = consumerMoneyEach
 
         self.environment_timesteps = self.environment_starter_timesteps
         
         return np.array(obs_arr).astype(np.float32)
 
+    def get_consumer_demand_schedule(self):
+        schedule = [
+            [0, 1000],
+            [10, 900],
+            [20, 800],
+            [30, 700],
+            [40, 600],
+            [50, 500],
+            [60, 400],
+            [70, 300],
+            [80, 200],
+            [90, 100],
+            [100, 0]
+        ]
+        return schedule
+    
+    def get_consumer_quantity_demanded_at_price(self, price):
+        dmnd_schedule = self.get_consumer_demand_schedule()
+        roundedPrice = round(price, -1)
+        for i in dmnd_schedule:
+            if i[0] == roundedPrice:
+                return i[1]
+
+        return 0
+      
+        # point_a = dmnd_schedule()[0]
+        # point_b = dmnd_schedule()[len(dmnd_schedule)]
+
+
+        # change_in_x = point_a[0] - point_b[0]
+        # change_in_y = point_a[1] - point_b[1]
+        # gradient = change_in_y / change_in_x
+
+
 
     def get_agent_default_observation_array(self):
         return [0.0, 0.0, 60.0, 5.0]
\ No newline at end of file

From 01b4c0a5ec7eb9fa320cd6824a051bb60ae3b051 Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Sun, 14 May 2023 11:31:22 +0100
Subject: [PATCH 14/40] corrects a bug where vendors were setting negative
 prices

---
 Macodiac.ML/multiagentenvironment.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index 7a6f73c..c8e80ad 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -17,6 +17,7 @@ def __init__(self):
         self.demand = 1
         self.utility = 0
         self.money = 0
+        self.total_consumed = 0
 
 class MultiAgentMacodiacEnvironment(Env):
     """
@@ -76,9 +77,10 @@ def set_agent_action(self, action, agent, actionSpace):
         # agent.state is the percentage price diff from the 
         # wholesale price
         agent.state = action - 100
-        agentBaseVendingPrice = self.env_wholesale_price * (agent.state / 100)
-        agentMarginalCostAddedVendingPrice = agentBaseVendingPrice + self.env_agent_marginal_cost
-        agent.vendingPrice = agentMarginalCostAddedVendingPrice
+        agentBaseVendingPriceAdjust = self.env_wholesale_price * (agent.state / 100)
+        baseAgentVendingPrice = self.env_wholesale_price + agentBaseVendingPriceAdjust
+        #agentMarginalCostAddedVendingPrice = agentBaseVendingPriceAdjust + self.env_agent_marginal_cost
+        agent.vendingPrice = baseAgentVendingPrice
         # print(f'agent vending price was {agent.vendingPrice}')
 
     def step_agent(self, agent):
@@ -152,14 +154,15 @@ def set_consumer_purchases(self, agents_arr, consumer):
         """
         lowestPriceAgnetIndex = 0
             
-        for i, agent in agents_arr:
-            if agent.vendingPrice < agents_arr[lowestPriceAgnetIndex]:
+        for i, agent in enumerate(agents_arr):
+            if agent.vendingPrice < agents_arr[lowestPriceAgnetIndex].vendingPrice:
                 lowestPriceAgnetIndex = i
 
         while consumer.money > 0:            
-            if lowestPriceAgnetIndex.vendingPrice < consumer.money:
+            if agents_arr[lowestPriceAgnetIndex].vendingPrice < consumer.money:
                 agents_arr[lowestPriceAgnetIndex].reward += agents_arr[lowestPriceAgnetIndex].vendingPrice
                 consumer.money -= agents_arr[lowestPriceAgnetIndex].vendingPrice
+                consumer.total_consumed += 1
             else:
                 # set the consumer's money to 0 if the vend price
                 # is less than the remaining money (stops infinite loop)

From 82c6d4e443add189dcdba9642d8f7c59a9d40c24 Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Sun, 14 May 2023 12:08:24 +0100
Subject: [PATCH 15/40] corrects a bug where vendors were giving away products
 for free

---
 Macodiac.ML/multiagent_main.py       | 4 ++--
 Macodiac.ML/multiagentenvironment.py | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/Macodiac.ML/multiagent_main.py b/Macodiac.ML/multiagent_main.py
index a3329d4..bf4bb18 100644
--- a/Macodiac.ML/multiagent_main.py
+++ b/Macodiac.ML/multiagent_main.py
@@ -25,7 +25,7 @@ def __init__(self):
         self.save_path =  os.path.join(filePath,'saved_models', 'model')
         self.save_path_intermittent =  os.path.join(filePath,'saved_models', 'intermittent_saved_models')
         self.numTrainingIterations = 1_000
-        self.numEpisodes = 15
+        self.numEpisodes = 150
         self.envTimesteps = 15
         self.numAgents = 10
 
@@ -87,7 +87,7 @@ def run_multiagent_project_with_rand_test(self, env:MultiAgentMacodiacEnvironmen
                 iterator+=1
                 print(f'iterator:{iterator}')
                 action_arr = env.action_space.sample()
-                
+
                 print(f'action for agents:\t{action_arr}')
                 
                 obs_arr, reward, isDone, info_arr = env.step(action_arr)
diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index c8e80ad..bcd543b 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -54,7 +54,7 @@ def __init__(self, envTimesteps:int, numAgents: int):
 
         self.action_space = spaces.MultiDiscrete(arr)
 
-        # the observation space is a nAgents by nActions array of float32 numbers between 0-100
+        # the observation space is a nAgents by nActions array of float32 numbers between -99-99
         # also contains the static value for marginal cost and wholesale price
         self.observation_space = spaces.Box(low=-100,high=100, shape=(numAgents, 4), dtype=np.float32)
 
@@ -80,7 +80,7 @@ def set_agent_action(self, action, agent, actionSpace):
         agentBaseVendingPriceAdjust = self.env_wholesale_price * (agent.state / 100)
         baseAgentVendingPrice = self.env_wholesale_price + agentBaseVendingPriceAdjust
         #agentMarginalCostAddedVendingPrice = agentBaseVendingPriceAdjust + self.env_agent_marginal_cost
-        agent.vendingPrice = baseAgentVendingPrice
+        agent.vendingPrice = max(baseAgentVendingPrice, 1)
         # print(f'agent vending price was {agent.vendingPrice}')
 
     def step_agent(self, agent):
@@ -268,4 +268,4 @@ def get_consumer_quantity_demanded_at_price(self, price):
 
 
     def get_agent_default_observation_array(self):
-        return [0.0, 0.0, 60.0, 5.0]
\ No newline at end of file
+        return [0.0, 0.0, self.env_wholesale_price, self.env_agent_marginal_cost]
\ No newline at end of file

From c6dd563071fdc8535c666805caa37976403292bb Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Sun, 14 May 2023 12:16:40 +0100
Subject: [PATCH 16/40] penalises the agents for charging less than the
 wholesale price

---
 Macodiac.ML/multiagent_main.py       | 12 +++++++-----
 Macodiac.ML/multiagentenvironment.py |  8 +-------
 2 files changed, 8 insertions(+), 12 deletions(-)

diff --git a/Macodiac.ML/multiagent_main.py b/Macodiac.ML/multiagent_main.py
index bf4bb18..6e53742 100644
--- a/Macodiac.ML/multiagent_main.py
+++ b/Macodiac.ML/multiagent_main.py
@@ -25,7 +25,7 @@ def __init__(self):
         self.save_path =  os.path.join(filePath,'saved_models', 'model')
         self.save_path_intermittent =  os.path.join(filePath,'saved_models', 'intermittent_saved_models')
         self.numTrainingIterations = 1_000
-        self.numEpisodes = 150
+        self.numEpisodes = 25
         self.envTimesteps = 15
         self.numAgents = 10
 
@@ -42,11 +42,11 @@ def __init__(self):
         self.__MODE_LOADMODEL__ = False
 
         # set to true if you want to train and then save the model
-        self.__MODE_TRAINMODEL__ = False
+        self.__MODE_TRAINMODEL__ = True
 
         # set to true to use the randomsample mode for testing, 
         # rather than the model version
-        self.__MODE_RANDOMSAMPLE__ = True
+        self.__MODE_RANDOMSAMPLE__ = False
 
 
     def Run(self):
@@ -54,7 +54,7 @@ def Run(self):
         Runs the project
         """
         if self.__MODE_RANDOMSAMPLE__:
-            self.run_multiagent_project_with_rand_test(self.env, 5)
+            self.run_multiagent_project_with_rand_test(self.env, self.numEpisodes)
 
         model = self.create_model(self.env, self.log_path)
 
@@ -97,8 +97,10 @@ def run_multiagent_project_with_rand_test(self, env:MultiAgentMacodiacEnvironmen
                 print(f'rewards for agents:\t{reward}')
                 print(f'obs for agents:\t{obs_arr}')
 
+                if done:
+                    print(f'is done')
                 done = isDone
-            print(f'Episode:{episode} | Aggregate agent scores:(Sum:{sum(agent_scores)})')
+            print(f'Episode:{episode} | \nAggregate agent scores:(Sum:{sum(agent_scores)})\n MeanAvg agent scores:({np.mean(agent_scores)})')
         env.close()
 
 
diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index bcd543b..31217f5 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -84,12 +84,6 @@ def set_agent_action(self, action, agent, actionSpace):
         # print(f'agent vending price was {agent.vendingPrice}')
 
     def step_agent(self, agent):
-        # if agent.state > 0:
-        #     reward = 1
-        # elif agent.state == 0:
-        #     reward = 0
-        # else:
-        #     reward = -1
         reward = agent.reward
         info = {}
         return agent.state, reward, False, info
@@ -160,7 +154,7 @@ def set_consumer_purchases(self, agents_arr, consumer):
 
         while consumer.money > 0:            
             if agents_arr[lowestPriceAgnetIndex].vendingPrice < consumer.money:
-                agents_arr[lowestPriceAgnetIndex].reward += agents_arr[lowestPriceAgnetIndex].vendingPrice
+                agents_arr[lowestPriceAgnetIndex].reward += (agents_arr[lowestPriceAgnetIndex].vendingPrice - self.env_wholesale_price)
                 consumer.money -= agents_arr[lowestPriceAgnetIndex].vendingPrice
                 consumer.total_consumed += 1
             else:

From 1d885a361aa0fdfdf3f2d79b335fba0a51944183 Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Sun, 14 May 2023 13:08:01 +0100
Subject: [PATCH 17/40] improves performance a little

---
 Macodiac.ML/multiagent_main.py       | 13 +++++-----
 Macodiac.ML/multiagentenvironment.py | 39 +++++++++++++++++++++-------
 2 files changed, 37 insertions(+), 15 deletions(-)

diff --git a/Macodiac.ML/multiagent_main.py b/Macodiac.ML/multiagent_main.py
index 6e53742..8d113ac 100644
--- a/Macodiac.ML/multiagent_main.py
+++ b/Macodiac.ML/multiagent_main.py
@@ -24,10 +24,10 @@ def __init__(self):
         self.log_path =  os.path.join(filePath,'Logs')
         self.save_path =  os.path.join(filePath,'saved_models', 'model')
         self.save_path_intermittent =  os.path.join(filePath,'saved_models', 'intermittent_saved_models')
-        self.numTrainingIterations = 1_000
-        self.numEpisodes = 25
-        self.envTimesteps = 15
-        self.numAgents = 10
+        self.numTrainingIterations = 20_000
+        self.numEpisodes = 20_000
+        self.envTimesteps = 5
+        self.numAgents = 1
 
         self.env = MultiAgentMacodiacEnvironment(envTimesteps=self.envTimesteps, numAgents=self.numAgents)
         check_env(self.env)
@@ -39,7 +39,7 @@ def __init__(self):
         # NOTES: 
         #   if loadmodel is set to false, and trainmodel is set to true, 
         #   the currently saved model is overwritten
-        self.__MODE_LOADMODEL__ = False
+        self.__MODE_LOADMODEL__ = True
 
         # set to true if you want to train and then save the model
         self.__MODE_TRAINMODEL__ = True
@@ -62,6 +62,7 @@ def Run(self):
             model = self.load_model(self.env, model, self.save_path)
 
         if self.__MODE_TRAINMODEL__: 
+
             model = self.train_model(model,
                                      self.numTrainingIterations, self.save_path_intermittent)
             self.save_model(model, self.save_path)
@@ -149,7 +150,7 @@ def run_project(self, env:MultiAgentMacodiacEnvironment, numEpisodes: int, model
 
             runningAvg = np.mean(scores)
 
-            print(f'Episode:{episode} \t| Score:{score} \t\t| RunningAvg: {round(runningAvg, 2)}')
+            print(f'Episode:\t{episode} \t| Score:\t{score} \t\t| RunningAvg: {round(runningAvg, 2)}')
         env.close()
 
 
diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index 31217f5..440211f 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -152,15 +152,36 @@ def set_consumer_purchases(self, agents_arr, consumer):
             if agent.vendingPrice < agents_arr[lowestPriceAgnetIndex].vendingPrice:
                 lowestPriceAgnetIndex = i
 
-        while consumer.money > 0:            
-            if agents_arr[lowestPriceAgnetIndex].vendingPrice < consumer.money:
-                agents_arr[lowestPriceAgnetIndex].reward += (agents_arr[lowestPriceAgnetIndex].vendingPrice - self.env_wholesale_price)
-                consumer.money -= agents_arr[lowestPriceAgnetIndex].vendingPrice
-                consumer.total_consumed += 1
-            else:
-                # set the consumer's money to 0 if the vend price
-                # is less than the remaining money (stops infinite loop)
-                consumer.money = 0
+        lowestAgentVendPrice = agents_arr[lowestPriceAgnetIndex].vendingPrice
+
+        # instead of this while loop, just return the 
+        quantityPurchasable = np.floor(consumer.money / lowestAgentVendPrice)
+        consumerConsumed = quantityPurchasable
+        tmpAgentRewardPerUnitSold = (lowestAgentVendPrice - self.env_wholesale_price)
+        agentReward = tmpAgentRewardPerUnitSold * consumerConsumed
+
+        consumer.money = 0
+        consumer.total_consumed += consumerConsumed
+        agents_arr[lowestPriceAgnetIndex].reward += agentReward
+
+        # agents_arr[lowestPriceAgnetIndex].reward = agentReward
+        # consumer.money = 0
+        # consumer.total_consumed = 
+
+        # consumerConsumedInWhile = 0
+        # agentRewardInWhile = 0
+        # while consumer.money > 0:            
+        #     if lowestAgentVendPrice <= consumer.money:
+        #         myAgentReward = (lowestAgentVendPrice - self.env_wholesale_price)
+        #         agents_arr[lowestPriceAgnetIndex].reward += myAgentReward
+        #         agentRewardInWhile += myAgentReward
+        #         consumer.money -= lowestAgentVendPrice
+        #         consumer.total_consumed += 1
+        #         consumerConsumedInWhile +=1
+        #     else:
+        #         # set the consumer's money to 0 if the vend price
+        #         # is less than the remaining money (stops infinite loop)
+        #         consumer.money = 0
 
 
 

From 8676b989434c925c690f4a6d743c38b667e69f9c Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Sun, 14 May 2023 13:27:34 +0100
Subject: [PATCH 18/40] reduces the problem space to ints instead of floats

---
 Macodiac.ML/multiagent_main.py       |  5 ++--
 Macodiac.ML/multiagentenvironment.py | 40 +++++++++-------------------
 2 files changed, 16 insertions(+), 29 deletions(-)

diff --git a/Macodiac.ML/multiagent_main.py b/Macodiac.ML/multiagent_main.py
index 8d113ac..a8488f8 100644
--- a/Macodiac.ML/multiagent_main.py
+++ b/Macodiac.ML/multiagent_main.py
@@ -25,7 +25,7 @@ def __init__(self):
         self.save_path =  os.path.join(filePath,'saved_models', 'model')
         self.save_path_intermittent =  os.path.join(filePath,'saved_models', 'intermittent_saved_models')
         self.numTrainingIterations = 20_000
-        self.numEpisodes = 20_000
+        self.numEpisodes = 200
         self.envTimesteps = 5
         self.numAgents = 1
 
@@ -39,7 +39,7 @@ def __init__(self):
         # NOTES: 
         #   if loadmodel is set to false, and trainmodel is set to true, 
         #   the currently saved model is overwritten
-        self.__MODE_LOADMODEL__ = True
+        self.__MODE_LOADMODEL__ = False
 
         # set to true if you want to train and then save the model
         self.__MODE_TRAINMODEL__ = True
@@ -97,6 +97,7 @@ def run_multiagent_project_with_rand_test(self, env:MultiAgentMacodiacEnvironmen
 
                 print(f'rewards for agents:\t{reward}')
                 print(f'obs for agents:\t{obs_arr}')
+                print(f'px for agents:\t{info_arr}')
 
                 if done:
                     print(f'is done')
diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index 440211f..5b3b475 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -11,6 +11,7 @@ def __init__(self):
         self.state = []
         self.vendingPrice = 0
         self.reward = 0
+        self.quantitySold = 0
 
 class ConsumerObject:
     def __init__(self):
@@ -29,8 +30,8 @@ class MultiAgentMacodiacEnvironment(Env):
     environment_starter_timesteps = 150
     env_wholesale_price = 50        # the price agents pay to purchase goods
     env_agent_marginal_cost = 0     # the marginal cost of vending
-    num_consumers = 2
-    consumer_total_money_per_turn = 25000
+    num_consumers = 20
+    consumer_total_money_per_turn = 2500
     consumers_arr = []
 
     def __init__(self, envTimesteps:int, numAgents: int):
@@ -56,7 +57,7 @@ def __init__(self, envTimesteps:int, numAgents: int):
 
         # the observation space is a nAgents by nActions array of float32 numbers between -99-99
         # also contains the static value for marginal cost and wholesale price
-        self.observation_space = spaces.Box(low=-100,high=100, shape=(numAgents, 4), dtype=np.float32)
+        self.observation_space = spaces.Box(low=-100,high=100, shape=(numAgents, 3), dtype=np.int32)
 
         print(f'obs_space.sample: {self.observation_space.sample()}')
 
@@ -76,7 +77,7 @@ def __init__(self, envTimesteps:int, numAgents: int):
     def set_agent_action(self, action, agent, actionSpace):
         # agent.state is the percentage price diff from the 
         # wholesale price
-        agent.state = action - 100
+        agent.state = action  - 100
         agentBaseVendingPriceAdjust = self.env_wholesale_price * (agent.state / 100)
         baseAgentVendingPrice = self.env_wholesale_price + agentBaseVendingPriceAdjust
         #agentMarginalCostAddedVendingPrice = agentBaseVendingPriceAdjust + self.env_agent_marginal_cost
@@ -135,7 +136,7 @@ def step(self, action_arr):
             partialObservationResult[1] = self._get_final_vend_price(agent) #The agent's result is present in the 0th element of its result
             tmpObsArray.append(partialObservationResult)
 
-        concatObsArray = np.array(tmpObsArray).astype(np.float32) 
+        concatObsArray = np.array(tmpObsArray).astype(np.int32) 
         return concatObsArray, sum(reward_arr), isTerminal,  info_arr
 
 
@@ -163,25 +164,9 @@ def set_consumer_purchases(self, agents_arr, consumer):
         consumer.money = 0
         consumer.total_consumed += consumerConsumed
         agents_arr[lowestPriceAgnetIndex].reward += agentReward
+        agents_arr[lowestPriceAgnetIndex].quantitySold += consumerConsumed
 
-        # agents_arr[lowestPriceAgnetIndex].reward = agentReward
-        # consumer.money = 0
-        # consumer.total_consumed = 
 
-        # consumerConsumedInWhile = 0
-        # agentRewardInWhile = 0
-        # while consumer.money > 0:            
-        #     if lowestAgentVendPrice <= consumer.money:
-        #         myAgentReward = (lowestAgentVendPrice - self.env_wholesale_price)
-        #         agents_arr[lowestPriceAgnetIndex].reward += myAgentReward
-        #         agentRewardInWhile += myAgentReward
-        #         consumer.money -= lowestAgentVendPrice
-        #         consumer.total_consumed += 1
-        #         consumerConsumedInWhile +=1
-        #     else:
-        #         # set the consumer's money to 0 if the vend price
-        #         # is less than the remaining money (stops infinite loop)
-        #         consumer.money = 0
 
 
 
@@ -213,7 +198,7 @@ def _get_info(self, agent):
         """
             accepts an Agent, and returns its info object
         """
-        return agent.info
+        return {'price:': agent.vendingPrice, 'sold': agent.quantitySold, 'reward': agent.reward}
 
 
     def render(self) -> None:
@@ -222,7 +207,7 @@ def render(self) -> None:
         """
         pass
 
-    def reset(self) -> float:
+    def reset(self): #-> float:
         """
         Sets the application to its initial conditions
 
@@ -232,12 +217,13 @@ def reset(self) -> float:
         for i in range(len(self.policy_agents)):
             self.policy_agents[i].state = np.array(
                                     self.get_agent_default_observation_array(), 
-                                     dtype=np.float32)
+                                     dtype=np.int32)
             obs_arr.append(self.policy_agents[i].state)
             self.policy_agents[i].reward = 0
             self.policy_agents[i].info = {}
             self.policy_agents[i].done = False
             self.policy_agents[i].vendingPrice = 0
+            self.policy_agents[i].quantitySold = 0
         
         consumerMoneyEach = self.consumer_total_money_per_turn / self.num_consumers
         for i in range(len(self.consumers_arr)):
@@ -245,7 +231,7 @@ def reset(self) -> float:
 
         self.environment_timesteps = self.environment_starter_timesteps
         
-        return np.array(obs_arr).astype(np.float32)
+        return np.array(obs_arr).astype(np.int32)
 
     def get_consumer_demand_schedule(self):
         schedule = [
@@ -283,4 +269,4 @@ def get_consumer_quantity_demanded_at_price(self, price):
 
 
     def get_agent_default_observation_array(self):
-        return [0.0, 0.0, self.env_wholesale_price, self.env_agent_marginal_cost]
\ No newline at end of file
+        return [0.0, 0.0, self.env_wholesale_price]
\ No newline at end of file

From 0cb69edfa41619373b0e450d96170e501bc32971 Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Sun, 14 May 2023 13:33:35 +0100
Subject: [PATCH 19/40] config changes

---
 Macodiac.ML/multiagent_main.py       | 6 +++---
 Macodiac.ML/multiagentenvironment.py | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Macodiac.ML/multiagent_main.py b/Macodiac.ML/multiagent_main.py
index a8488f8..529f7f6 100644
--- a/Macodiac.ML/multiagent_main.py
+++ b/Macodiac.ML/multiagent_main.py
@@ -24,9 +24,9 @@ def __init__(self):
         self.log_path =  os.path.join(filePath,'Logs')
         self.save_path =  os.path.join(filePath,'saved_models', 'model')
         self.save_path_intermittent =  os.path.join(filePath,'saved_models', 'intermittent_saved_models')
-        self.numTrainingIterations = 20_000
+        self.numTrainingIterations = 1_000_000
         self.numEpisodes = 200
-        self.envTimesteps = 5
+        self.envTimesteps = 10
         self.numAgents = 1
 
         self.env = MultiAgentMacodiacEnvironment(envTimesteps=self.envTimesteps, numAgents=self.numAgents)
@@ -39,7 +39,7 @@ def __init__(self):
         # NOTES: 
         #   if loadmodel is set to false, and trainmodel is set to true, 
         #   the currently saved model is overwritten
-        self.__MODE_LOADMODEL__ = False
+        self.__MODE_LOADMODEL__ = True
 
         # set to true if you want to train and then save the model
         self.__MODE_TRAINMODEL__ = True
diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index 5b3b475..f9a0741 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -31,7 +31,7 @@ class MultiAgentMacodiacEnvironment(Env):
     env_wholesale_price = 50        # the price agents pay to purchase goods
     env_agent_marginal_cost = 0     # the marginal cost of vending
     num_consumers = 20
-    consumer_total_money_per_turn = 2500
+    consumer_total_money_per_turn = 25000
     consumers_arr = []
 
     def __init__(self, envTimesteps:int, numAgents: int):

From 50bdadeb4148d97a20837c8f594cd530ca88753f Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Sun, 14 May 2023 15:28:06 +0100
Subject: [PATCH 20/40] oligopoly config

---
 Macodiac.ML/multiagent_main.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/Macodiac.ML/multiagent_main.py b/Macodiac.ML/multiagent_main.py
index 529f7f6..681dca0 100644
--- a/Macodiac.ML/multiagent_main.py
+++ b/Macodiac.ML/multiagent_main.py
@@ -24,10 +24,10 @@ def __init__(self):
         self.log_path =  os.path.join(filePath,'Logs')
         self.save_path =  os.path.join(filePath,'saved_models', 'model')
         self.save_path_intermittent =  os.path.join(filePath,'saved_models', 'intermittent_saved_models')
-        self.numTrainingIterations = 1_000_000
+        self.numTrainingIterations = 5_000_000
         self.numEpisodes = 200
-        self.envTimesteps = 10
-        self.numAgents = 1
+        self.envTimesteps = 2
+        self.numAgents = 5
 
         self.env = MultiAgentMacodiacEnvironment(envTimesteps=self.envTimesteps, numAgents=self.numAgents)
         check_env(self.env)
@@ -39,7 +39,7 @@ def __init__(self):
         # NOTES: 
         #   if loadmodel is set to false, and trainmodel is set to true, 
         #   the currently saved model is overwritten
-        self.__MODE_LOADMODEL__ = True
+        self.__MODE_LOADMODEL__ = False
 
         # set to true if you want to train and then save the model
         self.__MODE_TRAINMODEL__ = True
@@ -169,7 +169,7 @@ def train_model(self, model, numTimesteps: int, savePath:str):
         @param numTimesteps: the number of training iterations
         """
         
-        saveEveryNSteps = 1_000_000
+        saveEveryNSteps = 100_000
         
         if numTimesteps < saveEveryNSteps:
             model.learn(total_timesteps=numTimesteps)

From a597fcd72dd166dd369fbba274fb03a49c0d09a3 Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Mon, 15 May 2023 08:32:00 +0100
Subject: [PATCH 21/40] cleans up some code

---
 Macodiac.ML/multiagent_main.py       | 12 +++---
 Macodiac.ML/multiagentenvironment.py | 57 ++++++++--------------------
 2 files changed, 22 insertions(+), 47 deletions(-)

diff --git a/Macodiac.ML/multiagent_main.py b/Macodiac.ML/multiagent_main.py
index 681dca0..58f2569 100644
--- a/Macodiac.ML/multiagent_main.py
+++ b/Macodiac.ML/multiagent_main.py
@@ -24,10 +24,10 @@ def __init__(self):
         self.log_path =  os.path.join(filePath,'Logs')
         self.save_path =  os.path.join(filePath,'saved_models', 'model')
         self.save_path_intermittent =  os.path.join(filePath,'saved_models', 'intermittent_saved_models')
-        self.numTrainingIterations = 5_000_000
-        self.numEpisodes = 200
-        self.envTimesteps = 2
-        self.numAgents = 5
+        self.numTrainingIterations = 10_000
+        self.numEpisodes = 20
+        self.envTimesteps = 20
+        self.numAgents = 2
 
         self.env = MultiAgentMacodiacEnvironment(envTimesteps=self.envTimesteps, numAgents=self.numAgents)
         check_env(self.env)
@@ -156,7 +156,7 @@ def run_project(self, env:MultiAgentMacodiacEnvironment, numEpisodes: int, model
 
 
     def create_model(self, env: MultiAgentMacodiacEnvironment, log_path: str):
-        model = PPO('MlpPolicy', env, verbose=1, tensorboard_log=log_path)
+        model = PPO('MlpPolicy', env, verbose=1, tensorboard_log=log_path, device="cpu")
         return model
 
 
@@ -176,7 +176,7 @@ def train_model(self, model, numTimesteps: int, savePath:str):
 
         else:
             rangeUpper = int(numTimesteps / saveEveryNSteps)
-            for i in range(1,rangeUpper+1):
+            for i in range(9,rangeUpper+1):
                 model.learn(total_timesteps=saveEveryNSteps)
                 model.save(os.path.join(savePath, f'interim-{i}'))
 
diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index f9a0741..ef9db11 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -39,24 +39,30 @@ def __init__(self, envTimesteps:int, numAgents: int):
         Initialises the class
         """
         self.environment_starter_timesteps = envTimesteps
-        
         self.policy_agents = []
+        self.observation_space = []
+        # self.agents = [numAgents]
+        
+        # creates an array full of 200's shaped [200,200,200], with numAgents number of element
+
+
+        self.action_space = spaces.MultiDiscrete(np.full(numAgents, 20) )
+
         for i in range(numAgents):
             self.policy_agents.append(AgentObject())
         
         for i in range(self.num_consumers):
             self.consumers_arr.append(ConsumerObject())
 
-        self.observation_space = []
-        self.agents = [numAgents]
 
-        arr = np.full(numAgents, 200) # creates an array full of 200's shaped [200,200,200], with numAgents number of element
 
 
-        self.action_space = spaces.MultiDiscrete(arr)
 
         # the observation space is a nAgents by nActions array of float32 numbers between -99-99
-        # also contains the static value for marginal cost and wholesale price
+        # also contains the wholesale price
+        # Observations space:
+        # 0: 
+
         self.observation_space = spaces.Box(low=-100,high=100, shape=(numAgents, 3), dtype=np.int32)
 
         print(f'obs_space.sample: {self.observation_space.sample()}')
@@ -77,7 +83,7 @@ def __init__(self, envTimesteps:int, numAgents: int):
     def set_agent_action(self, action, agent, actionSpace):
         # agent.state is the percentage price diff from the 
         # wholesale price
-        agent.state = action  - 100
+        agent.state = (action * 10) - 100
         agentBaseVendingPriceAdjust = self.env_wholesale_price * (agent.state / 100)
         baseAgentVendingPrice = self.env_wholesale_price + agentBaseVendingPriceAdjust
         #agentMarginalCostAddedVendingPrice = agentBaseVendingPriceAdjust + self.env_agent_marginal_cost
@@ -233,40 +239,9 @@ def reset(self): #-> float:
         
         return np.array(obs_arr).astype(np.int32)
 
-    def get_consumer_demand_schedule(self):
-        schedule = [
-            [0, 1000],
-            [10, 900],
-            [20, 800],
-            [30, 700],
-            [40, 600],
-            [50, 500],
-            [60, 400],
-            [70, 300],
-            [80, 200],
-            [90, 100],
-            [100, 0]
-        ]
-        return schedule
-    
-    def get_consumer_quantity_demanded_at_price(self, price):
-        dmnd_schedule = self.get_consumer_demand_schedule()
-        roundedPrice = round(price, -1)
-        for i in dmnd_schedule:
-            if i[0] == roundedPrice:
-                return i[1]
-
-        return 0
-      
-        # point_a = dmnd_schedule()[0]
-        # point_b = dmnd_schedule()[len(dmnd_schedule)]
-
-
-        # change_in_x = point_a[0] - point_b[0]
-        # change_in_y = point_a[1] - point_b[1]
-        # gradient = change_in_y / change_in_x
-
-
 
     def get_agent_default_observation_array(self):
+        """
+        Gets a default observation for this space
+        """
         return [0.0, 0.0, self.env_wholesale_price]
\ No newline at end of file

From 269c413ef9ffc8d6d8c0fdb2abc396fa583f8fd6 Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Mon, 15 May 2023 09:33:17 +0100
Subject: [PATCH 22/40] massively reduces the problem-space

---
 Macodiac.ML/multiagent_main.py       |  6 ++--
 Macodiac.ML/multiagentenvironment.py | 45 +++++++++++++++-------------
 2 files changed, 27 insertions(+), 24 deletions(-)

diff --git a/Macodiac.ML/multiagent_main.py b/Macodiac.ML/multiagent_main.py
index 58f2569..5fdbf08 100644
--- a/Macodiac.ML/multiagent_main.py
+++ b/Macodiac.ML/multiagent_main.py
@@ -24,10 +24,10 @@ def __init__(self):
         self.log_path =  os.path.join(filePath,'Logs')
         self.save_path =  os.path.join(filePath,'saved_models', 'model')
         self.save_path_intermittent =  os.path.join(filePath,'saved_models', 'intermittent_saved_models')
-        self.numTrainingIterations = 10_000
+        self.numTrainingIterations = 1_000_000
         self.numEpisodes = 20
         self.envTimesteps = 20
-        self.numAgents = 2
+        self.numAgents = 10
 
         self.env = MultiAgentMacodiacEnvironment(envTimesteps=self.envTimesteps, numAgents=self.numAgents)
         check_env(self.env)
@@ -176,7 +176,7 @@ def train_model(self, model, numTimesteps: int, savePath:str):
 
         else:
             rangeUpper = int(numTimesteps / saveEveryNSteps)
-            for i in range(9,rangeUpper+1):
+            for i in range(1,rangeUpper+1):
                 model.learn(total_timesteps=saveEveryNSteps)
                 model.save(os.path.join(savePath, f'interim-{i}'))
 
diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index ef9db11..5ada05e 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -31,7 +31,7 @@ class MultiAgentMacodiacEnvironment(Env):
     env_wholesale_price = 50        # the price agents pay to purchase goods
     env_agent_marginal_cost = 0     # the marginal cost of vending
     num_consumers = 20
-    consumer_total_money_per_turn = 25000
+    consumer_total_money_per_turn = 2500
     consumers_arr = []
 
     def __init__(self, envTimesteps:int, numAgents: int):
@@ -41,12 +41,6 @@ def __init__(self, envTimesteps:int, numAgents: int):
         self.environment_starter_timesteps = envTimesteps
         self.policy_agents = []
         self.observation_space = []
-        # self.agents = [numAgents]
-        
-        # creates an array full of 200's shaped [200,200,200], with numAgents number of element
-
-
-        self.action_space = spaces.MultiDiscrete(np.full(numAgents, 20) )
 
         for i in range(numAgents):
             self.policy_agents.append(AgentObject())
@@ -54,16 +48,19 @@ def __init__(self, envTimesteps:int, numAgents: int):
         for i in range(self.num_consumers):
             self.consumers_arr.append(ConsumerObject())
 
-
-
+        
+        # creates an array full of 10's shaped [20,20,20], of length numAgents
+        self.action_space = spaces.MultiDiscrete(np.full(numAgents, 10) )
 
 
         # the observation space is a nAgents by nActions array of float32 numbers between -99-99
         # also contains the wholesale price
         # Observations space:
-        # 0: 
-
-        self.observation_space = spaces.Box(low=-100,high=100, shape=(numAgents, 3), dtype=np.int32)
+        # 0: agent's state, after the action has been applied
+        # 1: agent's vending price in this round
+        # 2: agent's count of sold items
+        # 3: the wholesale price in this round
+        self.observation_space = spaces.Box(low=0,high=200, shape=(numAgents, 3), dtype=np.int32)
 
         print(f'obs_space.sample: {self.observation_space.sample()}')
 
@@ -81,13 +78,14 @@ def __init__(self, envTimesteps:int, numAgents: int):
 
 
     def set_agent_action(self, action, agent, actionSpace):
-        # agent.state is the percentage price diff from the 
-        # wholesale price
-        agent.state = (action * 10) - 100
-        agentBaseVendingPriceAdjust = self.env_wholesale_price * (agent.state / 100)
-        baseAgentVendingPrice = self.env_wholesale_price + agentBaseVendingPriceAdjust
-        #agentMarginalCostAddedVendingPrice = agentBaseVendingPriceAdjust + self.env_agent_marginal_cost
-        agent.vendingPrice = max(baseAgentVendingPrice, 1)
+        # agent.state is the percentage price diff from the wholesale price
+        agent.state = action * 10
+        agent.vendingPrice = self.env_wholesale_price + agent.state
+
+        # agentBaseVendingPriceAdjust = self.env_wholesale_price * (agent.state / 100)
+        # baseAgentVendingPrice = self.env_wholesale_price + agentBaseVendingPriceAdjust
+        # #agentMarginalCostAddedVendingPrice = agentBaseVendingPriceAdjust + self.env_agent_marginal_cost
+        # agent.vendingPrice = max(baseAgentVendingPrice, 1)
         # print(f'agent vending price was {agent.vendingPrice}')
 
     def step_agent(self, agent):
@@ -140,6 +138,7 @@ def step(self, action_arr):
             partialObservationResult = self.get_agent_default_observation_array()
             partialObservationResult[0] = self._get_obs(agent) #The agent's result is present in the 0th element of its result
             partialObservationResult[1] = self._get_final_vend_price(agent) #The agent's result is present in the 0th element of its result
+            partialObservationResult[2] = self._get_quantity_sold(agent) #The agent's result is present in the 0th element of its result
             tmpObsArray.append(partialObservationResult)
 
         concatObsArray = np.array(tmpObsArray).astype(np.int32) 
@@ -174,7 +173,11 @@ def set_consumer_purchases(self, agents_arr, consumer):
 
 
 
-
+    def _get_quantity_sold(self, agent):
+        """
+            accepts an agent, and returns the number of items it sold
+        """
+        return agent.quantitySold
 
     def _get_final_vend_price(self, agent):
         """
@@ -244,4 +247,4 @@ def get_agent_default_observation_array(self):
         """
         Gets a default observation for this space
         """
-        return [0.0, 0.0, self.env_wholesale_price]
\ No newline at end of file
+        return [0.0, 0.0, 0]# , self.env_wholesale_price]
\ No newline at end of file

From 52f68f263fd8b47d76fc0a1a0fe4eeff27f52f4d Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Mon, 15 May 2023 11:43:25 +0100
Subject: [PATCH 23/40] starts logging custom vars to tensorboards

---
 Macodiac.ML/multiagent_main.py       | 10 +++++----
 Macodiac.ML/multiagentenvironment.py | 32 ++++++++++++++++++++++++++++
 2 files changed, 38 insertions(+), 4 deletions(-)

diff --git a/Macodiac.ML/multiagent_main.py b/Macodiac.ML/multiagent_main.py
index 5fdbf08..0407dd9 100644
--- a/Macodiac.ML/multiagent_main.py
+++ b/Macodiac.ML/multiagent_main.py
@@ -1,7 +1,7 @@
 import os
 import gymnasium as gym
 from gymnasium import Env
-from environment import MacodiacEnvironment
+from multiagentenvironment import TensorboardPriceCallback
 from multiagentenvironment import MultiAgentMacodiacEnvironment
 from stable_baselines3.common.evaluation import evaluate_policy
 from stable_baselines3.common.env_checker import check_env
@@ -27,7 +27,7 @@ def __init__(self):
         self.numTrainingIterations = 1_000_000
         self.numEpisodes = 20
         self.envTimesteps = 20
-        self.numAgents = 10
+        self.numAgents = 2
 
         self.env = MultiAgentMacodiacEnvironment(envTimesteps=self.envTimesteps, numAgents=self.numAgents)
         check_env(self.env)
@@ -172,12 +172,14 @@ def train_model(self, model, numTimesteps: int, savePath:str):
         saveEveryNSteps = 100_000
         
         if numTimesteps < saveEveryNSteps:
-            model.learn(total_timesteps=numTimesteps)
+            model.learn(total_timesteps=numTimesteps,
+                        callback=TensorboardPriceCallback())
 
         else:
             rangeUpper = int(numTimesteps / saveEveryNSteps)
             for i in range(1,rangeUpper+1):
-                model.learn(total_timesteps=saveEveryNSteps)
+                model.learn(total_timesteps=saveEveryNSteps,
+                            callback=TensorboardPriceCallback())
                 model.save(os.path.join(savePath, f'interim-{i}'))
 
         return model
diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index 5ada05e..6e33218 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -5,6 +5,7 @@
 from gym import spaces
 import numpy as np
 import random
+from stable_baselines3.common.callbacks import BaseCallback
 
 class AgentObject:
     def __init__(self):
@@ -20,6 +21,36 @@ def __init__(self):
         self.money = 0
         self.total_consumed = 0
 
+
+class TensorboardPriceCallback(BaseCallback):
+    """ 
+    custom logger to record the price charged by agents
+    """
+    def __init__(self, verbose=0):
+        super().__init__(verbose)
+
+    def _on_step(self) -> bool:
+        agent_arr = self.training_env.get_attr('policy_agents')[0]
+        pxList = []
+        acceptedVendedPx = 0
+        quantitySold = 0
+        vendorsMadeSale = 0
+        for agent in agent_arr:
+            if agent.quantitySold > 0:
+                acceptedVendedPx = agent.vendingPrice
+                quantitySold += agent.quantitySold
+                vendorsMadeSale += 1
+            pxList.append(agent.vendingPrice)
+
+        meanPxOffered = np.mean(pxList)
+
+        self.logger.record('vending/avgerage_offered_px_value', meanPxOffered)
+        self.logger.record('vending/actual_accepted_px_value', acceptedVendedPx)
+        self.logger.record('vending/quantity_sold_count', quantitySold)
+        self.logger.record('vending/vendors_made_sale_count', vendorsMadeSale)
+
+        return True
+
 class MultiAgentMacodiacEnvironment(Env):
     """
     Builds a profit maximising agent environment, supporting
@@ -34,6 +65,7 @@ class MultiAgentMacodiacEnvironment(Env):
     consumer_total_money_per_turn = 2500
     consumers_arr = []
 
+
     def __init__(self, envTimesteps:int, numAgents: int):
         """
         Initialises the class

From dc57b6e50f1279b944ac13db76f0ecf740c6cef0 Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Mon, 15 May 2023 13:05:32 +0100
Subject: [PATCH 24/40] WIP adding better logging

---
 Macodiac.ML/multiagentenvironment.py | 50 +++++++++++++++++++++++++---
 1 file changed, 45 insertions(+), 5 deletions(-)

diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index 6e33218..be16aa7 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -26,22 +26,57 @@ class TensorboardPriceCallback(BaseCallback):
     """ 
     custom logger to record the price charged by agents
     """
+    # iterator = 0
+    # offeredPxList = []
+    # acceptedPxList = []
+    # vendorsMadeSaleList = []
+    # quantitySoldList = []
+
+
+    runningAvgMeanPxOffered = 0
+    runningAvgAcceptedVendedPx = 0
+
     def __init__(self, verbose=0):
+        self.reset()
         super().__init__(verbose)
 
+    def reset(self):
+        self.iterator = 0
+        self.offeredPxList = []
+        self.acceptedPxList = []
+        self.vendorsMadeSaleList = []
+        self.quantitySoldList = []
+
+    def _on_rollout_end(self) -> None:
+        self.reset()
+        return super()._on_rollout_end()
+    
     def _on_step(self) -> bool:
+        self.iterator +=1
         agent_arr = self.training_env.get_attr('policy_agents')[0]
+        info_arr = self.locals['infos'][0]['n']
+
+        # print(f'iterator:{self.iterator}. len:{len(info_arr)}')
         pxList = []
+
+        for obj in info_arr:
+            self.offeredPxList.append(obj['price'])
+
         acceptedVendedPx = 0
         quantitySold = 0
         vendorsMadeSale = 0
+        agentsMadeSale = False
         for agent in agent_arr:
             if agent.quantitySold > 0:
                 acceptedVendedPx = agent.vendingPrice
                 quantitySold += agent.quantitySold
                 vendorsMadeSale += 1
+                agentsMadeSale = True
             pxList.append(agent.vendingPrice)
 
+        if agentsMadeSale == False:
+            print(f'error, no agents made a sale')
+
         meanPxOffered = np.mean(pxList)
 
         self.logger.record('vending/avgerage_offered_px_value', meanPxOffered)
@@ -59,10 +94,10 @@ class MultiAgentMacodiacEnvironment(Env):
     state = 0
     environment_timesteps = 0
     environment_starter_timesteps = 150
-    env_wholesale_price = 50        # the price agents pay to purchase goods
+    env_wholesale_price = 5        # the price agents pay to purchase goods
     env_agent_marginal_cost = 0     # the marginal cost of vending
-    num_consumers = 20
-    consumer_total_money_per_turn = 2500
+    num_consumers = 25
+    consumer_total_money_per_turn = 250
     consumers_arr = []
 
 
@@ -111,9 +146,14 @@ def __init__(self, envTimesteps:int, numAgents: int):
 
     def set_agent_action(self, action, agent, actionSpace):
         # agent.state is the percentage price diff from the wholesale price
-        agent.state = action * 10
+        agent.state = action
         agent.vendingPrice = self.env_wholesale_price + agent.state
 
+        if agent.vendingPrice == 0:
+            print(f'error')
+            agent.vendingPrice = max(1, agent.vendingPrice)
+
+
         # agentBaseVendingPriceAdjust = self.env_wholesale_price * (agent.state / 100)
         # baseAgentVendingPrice = self.env_wholesale_price + agentBaseVendingPriceAdjust
         # #agentMarginalCostAddedVendingPrice = agentBaseVendingPriceAdjust + self.env_agent_marginal_cost
@@ -239,7 +279,7 @@ def _get_info(self, agent):
         """
             accepts an Agent, and returns its info object
         """
-        return {'price:': agent.vendingPrice, 'sold': agent.quantitySold, 'reward': agent.reward}
+        return {"price": agent.vendingPrice, "sold": agent.quantitySold, "reward": agent.reward}
 
 
     def render(self) -> None:

From 5d3790c0304fbb9fd1ff96fa7d4aa1bc9b4d7b7d Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Mon, 15 May 2023 13:42:43 +0100
Subject: [PATCH 25/40] makes the logging clearer, and logs for each agent

---
 Macodiac.ML/multiagentenvironment.py | 74 +++++++++++++++++-----------
 1 file changed, 45 insertions(+), 29 deletions(-)

diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index be16aa7..cbc7205 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -42,47 +42,54 @@ def __init__(self, verbose=0):
 
     def reset(self):
         self.iterator = 0
-        self.offeredPxList = []
-        self.acceptedPxList = []
-        self.vendorsMadeSaleList = []
-        self.quantitySoldList = []
 
     def _on_rollout_end(self) -> None:
         self.reset()
         return super()._on_rollout_end()
     
     def _on_step(self) -> bool:
-        self.iterator +=1
-        agent_arr = self.training_env.get_attr('policy_agents')[0]
+        # self.iterator +=1
+        # agent_arr = self.training_env.get_attr('policy_agents')[0]
+        
+        ### generate a dict like:
+        # [{'agent_num': 0, 'price': 10, 'sold': 0.0, 'reward': 0.0}, 
+        # {'agent_num': 1, 'price': 10, 'sold': 50.0, 'reward': 0.0}]
         info_arr = self.locals['infos'][0]['n']
 
-        # print(f'iterator:{self.iterator}. len:{len(info_arr)}')
-        pxList = []
-
-        for obj in info_arr:
-            self.offeredPxList.append(obj['price'])
 
-        acceptedVendedPx = 0
-        quantitySold = 0
+        pxList = []
         vendorsMadeSale = 0
-        agentsMadeSale = False
-        for agent in agent_arr:
-            if agent.quantitySold > 0:
-                acceptedVendedPx = agent.vendingPrice
-                quantitySold += agent.quantitySold
-                vendorsMadeSale += 1
-                agentsMadeSale = True
-            pxList.append(agent.vendingPrice)
-
-        if agentsMadeSale == False:
-            print(f'error, no agents made a sale')
+        quantitySold = 0
+        countNoSale = 0
+        countWiSale = 0
+        acceptedVendedPx = 0
+        meanPxOffered = 0
 
+        for agentInfo in info_arr:
+            agent_sales = agentInfo['sold']
+            agent_vend_px = agentInfo['price']
+            pxList.append(agent_vend_px)
+            
+            if agent_sales > 0:
+                vendorsMadeSale += 1
+                quantitySold += agent_sales
+                countWiSale += 1
+                acceptedVendedPx = agent_vend_px
+            else:
+                countNoSale += 1
+            
+            self.logger.record(f'vending_agent_{agentInfo["agent_num"]}/offered_px',   agent_vend_px)
+            self.logger.record(f'vending_agent_{agentInfo["agent_num"]}/sales_complete',  agent_sales)
+            self.logger.record(f'vending_agent_{agentInfo["agent_num"]}/individual_reward',  agentInfo['reward'])
+            
         meanPxOffered = np.mean(pxList)
 
         self.logger.record('vending/avgerage_offered_px_value', meanPxOffered)
         self.logger.record('vending/actual_accepted_px_value', acceptedVendedPx)
         self.logger.record('vending/quantity_sold_count', quantitySold)
         self.logger.record('vending/vendors_made_sale_count', vendorsMadeSale)
+        self.logger.record('vending/count_no_sale', countNoSale)
+        self.logger.record('vending/count_wi_sale', countWiSale)
 
         return True
 
@@ -188,15 +195,24 @@ def step(self, action_arr):
 
         for i, consumer in enumerate(self.consumers_arr):
             self.set_consumer_purchases(self.policy_agents, consumer)
+        
+        anyConsumed = False
+        for consumer in self.consumers_arr:
+            if consumer.total_consumed > 0:
+                anyConsumed = True
+                break
+        if anyConsumed == False:
+            print(f'error, no consumption')
 
         for i, agent in enumerate(self.policy_agents):
             agent.state, agent.reward, agent.done, agent.info = self.step_agent(agent)
-
-        for agent in self.policy_agents:
             obs_arr.append(self._get_obs(agent))
             reward_arr.append(self._get_reward(agent))
             done_arr.append(self._get_done(agent))
-            info_arr['n'].append(self._get_info(agent))
+            info_arr['n'].append(self._get_info(agent, i))
+
+        # for agent in self.policy_agents:
+           
 
         if self.environment_timesteps <= 0:
             isTerminal = True
@@ -275,11 +291,11 @@ def _get_done(self, agent):
         """
         return agent.done
 
-    def _get_info(self, agent):
+    def _get_info(self, agent , i):
         """
             accepts an Agent, and returns its info object
         """
-        return {"price": agent.vendingPrice, "sold": agent.quantitySold, "reward": agent.reward}
+        return {"agent_num": i, "price": agent.vendingPrice, "sold": agent.quantitySold, "reward": agent.reward}
 
 
     def render(self) -> None:

From a77484560aa9473c7787ce709934c27ac5d2b59f Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Mon, 15 May 2023 14:24:33 +0100
Subject: [PATCH 26/40] logging debugging

---
 Macodiac.ML/multiagent_main.py       |  6 +++---
 Macodiac.ML/multiagentenvironment.py | 24 ++++++++++++++++--------
 2 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/Macodiac.ML/multiagent_main.py b/Macodiac.ML/multiagent_main.py
index 0407dd9..ec97a21 100644
--- a/Macodiac.ML/multiagent_main.py
+++ b/Macodiac.ML/multiagent_main.py
@@ -24,10 +24,10 @@ def __init__(self):
         self.log_path =  os.path.join(filePath,'Logs')
         self.save_path =  os.path.join(filePath,'saved_models', 'model')
         self.save_path_intermittent =  os.path.join(filePath,'saved_models', 'intermittent_saved_models')
-        self.numTrainingIterations = 1_000_000
+        self.numTrainingIterations = 100_000
         self.numEpisodes = 20
         self.envTimesteps = 20
-        self.numAgents = 2
+        self.numAgents = 3
 
         self.env = MultiAgentMacodiacEnvironment(envTimesteps=self.envTimesteps, numAgents=self.numAgents)
         check_env(self.env)
@@ -169,7 +169,7 @@ def train_model(self, model, numTimesteps: int, savePath:str):
         @param numTimesteps: the number of training iterations
         """
         
-        saveEveryNSteps = 100_000
+        saveEveryNSteps = 100_001
         
         if numTimesteps < saveEveryNSteps:
             model.learn(total_timesteps=numTimesteps,
diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index cbc7205..698b727 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -65,9 +65,11 @@ def _on_step(self) -> bool:
         acceptedVendedPx = 0
         meanPxOffered = 0
 
-        for agentInfo in info_arr:
-            agent_sales = agentInfo['sold']
-            agent_vend_px = agentInfo['price']
+        for i, agentInfo in enumerate(info_arr):
+            agent_sales = info_arr[i]['sold']
+            agent_vend_px = info_arr[i]['price']
+            agent_reward = info_arr[i]['reward']
+
             pxList.append(agent_vend_px)
             
             if agent_sales > 0:
@@ -80,7 +82,7 @@ def _on_step(self) -> bool:
             
             self.logger.record(f'vending_agent_{agentInfo["agent_num"]}/offered_px',   agent_vend_px)
             self.logger.record(f'vending_agent_{agentInfo["agent_num"]}/sales_complete',  agent_sales)
-            self.logger.record(f'vending_agent_{agentInfo["agent_num"]}/individual_reward',  agentInfo['reward'])
+            self.logger.record(f'vending_agent_{agentInfo["agent_num"]}/individual_reward',  agent_reward)
             
         meanPxOffered = np.mean(pxList)
 
@@ -104,7 +106,7 @@ class MultiAgentMacodiacEnvironment(Env):
     env_wholesale_price = 5        # the price agents pay to purchase goods
     env_agent_marginal_cost = 0     # the marginal cost of vending
     num_consumers = 25
-    consumer_total_money_per_turn = 250
+    consumer_total_money_per_turn = 500
     consumers_arr = []
 
 
@@ -194,8 +196,11 @@ def step(self, action_arr):
             self.set_agent_action(action_arr[i], agent, self.action_space[i])
 
         for i, consumer in enumerate(self.consumers_arr):
-            self.set_consumer_purchases(self.policy_agents, consumer)
+            lowestPriceAgnetIndex, lowestAgentVendPrice,vendingPrices_arr = self.set_consumer_purchases(self.policy_agents, consumer)
         
+        print(f'sold to agent:[{lowestPriceAgnetIndex}] with price [{lowestAgentVendPrice}]. Options were {vendingPrices_arr}.')# Agent reward is {self.policy_agents[lowestPriceAgnetIndex].reward}')
+
+
         anyConsumed = False
         for consumer in self.consumers_arr:
             if consumer.total_consumed > 0:
@@ -240,9 +245,12 @@ def set_consumer_purchases(self, agents_arr, consumer):
 
         Purchases as many items from the agent as possible
         """
+
         lowestPriceAgnetIndex = 0
+        vendingPrices = []
             
         for i, agent in enumerate(agents_arr):
+            vendingPrices.append(agent.vendingPrice)
             if agent.vendingPrice < agents_arr[lowestPriceAgnetIndex].vendingPrice:
                 lowestPriceAgnetIndex = i
 
@@ -254,11 +262,11 @@ def set_consumer_purchases(self, agents_arr, consumer):
         tmpAgentRewardPerUnitSold = (lowestAgentVendPrice - self.env_wholesale_price)
         agentReward = tmpAgentRewardPerUnitSold * consumerConsumed
 
-        consumer.money = 0
+        # consumer.money = 0
         consumer.total_consumed += consumerConsumed
         agents_arr[lowestPriceAgnetIndex].reward += agentReward
         agents_arr[lowestPriceAgnetIndex].quantitySold += consumerConsumed
-
+        return lowestPriceAgnetIndex, lowestAgentVendPrice, vendingPrices
 
 
     def _get_quantity_sold(self, agent):

From ac56de96ad2936c82dc4bf5d3411a615c3fb7177 Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Mon, 15 May 2023 14:50:13 +0100
Subject: [PATCH 27/40] clears the agent individual results between steps

---
 Macodiac.ML/multiagent_main.py       |  2 +-
 Macodiac.ML/multiagentenvironment.py | 21 +++++++++++++++++----
 2 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/Macodiac.ML/multiagent_main.py b/Macodiac.ML/multiagent_main.py
index ec97a21..81ffba3 100644
--- a/Macodiac.ML/multiagent_main.py
+++ b/Macodiac.ML/multiagent_main.py
@@ -26,7 +26,7 @@ def __init__(self):
         self.save_path_intermittent =  os.path.join(filePath,'saved_models', 'intermittent_saved_models')
         self.numTrainingIterations = 100_000
         self.numEpisodes = 20
-        self.envTimesteps = 20
+        self.envTimesteps = 50
         self.numAgents = 3
 
         self.env = MultiAgentMacodiacEnvironment(envTimesteps=self.envTimesteps, numAgents=self.numAgents)
diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index 698b727..43f57e3 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -48,6 +48,7 @@ def _on_rollout_end(self) -> None:
         return super()._on_rollout_end()
     
     def _on_step(self) -> bool:
+        self.reset()
         # self.iterator +=1
         # agent_arr = self.training_env.get_attr('policy_agents')[0]
         
@@ -58,12 +59,15 @@ def _on_step(self) -> bool:
 
 
         pxList = []
+        acceptedPxList = []
         vendorsMadeSale = 0
         quantitySold = 0
         countNoSale = 0
         countWiSale = 0
-        acceptedVendedPx = 0
         meanPxOffered = 0
+        agent_sales = 0
+        agent_vend_px = 0
+        agent_reward = 0
 
         for i, agentInfo in enumerate(info_arr):
             agent_sales = info_arr[i]['sold']
@@ -76,7 +80,7 @@ def _on_step(self) -> bool:
                 vendorsMadeSale += 1
                 quantitySold += agent_sales
                 countWiSale += 1
-                acceptedVendedPx = agent_vend_px
+                acceptedPxList.append(agent_vend_px)
             else:
                 countNoSale += 1
             
@@ -85,9 +89,10 @@ def _on_step(self) -> bool:
             self.logger.record(f'vending_agent_{agentInfo["agent_num"]}/individual_reward',  agent_reward)
             
         meanPxOffered = np.mean(pxList)
+        meanPxAccepted = np.mean(acceptedPxList)
 
         self.logger.record('vending/avgerage_offered_px_value', meanPxOffered)
-        self.logger.record('vending/actual_accepted_px_value', acceptedVendedPx)
+        self.logger.record('vending/average_accepted_px_value', meanPxAccepted)
         self.logger.record('vending/quantity_sold_count', quantitySold)
         self.logger.record('vending/vendors_made_sale_count', vendorsMadeSale)
         self.logger.record('vending/count_no_sale', countNoSale)
@@ -153,6 +158,12 @@ def __init__(self, envTimesteps:int, numAgents: int):
         print('-- ENV SETTINGS --')
 
 
+    def clear_agent_stats(self, agent):
+        # agent.state = []
+        agent.vendingPrice = 0
+        agent.reward = 0
+        agent.quantitySold = 0
+
     def set_agent_action(self, action, agent, actionSpace):
         # agent.state is the percentage price diff from the wholesale price
         agent.state = action
@@ -193,12 +204,13 @@ def step(self, action_arr):
         info_arr = {'n': []}
         
         for i, agent in enumerate(self.policy_agents):
+            self.clear_agent_stats(agent)
             self.set_agent_action(action_arr[i], agent, self.action_space[i])
 
         for i, consumer in enumerate(self.consumers_arr):
             lowestPriceAgnetIndex, lowestAgentVendPrice,vendingPrices_arr = self.set_consumer_purchases(self.policy_agents, consumer)
         
-        print(f'sold to agent:[{lowestPriceAgnetIndex}] with price [{lowestAgentVendPrice}]. Options were {vendingPrices_arr}.')# Agent reward is {self.policy_agents[lowestPriceAgnetIndex].reward}')
+        # print(f'sold to agent:[{lowestPriceAgnetIndex}] with price [{lowestAgentVendPrice}]. Options were {vendingPrices_arr}. Agent reward is {self.policy_agents[lowestPriceAgnetIndex].reward}')
 
 
         anyConsumed = False
@@ -216,6 +228,7 @@ def step(self, action_arr):
             done_arr.append(self._get_done(agent))
             info_arr['n'].append(self._get_info(agent, i))
 
+        # print(info_arr)
         # for agent in self.policy_agents:
            
 

From ca250d8d71816812a061483fa36aeec1e4bb53d3 Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Mon, 15 May 2023 14:55:06 +0100
Subject: [PATCH 28/40] updates logging paths

---
 Macodiac.ML/multiagent_main.py       |  4 ++--
 Macodiac.ML/multiagentenvironment.py | 18 +++++++++---------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/Macodiac.ML/multiagent_main.py b/Macodiac.ML/multiagent_main.py
index 81ffba3..2be7e86 100644
--- a/Macodiac.ML/multiagent_main.py
+++ b/Macodiac.ML/multiagent_main.py
@@ -24,9 +24,9 @@ def __init__(self):
         self.log_path =  os.path.join(filePath,'Logs')
         self.save_path =  os.path.join(filePath,'saved_models', 'model')
         self.save_path_intermittent =  os.path.join(filePath,'saved_models', 'intermittent_saved_models')
-        self.numTrainingIterations = 100_000
+        self.numTrainingIterations = 1_000_000
         self.numEpisodes = 20
-        self.envTimesteps = 50
+        self.envTimesteps = 25
         self.numAgents = 3
 
         self.env = MultiAgentMacodiacEnvironment(envTimesteps=self.envTimesteps, numAgents=self.numAgents)
diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index 43f57e3..11542ee 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -84,19 +84,19 @@ def _on_step(self) -> bool:
             else:
                 countNoSale += 1
             
-            self.logger.record(f'vending_agent_{agentInfo["agent_num"]}/offered_px',   agent_vend_px)
-            self.logger.record(f'vending_agent_{agentInfo["agent_num"]}/sales_complete',  agent_sales)
-            self.logger.record(f'vending_agent_{agentInfo["agent_num"]}/individual_reward',  agent_reward)
+            self.logger.record(f'a_vending_agent_{agentInfo["agent_num"]}/offered_px',   agent_vend_px)
+            self.logger.record(f'a_vending_agent_{agentInfo["agent_num"]}/sales_complete',  agent_sales)
+            self.logger.record(f'a_vending_agent_{agentInfo["agent_num"]}/individual_reward',  agent_reward)
             
         meanPxOffered = np.mean(pxList)
         meanPxAccepted = np.mean(acceptedPxList)
 
-        self.logger.record('vending/avgerage_offered_px_value', meanPxOffered)
-        self.logger.record('vending/average_accepted_px_value', meanPxAccepted)
-        self.logger.record('vending/quantity_sold_count', quantitySold)
-        self.logger.record('vending/vendors_made_sale_count', vendorsMadeSale)
-        self.logger.record('vending/count_no_sale', countNoSale)
-        self.logger.record('vending/count_wi_sale', countWiSale)
+        self.logger.record('a_vending/avgerage_offered_px_value', meanPxOffered)
+        self.logger.record('a_vending/average_accepted_px_value', meanPxAccepted)
+        self.logger.record('a_vending/quantity_sold_count', quantitySold)
+        self.logger.record('a_vending/vendors_made_sale_count', vendorsMadeSale)
+        self.logger.record('a_vending/count_no_sale', countNoSale)
+        self.logger.record('a_vending/count_wi_sale', countWiSale)
 
         return True
 

From b55c8b2640745cee4ca71dbe03b98c28f1c2e28f Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Mon, 15 May 2023 18:01:38 +0100
Subject: [PATCH 29/40] distributes sales across agents

---
 Macodiac.ML/multiagent_main.py       |  2 +-
 Macodiac.ML/multiagentenvironment.py | 84 +++++++++++++++++++++++-----
 2 files changed, 71 insertions(+), 15 deletions(-)

diff --git a/Macodiac.ML/multiagent_main.py b/Macodiac.ML/multiagent_main.py
index 2be7e86..34c3484 100644
--- a/Macodiac.ML/multiagent_main.py
+++ b/Macodiac.ML/multiagent_main.py
@@ -27,7 +27,7 @@ def __init__(self):
         self.numTrainingIterations = 1_000_000
         self.numEpisodes = 20
         self.envTimesteps = 25
-        self.numAgents = 3
+        self.numAgents = 2
 
         self.env = MultiAgentMacodiacEnvironment(envTimesteps=self.envTimesteps, numAgents=self.numAgents)
         check_env(self.env)
diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index 11542ee..592203d 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -5,6 +5,7 @@
 from gym import spaces
 import numpy as np
 import random
+from random import shuffle
 from stable_baselines3.common.callbacks import BaseCallback
 
 class AgentObject:
@@ -26,13 +27,6 @@ class TensorboardPriceCallback(BaseCallback):
     """ 
     custom logger to record the price charged by agents
     """
-    # iterator = 0
-    # offeredPxList = []
-    # acceptedPxList = []
-    # vendorsMadeSaleList = []
-    # quantitySoldList = []
-
-
     runningAvgMeanPxOffered = 0
     runningAvgAcceptedVendedPx = 0
 
@@ -157,6 +151,8 @@ def __init__(self, envTimesteps:int, numAgents: int):
         print(self.environment_timesteps)
         print('-- ENV SETTINGS --')
 
+    def clear_consumer_stats(self, consumer):
+        consumer.money = self.consumerMoneyEach
 
     def clear_agent_stats(self, agent):
         # agent.state = []
@@ -208,10 +204,9 @@ def step(self, action_arr):
             self.set_agent_action(action_arr[i], agent, self.action_space[i])
 
         for i, consumer in enumerate(self.consumers_arr):
-            lowestPriceAgnetIndex, lowestAgentVendPrice,vendingPrices_arr = self.set_consumer_purchases(self.policy_agents, consumer)
-        
-        # print(f'sold to agent:[{lowestPriceAgnetIndex}] with price [{lowestAgentVendPrice}]. Options were {vendingPrices_arr}. Agent reward is {self.policy_agents[lowestPriceAgnetIndex].reward}')
-
+            self.clear_consumer_stats(consumer)
+            self.alt_set_consumer_purchases(self.policy_agents, consumer)
+    
 
         anyConsumed = False
         for consumer in self.consumers_arr:
@@ -248,7 +243,67 @@ def step(self, action_arr):
             tmpObsArray.append(partialObservationResult)
 
         concatObsArray = np.array(tmpObsArray).astype(np.int32) 
-        return concatObsArray, sum(reward_arr), isTerminal,  info_arr
+        return concatObsArray, float(sum(reward_arr)), isTerminal,  info_arr
+
+    def alt_set_consumer_purchases(self, agents_arr, consumer):
+        """
+        So long as the consumer has money, loops through the agents, and selects the lowest
+        priced agent. 
+
+        if multiple agents share the same price points, distributes the sales across them all
+        """
+        lowestAbsolutePrice = 0
+        lowestPriceAgentIndexList = []
+        vendingPrices = []
+
+        for i, agent in enumerate(agents_arr):
+            vendingPrices.append(agent.vendingPrice)
+
+      
+        # print(f'prices are: {vendingPrices}')
+        lowestAbsolutePrice = min(vendingPrices)
+       
+        # gather all of the lowest price agents
+        for i, agent in enumerate(agents_arr):
+            if agent.vendingPrice == lowestAbsolutePrice:
+                lowestPriceAgentIndexList.append(i)
+
+        shuffle(lowestPriceAgentIndexList)
+
+        # while the consumer still has money, purchase
+        # items from the vendors
+        while consumer.money > 0:
+            # loop through each vendor, purchase one item from them
+            for agentIndex in lowestPriceAgentIndexList:
+                agentToPurchaseFrom = agents_arr[agentIndex]
+                
+                if agentToPurchaseFrom.vendingPrice != lowestAbsolutePrice:
+                    raise ValueError(f'agent vending price [{agentToPurchaseFrom.vendingPrice}] is not the same as lowestAbsPrice:[{lowestAbsolutePrice}]')
+
+                if consumer.money > agentToPurchaseFrom.vendingPrice:
+                    self.consumer_purchase_from_agent(consumer, agentToPurchaseFrom)
+                else:
+                    consumer.money = 0
+                    break
+        
+        # if vendingPrices[0] == vendingPrices[1]:
+        #     print(f'hit condition: [{vendingPrices[0]},{vendingPrices[1]}]')
+        # for i, agent in enumerate(agents_arr):
+        #     print(f'agentPrices:{agent.vendingPrice} | agentSold: {agent.quantitySold}')
+            
+
+
+    
+    def consumer_purchase_from_agent(self, consumer, agent):
+        if consumer.money < agent.vendingPrice:
+            raise ValueError(f'consumer with: [{consumer.money}] money attempted to purchase from agent charging: [{agent.vendingPrice}]')
+        consumer.money -= agent.vendingPrice
+        consumer.total_consumed += 1
+        agent.quantitySold += 1
+        agent.reward += (agent.vendingPrice - self.env_wholesale_price)
+        
+
+
 
 
     def set_consumer_purchases(self, agents_arr, consumer):
@@ -343,9 +398,10 @@ def reset(self): #-> float:
             self.policy_agents[i].vendingPrice = 0
             self.policy_agents[i].quantitySold = 0
         
-        consumerMoneyEach = self.consumer_total_money_per_turn / self.num_consumers
+        self.consumerMoneyEach = self.consumer_total_money_per_turn / self.num_consumers
         for i in range(len(self.consumers_arr)):
-            self.consumers_arr[i].money = consumerMoneyEach
+            self.clear_consumer_stats(self.consumers_arr[i])
+            # self.consumers_arr[i].money = consumerMoneyEach
 
         self.environment_timesteps = self.environment_starter_timesteps
         

From 4dfc6ff72e32252e48501a7a7458b2b8c81c015d Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Mon, 15 May 2023 18:04:13 +0100
Subject: [PATCH 30/40] configures for monop

---
 Macodiac.ML/multiagent_main.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Macodiac.ML/multiagent_main.py b/Macodiac.ML/multiagent_main.py
index 34c3484..8db0af4 100644
--- a/Macodiac.ML/multiagent_main.py
+++ b/Macodiac.ML/multiagent_main.py
@@ -24,10 +24,10 @@ def __init__(self):
         self.log_path =  os.path.join(filePath,'Logs')
         self.save_path =  os.path.join(filePath,'saved_models', 'model')
         self.save_path_intermittent =  os.path.join(filePath,'saved_models', 'intermittent_saved_models')
-        self.numTrainingIterations = 1_000_000
+        self.numTrainingIterations = 5_000_000
         self.numEpisodes = 20
         self.envTimesteps = 25
-        self.numAgents = 2
+        self.numAgents = 1
 
         self.env = MultiAgentMacodiacEnvironment(envTimesteps=self.envTimesteps, numAgents=self.numAgents)
         check_env(self.env)
@@ -169,7 +169,7 @@ def train_model(self, model, numTimesteps: int, savePath:str):
         @param numTimesteps: the number of training iterations
         """
         
-        saveEveryNSteps = 100_001
+        saveEveryNSteps = 1_000_000
         
         if numTimesteps < saveEveryNSteps:
             model.learn(total_timesteps=numTimesteps,

From 6a6dc105382b76d2a1037b0ed0ab7a790bb2904b Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Tue, 16 May 2023 11:13:10 +0100
Subject: [PATCH 31/40] sets up the model to loop through each market type

---
 Macodiac.ML/multiagent_main.py       | 72 ++++++++++++++++++----------
 Macodiac.ML/multiagentenvironment.py |  2 +-
 2 files changed, 49 insertions(+), 25 deletions(-)

diff --git a/Macodiac.ML/multiagent_main.py b/Macodiac.ML/multiagent_main.py
index 8db0af4..8bbcbbe 100644
--- a/Macodiac.ML/multiagent_main.py
+++ b/Macodiac.ML/multiagent_main.py
@@ -5,34 +5,17 @@
 from multiagentenvironment import MultiAgentMacodiacEnvironment
 from stable_baselines3.common.evaluation import evaluate_policy
 from stable_baselines3.common.env_checker import check_env
-
 import numpy as np
-
-
-
 from stable_baselines3 import PPO
 
 
 class MultiagentMain():
     isRunning = False
 
-    def __init__(self):
+    def __init__(self, mode):
         """
         init the class
         """
-        filePath = os.path.join('Macodiac.ML', 'training_multiagent','results')
-        self.log_path =  os.path.join(filePath,'Logs')
-        self.save_path =  os.path.join(filePath,'saved_models', 'model')
-        self.save_path_intermittent =  os.path.join(filePath,'saved_models', 'intermittent_saved_models')
-        self.numTrainingIterations = 5_000_000
-        self.numEpisodes = 20
-        self.envTimesteps = 25
-        self.numAgents = 1
-
-        self.env = MultiAgentMacodiacEnvironment(envTimesteps=self.envTimesteps, numAgents=self.numAgents)
-        check_env(self.env)
-
-
 
         # set to true if you want to load an existing model
         # model loading happens first, then training
@@ -48,6 +31,38 @@ def __init__(self):
         # rather than the model version
         self.__MODE_RANDOMSAMPLE__ = False
 
+        self.mode = mode
+
+
+        filePath = os.path.join('Macodiac.ML', 'training_multiagent','results')
+        self.log_path =  os.path.join(filePath,'Logs')
+        self.save_path =  os.path.join(filePath,'saved_models', self.mode)
+        self.save_path_intermittent =  os.path.join(filePath,'saved_models', 'intermittent_saved_models')
+
+        self.numEpisodes = 20
+        self.envTimesteps = 25
+
+        if self.mode == 'MONOPOLY':
+            self.numAgents = 1
+            self.numTrainingIterations = 3_000_000
+        elif self.mode == 'DUOPOLY':
+            self.numAgents = 2
+            self.numTrainingIterations = 3_000_000
+        elif self.mode == 'OLIGOPOLY':
+            self.numAgents = 5
+            self.numTrainingIterations = 5_000_000  
+        elif self.mode == 'PERFECT_COMP':
+            self.numAgents = 10
+            self.numTrainingIterations = 15_000_000
+        else:
+            raise ValueError(f'self.mode [{self.mode}] was not in mode options list [{self.__MODE_OPTIONS__}]')
+        
+        if self.numAgents == 0 or self.numTrainingIterations == 0:
+            raise ValueError('both numAgents and numTrainingItterations must be above 0')
+
+        self.env = MultiAgentMacodiacEnvironment(envTimesteps=self.envTimesteps, numAgents=self.numAgents)
+        check_env(self.env)
+        
 
     def Run(self):
         """
@@ -64,7 +79,9 @@ def Run(self):
         if self.__MODE_TRAINMODEL__: 
 
             model = self.train_model(model,
-                                     self.numTrainingIterations, self.save_path_intermittent)
+                                     self.numTrainingIterations, 
+                                     self.save_path_intermittent,
+                                     self.mode)
             self.save_model(model, self.save_path)
 
         if not self.__MODE_RANDOMSAMPLE__:
@@ -160,7 +177,7 @@ def create_model(self, env: MultiAgentMacodiacEnvironment, log_path: str):
         return model
 
 
-    def train_model(self, model, numTimesteps: int, savePath:str):
+    def train_model(self, model, numTimesteps: int, savePath:str, saveName: str):
         """
         Trains a model with the number of iterations in 
         numtimesteps. Creates a n intermediate save every 1m iterations
@@ -173,13 +190,15 @@ def train_model(self, model, numTimesteps: int, savePath:str):
         
         if numTimesteps < saveEveryNSteps:
             model.learn(total_timesteps=numTimesteps,
-                        callback=TensorboardPriceCallback())
+                        callback=TensorboardPriceCallback(), 
+                        tb_log_name=saveName)
 
         else:
             rangeUpper = int(numTimesteps / saveEveryNSteps)
             for i in range(1,rangeUpper+1):
                 model.learn(total_timesteps=saveEveryNSteps,
-                            callback=TensorboardPriceCallback())
+                            callback=TensorboardPriceCallback(),
+                            tb_log_name=saveName)
                 model.save(os.path.join(savePath, f'interim-{i}'))
 
         return model
@@ -219,5 +238,10 @@ def load_model(self, env: MultiAgentMacodiacEnvironment, model, modelPath: str):
         return model
 
 
-main = MultiagentMain()
-main.Run()
\ No newline at end of file
+
+
+
+__MODE_OPTIONS__ = ['MONOPOLY', 'DUOPOLY', 'OLIGOPOLY', 'PERFECT_COMP']
+for mode in __MODE_OPTIONS__:
+    main = MultiagentMain(mode)
+    main.Run()
\ No newline at end of file
diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index 592203d..3d64a32 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -102,7 +102,7 @@ class MultiAgentMacodiacEnvironment(Env):
     state = 0
     environment_timesteps = 0
     environment_starter_timesteps = 150
-    env_wholesale_price = 5        # the price agents pay to purchase goods
+    env_wholesale_price = 8        # the price agents pay to purchase goods
     env_agent_marginal_cost = 0     # the marginal cost of vending
     num_consumers = 25
     consumer_total_money_per_turn = 500

From 310a5b99c0107b8da2b986e5add0151697b85eca Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Tue, 16 May 2023 11:48:26 +0100
Subject: [PATCH 32/40] adjusts the scope of possible price points

---
 Macodiac.ML/multiagentenvironment.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index 3d64a32..76cacee 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -162,7 +162,7 @@ def clear_agent_stats(self, agent):
 
     def set_agent_action(self, action, agent, actionSpace):
         # agent.state is the percentage price diff from the wholesale price
-        agent.state = action
+        agent.state = (action * 1.5)
         agent.vendingPrice = self.env_wholesale_price + agent.state
 
         if agent.vendingPrice == 0:

From 0cb582b7f6f6e8ab0d3f8a0bfaa5ed6891163aec Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Tue, 16 May 2023 11:56:41 +0100
Subject: [PATCH 33/40] updates the agent action space to 15 instead of 10 px
 options

---
 Macodiac.ML/multiagentenvironment.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index 76cacee..a733396 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -125,7 +125,7 @@ def __init__(self, envTimesteps:int, numAgents: int):
 
         
         # creates an array full of 10's shaped [20,20,20], of length numAgents
-        self.action_space = spaces.MultiDiscrete(np.full(numAgents, 10) )
+        self.action_space = spaces.MultiDiscrete(np.full(numAgents, 15) )
 
 
         # the observation space is a nAgents by nActions array of float32 numbers between -99-99
@@ -162,7 +162,7 @@ def clear_agent_stats(self, agent):
 
     def set_agent_action(self, action, agent, actionSpace):
         # agent.state is the percentage price diff from the wholesale price
-        agent.state = (action * 1.5)
+        agent.state = action
         agent.vendingPrice = self.env_wholesale_price + agent.state
 
         if agent.vendingPrice == 0:

From 87eaa640a43a2c1836f3576235969fd37d31192e Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Tue, 16 May 2023 19:03:02 +0100
Subject: [PATCH 34/40] corrects a bug where too many products were being sold
 per loop

---
 Macodiac.ML/multiagentenvironment.py | 55 ++++++++++++++++------------
 1 file changed, 31 insertions(+), 24 deletions(-)

diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index a733396..1a31224 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -208,13 +208,13 @@ def step(self, action_arr):
             self.alt_set_consumer_purchases(self.policy_agents, consumer)
     
 
-        anyConsumed = False
-        for consumer in self.consumers_arr:
-            if consumer.total_consumed > 0:
-                anyConsumed = True
-                break
-        if anyConsumed == False:
-            print(f'error, no consumption')
+        # anyConsumed = False
+        # for consumer in self.consumers_arr:
+        #     if consumer.total_consumed > 0:
+        #         anyConsumed = True
+        #         break
+        # if anyConsumed == False:
+        #     print(f'error, no consumption')
 
         for i, agent in enumerate(self.policy_agents):
             agent.state, agent.reward, agent.done, agent.info = self.step_agent(agent)
@@ -275,16 +275,23 @@ def alt_set_consumer_purchases(self, agents_arr, consumer):
         while consumer.money > 0:
             # loop through each vendor, purchase one item from them
             for agentIndex in lowestPriceAgentIndexList:
-                agentToPurchaseFrom = agents_arr[agentIndex]
-                
-                if agentToPurchaseFrom.vendingPrice != lowestAbsolutePrice:
-                    raise ValueError(f'agent vending price [{agentToPurchaseFrom.vendingPrice}] is not the same as lowestAbsPrice:[{lowestAbsolutePrice}]')
-
-                if consumer.money > agentToPurchaseFrom.vendingPrice:
-                    self.consumer_purchase_from_agent(consumer, agentToPurchaseFrom)
-                else:
-                    consumer.money = 0
-                    break
+                if consumer.money > 0:
+                    agentToPurchaseFrom = agents_arr[agentIndex]
+                    
+                    if agentToPurchaseFrom.vendingPrice != lowestAbsolutePrice:
+                        raise ValueError(f'agent vending price [{agentToPurchaseFrom.vendingPrice}] is not the same as lowestAbsPrice:[{lowestAbsolutePrice}]')
+
+                    if consumer.money > agentToPurchaseFrom.vendingPrice:
+                        if consumer.money < agentToPurchaseFrom.vendingPrice:
+                            raise ValueError(f'consumer with: [{consumer.money}] money attempted to purchase from agent charging: [{agentToPurchaseFrom.vendingPrice}]')
+                        consumer.money -= agentToPurchaseFrom.vendingPrice
+                        consumer.total_consumed += 1
+                        agentToPurchaseFrom.quantitySold += 1
+                        agentToPurchaseFrom.reward += (agentToPurchaseFrom.vendingPrice - self.env_wholesale_price)
+                    else:
+                        # print(f'consumer money was {consumer.money}, setting to 0')
+                        consumer.money = 0
+                        break
         
         # if vendingPrices[0] == vendingPrices[1]:
         #     print(f'hit condition: [{vendingPrices[0]},{vendingPrices[1]}]')
@@ -294,13 +301,13 @@ def alt_set_consumer_purchases(self, agents_arr, consumer):
 
 
     
-    def consumer_purchase_from_agent(self, consumer, agent):
-        if consumer.money < agent.vendingPrice:
-            raise ValueError(f'consumer with: [{consumer.money}] money attempted to purchase from agent charging: [{agent.vendingPrice}]')
-        consumer.money -= agent.vendingPrice
-        consumer.total_consumed += 1
-        agent.quantitySold += 1
-        agent.reward += (agent.vendingPrice - self.env_wholesale_price)
+    # def consumer_purchase_from_agent(self, consumer, agentToPurchaseFrom):
+    #     if consumer.money < agentToPurchaseFrom.vendingPrice:
+    #         raise ValueError(f'consumer with: [{consumer.money}] money attempted to purchase from agent charging: [{agentToPurchaseFrom.vendingPrice}]')
+    #     consumer.money -= agentToPurchaseFrom.vendingPrice
+    #     consumer.total_consumed += 1
+    #     agentToPurchaseFrom.quantitySold += 1
+    #     agentToPurchaseFrom.reward += (agentToPurchaseFrom.vendingPrice - self.env_wholesale_price)
         
 
 

From a2fcc0b6a5754b7a216d68568cdf8a7bade81408 Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Wed, 17 May 2023 09:21:36 +0100
Subject: [PATCH 35/40] bug corrections

---
 Macodiac.ML/multiagent_main.py       | 34 ++++++++++++++----
 Macodiac.ML/multiagentenvironment.py | 52 +++++++---------------------
 2 files changed, 40 insertions(+), 46 deletions(-)

diff --git a/Macodiac.ML/multiagent_main.py b/Macodiac.ML/multiagent_main.py
index 8bbcbbe..5c0c980 100644
--- a/Macodiac.ML/multiagent_main.py
+++ b/Macodiac.ML/multiagent_main.py
@@ -25,11 +25,11 @@ def __init__(self, mode):
         self.__MODE_LOADMODEL__ = False
 
         # set to true if you want to train and then save the model
-        self.__MODE_TRAINMODEL__ = True
+        self.__MODE_TRAINMODEL__ = False
 
         # set to true to use the randomsample mode for testing, 
         # rather than the model version
-        self.__MODE_RANDOMSAMPLE__ = False
+        self.__MODE_RANDOMSAMPLE__ = True
 
         self.mode = mode
 
@@ -44,7 +44,7 @@ def __init__(self, mode):
 
         if self.mode == 'MONOPOLY':
             self.numAgents = 1
-            self.numTrainingIterations = 3_000_000
+            self.numTrainingIterations = 2_000_000
         elif self.mode == 'DUOPOLY':
             self.numAgents = 2
             self.numTrainingIterations = 3_000_000
@@ -103,8 +103,12 @@ def run_multiagent_project_with_rand_test(self, env:MultiAgentMacodiacEnvironmen
             while not done:
                 #env.render()
                 iterator+=1
-                print(f'iterator:{iterator}')
-                action_arr = env.action_space.sample()
+                #print(f'iterator:{iterator}')
+                # action_arr = env.action_space.sample()
+
+                action_arr = []
+                for i in range(self.numAgents):
+                    action_arr.append(11)
 
                 print(f'action for agents:\t{action_arr}')
                 
@@ -112,9 +116,25 @@ def run_multiagent_project_with_rand_test(self, env:MultiAgentMacodiacEnvironmen
                 
                 agent_scores.append(reward)
 
-                print(f'rewards for agents:\t{reward}')
-                print(f'obs for agents:\t{obs_arr}')
+                # print(f'rewards for agents:\t{reward}')
+                # print(f'obs for agents:\t{obs_arr}')
+
+                info_arr = info_arr['n']
                 print(f'px for agents:\t{info_arr}')
+                quantitySold = 0
+                moneySales = 0
+                for i, agentInfo in enumerate(info_arr):
+                    agent_sales = info_arr[i]['sold']
+                    agent_price = info_arr[i]['price']
+                    agent_sales_in_money = agent_sales * agent_price
+                    moneySales += agent_sales_in_money
+                    quantitySold += agent_sales
+                    
+                print(f'a_vending/quantity_sold_count: {quantitySold} at cost [{moneySales}]/[{env.peek_env_consumer_money()}. Consumer money per turn:{env.peek_env_consumer_money_each()}]')
+                if moneySales > env.peek_env_consumer_money():
+                    print(f'Money sales of [{moneySales}]/[{env.peek_env_consumer_money()}] were too high. Consumer money per turn:{env.peek_env_consumer_money_each()}')
+                    return # raise Exception(f'Money sales of [{moneySales}]/[{env.peek_env_consumer_money()}] were too high. Consumer money per turn:{env.peek_env_consumer_money_each()}')
+
 
                 if done:
                     print(f'is done')
diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index 1a31224..4c9ed7a 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -115,6 +115,7 @@ def __init__(self, envTimesteps:int, numAgents: int):
         """
         self.environment_starter_timesteps = envTimesteps
         self.policy_agents = []
+        self.consumers_arr = []
         self.observation_space = []
 
         for i in range(numAgents):
@@ -151,16 +152,20 @@ def __init__(self, envTimesteps:int, numAgents: int):
         print(self.environment_timesteps)
         print('-- ENV SETTINGS --')
 
+    def peek_env_consumer_money(self):
+        return self.consumer_total_money_per_turn
+    def peek_env_consumer_money_each(self):
+        return self.consumerMoneyEach
+
     def clear_consumer_stats(self, consumer):
         consumer.money = self.consumerMoneyEach
 
     def clear_agent_stats(self, agent):
-        # agent.state = []
         agent.vendingPrice = 0
         agent.reward = 0
         agent.quantitySold = 0
 
-    def set_agent_action(self, action, agent, actionSpace):
+    def set_agent_action(self, action, agent):
         # agent.state is the percentage price diff from the wholesale price
         agent.state = action
         agent.vendingPrice = self.env_wholesale_price + agent.state
@@ -201,31 +206,18 @@ def step(self, action_arr):
         
         for i, agent in enumerate(self.policy_agents):
             self.clear_agent_stats(agent)
-            self.set_agent_action(action_arr[i], agent, self.action_space[i])
+            self.set_agent_action(action_arr[i], agent)
 
         for i, consumer in enumerate(self.consumers_arr):
             self.clear_consumer_stats(consumer)
             self.alt_set_consumer_purchases(self.policy_agents, consumer)
-    
-
-        # anyConsumed = False
-        # for consumer in self.consumers_arr:
-        #     if consumer.total_consumed > 0:
-        #         anyConsumed = True
-        #         break
-        # if anyConsumed == False:
-        #     print(f'error, no consumption')
 
         for i, agent in enumerate(self.policy_agents):
             agent.state, agent.reward, agent.done, agent.info = self.step_agent(agent)
             obs_arr.append(self._get_obs(agent))
             reward_arr.append(self._get_reward(agent))
             done_arr.append(self._get_done(agent))
-            info_arr['n'].append(self._get_info(agent, i))
-
-        # print(info_arr)
-        # for agent in self.policy_agents:
-           
+            info_arr['n'].append(self._get_info(agent, i))        
 
         if self.environment_timesteps <= 0:
             isTerminal = True
@@ -281,10 +273,11 @@ def alt_set_consumer_purchases(self, agents_arr, consumer):
                     if agentToPurchaseFrom.vendingPrice != lowestAbsolutePrice:
                         raise ValueError(f'agent vending price [{agentToPurchaseFrom.vendingPrice}] is not the same as lowestAbsPrice:[{lowestAbsolutePrice}]')
 
-                    if consumer.money > agentToPurchaseFrom.vendingPrice:
+                    if consumer.money >= agentToPurchaseFrom.vendingPrice:
                         if consumer.money < agentToPurchaseFrom.vendingPrice:
                             raise ValueError(f'consumer with: [{consumer.money}] money attempted to purchase from agent charging: [{agentToPurchaseFrom.vendingPrice}]')
                         consumer.money -= agentToPurchaseFrom.vendingPrice
+                        # print(f'consumer money: {consumer.money}')
                         consumer.total_consumed += 1
                         agentToPurchaseFrom.quantitySold += 1
                         agentToPurchaseFrom.reward += (agentToPurchaseFrom.vendingPrice - self.env_wholesale_price)
@@ -292,25 +285,6 @@ def alt_set_consumer_purchases(self, agents_arr, consumer):
                         # print(f'consumer money was {consumer.money}, setting to 0')
                         consumer.money = 0
                         break
-        
-        # if vendingPrices[0] == vendingPrices[1]:
-        #     print(f'hit condition: [{vendingPrices[0]},{vendingPrices[1]}]')
-        # for i, agent in enumerate(agents_arr):
-        #     print(f'agentPrices:{agent.vendingPrice} | agentSold: {agent.quantitySold}')
-            
-
-
-    
-    # def consumer_purchase_from_agent(self, consumer, agentToPurchaseFrom):
-    #     if consumer.money < agentToPurchaseFrom.vendingPrice:
-    #         raise ValueError(f'consumer with: [{consumer.money}] money attempted to purchase from agent charging: [{agentToPurchaseFrom.vendingPrice}]')
-    #     consumer.money -= agentToPurchaseFrom.vendingPrice
-    #     consumer.total_consumed += 1
-    #     agentToPurchaseFrom.quantitySold += 1
-    #     agentToPurchaseFrom.reward += (agentToPurchaseFrom.vendingPrice - self.env_wholesale_price)
-        
-
-
 
 
     def set_consumer_purchases(self, agents_arr, consumer):
@@ -406,8 +380,8 @@ def reset(self): #-> float:
             self.policy_agents[i].quantitySold = 0
         
         self.consumerMoneyEach = self.consumer_total_money_per_turn / self.num_consumers
-        for i in range(len(self.consumers_arr)):
-            self.clear_consumer_stats(self.consumers_arr[i])
+        # for i in range(len(self.consumers_arr)):
+            # self.clear_consumer_stats(self.consumers_arr[i])
             # self.consumers_arr[i].money = consumerMoneyEach
 
         self.environment_timesteps = self.environment_starter_timesteps

From 7a28d72f6346f1c7de3a2c5045eb23181870b86a Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Wed, 17 May 2023 09:23:33 +0100
Subject: [PATCH 36/40] adds a new tracking param

---
 Macodiac.ML/multiagentenvironment.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index 4c9ed7a..4bbbefe 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -62,11 +62,14 @@ def _on_step(self) -> bool:
         agent_sales = 0
         agent_vend_px = 0
         agent_reward = 0
+        money_sales = 0
 
         for i, agentInfo in enumerate(info_arr):
             agent_sales = info_arr[i]['sold']
             agent_vend_px = info_arr[i]['price']
             agent_reward = info_arr[i]['reward']
+            agent_sales_in_money = agent_sales * agent_vend_px
+            money_sales += agent_sales_in_money
 
             pxList.append(agent_vend_px)
             
@@ -80,6 +83,7 @@ def _on_step(self) -> bool:
             
             self.logger.record(f'a_vending_agent_{agentInfo["agent_num"]}/offered_px',   agent_vend_px)
             self.logger.record(f'a_vending_agent_{agentInfo["agent_num"]}/sales_complete',  agent_sales)
+            self.logger.record(f'a_vending_agent_{agentInfo["agent_num"]}/sales_value',  agent_sales_in_money)
             self.logger.record(f'a_vending_agent_{agentInfo["agent_num"]}/individual_reward',  agent_reward)
             
         meanPxOffered = np.mean(pxList)
@@ -88,6 +92,7 @@ def _on_step(self) -> bool:
         self.logger.record('a_vending/avgerage_offered_px_value', meanPxOffered)
         self.logger.record('a_vending/average_accepted_px_value', meanPxAccepted)
         self.logger.record('a_vending/quantity_sold_count', quantitySold)
+        self.logger.record('a_vending/total_value_sold', money_sales)
         self.logger.record('a_vending/vendors_made_sale_count', vendorsMadeSale)
         self.logger.record('a_vending/count_no_sale', countNoSale)
         self.logger.record('a_vending/count_wi_sale', countWiSale)

From 76249b582853447870027a33a3bef47de656bac5 Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Wed, 17 May 2023 09:27:18 +0100
Subject: [PATCH 37/40] Update .gitignore

---
 Macodiac.ML/.gitignore | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/Macodiac.ML/.gitignore b/Macodiac.ML/.gitignore
index ea7e1d9..89a982f 100644
--- a/Macodiac.ML/.gitignore
+++ b/Macodiac.ML/.gitignore
@@ -6,6 +6,9 @@ training/results/saved_models/*
 !training/results/saved_models/.keep
 training_multiagent/results/saved_models/*
 !training_multiagent/results/saved_models/.keep
+training_multiagent_16may/results/logs/*
+training_multiagent_16may/results/saved_models/*
+
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

From 1e13d41c8a8aa4ba46a47dcaea6b38a0421481d3 Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Wed, 17 May 2023 09:29:03 +0100
Subject: [PATCH 38/40] Update multiagent_main.py

---
 Macodiac.ML/multiagent_main.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Macodiac.ML/multiagent_main.py b/Macodiac.ML/multiagent_main.py
index 5c0c980..2111aee 100644
--- a/Macodiac.ML/multiagent_main.py
+++ b/Macodiac.ML/multiagent_main.py
@@ -25,11 +25,11 @@ def __init__(self, mode):
         self.__MODE_LOADMODEL__ = False
 
         # set to true if you want to train and then save the model
-        self.__MODE_TRAINMODEL__ = False
+        self.__MODE_TRAINMODEL__ = True
 
         # set to true to use the randomsample mode for testing, 
         # rather than the model version
-        self.__MODE_RANDOMSAMPLE__ = True
+        self.__MODE_RANDOMSAMPLE__ = False
 
         self.mode = mode
 
@@ -261,7 +261,7 @@ def load_model(self, env: MultiAgentMacodiacEnvironment, model, modelPath: str):
 
 
 
-__MODE_OPTIONS__ = ['MONOPOLY', 'DUOPOLY', 'OLIGOPOLY', 'PERFECT_COMP']
+__MODE_OPTIONS__ = ['DUOPOLY', 'MONOPOLY', 'OLIGOPOLY', 'PERFECT_COMP']
 for mode in __MODE_OPTIONS__:
     main = MultiagentMain(mode)
     main.Run()
\ No newline at end of file

From af75efca515448b3ae38b696249bd7594b8139bf Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Wed, 17 May 2023 10:42:23 +0100
Subject: [PATCH 39/40] removes some NaN values from logging, adds MC pressure

---
 Macodiac.ML/multiagent_main.py       |  2 +-
 Macodiac.ML/multiagentenvironment.py | 61 ++++++++++++++++++++++++----
 2 files changed, 53 insertions(+), 10 deletions(-)

diff --git a/Macodiac.ML/multiagent_main.py b/Macodiac.ML/multiagent_main.py
index 2111aee..cfcd4d0 100644
--- a/Macodiac.ML/multiagent_main.py
+++ b/Macodiac.ML/multiagent_main.py
@@ -261,7 +261,7 @@ def load_model(self, env: MultiAgentMacodiacEnvironment, model, modelPath: str):
 
 
 
-__MODE_OPTIONS__ = ['DUOPOLY', 'MONOPOLY', 'OLIGOPOLY', 'PERFECT_COMP']
+__MODE_OPTIONS__ = ['MONOPOLY', 'DUOPOLY', 'OLIGOPOLY', 'PERFECT_COMP']
 for mode in __MODE_OPTIONS__:
     main = MultiagentMain(mode)
     main.Run()
\ No newline at end of file
diff --git a/Macodiac.ML/multiagentenvironment.py b/Macodiac.ML/multiagentenvironment.py
index 4bbbefe..ce0742d 100644
--- a/Macodiac.ML/multiagentenvironment.py
+++ b/Macodiac.ML/multiagentenvironment.py
@@ -11,9 +11,15 @@
 class AgentObject:
     def __init__(self):
         self.state = []
+        self.reset_values()
+    
+    def reset_values(self):
         self.vendingPrice = 0
         self.reward = 0
         self.quantitySold = 0
+        self.vendCost = 5
+        self.totalVendingCost = 0
+        self.vendCostTrend = 'down'
 
 class ConsumerObject:
     def __init__(self):
@@ -54,6 +60,7 @@ def _on_step(self) -> bool:
 
         pxList = []
         acceptedPxList = []
+        vendCostList = []
         vendorsMadeSale = 0
         quantitySold = 0
         countNoSale = 0
@@ -63,11 +70,18 @@ def _on_step(self) -> bool:
         agent_vend_px = 0
         agent_reward = 0
         money_sales = 0
-
+        agent_sales_in_money = 0
+        agent_total_vend_cost = 0
+        agent_final_vend_cost = 0
+        meanPxAccepted = 0
+        meanVendCost = 0
+         
         for i, agentInfo in enumerate(info_arr):
             agent_sales = info_arr[i]['sold']
             agent_vend_px = info_arr[i]['price']
             agent_reward = info_arr[i]['reward']
+            agent_final_vend_cost = info_arr[i]['vendCost']
+            agent_total_vend_cost = info_arr[i]['totalVendingCost']
             agent_sales_in_money = agent_sales * agent_vend_px
             money_sales += agent_sales_in_money
 
@@ -78,19 +92,28 @@ def _on_step(self) -> bool:
                 quantitySold += agent_sales
                 countWiSale += 1
                 acceptedPxList.append(agent_vend_px)
+                vendCostList.append(agent_final_vend_cost)
             else:
                 countNoSale += 1
             
             self.logger.record(f'a_vending_agent_{agentInfo["agent_num"]}/offered_px',   agent_vend_px)
             self.logger.record(f'a_vending_agent_{agentInfo["agent_num"]}/sales_complete',  agent_sales)
             self.logger.record(f'a_vending_agent_{agentInfo["agent_num"]}/sales_value',  agent_sales_in_money)
+            self.logger.record(f'a_vending_agent_{agentInfo["agent_num"]}/final_vend_cost',  agent_final_vend_cost)
+            self.logger.record(f'a_vending_agent_{agentInfo["agent_num"]}/total_vend_cost',  agent_total_vend_cost)
             self.logger.record(f'a_vending_agent_{agentInfo["agent_num"]}/individual_reward',  agent_reward)
             
-        meanPxOffered = np.mean(pxList)
-        meanPxAccepted = np.mean(acceptedPxList)
+        if len(pxList) > 0:
+            meanPxOffered = np.mean(pxList)
+        if len(acceptedPxList) > 0:
+            meanPxAccepted = np.mean(acceptedPxList)
+        if len(vendCostList) > 0:
+            meanVendCost = np.mean(vendCostList)
+
 
         self.logger.record('a_vending/avgerage_offered_px_value', meanPxOffered)
         self.logger.record('a_vending/average_accepted_px_value', meanPxAccepted)
+        self.logger.record('a_vending/average_final_vend_cost', meanVendCost)
         self.logger.record('a_vending/quantity_sold_count', quantitySold)
         self.logger.record('a_vending/total_value_sold', money_sales)
         self.logger.record('a_vending/vendors_made_sale_count', vendorsMadeSale)
@@ -110,7 +133,7 @@ class MultiAgentMacodiacEnvironment(Env):
     env_wholesale_price = 8        # the price agents pay to purchase goods
     env_agent_marginal_cost = 0     # the marginal cost of vending
     num_consumers = 25
-    consumer_total_money_per_turn = 500
+    consumer_total_money_per_turn = 475
     consumers_arr = []
 
 
@@ -166,9 +189,9 @@ def clear_consumer_stats(self, consumer):
         consumer.money = self.consumerMoneyEach
 
     def clear_agent_stats(self, agent):
-        agent.vendingPrice = 0
-        agent.reward = 0
-        agent.quantitySold = 0
+        agent.reset_values()
+        
+
 
     def set_agent_action(self, action, agent):
         # agent.state is the percentage price diff from the wholesale price
@@ -285,7 +308,17 @@ def alt_set_consumer_purchases(self, agents_arr, consumer):
                         # print(f'consumer money: {consumer.money}')
                         consumer.total_consumed += 1
                         agentToPurchaseFrom.quantitySold += 1
-                        agentToPurchaseFrom.reward += (agentToPurchaseFrom.vendingPrice - self.env_wholesale_price)
+
+                        # Marginal cost trends down towards 1, then increases upwards
+                        if agentToPurchaseFrom.vendCostTrend == 'up':
+                            agentToPurchaseFrom.vendCost += 0.66
+                        elif agentToPurchaseFrom.vendCostTrend == 'down':
+                            agentToPurchaseFrom.vendCost -= 0.66
+                            if agentToPurchaseFrom.vendCost < 1:
+                                agentToPurchaseFrom.vendCostTrend = 'up'
+                        
+                        agentToPurchaseFrom.totalVendingCost += agentToPurchaseFrom.vendCost 
+                        agentToPurchaseFrom.reward += (agentToPurchaseFrom.vendingPrice - self.env_wholesale_price - agentToPurchaseFrom.vendCost)
                     else:
                         # print(f'consumer money was {consumer.money}, setting to 0')
                         consumer.money = 0
@@ -357,7 +390,15 @@ def _get_info(self, agent , i):
         """
             accepts an Agent, and returns its info object
         """
-        return {"agent_num": i, "price": agent.vendingPrice, "sold": agent.quantitySold, "reward": agent.reward}
+        return {
+            "agent_num": i,
+            "price": agent.vendingPrice,
+            "sold": agent.quantitySold,
+            "reward": agent.reward,
+            "vendCost": agent.vendCost,
+            "totalVendingCost": agent.totalVendingCost
+            
+        }
 
 
     def render(self) -> None:
@@ -383,6 +424,8 @@ def reset(self): #-> float:
             self.policy_agents[i].done = False
             self.policy_agents[i].vendingPrice = 0
             self.policy_agents[i].quantitySold = 0
+            self.policy_agents[i].vendCost = 1
+
         
         self.consumerMoneyEach = self.consumer_total_money_per_turn / self.num_consumers
         # for i in range(len(self.consumers_arr)):

From 964ea861a2c610f9bfe35753f1d4165350f8e4bf Mon Sep 17 00:00:00 2001
From: Liam Laverty <liamlaverty@gmail.com>
Date: Fri, 19 May 2023 06:34:00 +0100
Subject: [PATCH 40/40] models for bertrand comp

---
 Macodiac.ML/.gitignore | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Macodiac.ML/.gitignore b/Macodiac.ML/.gitignore
index 89a982f..32fa32c 100644
--- a/Macodiac.ML/.gitignore
+++ b/Macodiac.ML/.gitignore
@@ -8,6 +8,8 @@ training_multiagent/results/saved_models/*
 !training_multiagent/results/saved_models/.keep
 training_multiagent_16may/results/logs/*
 training_multiagent_16may/results/saved_models/*
+training_multiagent/results_price_setter/logs/*
+training_multiagent/results_price_setter/saved_models/*
 
 
 # Byte-compiled / optimized / DLL files