From 57b70c5b678bd38f0b0a6006bde692ca0f78eb1d Mon Sep 17 00:00:00 2001 From: joe-justin369 Date: Tue, 21 Oct 2025 08:53:39 +0800 Subject: [PATCH 01/10] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E9=83=A8=E5=88=86?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=EF=BC=8C=E5=B9=B6=E6=88=90=E5=8A=9F=E8=BF=90?= =?UTF-8?q?=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/Environment.py | 373 ++++++++++++++++++ .../src/Hyperparameters.py | 69 ++++ src/Unmanned_vehicle_AD_DQN/src/Main.py | 153 +++++++ src/Unmanned_vehicle_AD_DQN/src/Model.py | 215 ++++++++++ src/Unmanned_vehicle_AD_DQN/src/Test.py | 77 ++++ 5 files changed, 887 insertions(+) create mode 100644 src/Unmanned_vehicle_AD_DQN/src/Environment.py create mode 100644 src/Unmanned_vehicle_AD_DQN/src/Hyperparameters.py create mode 100644 src/Unmanned_vehicle_AD_DQN/src/Main.py create mode 100644 src/Unmanned_vehicle_AD_DQN/src/Model.py create mode 100644 src/Unmanned_vehicle_AD_DQN/src/Test.py diff --git a/src/Unmanned_vehicle_AD_DQN/src/Environment.py b/src/Unmanned_vehicle_AD_DQN/src/Environment.py new file mode 100644 index 0000000000..eceba1b10e --- /dev/null +++ b/src/Unmanned_vehicle_AD_DQN/src/Environment.py @@ -0,0 +1,373 @@ +import glob +import os +import sys +import random +import time +import numpy as np +import cv2 +import math +from Hyperparameters import * + +avg_score = 0 +average_reward = 0 + +# 如果已使用 pip 安装 carla，则无需加载 egg +# try: +# sys.path.append(glob.glob('../carla/dist/carla-*%d.%d-%s.egg' % ( +# sys.version_info.major, +# sys.version_info.minor, +# 'win-amd64' if os.name == 'nt' else 'linux-x86_64'))[0]) +# except IndexError: +# pass + +import carla +from carla import ColorConverter + + +class CarEnv: + SHOW_CAM = SHOW_PREVIEW + # STEER_AMT = 1.0 + im_width = IM_WIDTH + im_height = IM_HEIGHT + + # front_camera = None + + def __init__(self): + self.actor_list = None + self.sem_cam = None + self.client = carla.Client("localhost", 2000) + self.client.set_timeout(20.0) + self.front_camera = None + + # self.world = self.client.get_world() + self.world = self.client.load_world('Town03') + + self.blueprint_library = self.world.get_blueprint_library() + self.model_3 = self.blueprint_library.filter("model3")[0] + + self.walker_list = [] + self.collision_history = [] + + self.slow_counter = 0 + + def spawn_pedestrians_general(self, number, isCross): + for i in range(number): + isLeft = random.choice([True, False]) + if isLeft: + self.spawn_pedestrians_left(isCross) + else: + self.spawn_pedestrians_right(isCross) + + def spawn_pedestrians_right(self, isCross): + + blueprints_walkers = self.world.get_blueprint_library().filter("walker.pedestrian.*") + walker_bp = random.choice(blueprints_walkers) + + # global walker_list + + for i in range(1): + walker_bp = random.choice(blueprints_walkers) + + min_x = -50 + max_x = 140 + min_y = -188 + max_y = -183 + + if isCross: + isFirstCross = random.choice([True, False]) + if isFirstCross: + min_x = -14 + max_x = -10.5 + else: + min_x = 17 + max_x = 20.5 + + # Randomly select the position for the pedestrian + x = random.uniform(min_x, max_x) + y = random.uniform(min_y, max_y) + + spawn_point = carla.Transform(carla.Location(x, y, 2.0)) + + while (-10 < spawn_point.location.x < 17) or (70 < spawn_point.location.x < 100): + x = random.uniform(min_x, max_x) + y = random.uniform(min_y, max_y) + spawn_point = carla.Transform(carla.Location(x, y, 2.0)) + + if spawn_point: + npc = self.world.try_spawn_actor(walker_bp, spawn_point) + + if npc is not None: + ped_control = carla.WalkerControl() + ped_control.speed = random.uniform(0.5, 1.0) + ped_control.direction.y = -1 + ped_control.direction.x = 0.15 + npc.apply_control(ped_control) + npc.set_simulate_physics(True) + # self.walker_list.append(npc) + + def spawn_pedestrians_left(self, isCross): + + blueprints_walkers = self.world.get_blueprint_library().filter("walker.pedestrian.*") + walker_bp = random.choice(blueprints_walkers) + + # global walker_list + + for i in range(1): + walker_bp = random.choice(blueprints_walkers) + # spawn_points = self.world.get_map().get_spawn_points() # Assuming spawn_points is defined elsewhere + # npc = self.world.try_spawn_actor(walker_bp, random.choice(spawn_points)) + + min_x = -50 + max_x = 140 + min_y = -216 + max_y = -210 + + if (isCross): + isFirstCross = random.choice([True, False]) + if isFirstCross: + min_x = -14 + max_x = -10.5 + else: + min_x = 17 + max_x = 20.5 + + # Randomly select the position for the pedestrian + x = random.uniform(min_x, max_x) + y = random.uniform(min_y, max_y) + + spawn_point = carla.Transform(carla.Location(x, y, 2.0)) + + while (-10 < spawn_point.location.x < 17) or (70 < spawn_point.location.x < 100): + x = random.uniform(min_x, max_x) + y = random.uniform(min_y, max_y) + spawn_point = carla.Transform(carla.Location(x, y, 2.0)) + + if spawn_point: + npc = self.world.try_spawn_actor(walker_bp, spawn_point) + + if npc is not None: + ped_control = carla.WalkerControl() + ped_control.speed = random.uniform(0.7, 1.3) + ped_control.direction.y = 1 + ped_control.direction.x = -0.05 + npc.apply_control(ped_control) + npc.set_simulate_physics(True) + # self.walker_list.append(npc) + + def reset(self): + + walkers = self.world.get_actors().filter('walker.*') + for walker in walkers: + walker.destroy() + + vehicles = self.world.get_actors().filter('vehicle.*') + for v in vehicles: + v.destroy() + + self.spawn_pedestrians_general(30, True) + self.spawn_pedestrians_general(10, False) + + self.collision_history = [] + self.actor_list = [] + + # self.isHit = False + self.slow_counter = 0 + + # self.transform = random.choice(self.world.get_map().get_spawn_points()) + + spawn_points = self.world.get_map().get_spawn_points() + spawn_point = random.choice(spawn_points) if spawn_points else carla.Transform() + spawn_point.location.x = -81.0 + spawn_point.location.y = -195.0 + spawn_point.location.z += 2.0 + spawn_point.rotation.roll = 0.0 + spawn_point.rotation.pitch = 0.0 + spawn_point.rotation.yaw = 0.0 + self.vehicle = self.world.spawn_actor(self.model_3, spawn_point) + + # self.vehicle = self.world.spawn_actor(self.model_3, self.transform) + self.actor_list.append(self.vehicle) + + # self.rgb_cam = self.blueprint_library.find('sensor.camera.rgb') + # self.rgb_cam.set_attribute("image_size_x", f"{self.im_width}") + # self.rgb_cam.set_attribute("image_size_y", f"{self.im_height}") + # self.rgb_cam.set_attribute("fov", f"110") + self.sem_cam = self.blueprint_library.find('sensor.camera.semantic_segmentation') + self.sem_cam.set_attribute("image_size_x", f"{self.im_width}") + self.sem_cam.set_attribute("image_size_y", f"{self.im_height}") + self.sem_cam.set_attribute("fov", f"110") + + transform = carla.Transform(carla.Location(x=2.5, z=0.7)) + # self.sensor = self.world.spawn_actor(self.rgb_cam, transform, attach_to=self.vehicle) + self.sensor = self.world.spawn_actor(self.sem_cam, transform, attach_to=self.vehicle) + self.actor_list.append(self.sensor) + self.sensor.listen(lambda data: self.process_img(data)) + + self.vehicle.apply_control(carla.VehicleControl(throttle=0.0, brake=0.0, steer=0.0)) + time.sleep(4) + + colsensor = self.blueprint_library.find("sensor.other.collision") + self.colsensor = self.world.spawn_actor(colsensor, transform, attach_to=self.vehicle) + self.actor_list.append(self.colsensor) + self.colsensor.listen(lambda event: self.collision_data(event)) + + while self.front_camera is None: + time.sleep(0.01) + + self.episode_start = time.time() + self.vehicle.apply_control(carla.VehicleControl(throttle=0.0, brake=0.0, steer=0.0)) + + return self.front_camera + + def collision_data(self, event): + # self.isHit = True + self.collision_history.append(event) + + def process_img(self, image): + + image.convert(carla.ColorConverter.CityScapesPalette) + + processed_image = np.array(image.raw_data) + processed_image = processed_image.reshape((self.im_height, self.im_width, 4)) + processed_image = processed_image[:, :, :3] + + if self.SHOW_CAM: + cv2.imshow("", processed_image) + cv2.waitKey(1) + + self.front_camera = processed_image + + def reward(self): + reward = 0 + done = False + + velocity = self.vehicle.get_velocity() + velocity_kmh = int(3.6 * math.sqrt(velocity.x ** 2 + velocity.y ** 2 + velocity.z ** 2)) + + distances = [] + walkers = self.world.get_actors().filter('walker.*') + for walker in walkers: + player_transform = walker.get_transform() + ped_location = player_transform.location + player_direction = walker.get_control().direction + if ped_location.y < -214 and player_direction.y == -1: + walker.destroy() + continue + elif ped_location.y > -191 and player_direction.y == 1: + walker.destroy() + continue + dx = self.vehicle.get_location().x - ped_location.x + dy = self.vehicle.get_location().y - ped_location.y + distance = math.sqrt((dx * dx) + (dy * dy)) + distances.append(distance) + + min_dist = min(distances) + + if len(self.collision_history) != 0: + reward = -5 + done = True + elif min_dist < 4: + reward = -2 + done = False + elif velocity_kmh == 0: + reward += -1 + done = False + elif 15 < velocity_kmh < 25: + reward += 1 + done = False + elif 35 < velocity_kmh < 45: + reward += 2 + done = False + + if self.vehicle.get_location().x > 155: + done = True + + return reward, done + + # def reward(self): +# + # reward = 0 + # done = False +# + # velocity = self.vehicle.get_velocity() + # velocity_kmh = int(3.6 * math.sqrt(velocity.x ** 2 + velocity.y ** 2 + velocity.z ** 2)) +# + # if velocity_kmh == 0: + # self.slow_counter += 1 + # else: + # self.slow_counter = 0 +# + # # if len(self.collision_history) != 0: + # # reward += -200 * len(self.collision_history) + # # self.collision_history = [] + # # done = True + # # elif velocity_kmh < 30: + # # reward += -1 + # # done = False + # # else: + # # reward += 1 + # # done = False +# + # if len(self.collision_history) != 0: + # reward += -300 * len(self.collision_history) + # self.collision_history = [] + # done = True + # elif velocity_kmh == 0: + # reward += -1 + # done = False + # elif 15 < velocity_kmh < 25: + # reward += 1 + # elif 35 < velocity_kmh < 45: + # reward += 2 + # done = False +# + # walkers = self.world.get_actors().filter('walker.*') + # for walker in walkers: +# + # player_transform = walker.get_transform() + # ped_location = player_transform.location + # player_direction = walker.get_control().direction +# + # if ped_location.y < -214 and player_direction.y == -1: + # walker.destroy() + # continue + # elif ped_location.y > -191 and player_direction.y == 1: + # walker.destroy() + # continue +# + # dx = self.vehicle.get_location().x - ped_location.x + # dy = self.vehicle.get_location().y - ped_location.y + # distance = math.sqrt((dx * dx) + (dy * dy)) +# + # # If the distance is lower than 8, give a negative reward of -20 + # if distance < 5: + # reward -= 2 +# + # # If there is a collision (vehicle position matches pedestrian position), give a negative reward of -200 + # if distance < 2: + # reward -= 25 +# + # reward += (80 / (166 - self.vehicle.get_location().x))**2 +# + # if self.vehicle.get_location().x > 155 or self.slow_counter == SLOW_COUNTER: # or self.episode_start + SECONDS_PER_EPISODE < time.time(): + # done = True +# + # return reward, done + + def step(self, action): + if action == 0: + # self.vehicle.apply_control(carla.VehicleControl(throttle=0.0, brake=0.6, steer=0.0)) + velocity = carla.Vector3D(x=0.00, y=0.0, z=0.0) + self.vehicle.set_target_velocity(velocity) + elif action == 1: + # self.vehicle.apply_control(carla.VehicleControl(throttle=0.0, brake=0.0, steer=0.0)) + velocity = carla.Vector3D(x=6.5, y=0.0, z=0.0) + self.vehicle.set_target_velocity(velocity) + elif action == 2: + # self.vehicle.apply_control(carla.VehicleControl(throttle=0.6, brake=0.0, steer=0.0)) + velocity = carla.Vector3D(x=12, y=0.0, z=0.0) + self.vehicle.set_target_velocity(velocity) + + reward, done = self.reward() + + return self.front_camera, reward, done, None \ No newline at end of file diff --git a/src/Unmanned_vehicle_AD_DQN/src/Hyperparameters.py b/src/Unmanned_vehicle_AD_DQN/src/Hyperparameters.py new file mode 100644 index 0000000000..923f4acf02 --- /dev/null +++ b/src/Unmanned_vehicle_AD_DQN/src/Hyperparameters.py @@ -0,0 +1,69 @@ +DISCOUNT = 0.99 +# Discount factor for future rewards in the RL algorithm. + +FPS = 60 +# Frames per second in the simulation. + +MEMORY_FRACTION = 0.35 +# Fraction of GPU memory allocated for training. + +REWARD_OFFSET = -100 +# Stops the simulation when reached + +MIN_REPLAY_MEMORY_SIZE = 1_000 +# Minimum size of the replay memory before training starts. + +REPLAY_MEMORY_SIZE = 5_000 +# Maximum capacity of the replay memory. + +MINIBATCH_SIZE = 64 +# Number of experiences sampled from the replay memory for each training iteration. + +PREDICTION_BATCH_SIZE = 1 +# Batch size used during the prediction phase. + +TRAINING_BATCH_SIZE = MINIBATCH_SIZE // 8 +# Batch size used during the training phase. + +EPISODES = 451 +# Number of episodes the agent will train on. + +# SECONDS_PER_EPISODE = 60 +SECONDS_PER_EPISODE = 45 +# Duration of each episode in seconds. + +MIN_EPSILON = 0.1 +EPSILON = 1.0 +# Exploration rates for the epsilon-greedy exploration strategy. + +EPSILON_DECAY = 0.9975 +# EPSILON_DECAY = 0.993 +# Decay rate of the exploration rate over time. + +MODEL_NAME = "YY" +# Name or identifier for the trained model.F + +MIN_REWARD = 100 +# MIN_REWARD = -1000 +# Minimum reward required for an experience to be considered "good" or "positive." + +UPDATE_TARGET_EVERY = 5 +# Frequency at which the target network is updated. + +AGGREGATE_STATS_EVERY = 10 +# Frequency at which statistics (e.g., average scores, rewards) are computed and aggregated. + +SHOW_PREVIEW = False +# Determines whether to show a preview window or not. + +IM_WIDTH = 640 +# Width of the image captured in the preview or simulation. + +IM_HEIGHT = 480 +# Height of the image captured in the preview or simulation. + +SLOW_COUNTER = 330 + +LOW_REWARD_THRESHOLD = -4 + +SUCCESSFUL_THRESHOLD = 1 diff --git a/src/Unmanned_vehicle_AD_DQN/src/Main.py b/src/Unmanned_vehicle_AD_DQN/src/Main.py new file mode 100644 index 0000000000..e661cabbb9 --- /dev/null +++ b/src/Unmanned_vehicle_AD_DQN/src/Main.py @@ -0,0 +1,153 @@ +import glob +import os +import sys +import random +import time +import numpy as np +import cv2 +import math +import matplotlib.pyplot as plt +from collections import deque +from tensorflow.keras.applications.xception import Xception +from tensorflow.keras.layers import Dense, GlobalAveragePooling2D +from tensorflow.keras.models import Sequential, Model +from tensorflow.keras.layers import Dense, GlobalAveragePooling2D, Input, Concatenate, Conv2D, AveragePooling2D, Activation, \ + Flatten +from tensorflow.keras.optimizers import Adam +from tensorflow.keras.models import Model +from tensorflow.keras.callbacks import TensorBoard +import tensorflow as tf +import tensorflow.keras.backend as backend +from threading import Thread + +from tqdm import tqdm + +import Hyperparameters +from Environment import * +from Model import * +from Hyperparameters import * + +if __name__ == '__main__': + + FPS = 60 + # For stats + ep_rewards = [-200] + + # For more repetitive results + # random.seed(1) + # np.random.seed(1) + # tf.compat.v1.set_random_seed(1) + + # Memory fraction, used mostly when training multiple agents + gpu_options = tf.compat.v1.GPUOptions(per_process_gpu_memory_fraction=MEMORY_FRACTION) + tf.compat.v1.keras.backend.set_session( + tf.compat.v1.Session(config=tf.compat.v1.ConfigProto(gpu_options=gpu_options))) + + # Create models folder + if not os.path.isdir('models'): + os.makedirs('models') + + # Create agent and environment + agent = DQNAgent() + env = CarEnv() + + # Start training thread and wait for training to be initialized + trainer_thread = Thread(target=agent.train_in_loop, daemon=True) + trainer_thread.start() + while not agent.training_initialized: + time.sleep(0.01) + + agent.get_qs(np.ones((env.im_height, env.im_width, 3))) + + # Iterate over episodes + epds = [] + scores = [] + avg_scores = [] + for episode in tqdm(range(1, EPISODES + 1), ascii=True, unit='episodes'): + + env.collision_hist = [] + agent.tensorboard.step = episode + + # Restarting episode - reset episode reward and step number + score = 0 + step = 1 + + + # Reset environment and get initial state + current_state = env.reset() + + # Reset flag and start iterating until episode ends + done = False + episode_start = time.time() + + # Play for given number of seconds only + while True: + + # This part stays mostly the same, the change is to query a model for Q values + if np.random.random() > Hyperparameters.EPSILON: + # Get action from Q table + qs = agent.get_qs(current_state) + action = np.argmax(qs) + print(f'Action: [{qs[0]:>5.2f}, {qs[1]:>5.2f}, {qs[2]:>5.2f}] {action}') + else: + # Get random action + action = np.random.randint(0, 3) + # This takes no time, so we add a delay matching 60 FPS (prediction above takes longer) + time.sleep(1 / FPS) + + new_state, reward, done, _ = env.step(action) + + # Transform new continuous state to new discrete state and count reward + score += reward + + if score < REWARD_OFFSET: + done = True + + # Every step we update replay memory + agent.update_replay_memory((current_state, action, reward, new_state, done)) + + current_state = new_state + step += 1 + + if done: + break + + # End of episode - destroy agents + for actor in env.actor_list: + actor.destroy() + + scores.append(score) + avg_scores.append(np.mean(scores[-10:])) + + if not episode % AGGREGATE_STATS_EVERY or episode == 1: + average_reward = np.mean(scores[-AGGREGATE_STATS_EVERY:]) + min_reward = min(scores[-AGGREGATE_STATS_EVERY:]) + max_reward = max(scores[-AGGREGATE_STATS_EVERY:]) + agent.tensorboard.update_stats(reward_avg=average_reward, reward_min=min_reward, reward_max=max_reward, + epsilon=Hyperparameters.EPSILON) + + # Save model, but only when min reward is greater or equal a set value + if min_reward >= MIN_REWARD and (episode not in epds): + agent.model.save( + f'models/{MODEL_NAME}__{max_reward:_>7.2f}max_{avg_score:_>7.2f}avg_{min_reward:_>7.2f}min__{int(time.time())}.model') + + epds.append(episode) + print('episode: ', episode, 'score %.2f' % score) + # Decay epsilon + if Hyperparameters.EPSILON > Hyperparameters.MIN_EPSILON: + Hyperparameters.EPSILON *= Hyperparameters.EPSILON_DECAY + Hyperparameters.EPSILON = max(Hyperparameters.MIN_EPSILON, Hyperparameters.EPSILON) + + # Set termination flag for training thread and wait for it to finish + agent.terminate = True + trainer_thread.join() + agent.model.save( + f'models/{MODEL_NAME}__{max_reward:_>7.2f}max_{avg_score:_>7.2f}avg_{min_reward:_>7.2f}min__{int(time.time())}.model') + + fig = plt.figure() + ax = fig.add_subplot(111) + plt.plot(scores) + plt.plot(avg_scores) + plt.ylabel('Score') + plt.xlabel('Episode #') + plt.show() \ No newline at end of file diff --git a/src/Unmanned_vehicle_AD_DQN/src/Model.py b/src/Unmanned_vehicle_AD_DQN/src/Model.py new file mode 100644 index 0000000000..0ac337d395 --- /dev/null +++ b/src/Unmanned_vehicle_AD_DQN/src/Model.py @@ -0,0 +1,215 @@ +import glob +import os +import sys +import random +import time +import numpy as np +import cv2 +import math +import matplotlib.pyplot as plt +from collections import deque +from tensorflow.keras.applications.xception import Xception +from tensorflow.keras.layers import Dense, GlobalAveragePooling2D +from tensorflow.keras.models import Sequential, Model +from tensorflow.keras.layers import Dense, GlobalAveragePooling2D, Input, Concatenate, Conv2D, AveragePooling2D, Activation, \ + Flatten, Dropout, BatchNormalization +from tensorflow.keras.optimizers import Adam +from tensorflow.keras.models import Model +from tensorflow.keras.callbacks import TensorBoard +import tensorflow as tf +import tensorflow.keras.backend as backend +from threading import Thread +from Environment import * + + +# Own Tensorboard class +class ModifiedTensorBoard(TensorBoard): + + # Overriding init to set initial step and writer (we want one log file for all .fit() calls) + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._log_write_dir = self.log_dir + self.step = 1 + self.writer = self.writer = tf.summary.create_file_writer(self.log_dir) + + # Overriding this method to stop creating default log writer + def set_model(self, model): + self.model = model + # + self._train_dir = os.path.join(self._log_write_dir, 'train') + self._train_step = self.model._train_counter + # + self._val_dir = os.path.join(self._log_write_dir, 'validation') + self._val_step = self.model._test_counter + # + self._should_write_train_graph = False + + # Overrided, saves logs with our step number + # (otherwise every .fit() will start writing from 0th step) + def on_epoch_end(self, epoch, logs=None): + self.update_stats(**logs) + + # Overrided + # We train for one batch only, no need to save anything at epoch end + def on_batch_end(self, batch, logs=None): + pass + + # Overrided, so won't close writer + def on_train_end(self, logs=None): + pass + + # Custom method for saving own metrics + # Creates writer, writes custom metrics and closes writer + def update_stats(self, **stats): + with self.writer.as_default(): + for key, value in stats.items(): + tf.summary.scalar(key, value, step=self.step) + self.writer.flush() + # self.step = self.step + 10 + + +class DQNAgent: + def __init__(self): + self.model = self.create_model() + self.target_model = self.create_model() + self.target_model.set_weights(self.model.get_weights()) + + self.replay_memory = deque(maxlen=REPLAY_MEMORY_SIZE) + + # Modified tensorboard + self.tensorboard = ModifiedTensorBoard(log_dir=f"logs/{MODEL_NAME}-{int(time.time())}") + self.target_update_counter = 0 + self.graph = tf.compat.v1.get_default_graph() + + self.terminate = False + self.last_logged_episode = 0 + self.training_initialized = False + + def create_model(self): + model = Sequential() + + model.add(Conv2D(64, (3, 3), input_shape=(IM_HEIGHT, IM_WIDTH, 3), padding='same')) + model.add(Activation('relu')) + model.add(AveragePooling2D(pool_size=(5, 5), strides=(3, 3), padding='same')) + + model.add(Conv2D(64, (3, 3), padding='same')) + model.add(Activation('relu')) + model.add(AveragePooling2D(pool_size=(5, 5), strides=(3, 3), padding='same')) + + model.add(Conv2D(64, (3, 3), padding='same')) + model.add(Activation('relu')) + model.add(AveragePooling2D(pool_size=(5, 5), strides=(3, 3), padding='same')) + + model.add(Flatten()) + + model.add(Dense(3, activation='softmax')) # Three output units for binary classification + + # model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) + model.compile(loss="mse", optimizer=Adam(lr=0.001), metrics=["accuracy"]) + return model + + def update_replay_memory(self, transition): + # transition = (current_state, action, reward, new_state, done) + self.replay_memory.append(transition) + + def minibatch_chooser(self): + # Initialize lists to store dangerous and non-dangerous samples + dangerous_samples = [] + non_dangerous_samples = [] + + # Iterate over the replay memory + for sample in self.replay_memory: + current_state, action, reward, new_state, done = sample + + # Check if the reward is considered low + if reward < LOW_REWARD_THRESHOLD: + # Append the sample to the dangerous samples list + dangerous_samples.append(sample) + else: + # Append the sample to the non-dangerous samples list + non_dangerous_samples.append(sample) + + # Shuffle the dangerous and non-dangerous samples + random.shuffle(dangerous_samples) + + successful_counter = 0 + successful_samples = [] + for sample in non_dangerous_samples: + if sample[2] == 2 and successful_counter < SUCCESSFUL_THRESHOLD: + successful_counter += 1 + successful_samples.append(sample) + + random.shuffle(non_dangerous_samples) + + # Determine the number of dangerous and non-dangerous samples to include in the minibatch + num_dangerous_samples = min(len(dangerous_samples), MINIBATCH_SIZE // 8) + remaining_samples = MINIBATCH_SIZE - num_dangerous_samples - successful_counter + + # Select dangerous and non-dangerous samples for the minibatch + minibatch = random.sample(dangerous_samples, num_dangerous_samples) + random.sample(self.replay_memory, + remaining_samples) + successful_samples + random.shuffle(minibatch) + return minibatch + + def train(self): + if len(self.replay_memory) < MIN_REPLAY_MEMORY_SIZE: + return + + minibatch = self.minibatch_chooser() + # minibatch = random.sample(self.replay_memory, MINIBATCH_SIZE) + print([transition[2] for transition in minibatch]) + + current_states = np.array([transition[0] for transition in minibatch]) / 255 + current_qs_list = self.model.predict(current_states, batch_size=PREDICTION_BATCH_SIZE) + + new_current_states = np.array([transition[3] for transition in minibatch]) / 255 + future_qs_list = self.target_model.predict(new_current_states, batch_size=PREDICTION_BATCH_SIZE) + + x = [] + y = [] + + for index, (current_state, action, reward, new_state, done) in enumerate(minibatch): + if not done: + max_future_q = np.max(future_qs_list[index]) + new_q = reward + DISCOUNT * max_future_q + else: + new_q = reward + + current_qs = current_qs_list[index] + current_qs[action] = new_q + + x.append(current_state) + y.append(current_qs) + + log_this_step = False + if self.tensorboard.step > self.last_logged_episode: + log_this_step = True + self.last_logged_episode = self.tensorboard.step + + self.model.fit(np.array(x) / 255, np.array(y), batch_size=TRAINING_BATCH_SIZE, verbose=0, shuffle=False, + callbacks=[self.tensorboard] if log_this_step else None) + + if log_this_step: + self.target_update_counter += 1 + + if self.target_update_counter > UPDATE_TARGET_EVERY: + print("Target is updated") + self.target_model.set_weights(self.model.get_weights()) + self.target_update_counter = 0 + + def train_in_loop(self): + x = np.random.uniform(size=(1, IM_HEIGHT, IM_WIDTH, 3)).astype(np.float32) + y = np.random.uniform(size=(1, 3)).astype(np.float32) + + self.model.fit(x, y, verbose=False, batch_size=1) + self.training_initialized = True + + while True: + if self.terminate: + return + # print("Memory: ", len(self.replay_memory)) + self.train() + time.sleep(0.01) + + def get_qs(self, state): + return self.model.predict(np.array(state).reshape(-1, *state.shape) / 255)[0] diff --git a/src/Unmanned_vehicle_AD_DQN/src/Test.py b/src/Unmanned_vehicle_AD_DQN/src/Test.py new file mode 100644 index 0000000000..84b1bb8e00 --- /dev/null +++ b/src/Unmanned_vehicle_AD_DQN/src/Test.py @@ -0,0 +1,77 @@ +import random +from collections import deque +import numpy as np +import cv2 +import time +import tensorflow as tf +import tensorflow.keras.backend as backend +from tensorflow.keras.models import load_model +from Environment import CarEnv, MEMORY_FRACTION +from Hyperparameters import* + + + +MODEL_PATH = 'models\YY____86.00max____0.00avg___19.00min__1760930742.model' + +if __name__ == '__main__': + + # Memory fraction + gpu_options = tf.compat.v1.GPUOptions(per_process_gpu_memory_fraction=MEMORY_FRACTION) + tf.compat.v1.keras.backend.set_session(tf.compat.v1.Session(config=tf.compat.v1.ConfigProto(gpu_options=gpu_options))) + + # Load the model + model = load_model(MODEL_PATH) + + # Create environment + env = CarEnv() + + # For agent speed measurements - keeps last 60 frametimes + fps_counter = deque(maxlen=60) + + # Initialize predictions - first prediction takes longer as of initialization that has to be done + # It's better to do a first prediction then before we start iterating over episode steps + model.predict(np.ones((1, env.im_height, env.im_width, 3))) + + # Loop over episodes + while True: + + print('Restarting episode') + + # Reset environment and get initial state + current_state = env.reset() + env.collision_hist = [] + + done = False + + # Loop over steps + while True: + + # For FPS counter + step_start = time.time() + + # Show current frame + cv2.imshow(f'Agent - preview', current_state) + cv2.waitKey(1) + + # Predict an action based on current observation space + qs = model.predict(np.array(current_state).reshape(-1, *current_state.shape)/255)[0] + action = np.argmax(qs) + + # Step environment (additional flag informs environment to not break an episode by time limit) + new_state, reward, done, _ = env.step(action) + + # Set current step for next loop iteration + current_state = new_state + + # If done - agent crashed, break an episode + if done: + break + + # Measure step time, append to a deque, then print mean FPS for last 60 frames, q values and taken action + frame_time = time.time() - step_start + fps_counter.append(frame_time) + print(f'Agent: {len(fps_counter)/sum(fps_counter):>4.1f} FPS | Action: [{qs[0]:>5.2f}, {qs[1]:>5.2f}, {qs[2]:>5.2f}] {action}') + + # Destroy an actor at end of episode + for actor in env.actor_list: + actor.destroy() From a478d247b88697be1ae58aafa569a247ab3801e8 Mon Sep 17 00:00:00 2001 From: joe-justin369 Date: Tue, 21 Oct 2025 09:50:57 +0800 Subject: [PATCH 02/10] Move AD_DQN source files from src/ subdirectory to root --- src/Unmanned_vehicle_AD_DQN/{src => }/Environment.py | 0 src/Unmanned_vehicle_AD_DQN/{src => }/Hyperparameters.py | 0 src/Unmanned_vehicle_AD_DQN/{src => }/Main.py | 0 src/Unmanned_vehicle_AD_DQN/{src => }/Model.py | 0 src/Unmanned_vehicle_AD_DQN/{src => }/Test.py | 2 +- 5 files changed, 1 insertion(+), 1 deletion(-) rename src/Unmanned_vehicle_AD_DQN/{src => }/Environment.py (100%) rename src/Unmanned_vehicle_AD_DQN/{src => }/Hyperparameters.py (100%) rename src/Unmanned_vehicle_AD_DQN/{src => }/Main.py (100%) rename src/Unmanned_vehicle_AD_DQN/{src => }/Model.py (100%) rename src/Unmanned_vehicle_AD_DQN/{src => }/Test.py (96%) diff --git a/src/Unmanned_vehicle_AD_DQN/src/Environment.py b/src/Unmanned_vehicle_AD_DQN/Environment.py similarity index 100% rename from src/Unmanned_vehicle_AD_DQN/src/Environment.py rename to src/Unmanned_vehicle_AD_DQN/Environment.py diff --git a/src/Unmanned_vehicle_AD_DQN/src/Hyperparameters.py b/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py similarity index 100% rename from src/Unmanned_vehicle_AD_DQN/src/Hyperparameters.py rename to src/Unmanned_vehicle_AD_DQN/Hyperparameters.py diff --git a/src/Unmanned_vehicle_AD_DQN/src/Main.py b/src/Unmanned_vehicle_AD_DQN/Main.py similarity index 100% rename from src/Unmanned_vehicle_AD_DQN/src/Main.py rename to src/Unmanned_vehicle_AD_DQN/Main.py diff --git a/src/Unmanned_vehicle_AD_DQN/src/Model.py b/src/Unmanned_vehicle_AD_DQN/Model.py similarity index 100% rename from src/Unmanned_vehicle_AD_DQN/src/Model.py rename to src/Unmanned_vehicle_AD_DQN/Model.py diff --git a/src/Unmanned_vehicle_AD_DQN/src/Test.py b/src/Unmanned_vehicle_AD_DQN/Test.py similarity index 96% rename from src/Unmanned_vehicle_AD_DQN/src/Test.py rename to src/Unmanned_vehicle_AD_DQN/Test.py index 84b1bb8e00..7117ff4c49 100644 --- a/src/Unmanned_vehicle_AD_DQN/src/Test.py +++ b/src/Unmanned_vehicle_AD_DQN/Test.py @@ -11,7 +11,7 @@ -MODEL_PATH = 'models\YY____86.00max____0.00avg___19.00min__1760930742.model' +MODEL_PATH = 'models\YY___153.00max____0.00avg__153.00min__1761010159.model' if __name__ == '__main__': From ab55863dcdbbc0aa28eaac6d548e9016aab6662c Mon Sep 17 00:00:00 2001 From: joe-justin369 Date: Mon, 27 Oct 2025 08:47:52 +0800 Subject: [PATCH 03/10] =?UTF-8?q?=E9=87=8D=E5=91=BD=E5=90=8DMain.py?= =?UTF-8?q?=E4=B8=BAmain.py=E5=B9=B6=E6=9B=B4=E6=96=B0README?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Unmanned_vehicle_AD_DQN/README.md | 27 +++++++++++++++++++ .../{Main.py => main.py} | 0 2 files changed, 27 insertions(+) rename src/Unmanned_vehicle_AD_DQN/{Main.py => main.py} (100%) diff --git a/src/Unmanned_vehicle_AD_DQN/README.md b/src/Unmanned_vehicle_AD_DQN/README.md index a329c8ad3f..7921a2bbd9 100644 --- a/src/Unmanned_vehicle_AD_DQN/README.md +++ b/src/Unmanned_vehicle_AD_DQN/README.md @@ -1,2 +1,29 @@ +# 基于DQN实现自动驾驶模型 + 这是深度 Q 网络 （DQN） 算法的实现，用于在 CARLA 中训练自动驾驶模型 +## 环境配置 + +1. python版本3.7.9 +2. 确保将carla在运行前配置到系统环境中 + +### carla环境配置 + +1. .egg文件的API请修改Environment.py文件中，删除位于import carla上方注释的# +2. .whl文件的API请在系统中下载对应文件，该部分的两种文件任选其一即可 + +## 使用步骤 + +确保carla在代码运行之前已经启动 + +### 训练 +训练模型请运行： + +``` +Main.py +``` +### 测试 +测试训练后的模型，请运行： +``` +Test.py +``` diff --git a/src/Unmanned_vehicle_AD_DQN/Main.py b/src/Unmanned_vehicle_AD_DQN/main.py similarity index 100% rename from src/Unmanned_vehicle_AD_DQN/Main.py rename to src/Unmanned_vehicle_AD_DQN/main.py From 15faa98106b6e0d8541a36dcef0b8a59caf3aa77 Mon Sep 17 00:00:00 2001 From: joe-justin369 Date: Mon, 27 Oct 2025 09:58:17 +0800 Subject: [PATCH 04/10] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E6=89=80=E9=9C=80=E4=BE=9D=E8=B5=96=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Unmanned_vehicle_AD_DQN/requirements.txt | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 src/Unmanned_vehicle_AD_DQN/requirements.txt diff --git a/src/Unmanned_vehicle_AD_DQN/requirements.txt b/src/Unmanned_vehicle_AD_DQN/requirements.txt new file mode 100644 index 0000000000..33f0018cf2 --- /dev/null +++ b/src/Unmanned_vehicle_AD_DQN/requirements.txt @@ -0,0 +1,6 @@ +carla==0.9.12 +numpy==1.19.5 +tensorflow==2.6.0 +tqdm==4.67.1 +matplotlib==3.5.3 +opencv-python==4.12.0.88 \ No newline at end of file From c8a4b97f0decc8b9ca60aab3ae1127dd66c034c0 Mon Sep 17 00:00:00 2001 From: joe-justin369 Date: Mon, 17 Nov 2025 10:55:14 +0800 Subject: [PATCH 05/10] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E6=A0=B8=E5=BF=83DQN?= =?UTF-8?q?=E7=AE=97=E6=B3=95=E6=96=87=E4=BB=B6=EF=BC=9A=E7=8E=AF=E5=A2=83?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E3=80=81=E8=B6=85=E5=8F=82=E6=95=B0=E3=80=81?= =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E3=80=81=E6=B5=8B=E8=AF=95=E5=92=8C=E4=B8=BB?= =?UTF-8?q?=E7=A8=8B=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Unmanned_vehicle_AD_DQN/Environment.py | 235 ++++++------------ .../Hyperparameters.py | 37 +-- src/Unmanned_vehicle_AD_DQN/Main.py | 153 ------------ src/Unmanned_vehicle_AD_DQN/Test.py | 10 +- src/Unmanned_vehicle_AD_DQN/main.py | 153 ------------ 5 files changed, 102 insertions(+), 486 deletions(-) delete mode 100644 src/Unmanned_vehicle_AD_DQN/Main.py delete mode 100644 src/Unmanned_vehicle_AD_DQN/main.py diff --git a/src/Unmanned_vehicle_AD_DQN/Environment.py b/src/Unmanned_vehicle_AD_DQN/Environment.py index eceba1b10e..34e8ab7a82 100644 --- a/src/Unmanned_vehicle_AD_DQN/Environment.py +++ b/src/Unmanned_vehicle_AD_DQN/Environment.py @@ -1,3 +1,4 @@ +# Environment.py import glob import os import sys @@ -11,27 +12,15 @@ avg_score = 0 average_reward = 0 -# 如果已使用 pip 安装 carla，则无需加载 egg -# try: -# sys.path.append(glob.glob('../carla/dist/carla-*%d.%d-%s.egg' % ( -# sys.version_info.major, -# sys.version_info.minor, -# 'win-amd64' if os.name == 'nt' else 'linux-x86_64'))[0]) -# except IndexError: -# pass - import carla from carla import ColorConverter class CarEnv: SHOW_CAM = SHOW_PREVIEW - # STEER_AMT = 1.0 im_width = IM_WIDTH im_height = IM_HEIGHT - # front_camera = None - def __init__(self): self.actor_list = None self.sem_cam = None @@ -39,15 +28,12 @@ def __init__(self): self.client.set_timeout(20.0) self.front_camera = None - # self.world = self.client.get_world() self.world = self.client.load_world('Town03') - self.blueprint_library = self.world.get_blueprint_library() self.model_3 = self.blueprint_library.filter("model3")[0] self.walker_list = [] self.collision_history = [] - self.slow_counter = 0 def spawn_pedestrians_general(self, number, isCross): @@ -59,12 +45,9 @@ def spawn_pedestrians_general(self, number, isCross): self.spawn_pedestrians_right(isCross) def spawn_pedestrians_right(self, isCross): - blueprints_walkers = self.world.get_blueprint_library().filter("walker.pedestrian.*") walker_bp = random.choice(blueprints_walkers) - # global walker_list - for i in range(1): walker_bp = random.choice(blueprints_walkers) @@ -82,7 +65,6 @@ def spawn_pedestrians_right(self, isCross): min_x = 17 max_x = 20.5 - # Randomly select the position for the pedestrian x = random.uniform(min_x, max_x) y = random.uniform(min_y, max_y) @@ -103,19 +85,13 @@ def spawn_pedestrians_right(self, isCross): ped_control.direction.x = 0.15 npc.apply_control(ped_control) npc.set_simulate_physics(True) - # self.walker_list.append(npc) def spawn_pedestrians_left(self, isCross): - blueprints_walkers = self.world.get_blueprint_library().filter("walker.pedestrian.*") walker_bp = random.choice(blueprints_walkers) - # global walker_list - for i in range(1): walker_bp = random.choice(blueprints_walkers) - # spawn_points = self.world.get_map().get_spawn_points() # Assuming spawn_points is defined elsewhere - # npc = self.world.try_spawn_actor(walker_bp, random.choice(spawn_points)) min_x = -50 max_x = 140 @@ -131,7 +107,6 @@ def spawn_pedestrians_left(self, isCross): min_x = 17 max_x = 20.5 - # Randomly select the position for the pedestrian x = random.uniform(min_x, max_x) y = random.uniform(min_y, max_y) @@ -152,10 +127,8 @@ def spawn_pedestrians_left(self, isCross): ped_control.direction.x = -0.05 npc.apply_control(ped_control) npc.set_simulate_physics(True) - # self.walker_list.append(npc) def reset(self): - walkers = self.world.get_actors().filter('walker.*') for walker in walkers: walker.destroy() @@ -164,41 +137,32 @@ def reset(self): for v in vehicles: v.destroy() + # 课程学习 - 根据训练阶段调整难度 self.spawn_pedestrians_general(30, True) self.spawn_pedestrians_general(10, False) self.collision_history = [] self.actor_list = [] - - # self.isHit = False self.slow_counter = 0 - # self.transform = random.choice(self.world.get_map().get_spawn_points()) - spawn_points = self.world.get_map().get_spawn_points() spawn_point = random.choice(spawn_points) if spawn_points else carla.Transform() spawn_point.location.x = -81.0 spawn_point.location.y = -195.0 - spawn_point.location.z += 2.0 + spawn_point.location.z = 2.0 spawn_point.rotation.roll = 0.0 spawn_point.rotation.pitch = 0.0 spawn_point.rotation.yaw = 0.0 + self.vehicle = self.world.spawn_actor(self.model_3, spawn_point) - - # self.vehicle = self.world.spawn_actor(self.model_3, self.transform) self.actor_list.append(self.vehicle) - # self.rgb_cam = self.blueprint_library.find('sensor.camera.rgb') - # self.rgb_cam.set_attribute("image_size_x", f"{self.im_width}") - # self.rgb_cam.set_attribute("image_size_y", f"{self.im_height}") - # self.rgb_cam.set_attribute("fov", f"110") self.sem_cam = self.blueprint_library.find('sensor.camera.semantic_segmentation') self.sem_cam.set_attribute("image_size_x", f"{self.im_width}") self.sem_cam.set_attribute("image_size_y", f"{self.im_height}") self.sem_cam.set_attribute("fov", f"110") transform = carla.Transform(carla.Location(x=2.5, z=0.7)) - # self.sensor = self.world.spawn_actor(self.rgb_cam, transform, attach_to=self.vehicle) self.sensor = self.world.spawn_actor(self.sem_cam, transform, attach_to=self.vehicle) self.actor_list.append(self.sensor) self.sensor.listen(lambda data: self.process_img(data)) @@ -220,11 +184,9 @@ def reset(self): return self.front_camera def collision_data(self, event): - # self.isHit = True self.collision_history.append(event) def process_img(self, image): - image.convert(carla.ColorConverter.CityScapesPalette) processed_image = np.array(image.raw_data) @@ -243,131 +205,90 @@ def reward(self): velocity = self.vehicle.get_velocity() velocity_kmh = int(3.6 * math.sqrt(velocity.x ** 2 + velocity.y ** 2 + velocity.z ** 2)) - - distances = [] + + # 获取车辆位置和前进方向 + vehicle_location = self.vehicle.get_location() + vehicle_rotation = self.vehicle.get_transform().rotation.yaw + + # 计算距离终点的进度奖励 + progress_reward = (vehicle_location.x + 81) / 236.0 # 从-81到155，总共236单位 + + # 速度奖励 - 更加平滑 + if velocity_kmh == 0: + reward -= 0.5 # 停车惩罚减少 + elif 20 <= velocity_kmh <= 40: # 理想速度区间 + reward += 0.8 + elif 10 <= velocity_kmh < 20 or 40 < velocity_kmh <= 50: + reward += 0.3 # 可接受速度区间 + else: + reward -= 0.2 # 不理想速度 + + # 方向奖励 - 确保车辆朝正确方向行驶 + if -45 <= vehicle_rotation <= 45: # 大致朝东方向 + reward += 0.2 + else: + reward -= 0.5 + + # 行人距离检测 + min_dist = float('inf') walkers = self.world.get_actors().filter('walker.*') for walker in walkers: - player_transform = walker.get_transform() - ped_location = player_transform.location + ped_location = walker.get_location() + dx = vehicle_location.x - ped_location.x + dy = vehicle_location.y - ped_location.y + distance = math.sqrt(dx**2 + dy**2) + min_dist = min(min_dist, distance) + + # 清理边界外的行人 player_direction = walker.get_control().direction - if ped_location.y < -214 and player_direction.y == -1: + if (ped_location.y < -214 and player_direction.y == -1) or \ + (ped_location.y > -191 and player_direction.y == 1): walker.destroy() - continue - elif ped_location.y > -191 and player_direction.y == 1: - walker.destroy() - continue - dx = self.vehicle.get_location().x - ped_location.x - dy = self.vehicle.get_location().y - ped_location.y - distance = math.sqrt((dx * dx) + (dy * dy)) - distances.append(distance) - - min_dist = min(distances) + # 基于行人距离的奖励 + if min_dist < 3.0: # 非常危险 + reward -= 3.0 + done = True + elif min_dist < 5.0: # 危险 + reward -= 1.0 + elif min_dist < 8.0: # 警告 + reward -= 0.3 + elif min_dist > 15.0: # 安全 + reward += 0.2 + + # 碰撞检测 if len(self.collision_history) != 0: - reward = -5 + reward = -10 # 增加碰撞惩罚 done = True - elif min_dist < 4: - reward = -2 - done = False - elif velocity_kmh == 0: - reward += -1 - done = False - elif 15 < velocity_kmh < 25: - reward += 1 - done = False - elif 35 < velocity_kmh < 45: - reward += 2 - done = False - - if self.vehicle.get_location().x > 155: + + # 进度奖励 + reward += progress_reward * 0.5 + + # 完成条件 + if vehicle_location.x > 155: + reward += 10 # 成功到达奖励 done = True - + elif vehicle_location.x < -90: # 倒退太多 + reward -= 5 + done = True + return reward, done - # def reward(self): -# - # reward = 0 - # done = False -# - # velocity = self.vehicle.get_velocity() - # velocity_kmh = int(3.6 * math.sqrt(velocity.x ** 2 + velocity.y ** 2 + velocity.z ** 2)) -# - # if velocity_kmh == 0: - # self.slow_counter += 1 - # else: - # self.slow_counter = 0 -# - # # if len(self.collision_history) != 0: - # # reward += -200 * len(self.collision_history) - # # self.collision_history = [] - # # done = True - # # elif velocity_kmh < 30: - # # reward += -1 - # # done = False - # # else: - # # reward += 1 - # # done = False -# - # if len(self.collision_history) != 0: - # reward += -300 * len(self.collision_history) - # self.collision_history = [] - # done = True - # elif velocity_kmh == 0: - # reward += -1 - # done = False - # elif 15 < velocity_kmh < 25: - # reward += 1 - # elif 35 < velocity_kmh < 45: - # reward += 2 - # done = False -# - # walkers = self.world.get_actors().filter('walker.*') - # for walker in walkers: -# - # player_transform = walker.get_transform() - # ped_location = player_transform.location - # player_direction = walker.get_control().direction -# - # if ped_location.y < -214 and player_direction.y == -1: - # walker.destroy() - # continue - # elif ped_location.y > -191 and player_direction.y == 1: - # walker.destroy() - # continue -# - # dx = self.vehicle.get_location().x - ped_location.x - # dy = self.vehicle.get_location().y - ped_location.y - # distance = math.sqrt((dx * dx) + (dy * dy)) -# - # # If the distance is lower than 8, give a negative reward of -20 - # if distance < 5: - # reward -= 2 -# - # # If there is a collision (vehicle position matches pedestrian position), give a negative reward of -200 - # if distance < 2: - # reward -= 25 -# - # reward += (80 / (166 - self.vehicle.get_location().x))**2 -# - # if self.vehicle.get_location().x > 155 or self.slow_counter == SLOW_COUNTER: # or self.episode_start + SECONDS_PER_EPISODE < time.time(): - # done = True -# - # return reward, done - def step(self, action): - if action == 0: - # self.vehicle.apply_control(carla.VehicleControl(throttle=0.0, brake=0.6, steer=0.0)) - velocity = carla.Vector3D(x=0.00, y=0.0, z=0.0) - self.vehicle.set_target_velocity(velocity) - elif action == 1: - # self.vehicle.apply_control(carla.VehicleControl(throttle=0.0, brake=0.0, steer=0.0)) - velocity = carla.Vector3D(x=6.5, y=0.0, z=0.0) - self.vehicle.set_target_velocity(velocity) - elif action == 2: - # self.vehicle.apply_control(carla.VehicleControl(throttle=0.6, brake=0.0, steer=0.0)) - velocity = carla.Vector3D(x=12, y=0.0, z=0.0) - self.vehicle.set_target_velocity(velocity) - + # 更平滑的控制 + if action == 0: # 减速 + self.vehicle.apply_control(carla.VehicleControl(throttle=0.0, brake=0.3)) + elif action == 1: # 保持/轻微加速 + self.vehicle.apply_control(carla.VehicleControl(throttle=0.3, brake=0.0)) + elif action == 2: # 加速 + self.vehicle.apply_control(carla.VehicleControl(throttle=0.7, brake=0.0)) + + # 等待物理更新 + time.sleep(0.05) + reward, done = self.reward() - + + # 限制极端奖励值 + reward = np.clip(reward, -10, 10) + return self.front_camera, reward, done, None \ No newline at end of file diff --git a/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py b/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py index 923f4acf02..3fae646c12 100644 --- a/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py +++ b/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py @@ -1,4 +1,5 @@ -DISCOUNT = 0.99 +# Hyperparameters.py +DISCOUNT = 0.95 # Discount factor for future rewards in the RL algorithm. FPS = 60 @@ -10,44 +11,41 @@ REWARD_OFFSET = -100 # Stops the simulation when reached -MIN_REPLAY_MEMORY_SIZE = 1_000 +MIN_REPLAY_MEMORY_SIZE = 2_000 # Minimum size of the replay memory before training starts. -REPLAY_MEMORY_SIZE = 5_000 +REPLAY_MEMORY_SIZE = 10_000 # Maximum capacity of the replay memory. -MINIBATCH_SIZE = 64 +MINIBATCH_SIZE = 32 # Number of experiences sampled from the replay memory for each training iteration. PREDICTION_BATCH_SIZE = 1 # Batch size used during the prediction phase. -TRAINING_BATCH_SIZE = MINIBATCH_SIZE // 8 +TRAINING_BATCH_SIZE = MINIBATCH_SIZE // 4 # Batch size used during the training phase. -EPISODES = 451 +EPISODES = 1000 # Number of episodes the agent will train on. -# SECONDS_PER_EPISODE = 60 -SECONDS_PER_EPISODE = 45 +SECONDS_PER_EPISODE = 60 # Duration of each episode in seconds. -MIN_EPSILON = 0.1 +MIN_EPSILON = 0.01 EPSILON = 1.0 # Exploration rates for the epsilon-greedy exploration strategy. -EPSILON_DECAY = 0.9975 -# EPSILON_DECAY = 0.993 +EPSILON_DECAY = 0.995 # Decay rate of the exploration rate over time. -MODEL_NAME = "YY" -# Name or identifier for the trained model.F +MODEL_NAME = "YY_Optimized" +# Name or identifier for the trained model. -MIN_REWARD = 100 -# MIN_REWARD = -1000 +MIN_REWARD = 5 # Minimum reward required for an experience to be considered "good" or "positive." -UPDATE_TARGET_EVERY = 5 +UPDATE_TARGET_EVERY = 10 # Frequency at which the target network is updated. AGGREGATE_STATS_EVERY = 10 @@ -64,6 +62,9 @@ SLOW_COUNTER = 330 -LOW_REWARD_THRESHOLD = -4 +LOW_REWARD_THRESHOLD = -2 -SUCCESSFUL_THRESHOLD = 1 +SUCCESSFUL_THRESHOLD = 3 + +LEARNING_RATE = 0.0001 +# Learning rate for the optimizer \ No newline at end of file diff --git a/src/Unmanned_vehicle_AD_DQN/Main.py b/src/Unmanned_vehicle_AD_DQN/Main.py deleted file mode 100644 index e661cabbb9..0000000000 --- a/src/Unmanned_vehicle_AD_DQN/Main.py +++ /dev/null @@ -1,153 +0,0 @@ -import glob -import os -import sys -import random -import time -import numpy as np -import cv2 -import math -import matplotlib.pyplot as plt -from collections import deque -from tensorflow.keras.applications.xception import Xception -from tensorflow.keras.layers import Dense, GlobalAveragePooling2D -from tensorflow.keras.models import Sequential, Model -from tensorflow.keras.layers import Dense, GlobalAveragePooling2D, Input, Concatenate, Conv2D, AveragePooling2D, Activation, \ - Flatten -from tensorflow.keras.optimizers import Adam -from tensorflow.keras.models import Model -from tensorflow.keras.callbacks import TensorBoard -import tensorflow as tf -import tensorflow.keras.backend as backend -from threading import Thread - -from tqdm import tqdm - -import Hyperparameters -from Environment import * -from Model import * -from Hyperparameters import * - -if __name__ == '__main__': - - FPS = 60 - # For stats - ep_rewards = [-200] - - # For more repetitive results - # random.seed(1) - # np.random.seed(1) - # tf.compat.v1.set_random_seed(1) - - # Memory fraction, used mostly when training multiple agents - gpu_options = tf.compat.v1.GPUOptions(per_process_gpu_memory_fraction=MEMORY_FRACTION) - tf.compat.v1.keras.backend.set_session( - tf.compat.v1.Session(config=tf.compat.v1.ConfigProto(gpu_options=gpu_options))) - - # Create models folder - if not os.path.isdir('models'): - os.makedirs('models') - - # Create agent and environment - agent = DQNAgent() - env = CarEnv() - - # Start training thread and wait for training to be initialized - trainer_thread = Thread(target=agent.train_in_loop, daemon=True) - trainer_thread.start() - while not agent.training_initialized: - time.sleep(0.01) - - agent.get_qs(np.ones((env.im_height, env.im_width, 3))) - - # Iterate over episodes - epds = [] - scores = [] - avg_scores = [] - for episode in tqdm(range(1, EPISODES + 1), ascii=True, unit='episodes'): - - env.collision_hist = [] - agent.tensorboard.step = episode - - # Restarting episode - reset episode reward and step number - score = 0 - step = 1 - - - # Reset environment and get initial state - current_state = env.reset() - - # Reset flag and start iterating until episode ends - done = False - episode_start = time.time() - - # Play for given number of seconds only - while True: - - # This part stays mostly the same, the change is to query a model for Q values - if np.random.random() > Hyperparameters.EPSILON: - # Get action from Q table - qs = agent.get_qs(current_state) - action = np.argmax(qs) - print(f'Action: [{qs[0]:>5.2f}, {qs[1]:>5.2f}, {qs[2]:>5.2f}] {action}') - else: - # Get random action - action = np.random.randint(0, 3) - # This takes no time, so we add a delay matching 60 FPS (prediction above takes longer) - time.sleep(1 / FPS) - - new_state, reward, done, _ = env.step(action) - - # Transform new continuous state to new discrete state and count reward - score += reward - - if score < REWARD_OFFSET: - done = True - - # Every step we update replay memory - agent.update_replay_memory((current_state, action, reward, new_state, done)) - - current_state = new_state - step += 1 - - if done: - break - - # End of episode - destroy agents - for actor in env.actor_list: - actor.destroy() - - scores.append(score) - avg_scores.append(np.mean(scores[-10:])) - - if not episode % AGGREGATE_STATS_EVERY or episode == 1: - average_reward = np.mean(scores[-AGGREGATE_STATS_EVERY:]) - min_reward = min(scores[-AGGREGATE_STATS_EVERY:]) - max_reward = max(scores[-AGGREGATE_STATS_EVERY:]) - agent.tensorboard.update_stats(reward_avg=average_reward, reward_min=min_reward, reward_max=max_reward, - epsilon=Hyperparameters.EPSILON) - - # Save model, but only when min reward is greater or equal a set value - if min_reward >= MIN_REWARD and (episode not in epds): - agent.model.save( - f'models/{MODEL_NAME}__{max_reward:_>7.2f}max_{avg_score:_>7.2f}avg_{min_reward:_>7.2f}min__{int(time.time())}.model') - - epds.append(episode) - print('episode: ', episode, 'score %.2f' % score) - # Decay epsilon - if Hyperparameters.EPSILON > Hyperparameters.MIN_EPSILON: - Hyperparameters.EPSILON *= Hyperparameters.EPSILON_DECAY - Hyperparameters.EPSILON = max(Hyperparameters.MIN_EPSILON, Hyperparameters.EPSILON) - - # Set termination flag for training thread and wait for it to finish - agent.terminate = True - trainer_thread.join() - agent.model.save( - f'models/{MODEL_NAME}__{max_reward:_>7.2f}max_{avg_score:_>7.2f}avg_{min_reward:_>7.2f}min__{int(time.time())}.model') - - fig = plt.figure() - ax = fig.add_subplot(111) - plt.plot(scores) - plt.plot(avg_scores) - plt.ylabel('Score') - plt.xlabel('Episode #') - plt.show() \ No newline at end of file diff --git a/src/Unmanned_vehicle_AD_DQN/Test.py b/src/Unmanned_vehicle_AD_DQN/Test.py index 7117ff4c49..b3ae126f59 100644 --- a/src/Unmanned_vehicle_AD_DQN/Test.py +++ b/src/Unmanned_vehicle_AD_DQN/Test.py @@ -1,3 +1,4 @@ +# Test.py import random from collections import deque import numpy as np @@ -7,11 +8,10 @@ import tensorflow.keras.backend as backend from tensorflow.keras.models import load_model from Environment import CarEnv, MEMORY_FRACTION -from Hyperparameters import* +from Hyperparameters import * - -MODEL_PATH = 'models\YY___153.00max____0.00avg__153.00min__1761010159.model' +MODEL_PATH = r'D:\Work\T_Unmanned_vehicle_AD_DQN\models\YY_best_74.00.model' # 请替换为实际的最佳模型路径 if __name__ == '__main__': @@ -70,8 +70,8 @@ # Measure step time, append to a deque, then print mean FPS for last 60 frames, q values and taken action frame_time = time.time() - step_start fps_counter.append(frame_time) - print(f'Agent: {len(fps_counter)/sum(fps_counter):>4.1f} FPS | Action: [{qs[0]:>5.2f}, {qs[1]:>5.2f}, {qs[2]:>5.2f}] {action}') + print(f'Agent: {len(fps_counter)/sum(fps_counter):>4.1f} FPS | Action: [{qs[0]:>5.2f}, {qs[1]:>5.2f}, {qs[2]:>5.2f}] {action} | Reward: {reward}') # Destroy an actor at end of episode for actor in env.actor_list: - actor.destroy() + actor.destroy() \ No newline at end of file diff --git a/src/Unmanned_vehicle_AD_DQN/main.py b/src/Unmanned_vehicle_AD_DQN/main.py deleted file mode 100644 index e661cabbb9..0000000000 --- a/src/Unmanned_vehicle_AD_DQN/main.py +++ /dev/null @@ -1,153 +0,0 @@ -import glob -import os -import sys -import random -import time -import numpy as np -import cv2 -import math -import matplotlib.pyplot as plt -from collections import deque -from tensorflow.keras.applications.xception import Xception -from tensorflow.keras.layers import Dense, GlobalAveragePooling2D -from tensorflow.keras.models import Sequential, Model -from tensorflow.keras.layers import Dense, GlobalAveragePooling2D, Input, Concatenate, Conv2D, AveragePooling2D, Activation, \ - Flatten -from tensorflow.keras.optimizers import Adam -from tensorflow.keras.models import Model -from tensorflow.keras.callbacks import TensorBoard -import tensorflow as tf -import tensorflow.keras.backend as backend -from threading import Thread - -from tqdm import tqdm - -import Hyperparameters -from Environment import * -from Model import * -from Hyperparameters import * - -if __name__ == '__main__': - - FPS = 60 - # For stats - ep_rewards = [-200] - - # For more repetitive results - # random.seed(1) - # np.random.seed(1) - # tf.compat.v1.set_random_seed(1) - - # Memory fraction, used mostly when training multiple agents - gpu_options = tf.compat.v1.GPUOptions(per_process_gpu_memory_fraction=MEMORY_FRACTION) - tf.compat.v1.keras.backend.set_session( - tf.compat.v1.Session(config=tf.compat.v1.ConfigProto(gpu_options=gpu_options))) - - # Create models folder - if not os.path.isdir('models'): - os.makedirs('models') - - # Create agent and environment - agent = DQNAgent() - env = CarEnv() - - # Start training thread and wait for training to be initialized - trainer_thread = Thread(target=agent.train_in_loop, daemon=True) - trainer_thread.start() - while not agent.training_initialized: - time.sleep(0.01) - - agent.get_qs(np.ones((env.im_height, env.im_width, 3))) - - # Iterate over episodes - epds = [] - scores = [] - avg_scores = [] - for episode in tqdm(range(1, EPISODES + 1), ascii=True, unit='episodes'): - - env.collision_hist = [] - agent.tensorboard.step = episode - - # Restarting episode - reset episode reward and step number - score = 0 - step = 1 - - - # Reset environment and get initial state - current_state = env.reset() - - # Reset flag and start iterating until episode ends - done = False - episode_start = time.time() - - # Play for given number of seconds only - while True: - - # This part stays mostly the same, the change is to query a model for Q values - if np.random.random() > Hyperparameters.EPSILON: - # Get action from Q table - qs = agent.get_qs(current_state) - action = np.argmax(qs) - print(f'Action: [{qs[0]:>5.2f}, {qs[1]:>5.2f}, {qs[2]:>5.2f}] {action}') - else: - # Get random action - action = np.random.randint(0, 3) - # This takes no time, so we add a delay matching 60 FPS (prediction above takes longer) - time.sleep(1 / FPS) - - new_state, reward, done, _ = env.step(action) - - # Transform new continuous state to new discrete state and count reward - score += reward - - if score < REWARD_OFFSET: - done = True - - # Every step we update replay memory - agent.update_replay_memory((current_state, action, reward, new_state, done)) - - current_state = new_state - step += 1 - - if done: - break - - # End of episode - destroy agents - for actor in env.actor_list: - actor.destroy() - - scores.append(score) - avg_scores.append(np.mean(scores[-10:])) - - if not episode % AGGREGATE_STATS_EVERY or episode == 1: - average_reward = np.mean(scores[-AGGREGATE_STATS_EVERY:]) - min_reward = min(scores[-AGGREGATE_STATS_EVERY:]) - max_reward = max(scores[-AGGREGATE_STATS_EVERY:]) - agent.tensorboard.update_stats(reward_avg=average_reward, reward_min=min_reward, reward_max=max_reward, - epsilon=Hyperparameters.EPSILON) - - # Save model, but only when min reward is greater or equal a set value - if min_reward >= MIN_REWARD and (episode not in epds): - agent.model.save( - f'models/{MODEL_NAME}__{max_reward:_>7.2f}max_{avg_score:_>7.2f}avg_{min_reward:_>7.2f}min__{int(time.time())}.model') - - epds.append(episode) - print('episode: ', episode, 'score %.2f' % score) - # Decay epsilon - if Hyperparameters.EPSILON > Hyperparameters.MIN_EPSILON: - Hyperparameters.EPSILON *= Hyperparameters.EPSILON_DECAY - Hyperparameters.EPSILON = max(Hyperparameters.MIN_EPSILON, Hyperparameters.EPSILON) - - # Set termination flag for training thread and wait for it to finish - agent.terminate = True - trainer_thread.join() - agent.model.save( - f'models/{MODEL_NAME}__{max_reward:_>7.2f}max_{avg_score:_>7.2f}avg_{min_reward:_>7.2f}min__{int(time.time())}.model') - - fig = plt.figure() - ax = fig.add_subplot(111) - plt.plot(scores) - plt.plot(avg_scores) - plt.ylabel('Score') - plt.xlabel('Episode #') - plt.show() \ No newline at end of file From 710db3814ede8e2a6647888167fce78fcd7f74c9 Mon Sep 17 00:00:00 2001 From: joe-justin369 Date: Mon, 17 Nov 2025 10:58:36 +0800 Subject: [PATCH 06/10] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E6=A0=B8=E5=BF=83DQN?= =?UTF-8?q?=E7=AE=97=E6=B3=95=E6=96=87=E4=BB=B6=EF=BC=9A=E7=8E=AF=E5=A2=83?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E3=80=81=E8=B6=85=E5=8F=82=E6=95=B0=E3=80=81?= =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E3=80=81=E6=B5=8B=E8=AF=95=E5=92=8C=E4=B8=BB?= =?UTF-8?q?=E7=A8=8B=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Unmanned_vehicle_AD_DQN/Model.py | 148 ++++++++++++----------- src/Unmanned_vehicle_AD_DQN/main.py | 173 +++++++++++++++++++++++++++ 2 files changed, 253 insertions(+), 68 deletions(-) create mode 100644 src/Unmanned_vehicle_AD_DQN/main.py diff --git a/src/Unmanned_vehicle_AD_DQN/Model.py b/src/Unmanned_vehicle_AD_DQN/Model.py index 0ac337d395..8fe6fedf8e 100644 --- a/src/Unmanned_vehicle_AD_DQN/Model.py +++ b/src/Unmanned_vehicle_AD_DQN/Model.py @@ -1,3 +1,4 @@ +# Model.py import glob import os import sys @@ -12,7 +13,7 @@ from tensorflow.keras.layers import Dense, GlobalAveragePooling2D from tensorflow.keras.models import Sequential, Model from tensorflow.keras.layers import Dense, GlobalAveragePooling2D, Input, Concatenate, Conv2D, AveragePooling2D, Activation, \ - Flatten, Dropout, BatchNormalization + Flatten, Dropout, BatchNormalization, MaxPooling2D from tensorflow.keras.optimizers import Adam from tensorflow.keras.models import Model from tensorflow.keras.callbacks import TensorBoard @@ -20,52 +21,39 @@ import tensorflow.keras.backend as backend from threading import Thread from Environment import * +from Hyperparameters import * # Own Tensorboard class class ModifiedTensorBoard(TensorBoard): - - # Overriding init to set initial step and writer (we want one log file for all .fit() calls) def __init__(self, **kwargs): super().__init__(**kwargs) self._log_write_dir = self.log_dir self.step = 1 self.writer = self.writer = tf.summary.create_file_writer(self.log_dir) - # Overriding this method to stop creating default log writer def set_model(self, model): self.model = model - # self._train_dir = os.path.join(self._log_write_dir, 'train') self._train_step = self.model._train_counter - # self._val_dir = os.path.join(self._log_write_dir, 'validation') self._val_step = self.model._test_counter - # self._should_write_train_graph = False - # Overrided, saves logs with our step number - # (otherwise every .fit() will start writing from 0th step) def on_epoch_end(self, epoch, logs=None): self.update_stats(**logs) - # Overrided - # We train for one batch only, no need to save anything at epoch end def on_batch_end(self, batch, logs=None): pass - # Overrided, so won't close writer def on_train_end(self, logs=None): pass - # Custom method for saving own metrics - # Creates writer, writes custom metrics and closes writer def update_stats(self, **stats): with self.writer.as_default(): for key, value in stats.items(): tf.summary.scalar(key, value, step=self.step) self.writer.flush() - # self.step = self.step + 10 class DQNAgent: @@ -86,26 +74,47 @@ def __init__(self): self.training_initialized = False def create_model(self): + # 使用更强大的网络架构 model = Sequential() - - model.add(Conv2D(64, (3, 3), input_shape=(IM_HEIGHT, IM_WIDTH, 3), padding='same')) + + # 第一卷积块 + model.add(Conv2D(32, (5, 5), strides=(2, 2), input_shape=(IM_HEIGHT, IM_WIDTH, 3), padding='same')) model.add(Activation('relu')) - model.add(AveragePooling2D(pool_size=(5, 5), strides=(3, 3), padding='same')) - + model.add(BatchNormalization()) + model.add(MaxPooling2D(pool_size=(2, 2))) + + # 第二卷积块 model.add(Conv2D(64, (3, 3), padding='same')) model.add(Activation('relu')) - model.add(AveragePooling2D(pool_size=(5, 5), strides=(3, 3), padding='same')) - - model.add(Conv2D(64, (3, 3), padding='same')) + model.add(BatchNormalization()) + model.add(MaxPooling2D(pool_size=(2, 2))) + + # 第三卷积块 + model.add(Conv2D(128, (3, 3), padding='same')) model.add(Activation('relu')) - model.add(AveragePooling2D(pool_size=(5, 5), strides=(3, 3), padding='same')) - + model.add(BatchNormalization()) + model.add(MaxPooling2D(pool_size=(2, 2))) + + # 第四卷积块 + model.add(Conv2D(256, (3, 3), padding='same')) + model.add(Activation('relu')) + model.add(BatchNormalization()) + model.add(Flatten()) - - model.add(Dense(3, activation='softmax')) # Three output units for binary classification - - # model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) - model.compile(loss="mse", optimizer=Adam(lr=0.001), metrics=["accuracy"]) + + # 全连接层 + model.add(Dense(512, activation='relu')) + model.add(Dropout(0.3)) + model.add(Dense(256, activation='relu')) + model.add(Dropout(0.3)) + model.add(Dense(128, activation='relu')) + model.add(Dropout(0.2)) + + # 输出层 - 使用线性激活函数用于Q值回归 + model.add(Dense(3, activation='linear')) + + # 使用更稳定的优化器配置 + model.compile(loss="huber", optimizer=Adam(lr=LEARNING_RATE), metrics=["mae"]) return model def update_replay_memory(self, transition): @@ -113,50 +122,54 @@ def update_replay_memory(self, transition): self.replay_memory.append(transition) def minibatch_chooser(self): - # Initialize lists to store dangerous and non-dangerous samples - dangerous_samples = [] - non_dangerous_samples = [] - - # Iterate over the replay memory + # 改进的经验采样策略 + if len(self.replay_memory) < MIN_REPLAY_MEMORY_SIZE: + return random.sample(self.replay_memory, min(len(self.replay_memory), MINIBATCH_SIZE)) + + # 分类经验 + positive_samples = [] # 高奖励 + negative_samples = [] # 负奖励/碰撞 + neutral_samples = [] # 中性奖励 + for sample in self.replay_memory: - current_state, action, reward, new_state, done = sample - - # Check if the reward is considered low - if reward < LOW_REWARD_THRESHOLD: - # Append the sample to the dangerous samples list - dangerous_samples.append(sample) - else: - # Append the sample to the non-dangerous samples list - non_dangerous_samples.append(sample) - - # Shuffle the dangerous and non-dangerous samples - random.shuffle(dangerous_samples) - - successful_counter = 0 - successful_samples = [] - for sample in non_dangerous_samples: - if sample[2] == 2 and successful_counter < SUCCESSFUL_THRESHOLD: - successful_counter += 1 - successful_samples.append(sample) - - random.shuffle(non_dangerous_samples) - - # Determine the number of dangerous and non-dangerous samples to include in the minibatch - num_dangerous_samples = min(len(dangerous_samples), MINIBATCH_SIZE // 8) - remaining_samples = MINIBATCH_SIZE - num_dangerous_samples - successful_counter - - # Select dangerous and non-dangerous samples for the minibatch - minibatch = random.sample(dangerous_samples, num_dangerous_samples) + random.sample(self.replay_memory, - remaining_samples) + successful_samples - random.shuffle(minibatch) - return minibatch + _, _, reward, _, done = sample + + if done and reward < -5: # 碰撞或严重错误 + negative_samples.append(sample) + elif reward > 1: # 积极经验 + positive_samples.append(sample) + else: # 中性经验 + neutral_samples.append(sample) + + # 平衡采样 + batch = [] + + # 采样负经验 (20%) + num_negative = min(len(negative_samples), MINIBATCH_SIZE // 5) + batch.extend(random.sample(negative_samples, num_negative)) + + # 采样正经验 (30%) + num_positive = min(len(positive_samples), MINIBATCH_SIZE // 3) + batch.extend(random.sample(positive_samples, num_positive)) + + # 用中性经验补全批次 + remaining = MINIBATCH_SIZE - len(batch) + if remaining > 0: + batch.extend(random.sample(neutral_samples, min(remaining, len(neutral_samples)))) + + # 从整个记忆库随机采样 + if len(batch) < MINIBATCH_SIZE: + additional = MINIBATCH_SIZE - len(batch) + batch.extend(random.sample(self.replay_memory, additional)) + + random.shuffle(batch) + return batch def train(self): if len(self.replay_memory) < MIN_REPLAY_MEMORY_SIZE: return minibatch = self.minibatch_chooser() - # minibatch = random.sample(self.replay_memory, MINIBATCH_SIZE) print([transition[2] for transition in minibatch]) current_states = np.array([transition[0] for transition in minibatch]) / 255 @@ -207,9 +220,8 @@ def train_in_loop(self): while True: if self.terminate: return - # print("Memory: ", len(self.replay_memory)) self.train() time.sleep(0.01) def get_qs(self, state): - return self.model.predict(np.array(state).reshape(-1, *state.shape) / 255)[0] + return self.model.predict(np.array(state).reshape(-1, *state.shape) / 255)[0] \ No newline at end of file diff --git a/src/Unmanned_vehicle_AD_DQN/main.py b/src/Unmanned_vehicle_AD_DQN/main.py new file mode 100644 index 0000000000..e66c08ffe5 --- /dev/null +++ b/src/Unmanned_vehicle_AD_DQN/main.py @@ -0,0 +1,173 @@ +# main.py +import glob +import os +import sys +import random +import time +import numpy as np +import cv2 +import math +import matplotlib.pyplot as plt +from collections import deque +from tensorflow.keras.applications.xception import Xception +from tensorflow.keras.layers import Dense, GlobalAveragePooling2D +from tensorflow.keras.models import Sequential, Model +from tensorflow.keras.layers import Dense, GlobalAveragePooling2D, Input, Concatenate, Conv2D, AveragePooling2D, Activation, \ + Flatten +from tensorflow.keras.optimizers import Adam +from tensorflow.keras.models import Model +from tensorflow.keras.callbacks import TensorBoard +import tensorflow as tf +import tensorflow.keras.backend as backend +from threading import Thread + +from tqdm import tqdm + +import Hyperparameters +from Environment import * +from Model import * +from Hyperparameters import * + +if __name__ == '__main__': + FPS = 60 + ep_rewards = [-200] + + # For more repetitive results + # random.seed(1) + # np.random.seed(1) + # tf.compat.v1.set_random_seed(1) + + # Memory fraction, used mostly when training multiple agents + gpu_options = tf.compat.v1.GPUOptions(per_process_gpu_memory_fraction=MEMORY_FRACTION) + tf.compat.v1.keras.backend.set_session( + tf.compat.v1.Session(config=tf.compat.v1.ConfigProto(gpu_options=gpu_options))) + + # Create models folder + if not os.path.isdir('models'): + os.makedirs('models') + + # Create agent and environment + agent = DQNAgent() + env = CarEnv() + + # Start training thread and wait for training to be initialized + trainer_thread = Thread(target=agent.train_in_loop, daemon=True) + trainer_thread.start() + while not agent.training_initialized: + time.sleep(0.01) + + agent.get_qs(np.ones((env.im_height, env.im_width, 3))) + + # 添加训练统计 + best_score = -float('inf') + success_count = 0 + scores = [] + avg_scores = [] + + # Iterate over episodes + epds = [] + for episode in tqdm(range(1, EPISODES + 1), ascii=True, unit='episodes'): + env.collision_hist = [] + agent.tensorboard.step = episode + + # 课程学习 - 随训练进度调整难度 + if episode > EPISODES // 2: + # 后期增加行人数量 + env.spawn_pedestrians_general(40, True) + env.spawn_pedestrians_general(15, False) + else: + # 前期减少行人数量 + env.spawn_pedestrians_general(25, True) + env.spawn_pedestrians_general(8, False) + + # Restarting episode - reset episode reward and step number + score = 0 + step = 1 + + # Reset environment and get initial state + current_state = env.reset() + + # Reset flag and start iterating until episode ends + done = False + episode_start = time.time() + + # 单次episode内的步数限制 + max_steps_per_episode = SECONDS_PER_EPISODE * FPS + + # Play for given number of seconds only + while not done and step < max_steps_per_episode: + + # This part stays mostly the same, the change is to query a model for Q values + if np.random.random() > Hyperparameters.EPSILON: + # Get action from Q table + qs = agent.get_qs(current_state) + action = np.argmax(qs) + print(f'Action: [{qs[0]:>5.2f}, {qs[1]:>5.2f}, {qs[2]:>5.2f}] {action}') + else: + # Get random action + action = np.random.randint(0, 3) + # This takes no time, so we add a delay matching 60 FPS (prediction above takes longer) + time.sleep(1 / FPS) + + # 更频繁的状态更新 + if step % 5 == 0: + new_state, reward, done, _ = env.step(action) + + score += reward + agent.update_replay_memory((current_state, action, reward, new_state, done)) + current_state = new_state + + step += 1 + + if done: + break + + # End of episode - destroy agents + for actor in env.actor_list: + actor.destroy() + + # 更新成功计数 + if score > 5: # 成功完成的阈值 + success_count += 1 + + # 动态保存最佳模型 + if score > best_score: + best_score = score + agent.model.save(f'models/{MODEL_NAME}_best_{score:.2f}.model') + + scores.append(score) + avg_scores.append(np.mean(scores[-10:])) + + if not episode % AGGREGATE_STATS_EVERY or episode == 1: + average_reward = np.mean(scores[-AGGREGATE_STATS_EVERY:]) + min_reward = min(scores[-AGGREGATE_STATS_EVERY:]) + max_reward = max(scores[-AGGREGATE_STATS_EVERY:]) + agent.tensorboard.update_stats(reward_avg=average_reward, reward_min=min_reward, reward_max=max_reward, + epsilon=Hyperparameters.EPSILON) + + # Save model, but only when min reward is greater or equal a set value + if min_reward >= MIN_REWARD and (episode not in epds): + agent.model.save( + f'models/{MODEL_NAME}__{max_reward:_>7.2f}max_{average_reward:_>7.2f}avg_{min_reward:_>7.2f}min__{int(time.time())}.model') + + epds.append(episode) + print('episode: ', episode, 'score %.2f' % score, 'success_count:', success_count) + + # Decay epsilon + if Hyperparameters.EPSILON > Hyperparameters.MIN_EPSILON: + Hyperparameters.EPSILON *= Hyperparameters.EPSILON_DECAY + Hyperparameters.EPSILON = max(Hyperparameters.MIN_EPSILON, Hyperparameters.EPSILON) + + # Set termination flag for training thread and wait for it to finish + agent.terminate = True + trainer_thread.join() + agent.model.save( + f'models/{MODEL_NAME}__{max_reward:_>7.2f}max_{average_reward:_>7.2f}avg_{min_reward:_>7.2f}min__{int(time.time())}.model') + + fig = plt.figure() + ax = fig.add_subplot(111) + plt.plot(scores) + plt.plot(avg_scores) + plt.ylabel('Score') + plt.xlabel('Episode #') + plt.show() \ No newline at end of file From 198eb2dafa0918d3ea64ad460593edf0b5d5c857 Mon Sep 17 00:00:00 2001 From: joe-justin369 Date: Mon, 17 Nov 2025 11:37:43 +0800 Subject: [PATCH 07/10] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=B3=A8=E9=87=8A?= =?UTF-8?q?=EF=BC=8C=E5=B9=B6=E5=85=A8=E9=83=A8=E6=9B=BF=E6=8D=A2=E4=B8=BA?= =?UTF-8?q?=E4=B8=AD=E6=96=87=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Unmanned_vehicle_AD_DQN/Environment.py | 111 ++++++++++++------ .../Hyperparameters.py | 49 ++++---- src/Unmanned_vehicle_AD_DQN/Model.py | 62 ++++++---- src/Unmanned_vehicle_AD_DQN/Test.py | 43 ++++--- src/Unmanned_vehicle_AD_DQN/main.py | 91 +++++++------- 5 files changed, 211 insertions(+), 145 deletions(-) diff --git a/src/Unmanned_vehicle_AD_DQN/Environment.py b/src/Unmanned_vehicle_AD_DQN/Environment.py index 34e8ab7a82..8d8a474af0 100644 --- a/src/Unmanned_vehicle_AD_DQN/Environment.py +++ b/src/Unmanned_vehicle_AD_DQN/Environment.py @@ -9,6 +9,7 @@ import math from Hyperparameters import * +# 全局统计变量 avg_score = 0 average_reward = 0 @@ -17,45 +18,51 @@ class CarEnv: - SHOW_CAM = SHOW_PREVIEW - im_width = IM_WIDTH - im_height = IM_HEIGHT + SHOW_CAM = SHOW_PREVIEW # 是否显示摄像头预览 + im_width = IM_WIDTH # 图像宽度 + im_height = IM_HEIGHT # 图像高度 def __init__(self): - self.actor_list = None - self.sem_cam = None - self.client = carla.Client("localhost", 2000) - self.client.set_timeout(20.0) - self.front_camera = None + self.actor_list = None # 存储所有actor的列表 + self.sem_cam = None # 语义分割摄像头 + self.client = carla.Client("localhost", 2000) # CARLA客户端 + self.client.set_timeout(20.0) # 连接超时设置 + self.front_camera = None # 前置摄像头图像 + # 加载世界和蓝图 self.world = self.client.load_world('Town03') self.blueprint_library = self.world.get_blueprint_library() - self.model_3 = self.blueprint_library.filter("model3")[0] + self.model_3 = self.blueprint_library.filter("model3")[0] # Tesla Model3车辆 + # 行人列表和碰撞历史 self.walker_list = [] self.collision_history = [] - self.slow_counter = 0 + self.slow_counter = 0 # 慢速计数器 def spawn_pedestrians_general(self, number, isCross): + """生成指定数量的行人""" for i in range(number): - isLeft = random.choice([True, False]) + isLeft = random.choice([True, False]) # 随机选择左右侧 if isLeft: self.spawn_pedestrians_left(isCross) else: self.spawn_pedestrians_right(isCross) def spawn_pedestrians_right(self, isCross): + """在右侧生成行人""" blueprints_walkers = self.world.get_blueprint_library().filter("walker.pedestrian.*") walker_bp = random.choice(blueprints_walkers) for i in range(1): walker_bp = random.choice(blueprints_walkers) + # 设置生成区域 min_x = -50 max_x = 140 min_y = -188 max_y = -183 + # 如果是十字路口，调整生成位置 if isCross: isFirstCross = random.choice([True, False]) if isFirstCross: @@ -65,39 +72,46 @@ def spawn_pedestrians_right(self, isCross): min_x = 17 max_x = 20.5 + # 随机生成位置 x = random.uniform(min_x, max_x) y = random.uniform(min_y, max_y) spawn_point = carla.Transform(carla.Location(x, y, 2.0)) + # 避免在特定区域生成 while (-10 < spawn_point.location.x < 17) or (70 < spawn_point.location.x < 100): x = random.uniform(min_x, max_x) y = random.uniform(min_y, max_y) spawn_point = carla.Transform(carla.Location(x, y, 2.0)) + # 尝试生成行人 if spawn_point: npc = self.world.try_spawn_actor(walker_bp, spawn_point) if npc is not None: + # 设置行人控制参数 ped_control = carla.WalkerControl() - ped_control.speed = random.uniform(0.5, 1.0) - ped_control.direction.y = -1 - ped_control.direction.x = 0.15 + ped_control.speed = random.uniform(0.5, 1.0) # 随机速度 + ped_control.direction.y = -1 # 主要移动方向 + ped_control.direction.x = 0.15 # 轻微横向移动 npc.apply_control(ped_control) - npc.set_simulate_physics(True) + npc.set_simulate_physics(True) # 启用物理模拟 def spawn_pedestrians_left(self, isCross): + """在左侧生成行人""" blueprints_walkers = self.world.get_blueprint_library().filter("walker.pedestrian.*") walker_bp = random.choice(blueprints_walkers) for i in range(1): walker_bp = random.choice(blueprints_walkers) + # 设置生成区域 min_x = -50 max_x = 140 min_y = -216 max_y = -210 + # 如果是十字路口，调整生成位置 if (isCross): isFirstCross = random.choice([True, False]) if isFirstCross: @@ -107,28 +121,34 @@ def spawn_pedestrians_left(self, isCross): min_x = 17 max_x = 20.5 + # 随机生成位置 x = random.uniform(min_x, max_x) y = random.uniform(min_y, max_y) spawn_point = carla.Transform(carla.Location(x, y, 2.0)) + # 避免在特定区域生成 while (-10 < spawn_point.location.x < 17) or (70 < spawn_point.location.x < 100): x = random.uniform(min_x, max_x) y = random.uniform(min_y, max_y) spawn_point = carla.Transform(carla.Location(x, y, 2.0)) + # 尝试生成行人 if spawn_point: npc = self.world.try_spawn_actor(walker_bp, spawn_point) if npc is not None: + # 设置行人控制参数 ped_control = carla.WalkerControl() - ped_control.speed = random.uniform(0.7, 1.3) - ped_control.direction.y = 1 - ped_control.direction.x = -0.05 + ped_control.speed = random.uniform(0.7, 1.3) # 随机速度 + ped_control.direction.y = 1 # 主要移动方向 + ped_control.direction.x = -0.05 # 轻微横向移动 npc.apply_control(ped_control) - npc.set_simulate_physics(True) + npc.set_simulate_physics(True) # 启用物理模拟 def reset(self): + """重置环境""" + # 清理现有的行人和车辆 walkers = self.world.get_actors().filter('walker.*') for walker in walkers: walker.destroy() @@ -141,10 +161,12 @@ def reset(self): self.spawn_pedestrians_general(30, True) self.spawn_pedestrians_general(10, False) + # 重置状态变量 self.collision_history = [] self.actor_list = [] self.slow_counter = 0 + # 设置车辆生成点 spawn_points = self.world.get_map().get_spawn_points() spawn_point = random.choice(spawn_points) if spawn_points else carla.Transform() spawn_point.location.x = -81.0 @@ -154,59 +176,72 @@ def reset(self): spawn_point.rotation.pitch = 0.0 spawn_point.rotation.yaw = 0.0 + # 生成主车辆 self.vehicle = self.world.spawn_actor(self.model_3, spawn_point) self.actor_list.append(self.vehicle) + # 设置语义分割摄像头 self.sem_cam = self.blueprint_library.find('sensor.camera.semantic_segmentation') self.sem_cam.set_attribute("image_size_x", f"{self.im_width}") self.sem_cam.set_attribute("image_size_y", f"{self.im_height}") - self.sem_cam.set_attribute("fov", f"110") + self.sem_cam.set_attribute("fov", f"110") # 视野角度 + # 安装摄像头传感器 transform = carla.Transform(carla.Location(x=2.5, z=0.7)) self.sensor = self.world.spawn_actor(self.sem_cam, transform, attach_to=self.vehicle) self.actor_list.append(self.sensor) - self.sensor.listen(lambda data: self.process_img(data)) + self.sensor.listen(lambda data: self.process_img(data)) # 设置图像处理回调 + # 初始化车辆控制 self.vehicle.apply_control(carla.VehicleControl(throttle=0.0, brake=0.0, steer=0.0)) - time.sleep(4) + time.sleep(4) # 等待环境稳定 + # 设置碰撞传感器 colsensor = self.blueprint_library.find("sensor.other.collision") self.colsensor = self.world.spawn_actor(colsensor, transform, attach_to=self.vehicle) self.actor_list.append(self.colsensor) - self.colsensor.listen(lambda event: self.collision_data(event)) + self.colsensor.listen(lambda event: self.collision_data(event)) # 设置碰撞检测回调 + # 等待摄像头初始化完成 while self.front_camera is None: time.sleep(0.01) + # 记录episode开始时间并重置控制 self.episode_start = time.time() self.vehicle.apply_control(carla.VehicleControl(throttle=0.0, brake=0.0, steer=0.0)) return self.front_camera def collision_data(self, event): + """处理碰撞事件""" self.collision_history.append(event) def process_img(self, image): - image.convert(carla.ColorConverter.CityScapesPalette) + """处理摄像头图像""" + image.convert(carla.ColorConverter.CityScapesPalette) # 转换为CityScapes调色板 + # 处理原始图像数据 processed_image = np.array(image.raw_data) processed_image = processed_image.reshape((self.im_height, self.im_width, 4)) - processed_image = processed_image[:, :, :3] + processed_image = processed_image[:, :, :3] # 移除alpha通道 + # 显示预览（如果启用） if self.SHOW_CAM: cv2.imshow("", processed_image) cv2.waitKey(1) - self.front_camera = processed_image + self.front_camera = processed_image # 更新前置摄像头图像 def reward(self): + """计算奖励函数""" reward = 0 done = False + # 计算车辆速度 velocity = self.vehicle.get_velocity() velocity_kmh = int(3.6 * math.sqrt(velocity.x ** 2 + velocity.y ** 2 + velocity.z ** 2)) - # 获取车辆位置和前进方向 + # 获取车辆位置和方向 vehicle_location = self.vehicle.get_location() vehicle_rotation = self.vehicle.get_transform().rotation.yaw @@ -230,14 +265,14 @@ def reward(self): reward -= 0.5 # 行人距离检测 - min_dist = float('inf') + min_dist = float('inf') # 最小距离初始化为无穷大 walkers = self.world.get_actors().filter('walker.*') for walker in walkers: ped_location = walker.get_location() dx = vehicle_location.x - ped_location.x dy = vehicle_location.y - ped_location.y distance = math.sqrt(dx**2 + dy**2) - min_dist = min(min_dist, distance) + min_dist = min(min_dist, distance) # 更新最小距离 # 清理边界外的行人 player_direction = walker.get_control().direction @@ -246,26 +281,26 @@ def reward(self): walker.destroy() # 基于行人距离的奖励 - if min_dist < 3.0: # 非常危险 + if min_dist < 3.0: # 非常危险距离 reward -= 3.0 done = True - elif min_dist < 5.0: # 危险 + elif min_dist < 5.0: # 危险距离 reward -= 1.0 - elif min_dist < 8.0: # 警告 + elif min_dist < 8.0: # 警告距离 reward -= 0.3 - elif min_dist > 15.0: # 安全 + elif min_dist > 15.0: # 安全距离 reward += 0.2 # 碰撞检测 if len(self.collision_history) != 0: - reward = -10 # 增加碰撞惩罚 + reward = -10 # 碰撞惩罚 done = True # 进度奖励 reward += progress_reward * 0.5 - # 完成条件 - if vehicle_location.x > 155: + # 完成条件判断 + if vehicle_location.x > 155: # 成功到达终点 reward += 10 # 成功到达奖励 done = True elif vehicle_location.x < -90: # 倒退太多 @@ -275,7 +310,8 @@ def reward(self): return reward, done def step(self, action): - # 更平滑的控制 + """执行动作并返回新状态""" + # 更平滑的控制策略 if action == 0: # 减速 self.vehicle.apply_control(carla.VehicleControl(throttle=0.0, brake=0.3)) elif action == 1: # 保持/轻微加速 @@ -286,6 +322,7 @@ def step(self, action): # 等待物理更新 time.sleep(0.05) + # 计算奖励和完成状态 reward, done = self.reward() # 限制极端奖励值 diff --git a/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py b/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py index 3fae646c12..920422c1b2 100644 --- a/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py +++ b/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py @@ -1,70 +1,77 @@ # Hyperparameters.py +# 深度强化学习超参数配置 + DISCOUNT = 0.95 -# Discount factor for future rewards in the RL algorithm. +# 未来奖励的折扣因子 FPS = 60 -# Frames per second in the simulation. +# 模拟环境的帧率 MEMORY_FRACTION = 0.35 -# Fraction of GPU memory allocated for training. +# GPU内存分配比例 REWARD_OFFSET = -100 -# Stops the simulation when reached +# 停止模拟的奖励阈值 MIN_REPLAY_MEMORY_SIZE = 2_000 -# Minimum size of the replay memory before training starts. +# 开始训练前经验回放缓冲区的最小大小 REPLAY_MEMORY_SIZE = 10_000 -# Maximum capacity of the replay memory. +# 经验回放缓冲区的最大容量 MINIBATCH_SIZE = 32 -# Number of experiences sampled from the replay memory for each training iteration. +# 每次训练从经验回放中采样的经验数量 PREDICTION_BATCH_SIZE = 1 -# Batch size used during the prediction phase. +# 预测阶段使用的批次大小 TRAINING_BATCH_SIZE = MINIBATCH_SIZE // 4 -# Batch size used during the training phase. +# 训练阶段使用的批次大小 EPISODES = 1000 -# Number of episodes the agent will train on. +# 智能体训练的总轮次数 SECONDS_PER_EPISODE = 60 -# Duration of each episode in seconds. +# 每轮训练的秒数 MIN_EPSILON = 0.01 +# 最小探索率 + EPSILON = 1.0 -# Exploration rates for the epsilon-greedy exploration strategy. +# 初始探索率 EPSILON_DECAY = 0.995 -# Decay rate of the exploration rate over time. +# 探索率的衰减率 MODEL_NAME = "YY_Optimized" -# Name or identifier for the trained model. +# 训练模型的名称标识 MIN_REWARD = 5 -# Minimum reward required for an experience to be considered "good" or "positive." +# 被认为是"良好"或"积极"经验的最小奖励值 UPDATE_TARGET_EVERY = 10 -# Frequency at which the target network is updated. +# 目标网络更新的频率 AGGREGATE_STATS_EVERY = 10 -# Frequency at which statistics (e.g., average scores, rewards) are computed and aggregated. +# 计算和聚合统计信息（如平均得分、奖励）的频率 SHOW_PREVIEW = False -# Determines whether to show a preview window or not. +# 是否显示预览窗口 IM_WIDTH = 640 -# Width of the image captured in the preview or simulation. +# 预览或模拟中捕获图像的宽度 IM_HEIGHT = 480 -# Height of the image captured in the preview or simulation. +# 预览或模拟中捕获图像的高度 SLOW_COUNTER = 330 +# 慢速计数器阈值 LOW_REWARD_THRESHOLD = -2 +# 低奖励阈值 SUCCESSFUL_THRESHOLD = 3 +# 成功阈值 LEARNING_RATE = 0.0001 -# Learning rate for the optimizer \ No newline at end of file +# 优化器的学习率 \ No newline at end of file diff --git a/src/Unmanned_vehicle_AD_DQN/Model.py b/src/Unmanned_vehicle_AD_DQN/Model.py index 8fe6fedf8e..13fc689eb7 100644 --- a/src/Unmanned_vehicle_AD_DQN/Model.py +++ b/src/Unmanned_vehicle_AD_DQN/Model.py @@ -24,7 +24,7 @@ from Hyperparameters import * -# Own Tensorboard class +# 自定义TensorBoard类 class ModifiedTensorBoard(TensorBoard): def __init__(self, **kwargs): super().__init__(**kwargs) @@ -56,31 +56,35 @@ def update_stats(self, **stats): self.writer.flush() +# DQN智能体类 class DQNAgent: def __init__(self): + # 创建主网络和目标网络 self.model = self.create_model() self.target_model = self.create_model() self.target_model.set_weights(self.model.get_weights()) + # 经验回放缓冲区 self.replay_memory = deque(maxlen=REPLAY_MEMORY_SIZE) - # Modified tensorboard + # 自定义TensorBoard self.tensorboard = ModifiedTensorBoard(log_dir=f"logs/{MODEL_NAME}-{int(time.time())}") - self.target_update_counter = 0 + self.target_update_counter = 0 # 目标网络更新计数器 self.graph = tf.compat.v1.get_default_graph() + # 训练控制标志 self.terminate = False self.last_logged_episode = 0 self.training_initialized = False def create_model(self): - # 使用更强大的网络架构 + """创建深度Q网络模型""" model = Sequential() # 第一卷积块 model.add(Conv2D(32, (5, 5), strides=(2, 2), input_shape=(IM_HEIGHT, IM_WIDTH, 3), padding='same')) model.add(Activation('relu')) - model.add(BatchNormalization()) + model.add(BatchNormalization()) # 批归一化 model.add(MaxPooling2D(pool_size=(2, 2))) # 第二卷积块 @@ -100,11 +104,12 @@ def create_model(self): model.add(Activation('relu')) model.add(BatchNormalization()) + # 展平层 model.add(Flatten()) # 全连接层 model.add(Dense(512, activation='relu')) - model.add(Dropout(0.3)) + model.add(Dropout(0.3)) # 防止过拟合 model.add(Dense(256, activation='relu')) model.add(Dropout(0.3)) model.add(Dense(128, activation='relu')) @@ -113,23 +118,24 @@ def create_model(self): # 输出层 - 使用线性激活函数用于Q值回归 model.add(Dense(3, activation='linear')) - # 使用更稳定的优化器配置 + # 编译模型，使用Huber损失和Adam优化器 model.compile(loss="huber", optimizer=Adam(lr=LEARNING_RATE), metrics=["mae"]) return model def update_replay_memory(self, transition): - # transition = (current_state, action, reward, new_state, done) + """更新经验回放缓冲区""" + # transition = (当前状态, 动作, 奖励, 新状态, 完成标志) self.replay_memory.append(transition) def minibatch_chooser(self): - # 改进的经验采样策略 + """改进的经验采样策略""" if len(self.replay_memory) < MIN_REPLAY_MEMORY_SIZE: return random.sample(self.replay_memory, min(len(self.replay_memory), MINIBATCH_SIZE)) - # 分类经验 - positive_samples = [] # 高奖励 - negative_samples = [] # 负奖励/碰撞 - neutral_samples = [] # 中性奖励 + # 分类经验样本 + positive_samples = [] # 高奖励经验 + negative_samples = [] # 负奖励/碰撞经验 + neutral_samples = [] # 中性奖励经验 for sample in self.replay_memory: _, _, reward, _, done = sample @@ -157,71 +163,83 @@ def minibatch_chooser(self): if remaining > 0: batch.extend(random.sample(neutral_samples, min(remaining, len(neutral_samples)))) - # 从整个记忆库随机采样 + # 如果还不够，从整个记忆库随机采样 if len(batch) < MINIBATCH_SIZE: additional = MINIBATCH_SIZE - len(batch) batch.extend(random.sample(self.replay_memory, additional)) - random.shuffle(batch) + random.shuffle(batch) # 打乱批次 return batch def train(self): + """训练DQN网络""" if len(self.replay_memory) < MIN_REPLAY_MEMORY_SIZE: return + # 选择小批量经验 minibatch = self.minibatch_chooser() - print([transition[2] for transition in minibatch]) + print([transition[2] for transition in minibatch]) # 打印奖励值 + # 准备训练数据 current_states = np.array([transition[0] for transition in minibatch]) / 255 current_qs_list = self.model.predict(current_states, batch_size=PREDICTION_BATCH_SIZE) new_current_states = np.array([transition[3] for transition in minibatch]) / 255 future_qs_list = self.target_model.predict(new_current_states, batch_size=PREDICTION_BATCH_SIZE) - x = [] - y = [] + x = [] # 输入状态 + y = [] # 目标Q值 + # 计算目标Q值 for index, (current_state, action, reward, new_state, done) in enumerate(minibatch): if not done: + # 使用贝尔曼方程计算目标Q值 max_future_q = np.max(future_qs_list[index]) new_q = reward + DISCOUNT * max_future_q else: - new_q = reward + new_q = reward # 终止状态 current_qs = current_qs_list[index] - current_qs[action] = new_q + current_qs[action] = new_q # 更新对应动作的Q值 x.append(current_state) y.append(current_qs) + # 记录日志判断 log_this_step = False if self.tensorboard.step > self.last_logged_episode: log_this_step = True self.last_logged_episode = self.tensorboard.step + # 训练模型 self.model.fit(np.array(x) / 255, np.array(y), batch_size=TRAINING_BATCH_SIZE, verbose=0, shuffle=False, callbacks=[self.tensorboard] if log_this_step else None) + # 更新目标网络 if log_this_step: self.target_update_counter += 1 if self.target_update_counter > UPDATE_TARGET_EVERY: - print("Target is updated") + print("目标网络已更新") self.target_model.set_weights(self.model.get_weights()) self.target_update_counter = 0 def train_in_loop(self): + """在单独线程中持续训练""" + # 预热训练 x = np.random.uniform(size=(1, IM_HEIGHT, IM_WIDTH, 3)).astype(np.float32) y = np.random.uniform(size=(1, 3)).astype(np.float32) self.model.fit(x, y, verbose=False, batch_size=1) self.training_initialized = True + # 持续训练循环 while True: if self.terminate: return self.train() - time.sleep(0.01) + time.sleep(0.01) # 控制训练频率 def get_qs(self, state): + """获取状态的Q值""" return self.model.predict(np.array(state).reshape(-1, *state.shape) / 255)[0] \ No newline at end of file diff --git a/src/Unmanned_vehicle_AD_DQN/Test.py b/src/Unmanned_vehicle_AD_DQN/Test.py index b3ae126f59..ef250e7dda 100644 --- a/src/Unmanned_vehicle_AD_DQN/Test.py +++ b/src/Unmanned_vehicle_AD_DQN/Test.py @@ -15,63 +15,62 @@ if __name__ == '__main__': - # Memory fraction + # GPU内存配置 gpu_options = tf.compat.v1.GPUOptions(per_process_gpu_memory_fraction=MEMORY_FRACTION) tf.compat.v1.keras.backend.set_session(tf.compat.v1.Session(config=tf.compat.v1.ConfigProto(gpu_options=gpu_options))) - # Load the model + # 加载训练好的模型 model = load_model(MODEL_PATH) - # Create environment + # 创建测试环境 env = CarEnv() - # For agent speed measurements - keeps last 60 frametimes + # FPS计数器 - 保存最近60帧的时间 fps_counter = deque(maxlen=60) - # Initialize predictions - first prediction takes longer as of initialization that has to be done - # It's better to do a first prediction then before we start iterating over episode steps + # 初始化预测 - 第一次预测需要较长时间进行初始化 model.predict(np.ones((1, env.im_height, env.im_width, 3))) - # Loop over episodes + # 循环测试多个episode while True: - print('Restarting episode') + print('开始新的测试轮次') - # Reset environment and get initial state + # 重置环境并获取初始状态 current_state = env.reset() - env.collision_hist = [] + env.collision_hist = [] # 重置碰撞历史 done = False - # Loop over steps + # 单次episode内的循环 while True: - # For FPS counter + # FPS计数开始 step_start = time.time() - # Show current frame - cv2.imshow(f'Agent - preview', current_state) + # 显示当前帧 + cv2.imshow(f'智能体预览', current_state) cv2.waitKey(1) - # Predict an action based on current observation space + # 基于当前观察空间预测动作 qs = model.predict(np.array(current_state).reshape(-1, *current_state.shape)/255)[0] - action = np.argmax(qs) + action = np.argmax(qs) # 选择Q值最大的动作 - # Step environment (additional flag informs environment to not break an episode by time limit) + # 执行环境步进 new_state, reward, done, _ = env.step(action) - # Set current step for next loop iteration + # 更新当前状态 current_state = new_state - # If done - agent crashed, break an episode + # 如果完成（碰撞等），结束当前episode if done: break - # Measure step time, append to a deque, then print mean FPS for last 60 frames, q values and taken action + # 计算帧时间，更新FPS计数器，打印统计信息 frame_time = time.time() - step_start fps_counter.append(frame_time) - print(f'Agent: {len(fps_counter)/sum(fps_counter):>4.1f} FPS | Action: [{qs[0]:>5.2f}, {qs[1]:>5.2f}, {qs[2]:>5.2f}] {action} | Reward: {reward}') + print(f'智能体: {len(fps_counter)/sum(fps_counter):>4.1f} FPS | 动作: [{qs[0]:>5.2f}, {qs[1]:>5.2f}, {qs[2]:>5.2f}] {action} | 奖励: {reward}') - # Destroy an actor at end of episode + # episode结束时销毁所有actor for actor in env.actor_list: actor.destroy() \ No newline at end of file diff --git a/src/Unmanned_vehicle_AD_DQN/main.py b/src/Unmanned_vehicle_AD_DQN/main.py index e66c08ffe5..0fdfcc5c29 100644 --- a/src/Unmanned_vehicle_AD_DQN/main.py +++ b/src/Unmanned_vehicle_AD_DQN/main.py @@ -29,100 +29,101 @@ from Hyperparameters import * if __name__ == '__main__': - FPS = 60 - ep_rewards = [-200] + FPS = 60 # 帧率 + ep_rewards = [-200] # 存储每轮奖励 - # For more repetitive results + # 为了结果可重复性（注释掉） # random.seed(1) # np.random.seed(1) # tf.compat.v1.set_random_seed(1) - # Memory fraction, used mostly when training multiple agents + # GPU内存配置，主要用于多智能体训练 gpu_options = tf.compat.v1.GPUOptions(per_process_gpu_memory_fraction=MEMORY_FRACTION) tf.compat.v1.keras.backend.set_session( tf.compat.v1.Session(config=tf.compat.v1.ConfigProto(gpu_options=gpu_options))) - # Create models folder + # 创建模型保存目录 if not os.path.isdir('models'): os.makedirs('models') - # Create agent and environment + # 创建智能体和环境 agent = DQNAgent() env = CarEnv() - # Start training thread and wait for training to be initialized + # 启动训练线程并等待训练初始化完成 trainer_thread = Thread(target=agent.train_in_loop, daemon=True) trainer_thread.start() while not agent.training_initialized: time.sleep(0.01) + # 预热Q网络 agent.get_qs(np.ones((env.im_height, env.im_width, 3))) - # 添加训练统计 - best_score = -float('inf') - success_count = 0 - scores = [] - avg_scores = [] + # 训练统计变量 + best_score = -float('inf') # 最佳得分 + success_count = 0 # 成功次数计数 + scores = [] # 存储每轮得分 + avg_scores = [] # 存储平均得分 - # Iterate over episodes + # 迭代训练轮次 epds = [] for episode in tqdm(range(1, EPISODES + 1), ascii=True, unit='episodes'): - env.collision_hist = [] - agent.tensorboard.step = episode + env.collision_hist = [] # 重置碰撞历史 + agent.tensorboard.step = episode # 设置TensorBoard步数 # 课程学习 - 随训练进度调整难度 if episode > EPISODES // 2: - # 后期增加行人数量 + # 训练后期增加行人数量以提高难度 env.spawn_pedestrians_general(40, True) env.spawn_pedestrians_general(15, False) else: - # 前期减少行人数量 + # 训练前期减少行人数量以降低难度 env.spawn_pedestrians_general(25, True) env.spawn_pedestrians_general(8, False) - # Restarting episode - reset episode reward and step number + # 重置每轮统计 - 重置得分和步数 score = 0 step = 1 - # Reset environment and get initial state + # 重置环境并获取初始状态 current_state = env.reset() - # Reset flag and start iterating until episode ends + # 重置完成标志并开始迭代直到本轮结束 done = False episode_start = time.time() - # 单次episode内的步数限制 + # 单次episode内的最大步数限制 max_steps_per_episode = SECONDS_PER_EPISODE * FPS - # Play for given number of seconds only + # 仅在给定秒数内运行 while not done and step < max_steps_per_episode: - # This part stays mostly the same, the change is to query a model for Q values + # 选择动作策略 if np.random.random() > Hyperparameters.EPSILON: - # Get action from Q table + # 从Q网络获取动作（利用） qs = agent.get_qs(current_state) action = np.argmax(qs) - print(f'Action: [{qs[0]:>5.2f}, {qs[1]:>5.2f}, {qs[2]:>5.2f}] {action}') + print(f'动作: [{qs[0]:>5.2f}, {qs[1]:>5.2f}, {qs[2]:>5.2f}] {action}') else: - # Get random action + # 随机选择动作（探索） action = np.random.randint(0, 3) - # This takes no time, so we add a delay matching 60 FPS (prediction above takes longer) + # 添加延迟以匹配60FPS time.sleep(1 / FPS) # 更频繁的状态更新 if step % 5 == 0: new_state, reward, done, _ = env.step(action) - score += reward - agent.update_replay_memory((current_state, action, reward, new_state, done)) - current_state = new_state + score += reward # 累加奖励 + agent.update_replay_memory((current_state, action, reward, new_state, done)) # 更新经验回放 + current_state = new_state # 更新当前状态 step += 1 if done: break - # End of episode - destroy agents + # 本轮结束 - 销毁所有actor for actor in env.actor_list: actor.destroy() @@ -135,39 +136,43 @@ best_score = score agent.model.save(f'models/{MODEL_NAME}_best_{score:.2f}.model') + # 记录得分统计 scores.append(score) - avg_scores.append(np.mean(scores[-10:])) + avg_scores.append(np.mean(scores[-10:])) # 计算最近10轮平均分 + # 定期聚合统计信息 if not episode % AGGREGATE_STATS_EVERY or episode == 1: - average_reward = np.mean(scores[-AGGREGATE_STATS_EVERY:]) - min_reward = min(scores[-AGGREGATE_STATS_EVERY:]) - max_reward = max(scores[-AGGREGATE_STATS_EVERY:]) + average_reward = np.mean(scores[-AGGREGATE_STATS_EVERY:]) # 平均奖励 + min_reward = min(scores[-AGGREGATE_STATS_EVERY:]) # 最小奖励 + max_reward = max(scores[-AGGREGATE_STATS_EVERY:]) # 最大奖励 agent.tensorboard.update_stats(reward_avg=average_reward, reward_min=min_reward, reward_max=max_reward, epsilon=Hyperparameters.EPSILON) - # Save model, but only when min reward is greater or equal a set value + # 保存模型，仅当最小奖励达到设定值时 if min_reward >= MIN_REWARD and (episode not in epds): agent.model.save( f'models/{MODEL_NAME}__{max_reward:_>7.2f}max_{average_reward:_>7.2f}avg_{min_reward:_>7.2f}min__{int(time.time())}.model') epds.append(episode) - print('episode: ', episode, 'score %.2f' % score, 'success_count:', success_count) + print('轮次: ', episode, '得分 %.2f' % score, '成功次数:', success_count) - # Decay epsilon + # 衰减探索率 if Hyperparameters.EPSILON > Hyperparameters.MIN_EPSILON: Hyperparameters.EPSILON *= Hyperparameters.EPSILON_DECAY Hyperparameters.EPSILON = max(Hyperparameters.MIN_EPSILON, Hyperparameters.EPSILON) - # Set termination flag for training thread and wait for it to finish + # 设置训练线程终止标志并等待其结束 agent.terminate = True trainer_thread.join() + # 保存最终模型 agent.model.save( f'models/{MODEL_NAME}__{max_reward:_>7.2f}max_{average_reward:_>7.2f}avg_{min_reward:_>7.2f}min__{int(time.time())}.model') + # 绘制训练曲线 fig = plt.figure() ax = fig.add_subplot(111) - plt.plot(scores) - plt.plot(avg_scores) - plt.ylabel('Score') - plt.xlabel('Episode #') + plt.plot(scores) # 得分曲线 + plt.plot(avg_scores) # 平均得分曲线 + plt.ylabel('得分') + plt.xlabel('训练轮次') plt.show() \ No newline at end of file From c048059f92b12595543f8188f1986323efb78b3a Mon Sep 17 00:00:00 2001 From: joe-justin369 Date: Mon, 1 Dec 2025 08:59:47 +0800 Subject: [PATCH 08/10] Signed-off-by: joe-justin369 --- src/Unmanned_vehicle_AD_DQN/Environment.py | 51 ++++++++- .../Hyperparameters.py | 2 +- src/Unmanned_vehicle_AD_DQN/Test.py | 103 +++++++++++------- 3 files changed, 110 insertions(+), 46 deletions(-) diff --git a/src/Unmanned_vehicle_AD_DQN/Environment.py b/src/Unmanned_vehicle_AD_DQN/Environment.py index 8d8a474af0..fbf3bff9aa 100644 --- a/src/Unmanned_vehicle_AD_DQN/Environment.py +++ b/src/Unmanned_vehicle_AD_DQN/Environment.py @@ -9,10 +9,6 @@ import math from Hyperparameters import * -# 全局统计变量 -avg_score = 0 -average_reward = 0 - import carla from carla import ColorConverter @@ -31,6 +27,10 @@ def __init__(self): # 加载世界和蓝图 self.world = self.client.load_world('Town03') + + # 设置观察者视角，让CARLA窗口显示 + self.setup_observer_view() + self.blueprint_library = self.world.get_blueprint_library() self.model_3 = self.blueprint_library.filter("model3")[0] # Tesla Model3车辆 @@ -39,6 +39,29 @@ def __init__(self): self.collision_history = [] self.slow_counter = 0 # 慢速计数器 + def setup_observer_view(self): + """设置观察者视角，让用户可以在CARLA窗口中看到场景""" + try: + # 获取当前地图的生成点 + spawn_points = self.world.get_map().get_spawn_points() + if spawn_points: + # 选择一个合适的观察者位置 + spectator = self.world.get_spectator() + + # 设置观察者位置在车辆起始位置附近 + transform = carla.Transform() + transform.location.x = -81.0 + transform.location.y = -195.0 + transform.location.z = 15.0 # 提高视角高度 + transform.rotation.pitch = -45.0 # 向下倾斜视角 + transform.rotation.yaw = 0.0 + transform.rotation.roll = 0.0 + + spectator.set_transform(transform) + print("观察者视角已设置") + except Exception as e: + print(f"设置观察者视角时出错: {e}") + def spawn_pedestrians_general(self, number, isCross): """生成指定数量的行人""" for i in range(number): @@ -206,12 +229,32 @@ def reset(self): while self.front_camera is None: time.sleep(0.01) + # 设置跟随相机（用于观察） + self.setup_follow_camera() + # 记录episode开始时间并重置控制 self.episode_start = time.time() self.vehicle.apply_control(carla.VehicleControl(throttle=0.0, brake=0.0, steer=0.0)) return self.front_camera + def setup_follow_camera(self): + """设置跟随车辆的相机，用于在CARLA窗口中观察""" + try: + # 创建RGB相机 + camera_bp = self.blueprint_library.find('sensor.camera.rgb') + camera_bp.set_attribute('image_size_x', '800') + camera_bp.set_attribute('image_size_y', '600') + camera_bp.set_attribute('fov', '110') + + # 相机位置相对于车辆（后方上方） + camera_transform = carla.Transform(carla.Location(x=-8, z=6), carla.Rotation(pitch=-20)) + follow_camera = self.world.spawn_actor(camera_bp, camera_transform, attach_to=self.vehicle) + self.actor_list.append(follow_camera) + print("跟随相机已设置") + except Exception as e: + print(f"设置跟随相机时出错: {e}") + def collision_data(self, event): """处理碰撞事件""" self.collision_history.append(event) diff --git a/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py b/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py index 920422c1b2..270b43a586 100644 --- a/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py +++ b/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py @@ -56,7 +56,7 @@ # 计算和聚合统计信息（如平均得分、奖励）的频率 SHOW_PREVIEW = False -# 是否显示预览窗口 +# 是否显示预览窗口 - 测试时设为False以显示CARLA主窗口 IM_WIDTH = 640 # 预览或模拟中捕获图像的宽度 diff --git a/src/Unmanned_vehicle_AD_DQN/Test.py b/src/Unmanned_vehicle_AD_DQN/Test.py index ef250e7dda..b3f3945ff8 100644 --- a/src/Unmanned_vehicle_AD_DQN/Test.py +++ b/src/Unmanned_vehicle_AD_DQN/Test.py @@ -11,7 +11,7 @@ from Hyperparameters import * -MODEL_PATH = r'D:\Work\T_Unmanned_vehicle_AD_DQN\models\YY_best_74.00.model' # 请替换为实际的最佳模型路径 +MODEL_PATH = r'D:\Work\T_Unmanned_vehicle_AD_DQN\models\YY_Optimized_best_240.26.model' # 请替换为实际的最佳模型路径 if __name__ == '__main__': @@ -22,8 +22,9 @@ # 加载训练好的模型 model = load_model(MODEL_PATH) - # 创建测试环境 + # 创建测试环境 - 禁用摄像头预览，让CARLA主窗口显示 env = CarEnv() + env.SHOW_CAM = False # 关闭小窗口预览 # FPS计数器 - 保存最近60帧的时间 fps_counter = deque(maxlen=60) @@ -31,46 +32,66 @@ # 初始化预测 - 第一次预测需要较长时间进行初始化 model.predict(np.ones((1, env.im_height, env.im_width, 3))) - # 循环测试多个episode - while True: - - print('开始新的测试轮次') - - # 重置环境并获取初始状态 - current_state = env.reset() - env.collision_hist = [] # 重置碰撞历史 - - done = False + print("开始测试！请查看CARLA窗口观看智能体运行...") + print("按Ctrl+C停止测试") - # 单次episode内的循环 + # 循环测试多个episode + episode_count = 0 + try: while True: - - # FPS计数开始 - step_start = time.time() - - # 显示当前帧 - cv2.imshow(f'智能体预览', current_state) - cv2.waitKey(1) - - # 基于当前观察空间预测动作 - qs = model.predict(np.array(current_state).reshape(-1, *current_state.shape)/255)[0] - action = np.argmax(qs) # 选择Q值最大的动作 - - # 执行环境步进 - new_state, reward, done, _ = env.step(action) - - # 更新当前状态 - current_state = new_state - - # 如果完成（碰撞等），结束当前episode - if done: - break - - # 计算帧时间，更新FPS计数器，打印统计信息 - frame_time = time.time() - step_start - fps_counter.append(frame_time) - print(f'智能体: {len(fps_counter)/sum(fps_counter):>4.1f} FPS | 动作: [{qs[0]:>5.2f}, {qs[1]:>5.2f}, {qs[2]:>5.2f}] {action} | 奖励: {reward}') - - # episode结束时销毁所有actor + episode_count += 1 + print(f'\n开始第 {episode_count} 个测试轮次') + + # 重置环境并获取初始状态 + current_state = env.reset() + env.collision_hist = [] # 重置碰撞历史 + + done = False + total_reward = 0 + step_count = 0 + + # 单次episode内的循环 + while True: + + # FPS计数开始 + step_start = time.time() + + # 基于当前观察空间预测动作 + qs = model.predict(np.array(current_state).reshape(-1, *current_state.shape)/255)[0] + action = np.argmax(qs) # 选择Q值最大的动作 + + # 执行环境步进 + new_state, reward, done, _ = env.step(action) + + # 更新当前状态 + current_state = new_state + total_reward += reward + step_count += 1 + + # 如果完成（碰撞等），结束当前episode + if done: + break + + # 计算帧时间，更新FPS计数器，打印统计信息 + frame_time = time.time() - step_start + fps_counter.append(frame_time) + if step_count % 10 == 0: # 每10步打印一次信息 + print(f'轮次 {episode_count} | 步数: {step_count} | FPS: {len(fps_counter)/sum(fps_counter):>4.1f} | 动作: [{qs[0]:>5.2f}, {qs[1]:>5.2f}, {qs[2]:>5.2f}] {action} | 奖励: {reward:.2f} | 累计奖励: {total_reward:.2f}') + + # episode结束时显示结果并销毁所有actor + result = "成功到达终点!" if reward > 5 else "发生碰撞或失败" + print(f'第 {episode_count} 轮结束: {result} | 总步数: {step_count} | 总奖励: {total_reward:.2f}') + + for actor in env.actor_list: + actor.destroy() + + # 短暂暂停后开始下一轮 + time.sleep(2) + + except KeyboardInterrupt: + print("\n测试被用户中断") + finally: + # 清理环境 + print("清理环境...") for actor in env.actor_list: actor.destroy() \ No newline at end of file From 4e9bfa5051b1bcc8c1b32ee8ee3e15ac75421371 Mon Sep 17 00:00:00 2001 From: joe-justin369 Date: Mon, 1 Dec 2025 10:03:29 +0800 Subject: [PATCH 09/10] =?UTF-8?q?=E4=B8=BA=E6=A8=A1=E5=9E=8B=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E8=BD=AC=E5=90=91=E5=8A=9F=E8=83=BD=EF=BC=8C=E7=94=A8?= =?UTF-8?q?=E4=BA=8E=E8=BA=B2=E9=81=BF=E9=9A=9C=E7=A2=8D=E6=88=96=E8=A1=8C?= =?UTF-8?q?=E4=BA=BA=EF=BC=8C=E5=B9=B6=E4=BC=98=E5=8C=96=E6=A8=A1=E6=8B=9F?= =?UTF-8?q?=E7=8E=AF=E5=A2=83=E4=B8=AD=E7=9A=84=E4=BA=BA=E5=91=98=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E6=83=85=E5=86=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Unmanned_vehicle_AD_DQN/Environment.py | 290 +++++++++++++-------- src/Unmanned_vehicle_AD_DQN/Model.py | 11 +- src/Unmanned_vehicle_AD_DQN/Test.py | 10 +- src/Unmanned_vehicle_AD_DQN/main.py | 17 +- 4 files changed, 196 insertions(+), 132 deletions(-) diff --git a/src/Unmanned_vehicle_AD_DQN/Environment.py b/src/Unmanned_vehicle_AD_DQN/Environment.py index fbf3bff9aa..9f0becb248 100644 --- a/src/Unmanned_vehicle_AD_DQN/Environment.py +++ b/src/Unmanned_vehicle_AD_DQN/Environment.py @@ -19,7 +19,7 @@ class CarEnv: im_height = IM_HEIGHT # 图像高度 def __init__(self): - self.actor_list = None # 存储所有actor的列表 + self.actor_list = [] # 存储所有actor的列表 self.sem_cam = None # 语义分割摄像头 self.client = carla.Client("localhost", 2000) # CARLA客户端 self.client.set_timeout(20.0) # 连接超时设置 @@ -38,6 +38,7 @@ def __init__(self): self.walker_list = [] self.collision_history = [] self.slow_counter = 0 # 慢速计数器 + self.steer_counter = 0 # 转向计数器，用于限制过度转向 def setup_observer_view(self): """设置观察者视角，让用户可以在CARLA窗口中看到场景""" @@ -63,7 +64,10 @@ def setup_observer_view(self): print(f"设置观察者视角时出错: {e}") def spawn_pedestrians_general(self, number, isCross): - """生成指定数量的行人""" + """生成指定数量的行人 - 大幅减少数量""" + # 限制最大生成数量 + number = min(number, 8) # 最多8个行人 + for i in range(number): isLeft = random.choice([True, False]) # 随机选择左右侧 if isLeft: @@ -74,124 +78,115 @@ def spawn_pedestrians_general(self, number, isCross): def spawn_pedestrians_right(self, isCross): """在右侧生成行人""" blueprints_walkers = self.world.get_blueprint_library().filter("walker.pedestrian.*") - walker_bp = random.choice(blueprints_walkers) + + # 设置生成区域 + min_x = -50 + max_x = 140 + min_y = -188 + max_y = -183 + + # 如果是十字路口，调整生成位置 + if isCross: + isFirstCross = random.choice([True, False]) + if isFirstCross: + min_x = -14 + max_x = -10.5 + else: + min_x = 17 + max_x = 20.5 + + # 随机生成位置 + x = random.uniform(min_x, max_x) + y = random.uniform(min_y, max_y) + + spawn_point = carla.Transform(carla.Location(x, y, 2.0)) - for i in range(1): - walker_bp = random.choice(blueprints_walkers) - - # 设置生成区域 - min_x = -50 - max_x = 140 - min_y = -188 - max_y = -183 - - # 如果是十字路口，调整生成位置 - if isCross: - isFirstCross = random.choice([True, False]) - if isFirstCross: - min_x = -14 - max_x = -10.5 - else: - min_x = 17 - max_x = 20.5 - - # 随机生成位置 + # 避免在特定区域生成 + while (-10 < spawn_point.location.x < 17) or (70 < spawn_point.location.x < 100): x = random.uniform(min_x, max_x) y = random.uniform(min_y, max_y) - spawn_point = carla.Transform(carla.Location(x, y, 2.0)) - # 避免在特定区域生成 - while (-10 < spawn_point.location.x < 17) or (70 < spawn_point.location.x < 100): - x = random.uniform(min_x, max_x) - y = random.uniform(min_y, max_y) - spawn_point = carla.Transform(carla.Location(x, y, 2.0)) - - # 尝试生成行人 - if spawn_point: - npc = self.world.try_spawn_actor(walker_bp, spawn_point) - - if npc is not None: - # 设置行人控制参数 - ped_control = carla.WalkerControl() - ped_control.speed = random.uniform(0.5, 1.0) # 随机速度 - ped_control.direction.y = -1 # 主要移动方向 - ped_control.direction.x = 0.15 # 轻微横向移动 - npc.apply_control(ped_control) - npc.set_simulate_physics(True) # 启用物理模拟 + # 尝试生成行人 + walker_bp = random.choice(blueprints_walkers) + npc = self.world.try_spawn_actor(walker_bp, spawn_point) + + if npc is not None: + # 设置行人控制参数 + ped_control = carla.WalkerControl() + ped_control.speed = random.uniform(0.5, 1.0) # 随机速度 + ped_control.direction.y = -1 # 主要移动方向 + ped_control.direction.x = 0.15 # 轻微横向移动 + npc.apply_control(ped_control) + npc.set_simulate_physics(True) # 启用物理模拟 + self.walker_list.append(npc) # 添加到行人列表 def spawn_pedestrians_left(self, isCross): """在左侧生成行人""" blueprints_walkers = self.world.get_blueprint_library().filter("walker.pedestrian.*") - walker_bp = random.choice(blueprints_walkers) + + # 设置生成区域 + min_x = -50 + max_x = 140 + min_y = -216 + max_y = -210 + + # 如果是十字路口，调整生成位置 + if isCross: + isFirstCross = random.choice([True, False]) + if isFirstCross: + min_x = -14 + max_x = -10.5 + else: + min_x = 17 + max_x = 20.5 - for i in range(1): - walker_bp = random.choice(blueprints_walkers) - - # 设置生成区域 - min_x = -50 - max_x = 140 - min_y = -216 - max_y = -210 - - # 如果是十字路口，调整生成位置 - if (isCross): - isFirstCross = random.choice([True, False]) - if isFirstCross: - min_x = -14 - max_x = -10.5 - else: - min_x = 17 - max_x = 20.5 - - # 随机生成位置 + # 随机生成位置 + x = random.uniform(min_x, max_x) + y = random.uniform(min_y, max_y) + + spawn_point = carla.Transform(carla.Location(x, y, 2.0)) + + # 避免在特定区域生成 + while (-10 < spawn_point.location.x < 17) or (70 < spawn_point.location.x < 100): x = random.uniform(min_x, max_x) y = random.uniform(min_y, max_y) - spawn_point = carla.Transform(carla.Location(x, y, 2.0)) - # 避免在特定区域生成 - while (-10 < spawn_point.location.x < 17) or (70 < spawn_point.location.x < 100): - x = random.uniform(min_x, max_x) - y = random.uniform(min_y, max_y) - spawn_point = carla.Transform(carla.Location(x, y, 2.0)) - - # 尝试生成行人 - if spawn_point: - npc = self.world.try_spawn_actor(walker_bp, spawn_point) - - if npc is not None: - # 设置行人控制参数 - ped_control = carla.WalkerControl() - ped_control.speed = random.uniform(0.7, 1.3) # 随机速度 - ped_control.direction.y = 1 # 主要移动方向 - ped_control.direction.x = -0.05 # 轻微横向移动 - npc.apply_control(ped_control) - npc.set_simulate_physics(True) # 启用物理模拟 + # 尝试生成行人 + walker_bp = random.choice(blueprints_walkers) + npc = self.world.try_spawn_actor(walker_bp, spawn_point) + + if npc is not None: + # 设置行人控制参数 + ped_control = carla.WalkerControl() + ped_control.speed = random.uniform(0.7, 1.3) # 随机速度 + ped_control.direction.y = 1 # 主要移动方向 + ped_control.direction.x = -0.05 # 轻微横向移动 + npc.apply_control(ped_control) + npc.set_simulate_physics(True) # 启用物理模拟 + self.walker_list.append(npc) # 添加到行人列表 def reset(self): """重置环境""" # 清理现有的行人和车辆 - walkers = self.world.get_actors().filter('walker.*') - for walker in walkers: - walker.destroy() - - vehicles = self.world.get_actors().filter('vehicle.*') - for v in vehicles: - v.destroy() + self.cleanup_actors() + + # 重置行人列表 + self.walker_list = [] - # 课程学习 - 根据训练阶段调整难度 - self.spawn_pedestrians_general(30, True) - self.spawn_pedestrians_general(10, False) + # 大幅减少行人数量 - 从30+10减少到8+4 + self.spawn_pedestrians_general(8, True) + self.spawn_pedestrians_general(4, False) # 重置状态变量 self.collision_history = [] self.actor_list = [] self.slow_counter = 0 + self.steer_counter = 0 # 设置车辆生成点 - spawn_points = self.world.get_map().get_spawn_points() - spawn_point = random.choice(spawn_points) if spawn_points else carla.Transform() + spawn_point = carla.Transform() spawn_point.location.x = -81.0 spawn_point.location.y = -195.0 spawn_point.location.z = 2.0 @@ -217,7 +212,7 @@ def reset(self): # 初始化车辆控制 self.vehicle.apply_control(carla.VehicleControl(throttle=0.0, brake=0.0, steer=0.0)) - time.sleep(4) # 等待环境稳定 + time.sleep(2) # 等待环境稳定 # 设置碰撞传感器 colsensor = self.blueprint_library.find("sensor.other.collision") @@ -238,6 +233,27 @@ def reset(self): return self.front_camera + def cleanup_actors(self): + """清理所有actors""" + # 清理车辆 + vehicles = self.world.get_actors().filter('vehicle.*') + for vehicle in vehicles: + if vehicle.is_alive: + vehicle.destroy() + + # 清理行人 + walkers = self.world.get_actors().filter('walker.*') + for walker in walkers: + if walker.is_alive: + walker.destroy() + + # 清理传感器 + for actor in self.actor_list: + if actor.is_alive: + actor.destroy() + + self.actor_list = [] + def setup_follow_camera(self): """设置跟随车辆的相机，用于在CARLA窗口中观察""" try: @@ -276,7 +292,7 @@ def process_img(self, image): self.front_camera = processed_image # 更新前置摄像头图像 def reward(self): - """计算奖励函数""" + """计算奖励函数 - 增强方向控制奖励""" reward = 0 done = False @@ -301,16 +317,22 @@ def reward(self): else: reward -= 0.2 # 不理想速度 - # 方向奖励 - 确保车辆朝正确方向行驶 - if -45 <= vehicle_rotation <= 45: # 大致朝东方向 - reward += 0.2 + # 增强方向奖励 - 确保车辆朝正确方向行驶 + if -20 <= vehicle_rotation <= 20: # 严格限制在正东方向附近 + reward += 0.5 # 增加直行奖励 + self.steer_counter = max(0, self.steer_counter - 1) # 减少转向计数 + elif -45 <= vehicle_rotation <= 45: + reward += 0.1 # 较小奖励 else: - reward -= 0.5 + reward -= 1.0 # 严重偏离惩罚 + self.steer_counter += 2 # 增加转向计数 # 行人距离检测 min_dist = float('inf') # 最小距离初始化为无穷大 - walkers = self.world.get_actors().filter('walker.*') - for walker in walkers: + for walker in self.walker_list: + if not walker.is_alive: + continue + ped_location = walker.get_location() dx = vehicle_location.x - ped_location.x dy = vehicle_location.y - ped_location.y @@ -318,10 +340,14 @@ def reward(self): min_dist = min(min_dist, distance) # 更新最小距离 # 清理边界外的行人 - player_direction = walker.get_control().direction - if (ped_location.y < -214 and player_direction.y == -1) or \ - (ped_location.y > -191 and player_direction.y == 1): - walker.destroy() + try: + player_direction = walker.get_control().direction + if (ped_location.y < -214 and player_direction.y == -1) or \ + (ped_location.y > -191 and player_direction.y == 1): + if walker.is_alive: + walker.destroy() + except: + pass # 如果无法获取控制信息，跳过 # 基于行人距离的奖励 if min_dist < 3.0: # 非常危险距离 @@ -353,14 +379,56 @@ def reward(self): return reward, done def step(self, action): - """执行动作并返回新状态""" - # 更平滑的控制策略 + """执行动作并返回新状态 - 扩展为5个动作包含转向""" + # 扩展的动作空间: 0-减速, 1-保持, 2-加速, 3-左转, 4-右转 + + # 限制连续转向次数，避免过度转向 + max_continuous_steer = 3 + current_steer = 0.0 + + if action == 3: # 左转 + if self.steer_counter < max_continuous_steer: + current_steer = -0.3 # 小角度左转 + self.steer_counter += 1 + else: + # 强制直行一段时间 + current_steer = 0.0 + action = 1 # 改为保持动作 + elif action == 4: # 右转 + if self.steer_counter < max_continuous_steer: + current_steer = 0.3 # 小角度右转 + self.steer_counter += 1 + else: + # 强制直行一段时间 + current_steer = 0.0 + action = 1 # 改为保持动作 + else: + # 非转向动作时逐渐减少转向计数 + self.steer_counter = max(0, self.steer_counter - 0.5) + + # 速度控制 + throttle = 0.0 + brake = 0.0 + if action == 0: # 减速 - self.vehicle.apply_control(carla.VehicleControl(throttle=0.0, brake=0.3)) + throttle = 0.0 + brake = 0.3 elif action == 1: # 保持/轻微加速 - self.vehicle.apply_control(carla.VehicleControl(throttle=0.3, brake=0.0)) + throttle = 0.3 + brake = 0.0 elif action == 2: # 加速 - self.vehicle.apply_control(carla.VehicleControl(throttle=0.7, brake=0.0)) + throttle = 0.7 + brake = 0.0 + elif action == 3 or action == 4: # 转向时保持适中速度 + throttle = 0.4 + brake = 0.0 + + # 应用控制 + self.vehicle.apply_control(carla.VehicleControl( + throttle=throttle, + brake=brake, + steer=current_steer + )) # 等待物理更新 time.sleep(0.05) diff --git a/src/Unmanned_vehicle_AD_DQN/Model.py b/src/Unmanned_vehicle_AD_DQN/Model.py index 13fc689eb7..f62147b7e2 100644 --- a/src/Unmanned_vehicle_AD_DQN/Model.py +++ b/src/Unmanned_vehicle_AD_DQN/Model.py @@ -30,7 +30,7 @@ def __init__(self, **kwargs): super().__init__(**kwargs) self._log_write_dir = self.log_dir self.step = 1 - self.writer = self.writer = tf.summary.create_file_writer(self.log_dir) + self.writer = tf.summary.create_file_writer(self.log_dir) def set_model(self, model): self.model = model @@ -78,7 +78,7 @@ def __init__(self): self.training_initialized = False def create_model(self): - """创建深度Q网络模型""" + """创建深度Q网络模型 - 扩展为5个输出""" model = Sequential() # 第一卷积块 @@ -115,8 +115,8 @@ def create_model(self): model.add(Dense(128, activation='relu')) model.add(Dropout(0.2)) - # 输出层 - 使用线性激活函数用于Q值回归 - model.add(Dense(3, activation='linear')) + # 输出层 - 扩展为5个动作: 0-减速, 1-保持, 2-加速, 3-左转, 4-右转 + model.add(Dense(5, activation='linear')) # 编译模型，使用Huber损失和Adam优化器 model.compile(loss="huber", optimizer=Adam(lr=LEARNING_RATE), metrics=["mae"]) @@ -178,7 +178,6 @@ def train(self): # 选择小批量经验 minibatch = self.minibatch_chooser() - print([transition[2] for transition in minibatch]) # 打印奖励值 # 准备训练数据 current_states = np.array([transition[0] for transition in minibatch]) / 255 @@ -228,7 +227,7 @@ def train_in_loop(self): """在单独线程中持续训练""" # 预热训练 x = np.random.uniform(size=(1, IM_HEIGHT, IM_WIDTH, 3)).astype(np.float32) - y = np.random.uniform(size=(1, 3)).astype(np.float32) + y = np.random.uniform(size=(1, 5)).astype(np.float32) # 改为5个输出 self.model.fit(x, y, verbose=False, batch_size=1) self.training_initialized = True diff --git a/src/Unmanned_vehicle_AD_DQN/Test.py b/src/Unmanned_vehicle_AD_DQN/Test.py index b3f3945ff8..2fd5906018 100644 --- a/src/Unmanned_vehicle_AD_DQN/Test.py +++ b/src/Unmanned_vehicle_AD_DQN/Test.py @@ -11,7 +11,7 @@ from Hyperparameters import * -MODEL_PATH = r'D:\Work\T_Unmanned_vehicle_AD_DQN\models\YY_Optimized_best_240.26.model' # 请替换为实际的最佳模型路径 +MODEL_PATH = r'D:\Work\T_Unmanned_vehicle_AD_DQN\models\YY_Optimized___290.14max___97.16avg___13.42min__1764553908.model' # 请替换为实际的最佳模型路径 if __name__ == '__main__': @@ -76,14 +76,13 @@ frame_time = time.time() - step_start fps_counter.append(frame_time) if step_count % 10 == 0: # 每10步打印一次信息 - print(f'轮次 {episode_count} | 步数: {step_count} | FPS: {len(fps_counter)/sum(fps_counter):>4.1f} | 动作: [{qs[0]:>5.2f}, {qs[1]:>5.2f}, {qs[2]:>5.2f}] {action} | 奖励: {reward:.2f} | 累计奖励: {total_reward:.2f}') + print(f'轮次 {episode_count} | 步数: {step_count} | FPS: {len(fps_counter)/sum(fps_counter):>4.1f} | 动作: [{qs[0]:>5.2f}, {qs[1]:>5.2f}, {qs[2]:>5.2f}, {qs[3]:>5.2f}, {qs[4]:>5.2f}] {action} | 奖励: {reward:.2f} | 累计奖励: {total_reward:.2f}') # episode结束时显示结果并销毁所有actor result = "成功到达终点!" if reward > 5 else "发生碰撞或失败" print(f'第 {episode_count} 轮结束: {result} | 总步数: {step_count} | 总奖励: {total_reward:.2f}') - for actor in env.actor_list: - actor.destroy() + env.cleanup_actors() # 短暂暂停后开始下一轮 time.sleep(2) @@ -93,5 +92,4 @@ finally: # 清理环境 print("清理环境...") - for actor in env.actor_list: - actor.destroy() \ No newline at end of file + env.cleanup_actors() \ No newline at end of file diff --git a/src/Unmanned_vehicle_AD_DQN/main.py b/src/Unmanned_vehicle_AD_DQN/main.py index 0fdfcc5c29..9bf9ca442f 100644 --- a/src/Unmanned_vehicle_AD_DQN/main.py +++ b/src/Unmanned_vehicle_AD_DQN/main.py @@ -74,12 +74,12 @@ # 课程学习 - 随训练进度调整难度 if episode > EPISODES // 2: # 训练后期增加行人数量以提高难度 - env.spawn_pedestrians_general(40, True) - env.spawn_pedestrians_general(15, False) + env.spawn_pedestrians_general(8, True) # 减少数量 + env.spawn_pedestrians_general(4, False) # 减少数量 else: # 训练前期减少行人数量以降低难度 - env.spawn_pedestrians_general(25, True) - env.spawn_pedestrians_general(8, False) + env.spawn_pedestrians_general(6, True) # 减少数量 + env.spawn_pedestrians_general(3, False) # 减少数量 # 重置每轮统计 - 重置得分和步数 score = 0 @@ -103,10 +103,10 @@ # 从Q网络获取动作（利用） qs = agent.get_qs(current_state) action = np.argmax(qs) - print(f'动作: [{qs[0]:>5.2f}, {qs[1]:>5.2f}, {qs[2]:>5.2f}] {action}') + print(f'动作: [{qs[0]:>5.2f}, {qs[1]:>5.2f}, {qs[2]:>5.2f}, {qs[3]:>5.2f}, {qs[4]:>5.2f}] {action}') else: - # 随机选择动作（探索） - action = np.random.randint(0, 3) + # 随机选择动作（探索）- 扩展为5个动作 + action = np.random.randint(0, 5) # 添加延迟以匹配60FPS time.sleep(1 / FPS) @@ -124,8 +124,7 @@ break # 本轮结束 - 销毁所有actor - for actor in env.actor_list: - actor.destroy() + env.cleanup_actors() # 更新成功计数 if score > 5: # 成功完成的阈值 From fb36a8d8be32626323f162b791acac39fb08981c Mon Sep 17 00:00:00 2001 From: joe-justin369 Date: Mon, 15 Dec 2025 10:53:09 +0800 Subject: [PATCH 10/10] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E8=A1=8C=E4=BA=BA?= =?UTF-8?q?=E9=81=BF=E9=9A=9C=E9=97=AE=E9=A2=98=EF=BC=8C=E5=90=8C=E6=97=B6?= =?UTF-8?q?=E4=BF=9D=E6=8C=81=E8=BD=A6=E8=BE=86=E6=B2=BF=E9=81=93=E8=B7=AF?= =?UTF-8?q?=E8=A1=8C=E9=A9=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Unmanned_vehicle_AD_DQN/Environment.py | 414 ++++++++++-------- .../Hyperparameters.py | 26 +- src/Unmanned_vehicle_AD_DQN/Model.py | 69 +-- src/Unmanned_vehicle_AD_DQN/Test.py | 34 +- src/Unmanned_vehicle_AD_DQN/main.py | 31 +- 5 files changed, 334 insertions(+), 240 deletions(-) diff --git a/src/Unmanned_vehicle_AD_DQN/Environment.py b/src/Unmanned_vehicle_AD_DQN/Environment.py index 092655362d..5929383d5f 100644 --- a/src/Unmanned_vehicle_AD_DQN/Environment.py +++ b/src/Unmanned_vehicle_AD_DQN/Environment.py @@ -24,6 +24,14 @@ def __init__(self): self.client = carla.Client("localhost", 2000) # CARLA客户端 self.client.set_timeout(20.0) # 连接超时设置 self.front_camera = None # 前置摄像头图像 + + # 新增变量 + self.last_action = 1 # 上一个动作，默认保持 + self.same_steer_counter = 0 # 连续同向转向计数器 + self.suggested_action = None # 建议的避让动作 + self.episode_start_time = None # 每轮开始时间 + self.last_ped_distance = float('inf') # 上次最近行人距离 + self.current_episode = 1 # 当前episode编号 # 加载世界和蓝图 self.world = self.client.load_world('Town03') @@ -63,40 +71,36 @@ def setup_observer_view(self): except Exception as e: print(f"设置观察者视角时出错: {e}") - def setup_observer_view(self): - """设置观察者视角，让用户可以在CARLA窗口中看到场景""" - try: - # 获取当前地图的生成点 - spawn_points = self.world.get_map().get_spawn_points() - if spawn_points: - # 选择一个合适的观察者位置 - spectator = self.world.get_spectator() - - # 设置观察者位置在车辆起始位置附近 - transform = carla.Transform() - transform.location.x = -81.0 - transform.location.y = -195.0 - transform.location.z = 15.0 # 提高视角高度 - transform.rotation.pitch = -45.0 # 向下倾斜视角 - transform.rotation.yaw = 0.0 - transform.rotation.roll = 0.0 - - spectator.set_transform(transform) - print("观察者视角已设置") - except Exception as e: - print(f"设置观察者视角时出错: {e}") - def spawn_pedestrians_general(self, number, isCross): - """生成指定数量的行人 - 大幅减少数量""" - # 限制最大生成数量 - number = min(number, 8) # 最多8个行人 + """生成指定数量的行人""" + # 记录要生成的数量 + target_number = number + + # 记录成功生成的数量 + success_count = 0 - for i in range(number): + # 尝试生成指定数量的行人 + attempts = 0 + max_attempts = number * 3 # 最多尝试3倍次数 + + while success_count < target_number and attempts < max_attempts: + attempts += 1 isLeft = random.choice([True, False]) # 随机选择左右侧 - if isLeft: - self.spawn_pedestrians_left(isCross) - else: - self.spawn_pedestrians_right(isCross) + + try: + if isLeft: + if self.spawn_pedestrians_left(isCross): + success_count += 1 + else: + if self.spawn_pedestrians_right(isCross): + success_count += 1 + except Exception as e: + # 如果生成失败，继续尝试 + print(f"生成行人失败: {e}") + continue + + print(f"成功生成 {success_count}/{target_number} 个行人 (isCross={isCross})") + return success_count def spawn_pedestrians_right(self, isCross): """在右侧生成行人""" @@ -118,31 +122,40 @@ def spawn_pedestrians_right(self, isCross): min_x = 17 max_x = 20.5 - # 随机生成位置 - x = random.uniform(min_x, max_x) - y = random.uniform(min_y, max_y) - - spawn_point = carla.Transform(carla.Location(x, y, 2.0)) - - # 避免在特定区域生成 - while (-10 < spawn_point.location.x < 17) or (70 < spawn_point.location.x < 100): + # 尝试多次生成直到成功 + for attempt in range(3): # 尝试3次 + # 随机生成位置 x = random.uniform(min_x, max_x) y = random.uniform(min_y, max_y) + spawn_point = carla.Transform(carla.Location(x, y, 2.0)) - # 尝试生成行人 - walker_bp = random.choice(blueprints_walkers) - npc = self.world.try_spawn_actor(walker_bp, spawn_point) + # 避免在特定区域生成 + while (-10 < spawn_point.location.x < 17) or (70 < spawn_point.location.x < 100): + x = random.uniform(min_x, max_x) + y = random.uniform(min_y, max_y) + spawn_point = carla.Transform(carla.Location(x, y, 2.0)) - if npc is not None: - # 设置行人控制参数 - ped_control = carla.WalkerControl() - ped_control.speed = random.uniform(0.5, 1.0) # 随机速度 - ped_control.direction.y = -1 # 主要移动方向 - ped_control.direction.x = 0.15 # 轻微横向移动 - npc.apply_control(ped_control) - npc.set_simulate_physics(True) # 启用物理模拟 - self.walker_list.append(npc) # 添加到行人列表 + # 尝试生成行人 + try: + walker_bp = random.choice(blueprints_walkers) + npc = self.world.try_spawn_actor(walker_bp, spawn_point) + + if npc is not None: + # 设置行人控制参数 + ped_control = carla.WalkerControl() + ped_control.speed = random.uniform(0.5, 1.0) # 随机速度 + ped_control.direction.y = -1 # 主要移动方向 + ped_control.direction.x = 0.15 # 轻微横向移动 + npc.apply_control(ped_control) + npc.set_simulate_physics(True) # 启用物理模拟 + self.walker_list.append(npc) # 添加到行人列表 + return True # 生成成功 + except Exception as e: + print(f"生成右侧行人失败 (尝试 {attempt+1}): {e}") + continue + + return False # 生成失败 def spawn_pedestrians_left(self, isCross): """在左侧生成行人""" @@ -164,49 +177,75 @@ def spawn_pedestrians_left(self, isCross): min_x = 17 max_x = 20.5 - # 随机生成位置 - x = random.uniform(min_x, max_x) - y = random.uniform(min_y, max_y) - - spawn_point = carla.Transform(carla.Location(x, y, 2.0)) - - # 避免在特定区域生成 - while (-10 < spawn_point.location.x < 17) or (70 < spawn_point.location.x < 100): + # 尝试多次生成直到成功 + for attempt in range(3): # 尝试3次 + # 随机生成位置 x = random.uniform(min_x, max_x) y = random.uniform(min_y, max_y) + spawn_point = carla.Transform(carla.Location(x, y, 2.0)) - # 尝试生成行人 - walker_bp = random.choice(blueprints_walkers) - npc = self.world.try_spawn_actor(walker_bp, spawn_point) - - if npc is not None: - # 设置行人控制参数 - ped_control = carla.WalkerControl() - ped_control.speed = random.uniform(0.7, 1.3) # 随机速度 - ped_control.direction.y = 1 # 主要移动方向 - ped_control.direction.x = -0.05 # 轻微横向移动 - npc.apply_control(ped_control) - npc.set_simulate_physics(True) # 启用物理模拟 - self.walker_list.append(npc) # 添加到行人列表 - - def reset(self): - """重置环境""" + # 避免在特定区域生成 + while (-10 < spawn_point.location.x < 17) or (70 < spawn_point.location.x < 100): + x = random.uniform(min_x, max_x) + y = random.uniform(min_y, max_y) + spawn_point = carla.Transform(carla.Location(x, y, 2.0)) + + # 尝试生成行人 + try: + walker_bp = random.choice(blueprints_walkers) + npc = self.world.try_spawn_actor(walker_bp, spawn_point) + + if npc is not None: + # 设置行人控制参数 + ped_control = carla.WalkerControl() + ped_control.speed = random.uniform(0.7, 1.3) # 随机速度 + ped_control.direction.y = 1 # 主要移动方向 + ped_control.direction.x = -0.05 # 轻微横向移动 + npc.apply_control(ped_control) + npc.set_simulate_physics(True) # 启用物理模拟 + self.walker_list.append(npc) # 添加到行人列表 + return True # 生成成功 + except Exception as e: + print(f"生成左侧行人失败 (尝试 {attempt+1}): {e}") + continue + + return False # 生成失败 + + def reset(self, episode=1): + """重置环境，根据episode参数生成不同数量的行人""" + self.current_episode = episode + # 清理现有的行人和车辆 self.cleanup_actors() # 重置行人列表 self.walker_list = [] - # 大幅减少行人数量 - 从30+10减少到8+4 - self.spawn_pedestrians_general(8, True) - self.spawn_pedestrians_general(4, False) + # 根据训练阶段生成不同数量的行人 + if episode < 100: # 第一阶段：少量行人 + print(f"Episode {episode}: 生成少量行人 (4十字路口 + 2非十字路口)") + self.spawn_pedestrians_general(4, True) # 十字路口行人 + self.spawn_pedestrians_general(2, False) # 非十字路口行人 + elif episode < 400: # 第二阶段：中等数量行人 + print(f"Episode {episode}: 生成中等数量行人 (6十字路口 + 3非十字路口)") + self.spawn_pedestrians_general(6, True) + self.spawn_pedestrians_general(3, False) + else: # 第三阶段：正常难度 + print(f"Episode {episode}: 生成正常数量行人 (8十字路口 + 4非十字路口)") + self.spawn_pedestrians_general(8, True) + self.spawn_pedestrians_general(4, False) # 重置状态变量 self.collision_history = [] self.actor_list = [] self.slow_counter = 0 self.steer_counter = 0 + self.same_steer_counter = 0 + self.suggested_action = None + self.last_action = 1 + self.episode_start_time = time.time() + self.last_ped_distance = float('inf') # 设置车辆生成点 spawn_point = carla.Transform() @@ -276,6 +315,7 @@ def cleanup_actors(self): actor.destroy() self.actor_list = [] + self.walker_list = [] # 清空行人列表 def setup_follow_camera(self): """设置跟随车辆的相机，用于在CARLA窗口中观察""" @@ -314,152 +354,182 @@ def process_img(self, image): self.front_camera = processed_image # 更新前置摄像头图像 - def reward(self): - """计算奖励函数 - 增强方向控制奖励""" + def reward(self, speed_kmh, current_steer): + """增强的奖励函数 - 特别强调行人避障""" reward = 0 done = False - - # 计算车辆速度 - velocity = self.vehicle.get_velocity() - velocity_kmh = int(3.6 * math.sqrt(velocity.x ** 2 + velocity.y ** 2 + velocity.z ** 2)) - # 获取车辆位置和方向 + # 获取车辆状态 vehicle_location = self.vehicle.get_location() vehicle_rotation = self.vehicle.get_transform().rotation.yaw - # 计算距离终点的进度奖励 - progress_reward = (vehicle_location.x + 81) / 236.0 # 从-81到155，总共236单位 + # 1. 道路保持奖励（重要的基础） + heading_error = abs(vehicle_rotation) + + if heading_error < 5: # 完美保持方向 + reward += 0.8 # 略微降低权重，给行人避障更多空间 + elif heading_error < 15: # 良好保持 + reward += 0.4 + elif heading_error < 30: # 可接受 + reward += 0.1 + else: # 方向偏差过大 + reward -= 0.3 * (heading_error / 30.0) # 降低惩罚强度 + + # 2. 行人避障（最高优先级） - 增强奖励机制 + min_ped_distance = float('inf') + closest_pedestrian = None + + # 统计有效行人数量 + active_pedestrians = 0 - # 速度奖励 - 更加平滑 - if velocity_kmh == 0: - reward -= 0.5 # 停车惩罚减少 - elif 20 <= velocity_kmh <= 40: # 理想速度区间 - reward += 0.8 - elif 10 <= velocity_kmh < 20 or 40 < velocity_kmh <= 50: - reward += 0.3 # 可接受速度区间 - else: - reward -= 0.2 # 不理想速度 - - # 增强方向奖励 - 确保车辆朝正确方向行驶 - if -20 <= vehicle_rotation <= 20: # 严格限制在正东方向附近 - reward += 0.5 # 增加直行奖励 - self.steer_counter = max(0, self.steer_counter - 1) # 减少转向计数 - elif -45 <= vehicle_rotation <= 45: - reward += 0.1 # 较小奖励 - else: - reward -= 1.0 # 严重偏离惩罚 - self.steer_counter += 2 # 增加转向计数 - - # 行人距离检测 - min_dist = float('inf') # 最小距离初始化为无穷大 for walker in self.walker_list: if not walker.is_alive: continue + active_pedestrians += 1 ped_location = walker.get_location() dx = vehicle_location.x - ped_location.x dy = vehicle_location.y - ped_location.y distance = math.sqrt(dx**2 + dy**2) - min_dist = min(min_dist, distance) # 更新最小距离 - # 清理边界外的行人 - try: - player_direction = walker.get_control().direction - if (ped_location.y < -214 and player_direction.y == -1) or \ - (ped_location.y > -191 and player_direction.y == 1): - if walker.is_alive: - walker.destroy() - except: - pass # 如果无法获取控制信息，跳过 - - # 基于行人距离的奖励 - if min_dist < 3.0: # 非常危险距离 - reward -= 3.0 - done = True - elif min_dist < 5.0: # 危险距离 - reward -= 1.0 - elif min_dist < 8.0: # 警告距离 - reward -= 0.3 - elif min_dist > 15.0: # 安全距离 + if distance < min_ped_distance: + min_ped_distance = distance + closest_pedestrian = walker + + # 如果当前没有有效行人，重置距离 + if active_pedestrians == 0: + min_ped_distance = float('inf') + + # 行人距离分级奖励 - 增强避障激励 + if min_ped_distance < 100: # 只考虑100米内的行人 + if min_ped_distance < 3.0: # 紧急避让距离 + reward -= 8.0 # 增加惩罚 + done = True + print(f"Episode {self.current_episode}: 与行人距离过近 ({min_ped_distance:.1f}m)!") + elif min_ped_distance < 5.0: # 危险距离 + reward -= 3.0 # 增加惩罚 + # 计算避让方向 + if closest_pedestrian: + ped_y = closest_pedestrian.get_location().y + veh_y = vehicle_location.y + # 如果行人在车辆左侧，鼓励右转；反之鼓励左转 + if ped_y < veh_y: # 行人在左侧 + self.suggested_action = 4 # 右转 + else: # 行人在右侧 + self.suggested_action = 3 # 左转 + elif min_ped_distance < 8.0: # 预警距离 + reward -= 0.8 + elif min_ped_distance < 12.0: # 安全距离 + reward += 0.5 # 增加安全距离奖励 + else: # 非常安全 + reward += 0.2 + + # 3. 成功避障奖励 - 当成功避开行人后给予额外奖励 + if self.last_ped_distance < 8.0 and min_ped_distance > self.last_ped_distance: + # 如果上次距离危险，这次距离更远了，说明成功避让 + reward += 0.3 + + self.last_ped_distance = min_ped_distance # 保存当前距离 + + # 4. 速度奖励（平衡避障和前进） + if 15 <= speed_kmh <= 35: # 理想速度区间 + reward += 0.4 + elif 5 <= speed_kmh < 15: # 较慢但安全（避障时可能需要减速） reward += 0.2 - - # 碰撞检测 + elif 35 < speed_kmh <= 45: # 稍快 + reward += 0.1 + elif speed_kmh > 45: # 过快，在行人环境中危险 + reward -= 0.5 # 增加惩罚 + else: # 停车或极慢 + reward -= 0.05 # 轻微惩罚 + + # 5. 转向平滑性奖励（避障时可能需要适当转向） + steer_penalty = abs(current_steer) * 0.3 # 降低惩罚，给避障转向更多空间 + reward -= steer_penalty + + # 6. 碰撞检测 if len(self.collision_history) != 0: - reward = -10 # 碰撞惩罚 + reward = -15 # 增加碰撞惩罚 done = True - - # 进度奖励 - reward += progress_reward * 0.5 + print(f"Episode {self.current_episode}: 发生碰撞!") + + # 7. 进度奖励（次要于避障） + progress = (vehicle_location.x + 81) / 236.0 # 从-81到155 + reward += progress * 0.2 # 降低进度权重 - # 完成条件判断 + # 8. 边界检查 if vehicle_location.x > 155: # 成功到达终点 - reward += 10 # 成功到达奖励 + reward += 20 # 增加到达终点的奖励 done = True - elif vehicle_location.x < -90: # 倒退太多 - reward -= 5 + print(f"Episode {self.current_episode}: 成功到达终点!") + elif vehicle_location.x < -90 or abs(vehicle_location.y + 195) > 30: # 偏离道路 + reward -= 3 # 降低偏离惩罚 done = True - + print(f"Episode {self.current_episode}: 偏离道路!") + return reward, done def step(self, action): - """执行动作并返回新状态 - 扩展为5个动作包含转向""" - # 扩展的动作空间: 0-减速, 1-保持, 2-加速, 3-左转, 4-右转 + """执行动作并返回新状态 - 增强平滑性和安全性""" + # 获取当前速度 + velocity = self.vehicle.get_velocity() + speed_kmh = 3.6 * math.sqrt(velocity.x**2 + velocity.y**2 + velocity.z**2) - # 限制连续转向次数，避免过度转向 - max_continuous_steer = 3 - current_steer = 0.0 + # 5个动作: 0-减速, 1-保持, 2-加速, 3-左转, 4-右转 - if action == 3: # 左转 - if self.steer_counter < max_continuous_steer: - current_steer = -0.3 # 小角度左转 - self.steer_counter += 1 - else: - # 强制直行一段时间 - current_steer = 0.0 - action = 1 # 改为保持动作 - elif action == 4: # 右转 - if self.steer_counter < max_continuous_steer: - current_steer = 0.3 # 小角度右转 - self.steer_counter += 1 - else: - # 强制直行一段时间 - current_steer = 0.0 - action = 1 # 改为保持动作 - else: - # 非转向动作时逐渐减少转向计数 - self.steer_counter = max(0, self.steer_counter - 0.5) + # 根据速度调整转向幅度 - 速度越快，转向越平滑 + speed_factor = max(0.5, min(1.0, 30.0 / max(1.0, speed_kmh))) - # 速度控制 + # 基础控制参数 throttle = 0.0 brake = 0.0 + steer = 0.0 + # 速度控制 if action == 0: # 减速 throttle = 0.0 - brake = 0.3 + brake = 0.5 elif action == 1: # 保持/轻微加速 throttle = 0.3 brake = 0.0 elif action == 2: # 加速 throttle = 0.7 brake = 0.0 - elif action == 3 or action == 4: # 转向时保持适中速度 + elif action == 3: # 左转 throttle = 0.4 brake = 0.0 - + steer = -0.2 * speed_factor # 速度相关的转向幅度 + elif action == 4: # 右转 + throttle = 0.4 + brake = 0.0 + steer = 0.2 * speed_factor # 速度相关的转向幅度 + + # 限制连续同向转向 - 防止过度转向 + if (action == 3 and self.last_action == 3) or (action == 4 and self.last_action == 4): + self.same_steer_counter += 1 + if self.same_steer_counter > 3: # 连续3次同向转向后强制回正 + steer *= 0.5 # 减小转向幅度 + throttle *= 0.8 # 减速 + else: + self.same_steer_counter = 0 + + # 记录上一个动作 + self.last_action = action + # 应用控制 self.vehicle.apply_control(carla.VehicleControl( throttle=throttle, brake=brake, - steer=current_steer + steer=steer )) - + # 等待物理更新 time.sleep(0.05) # 计算奖励和完成状态 - reward, done = self.reward() + reward, done = self.reward(speed_kmh, steer) # 限制极端奖励值 - reward = np.clip(reward, -10, 10) + reward = np.clip(reward, -15, 20) return self.front_camera, reward, done, None \ No newline at end of file diff --git a/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py b/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py index 270b43a586..717228cf84 100644 --- a/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py +++ b/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py @@ -1,8 +1,8 @@ # Hyperparameters.py # 深度强化学习超参数配置 -DISCOUNT = 0.95 -# 未来奖励的折扣因子 +DISCOUNT = 0.97 +# 未来奖励的折扣因子 - 提高未来奖励的重要性 FPS = 60 # 模拟环境的帧率 @@ -13,14 +13,14 @@ REWARD_OFFSET = -100 # 停止模拟的奖励阈值 -MIN_REPLAY_MEMORY_SIZE = 2_000 -# 开始训练前经验回放缓冲区的最小大小 +MIN_REPLAY_MEMORY_SIZE = 3_000 +# 开始训练前经验回放缓冲区的最小大小 - 增加以获得更稳定训练 REPLAY_MEMORY_SIZE = 10_000 # 经验回放缓冲区的最大容量 -MINIBATCH_SIZE = 32 -# 每次训练从经验回放中采样的经验数量 +MINIBATCH_SIZE = 64 +# 每次训练从经验回放中采样的经验数量 - 增加批次大小 PREDICTION_BATCH_SIZE = 1 # 预测阶段使用的批次大小 @@ -28,7 +28,7 @@ TRAINING_BATCH_SIZE = MINIBATCH_SIZE // 4 # 训练阶段使用的批次大小 -EPISODES = 1000 +EPISODES = 800 # 从1000减少到800，因为减少了无行人阶段 # 智能体训练的总轮次数 SECONDS_PER_EPISODE = 60 @@ -40,8 +40,8 @@ EPSILON = 1.0 # 初始探索率 -EPSILON_DECAY = 0.995 -# 探索率的衰减率 +EPSILON_DECAY = 0.998 +# 探索率的衰减率 - 减缓衰减速度 MODEL_NAME = "YY_Optimized" # 训练模型的名称标识 @@ -49,8 +49,8 @@ MIN_REWARD = 5 # 被认为是"良好"或"积极"经验的最小奖励值 -UPDATE_TARGET_EVERY = 10 -# 目标网络更新的频率 +UPDATE_TARGET_EVERY = 20 +# 目标网络更新的频率 - 增加以获得更稳定训练 AGGREGATE_STATS_EVERY = 10 # 计算和聚合统计信息（如平均得分、奖励）的频率 @@ -73,5 +73,5 @@ SUCCESSFUL_THRESHOLD = 3 # 成功阈值 -LEARNING_RATE = 0.0001 -# 优化器的学习率 \ No newline at end of file +LEARNING_RATE = 0.00005 +# 优化器的学习率 - 降低以获得更稳定训练 \ No newline at end of file diff --git a/src/Unmanned_vehicle_AD_DQN/Model.py b/src/Unmanned_vehicle_AD_DQN/Model.py index f62147b7e2..5adcb42262 100644 --- a/src/Unmanned_vehicle_AD_DQN/Model.py +++ b/src/Unmanned_vehicle_AD_DQN/Model.py @@ -10,12 +10,10 @@ import matplotlib.pyplot as plt from collections import deque from tensorflow.keras.applications.xception import Xception -from tensorflow.keras.layers import Dense, GlobalAveragePooling2D -from tensorflow.keras.models import Sequential, Model from tensorflow.keras.layers import Dense, GlobalAveragePooling2D, Input, Concatenate, Conv2D, AveragePooling2D, Activation, \ - Flatten, Dropout, BatchNormalization, MaxPooling2D + Flatten, Dropout, BatchNormalization, MaxPooling2D, Multiply from tensorflow.keras.optimizers import Adam -from tensorflow.keras.models import Model +from tensorflow.keras.models import Sequential, Model from tensorflow.keras.callbacks import TensorBoard import tensorflow as tf import tensorflow.keras.backend as backend @@ -78,47 +76,52 @@ def __init__(self): self.training_initialized = False def create_model(self): - """创建深度Q网络模型 - 扩展为5个输出""" - model = Sequential() + """创建深度Q网络模型 - 优化网络结构""" + # 使用函数式API以支持注意力机制 + inputs = Input(shape=(IM_HEIGHT, IM_WIDTH, 3)) # 第一卷积块 - model.add(Conv2D(32, (5, 5), strides=(2, 2), input_shape=(IM_HEIGHT, IM_WIDTH, 3), padding='same')) - model.add(Activation('relu')) - model.add(BatchNormalization()) # 批归一化 - model.add(MaxPooling2D(pool_size=(2, 2))) + x = Conv2D(32, (5, 5), strides=(2, 2), padding='same')(inputs) + x = Activation('relu')(x) + x = BatchNormalization()(x) + x = MaxPooling2D(pool_size=(2, 2))(x) # 第二卷积块 - model.add(Conv2D(64, (3, 3), padding='same')) - model.add(Activation('relu')) - model.add(BatchNormalization()) - model.add(MaxPooling2D(pool_size=(2, 2))) + x = Conv2D(64, (3, 3), padding='same')(x) + x = Activation('relu')(x) + x = BatchNormalization()(x) + x = MaxPooling2D(pool_size=(2, 2))(x) # 第三卷积块 - model.add(Conv2D(128, (3, 3), padding='same')) - model.add(Activation('relu')) - model.add(BatchNormalization()) - model.add(MaxPooling2D(pool_size=(2, 2))) + x = Conv2D(128, (3, 3), padding='same')(x) + x = Activation('relu')(x) + x = BatchNormalization()(x) + x = MaxPooling2D(pool_size=(2, 2))(x) - # 第四卷积块 - model.add(Conv2D(256, (3, 3), padding='same')) - model.add(Activation('relu')) - model.add(BatchNormalization()) + # 空间注意力机制 + attention = Conv2D(1, (1, 1), padding='same', activation='sigmoid')(x) + x = Multiply()([x, attention]) # 展平层 - model.add(Flatten()) + x = Flatten()(x) + + # 全连接层 - 增加层深度 + x = Dense(512, activation='relu')(x) + x = Dropout(0.3)(x) + x = Dense(256, activation='relu')(x) + x = Dropout(0.3)(x) + x = Dense(128, activation='relu')(x) + x = Dropout(0.2)(x) + x = Dense(64, activation='relu')(x) + x = Dropout(0.1)(x) - # 全连接层 - model.add(Dense(512, activation='relu')) - model.add(Dropout(0.3)) # 防止过拟合 - model.add(Dense(256, activation='relu')) - model.add(Dropout(0.3)) - model.add(Dense(128, activation='relu')) - model.add(Dropout(0.2)) + # 输出层 - 5个动作 + outputs = Dense(5, activation='linear')(x) - # 输出层 - 扩展为5个动作: 0-减速, 1-保持, 2-加速, 3-左转, 4-右转 - model.add(Dense(5, activation='linear')) + # 创建模型 + model = Model(inputs=inputs, outputs=outputs) - # 编译模型，使用Huber损失和Adam优化器 + # 编译模型 model.compile(loss="huber", optimizer=Adam(lr=LEARNING_RATE), metrics=["mae"]) return model diff --git a/src/Unmanned_vehicle_AD_DQN/Test.py b/src/Unmanned_vehicle_AD_DQN/Test.py index 7e00fe6435..9ee3f92184 100644 --- a/src/Unmanned_vehicle_AD_DQN/Test.py +++ b/src/Unmanned_vehicle_AD_DQN/Test.py @@ -11,7 +11,26 @@ from Hyperparameters import * -MODEL_PATH = r'D:\Work\T_Unmanned_vehicle_AD_DQN\models\YY_Optimized___290.14max___97.16avg___13.42min__1764553908.model' # 请替换为实际的最佳模型路径 +MODEL_PATH = r'D:\Work\T_Unmanned_vehicle_AD_DQN\models\YY_Optimized_best_363.53.model' # 请替换为实际的最佳模型路径 + +def get_safe_action(model, state, env, previous_action): + """结合模型预测和安全规则的混合动作选择""" + # 模型预测 + qs = model.predict(np.array(state).reshape(-1, *state.shape)/255)[0] + + # 如果有建议的避让动作（来自环境），优先考虑 + if hasattr(env, 'suggested_action') and env.suggested_action is not None: + suggested_q = qs[env.suggested_action] + qs[env.suggested_action] += 1.0 # 提高建议动作的Q值 + env.suggested_action = None # 重置 + + # 避免频繁切换动作（平滑性） + if previous_action in [3, 4]: # 如果是转向动作 + qs[previous_action] += 0.3 # 稍微提高继续当前转向的倾向 + + # 选择动作 + action = np.argmax(qs) + return action, qs if __name__ == '__main__': @@ -37,13 +56,16 @@ # 循环测试多个episode episode_count = 0 + previous_action = 1 # 初始动作为保持 + try: while True: episode_count += 1 print(f'\n开始第 {episode_count} 个测试轮次') # 重置环境并获取初始状态 - current_state = env.reset() + # 测试时使用正常难度（相当于训练的第3阶段） + current_state = env.reset(401) # 401表示使用正常难度 env.collision_hist = [] # 重置碰撞历史 done = False @@ -56,9 +78,9 @@ # FPS计数开始 step_start = time.time() - # 基于当前观察空间预测动作 - qs = model.predict(np.array(current_state).reshape(-1, *current_state.shape)/255)[0] - action = np.argmax(qs) # 选择Q值最大的动作 + # 基于当前观察空间预测动作（使用安全版本） + action, qs = get_safe_action(model, current_state, env, previous_action) + previous_action = action # 执行环境步进 new_state, reward, done, _ = env.step(action) @@ -92,4 +114,4 @@ finally: # 清理环境 print("清理环境...") - env.cleanup_actors() + env.cleanup_actors() \ No newline at end of file diff --git a/src/Unmanned_vehicle_AD_DQN/main.py b/src/Unmanned_vehicle_AD_DQN/main.py index 9bf9ca442f..3dadcf5b39 100644 --- a/src/Unmanned_vehicle_AD_DQN/main.py +++ b/src/Unmanned_vehicle_AD_DQN/main.py @@ -71,22 +71,13 @@ env.collision_hist = [] # 重置碰撞历史 agent.tensorboard.step = episode # 设置TensorBoard步数 - # 课程学习 - 随训练进度调整难度 - if episode > EPISODES // 2: - # 训练后期增加行人数量以提高难度 - env.spawn_pedestrians_general(8, True) # 减少数量 - env.spawn_pedestrians_general(4, False) # 减少数量 - else: - # 训练前期减少行人数量以降低难度 - env.spawn_pedestrians_general(6, True) # 减少数量 - env.spawn_pedestrians_general(3, False) # 减少数量 - # 重置每轮统计 - 重置得分和步数 score = 0 step = 1 # 重置环境并获取初始状态 - current_state = env.reset() + # 将episode编号传递给环境，环境会根据episode生成相应数量的行人 + current_state = env.reset(episode) # 重置完成标志并开始迭代直到本轮结束 done = False @@ -153,7 +144,7 @@ f'models/{MODEL_NAME}__{max_reward:_>7.2f}max_{average_reward:_>7.2f}avg_{min_reward:_>7.2f}min__{int(time.time())}.model') epds.append(episode) - print('轮次: ', episode, '得分 %.2f' % score, '成功次数:', success_count) + print(f'轮次: {episode}, 得分: {score:.2f}, 成功次数: {success_count}') # 衰减探索率 if Hyperparameters.EPSILON > Hyperparameters.MIN_EPSILON: @@ -163,15 +154,23 @@ # 设置训练线程终止标志并等待其结束 agent.terminate = True trainer_thread.join() + # 保存最终模型 - agent.model.save( - f'models/{MODEL_NAME}__{max_reward:_>7.2f}max_{average_reward:_>7.2f}avg_{min_reward:_>7.2f}min__{int(time.time())}.model') + if len(scores) > 0: + final_max_reward = max(scores[-AGGREGATE_STATS_EVERY:] if len(scores) >= AGGREGATE_STATS_EVERY else scores) + final_avg_reward = np.mean(scores[-AGGREGATE_STATS_EVERY:] if len(scores) >= AGGREGATE_STATS_EVERY else scores) + final_min_reward = min(scores[-AGGREGATE_STATS_EVERY:] if len(scores) >= AGGREGATE_STATS_EVERY else scores) + agent.model.save( + f'models/{MODEL_NAME}__{final_max_reward:_>7.2f}max_{final_avg_reward:_>7.2f}avg_{final_min_reward:_>7.2f}min__{int(time.time())}.model') # 绘制训练曲线 fig = plt.figure() ax = fig.add_subplot(111) - plt.plot(scores) # 得分曲线 - plt.plot(avg_scores) # 平均得分曲线 + plt.plot(scores, label='每轮得分') + plt.plot(avg_scores, label='平均得分(最近10轮)', linewidth=2) plt.ylabel('得分') plt.xlabel('训练轮次') + plt.title('训练进度') + plt.legend() + plt.grid(True, alpha=0.3) plt.show() \ No newline at end of file