diff --git a/examples/2DTerrainChallenge/2DTerrainChallenge.csproj b/examples/2DTerrainChallenge/2DTerrainChallenge.csproj new file mode 100644 index 0000000..d3eefe7 --- /dev/null +++ b/examples/2DTerrainChallenge/2DTerrainChallenge.csproj @@ -0,0 +1,12 @@ + + + net8.0 + net7.0 + net8.0 + true + DTerrainChallenge + + + + + \ No newline at end of file diff --git a/examples/2DTerrainChallenge/2DTerrainChallenge.sln b/examples/2DTerrainChallenge/2DTerrainChallenge.sln new file mode 100644 index 0000000..1a3d355 --- /dev/null +++ b/examples/2DTerrainChallenge/2DTerrainChallenge.sln @@ -0,0 +1,19 @@ +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 2012 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "2DTerrainChallenge", "2DTerrainChallenge.csproj", "{DA83C40B-57E5-42A0-8257-D9FC0F17274A}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + ExportDebug|Any CPU = ExportDebug|Any CPU + ExportRelease|Any CPU = ExportRelease|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {DA83C40B-57E5-42A0-8257-D9FC0F17274A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {DA83C40B-57E5-42A0-8257-D9FC0F17274A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DA83C40B-57E5-42A0-8257-D9FC0F17274A}.ExportDebug|Any CPU.ActiveCfg = ExportDebug|Any CPU + {DA83C40B-57E5-42A0-8257-D9FC0F17274A}.ExportDebug|Any CPU.Build.0 = ExportDebug|Any CPU + {DA83C40B-57E5-42A0-8257-D9FC0F17274A}.ExportRelease|Any CPU.ActiveCfg = ExportRelease|Any CPU + {DA83C40B-57E5-42A0-8257-D9FC0F17274A}.ExportRelease|Any CPU.Build.0 = ExportRelease|Any CPU + EndGlobalSection +EndGlobal diff --git a/examples/2DTerrainChallenge/addons/godot_rl_agents/controller/ai_controller_2d.gd b/examples/2DTerrainChallenge/addons/godot_rl_agents/controller/ai_controller_2d.gd new file mode 100644 index 0000000..5247135 --- /dev/null +++ b/examples/2DTerrainChallenge/addons/godot_rl_agents/controller/ai_controller_2d.gd @@ -0,0 +1,130 @@ +extends Node2D +class_name AIController2D + +enum ControlModes { INHERIT_FROM_SYNC, HUMAN, TRAINING, ONNX_INFERENCE, RECORD_EXPERT_DEMOS } ## Inherit setting from sync node ## Test the environment manually ## Train a model ## Load a pretrained model using an .onnx file ## Record observations and actions for expert demonstrations +@export var control_mode: ControlModes = ControlModes.INHERIT_FROM_SYNC +## The path to a trained .onnx model file to use for inference (overrides the path set in sync node). +@export var onnx_model_path := "" +## Once the number of steps has passed, the flag 'needs_reset' will be set to 'true' for this instance. +@export var reset_after := 1000 + +@export_group("Record expert demos mode options") +## Path where the demos will be saved. The file can later be used for imitation learning. +@export var expert_demo_save_path: String +## The action that erases the last recorded episode from the currently recorded data. +@export var remove_last_episode_key: InputEvent +## Action will be repeated for n frames. Will introduce control lag if larger than 1. +## Can be used to ensure that action_repeat on inference and training matches +## the recorded demonstrations. +@export var action_repeat: int = 1 + +@export_group("Multi-policy mode options") +## Allows you to set certain agents to use different policies. +## Changing has no effect with default SB3 training. Works with Rllib example. +## Tutorial: https://github.com/edbeeching/godot_rl_agents/blob/main/docs/TRAINING_MULTIPLE_POLICIES.md +@export var policy_name: String = "shared_policy" + +var onnx_model: ONNXModel + +var heuristic := "human" +var done := false +var reward := 0.0 +var n_steps := 0 +var needs_reset := false + +var _player: Node2D + + +func _ready(): + add_to_group("AGENT") + + +func init(player: Node2D): + _player = player + + +#region Methods that need implementing using the "extend script" option in Godot +func get_obs() -> Dictionary: + assert(false, "the get_obs method is not implemented when extending from ai_controller") + return {"obs": []} + + +func get_reward() -> float: + assert(false, "the get_reward method is not implemented when extending from ai_controller") + return 0.0 + + +func get_action_space() -> Dictionary: + assert( + false, "the get_action_space method is not implemented when extending from ai_controller" + ) + return { + "example_actions_continous": {"size": 2, "action_type": "continuous"}, + "example_actions_discrete": {"size": 2, "action_type": "discrete"}, + } + + +func set_action(action) -> void: + assert(false, "the set_action method is not implemented when extending from ai_controller") + + +#endregion + + +#region Methods that sometimes need implementing using the "extend script" option in Godot +# Only needed if you are recording expert demos with this AIController +func get_action() -> Array: + assert( + false, + "the get_action method is not implemented in extended AIController but demo_recorder is used" + ) + return [] + + +# For providing additional info (e.g. `is_success` for SB3 training) +func get_info() -> Dictionary: + return {} + + +#endregion + + +func _physics_process(delta): + n_steps += 1 + if n_steps > reset_after: + needs_reset = true + + +func get_obs_space(): + # may need overriding if the obs space is complex + var obs = get_obs() + return { + "obs": {"size": [len(obs["obs"])], "space": "box"}, + } + + +func reset(): + n_steps = 0 + needs_reset = false + + +func reset_if_done(): + if done: + reset() + + +func set_heuristic(h): + # sets the heuristic from "human" or "model" nothing to change here + heuristic = h + + +func get_done(): + return done + + +func set_done_false(): + done = false + + +func zero_reward(): + reward = 0.0 diff --git a/examples/2DTerrainChallenge/addons/godot_rl_agents/controller/ai_controller_3d.gd b/examples/2DTerrainChallenge/addons/godot_rl_agents/controller/ai_controller_3d.gd new file mode 100644 index 0000000..4984b18 --- /dev/null +++ b/examples/2DTerrainChallenge/addons/godot_rl_agents/controller/ai_controller_3d.gd @@ -0,0 +1,130 @@ +extends Node3D +class_name AIController3D + +enum ControlModes { INHERIT_FROM_SYNC, HUMAN, TRAINING, ONNX_INFERENCE, RECORD_EXPERT_DEMOS } ## Inherit setting from sync node ## Test the environment manually ## Train a model ## Load a pretrained model using an .onnx file ## Record observations and actions for expert demonstrations +@export var control_mode: ControlModes = ControlModes.INHERIT_FROM_SYNC +## The path to a trained .onnx model file to use for inference (overrides the path set in sync node). +@export var onnx_model_path := "" +## Once the number of steps has passed, the flag 'needs_reset' will be set to 'true' for this instance. +@export var reset_after := 1000 + +@export_group("Record expert demos mode options") +## Path where the demos will be saved. The file can later be used for imitation learning. +@export var expert_demo_save_path: String +## The action that erases the last recorded episode from the currently recorded data. +@export var remove_last_episode_key: InputEvent +## Action will be repeated for n frames. Will introduce control lag if larger than 1. +## Can be used to ensure that action_repeat on inference and training matches +## the recorded demonstrations. +@export var action_repeat: int = 1 + +@export_group("Multi-policy mode options") +## Allows you to set certain agents to use different policies. +## Changing has no effect with default SB3 training. Works with Rllib example. +## Tutorial: https://github.com/edbeeching/godot_rl_agents/blob/main/docs/TRAINING_MULTIPLE_POLICIES.md +@export var policy_name: String = "shared_policy" + +var onnx_model: ONNXModel + +var heuristic := "human" +var done := false +var reward := 0.0 +var n_steps := 0 +var needs_reset := false + +var _player: Node3D + + +func _ready(): + add_to_group("AGENT") + + +func init(player: Node3D): + _player = player + + +#region Methods that need implementing using the "extend script" option in Godot +func get_obs() -> Dictionary: + assert(false, "the get_obs method is not implemented when extending from ai_controller") + return {"obs": []} + + +func get_reward() -> float: + assert(false, "the get_reward method is not implemented when extending from ai_controller") + return 0.0 + + +func get_action_space() -> Dictionary: + assert( + false, "the get_action_space method is not implemented when extending from ai_controller" + ) + return { + "example_actions_continous": {"size": 2, "action_type": "continuous"}, + "example_actions_discrete": {"size": 2, "action_type": "discrete"}, + } + + +func set_action(action) -> void: + assert(false, "the set_action method is not implemented when extending from ai_controller") + + +#endregion + + +#region Methods that sometimes need implementing using the "extend script" option in Godot +# Only needed if you are recording expert demos with this AIController +func get_action() -> Array: + assert( + false, + "the get_action method is not implemented in extended AIController but demo_recorder is used" + ) + return [] + + +# For providing additional info (e.g. `is_success` for SB3 training) +func get_info() -> Dictionary: + return {} + + +#endregion + + +func _physics_process(delta): + n_steps += 1 + if n_steps > reset_after: + needs_reset = true + + +func get_obs_space(): + # may need overriding if the obs space is complex + var obs = get_obs() + return { + "obs": {"size": [len(obs["obs"])], "space": "box"}, + } + + +func reset(): + n_steps = 0 + needs_reset = false + + +func reset_if_done(): + if done: + reset() + + +func set_heuristic(h): + # sets the heuristic from "human" or "model" nothing to change here + heuristic = h + + +func get_done(): + return done + + +func set_done_false(): + done = false + + +func zero_reward(): + reward = 0.0 diff --git a/examples/2DTerrainChallenge/addons/godot_rl_agents/godot_rl_agents.gd b/examples/2DTerrainChallenge/addons/godot_rl_agents/godot_rl_agents.gd new file mode 100644 index 0000000..e4fe136 --- /dev/null +++ b/examples/2DTerrainChallenge/addons/godot_rl_agents/godot_rl_agents.gd @@ -0,0 +1,16 @@ +@tool +extends EditorPlugin + + +func _enter_tree(): + # Initialization of the plugin goes here. + # Add the new type with a name, a parent type, a script and an icon. + add_custom_type("Sync", "Node", preload("sync.gd"), preload("icon.png")) + #add_custom_type("RaycastSensor2D2", "Node", preload("raycast_sensor_2d.gd"), preload("icon.png")) + + +func _exit_tree(): + # Clean-up of the plugin goes here. + # Always remember to remove it from the engine when deactivated. + remove_custom_type("Sync") + #remove_custom_type("RaycastSensor2D2") diff --git a/examples/2DTerrainChallenge/addons/godot_rl_agents/icon.png b/examples/2DTerrainChallenge/addons/godot_rl_agents/icon.png new file mode 100644 index 0000000..fd8190e Binary files /dev/null and b/examples/2DTerrainChallenge/addons/godot_rl_agents/icon.png differ diff --git a/examples/2DTerrainChallenge/addons/godot_rl_agents/onnx/csharp/ONNXInference.cs b/examples/2DTerrainChallenge/addons/godot_rl_agents/onnx/csharp/ONNXInference.cs new file mode 100644 index 0000000..6dcfa18 --- /dev/null +++ b/examples/2DTerrainChallenge/addons/godot_rl_agents/onnx/csharp/ONNXInference.cs @@ -0,0 +1,109 @@ +using Godot; +using Microsoft.ML.OnnxRuntime; +using Microsoft.ML.OnnxRuntime.Tensors; +using System.Collections.Generic; +using System.Linq; + +namespace GodotONNX +{ + /// + public partial class ONNXInference : GodotObject + { + + private InferenceSession session; + /// + /// Path to the ONNX model. Use Initialize to change it. + /// + private string modelPath; + private int batchSize; + + private SessionOptions SessionOpt; + + /// + /// init function + /// + /// + /// + /// Returns the output size of the model + public int Initialize(string Path, int BatchSize) + { + modelPath = Path; + batchSize = BatchSize; + SessionOpt = SessionConfigurator.MakeConfiguredSessionOptions(); + session = LoadModel(modelPath); + return session.OutputMetadata["output"].Dimensions[1]; + } + + + /// + public Godot.Collections.Dictionary> RunInference(Godot.Collections.Array obs, int state_ins) + { + //Current model: Any (Godot Rl Agents) + //Expects a tensor of shape [batch_size, input_size] type float named obs and a tensor of shape [batch_size] type float named state_ins + + //Fill the input tensors + // create span from inputSize + var span = new float[obs.Count]; //There's probably a better way to do this + for (int i = 0; i < obs.Count; i++) + { + span[i] = obs[i]; + } + + IReadOnlyCollection inputs = new List + { + NamedOnnxValue.CreateFromTensor("obs", new DenseTensor(span, new int[] { batchSize, obs.Count })), + NamedOnnxValue.CreateFromTensor("state_ins", new DenseTensor(new float[] { state_ins }, new int[] { batchSize })) + }; + IReadOnlyCollection outputNames = new List { "output", "state_outs" }; //ONNX is sensible to these names, as well as the input names + + IDisposableReadOnlyCollection results; + //We do not use "using" here so we get a better exception explaination later + try + { + results = session.Run(inputs, outputNames); + } + catch (OnnxRuntimeException e) + { + //This error usually means that the model is not compatible with the input, beacause of the input shape (size) + GD.Print("Error at inference: ", e); + return null; + } + //Can't convert IEnumerable to Variant, so we have to convert it to an array or something + Godot.Collections.Dictionary> output = new Godot.Collections.Dictionary>(); + DisposableNamedOnnxValue output1 = results.First(); + DisposableNamedOnnxValue output2 = results.Last(); + Godot.Collections.Array output1Array = new Godot.Collections.Array(); + Godot.Collections.Array output2Array = new Godot.Collections.Array(); + + foreach (float f in output1.AsEnumerable()) + { + output1Array.Add(f); + } + + foreach (float f in output2.AsEnumerable()) + { + output2Array.Add(f); + } + + output.Add(output1.Name, output1Array); + output.Add(output2.Name, output2Array); + + //Output is a dictionary of arrays, ex: { "output" : [0.1, 0.2, 0.3, 0.4, ...], "state_outs" : [0.5, ...]} + results.Dispose(); + return output; + } + /// + public InferenceSession LoadModel(string Path) + { + using Godot.FileAccess file = FileAccess.Open(Path, Godot.FileAccess.ModeFlags.Read); + byte[] model = file.GetBuffer((int)file.GetLength()); + //file.Close(); file.Dispose(); //Close the file, then dispose the reference. + return new InferenceSession(model, SessionOpt); //Load the model + } + public void FreeDisposables() + { + session.Dispose(); + SessionOpt.Dispose(); + } + } +} diff --git a/examples/2DTerrainChallenge/addons/godot_rl_agents/onnx/csharp/SessionConfigurator.cs b/examples/2DTerrainChallenge/addons/godot_rl_agents/onnx/csharp/SessionConfigurator.cs new file mode 100644 index 0000000..ad7a41c --- /dev/null +++ b/examples/2DTerrainChallenge/addons/godot_rl_agents/onnx/csharp/SessionConfigurator.cs @@ -0,0 +1,131 @@ +using Godot; +using Microsoft.ML.OnnxRuntime; + +namespace GodotONNX +{ + /// + + public static class SessionConfigurator + { + public enum ComputeName + { + CUDA, + ROCm, + DirectML, + CoreML, + CPU + } + + /// + public static SessionOptions MakeConfiguredSessionOptions() + { + SessionOptions sessionOptions = new(); + SetOptions(sessionOptions); + return sessionOptions; + } + + private static void SetOptions(SessionOptions sessionOptions) + { + sessionOptions.LogSeverityLevel = OrtLoggingLevel.ORT_LOGGING_LEVEL_WARNING; + ApplySystemSpecificOptions(sessionOptions); + } + + /// + static public void ApplySystemSpecificOptions(SessionOptions sessionOptions) + { + //Most code for this function is verbose only, the only reason it exists is to track + //implementation progress of the different compute APIs. + + //December 2022: CUDA is not working. + + string OSName = OS.GetName(); //Get OS Name + + //ComputeName ComputeAPI = ComputeCheck(); //Get Compute API + // //TODO: Get CPU architecture + + //Linux can use OpenVINO (C#) on x64 and ROCm on x86 (GDNative/C++) + //Windows can use OpenVINO (C#) on x64 + //TODO: try TensorRT instead of CUDA + //TODO: Use OpenVINO for Intel Graphics + + // Temporarily using CPU on all platforms to avoid errors detected with DML + ComputeName ComputeAPI = ComputeName.CPU; + + //match OS and Compute API + GD.Print($"OS: {OSName} Compute API: {ComputeAPI}"); + + // CPU is set by default without appending necessary + // sessionOptions.AppendExecutionProvider_CPU(0); + + /* + switch (OSName) + { + case "Windows": //Can use CUDA, DirectML + if (ComputeAPI is ComputeName.CUDA) + { + //CUDA + //sessionOptions.AppendExecutionProvider_CUDA(0); + //sessionOptions.AppendExecutionProvider_DML(0); + } + else if (ComputeAPI is ComputeName.DirectML) + { + //DirectML + //sessionOptions.AppendExecutionProvider_DML(0); + } + break; + case "X11": //Can use CUDA, ROCm + if (ComputeAPI is ComputeName.CUDA) + { + //CUDA + //sessionOptions.AppendExecutionProvider_CUDA(0); + } + if (ComputeAPI is ComputeName.ROCm) + { + //ROCm, only works on x86 + //Research indicates that this has to be compiled as a GDNative plugin + //GD.Print("ROCm not supported yet, using CPU."); + //sessionOptions.AppendExecutionProvider_CPU(0); + } + break; + case "macOS": //Can use CoreML + if (ComputeAPI is ComputeName.CoreML) + { //CoreML + //TODO: Needs testing + //sessionOptions.AppendExecutionProvider_CoreML(0); + //CoreML on ARM64, out of the box, on x64 needs .tar file from GitHub + } + break; + default: + GD.Print("OS not Supported."); + break; + } + */ + } + + + /// + public static ComputeName ComputeCheck() + { + string adapterName = Godot.RenderingServer.GetVideoAdapterName(); + //string adapterVendor = Godot.RenderingServer.GetVideoAdapterVendor(); + adapterName = adapterName.ToUpper(new System.Globalization.CultureInfo("")); + //TODO: GPU vendors for MacOS, what do they even use these days? + + if (adapterName.Contains("INTEL")) + { + return ComputeName.DirectML; + } + if (adapterName.Contains("AMD") || adapterName.Contains("RADEON")) + { + return ComputeName.DirectML; + } + if (adapterName.Contains("NVIDIA")) + { + return ComputeName.CUDA; + } + + GD.Print("Graphics Card not recognized."); //Should use CPU + return ComputeName.CPU; + } + } +} diff --git a/examples/2DTerrainChallenge/addons/godot_rl_agents/onnx/csharp/docs/ONNXInference.xml b/examples/2DTerrainChallenge/addons/godot_rl_agents/onnx/csharp/docs/ONNXInference.xml new file mode 100644 index 0000000..91b07d6 --- /dev/null +++ b/examples/2DTerrainChallenge/addons/godot_rl_agents/onnx/csharp/docs/ONNXInference.xml @@ -0,0 +1,31 @@ + + + + + The main ONNXInference Class that handles the inference process. + + + + + Starts the inference process. + + Path to the ONNX model, expects a path inside resources. + How many observations will the model recieve. + + + + Runs the given input through the model and returns the output. + + Dictionary containing all observations. + How many different agents are creating these observations. + A Dictionary of arrays, containing instructions based on the observations. + + + + Loads the given model into the inference process, using the best Execution provider available. + + Path to the ONNX model, expects a path inside resources. + InferenceSession ready to run. + + + \ No newline at end of file diff --git a/examples/2DTerrainChallenge/addons/godot_rl_agents/onnx/csharp/docs/SessionConfigurator.xml b/examples/2DTerrainChallenge/addons/godot_rl_agents/onnx/csharp/docs/SessionConfigurator.xml new file mode 100644 index 0000000..f160c02 --- /dev/null +++ b/examples/2DTerrainChallenge/addons/godot_rl_agents/onnx/csharp/docs/SessionConfigurator.xml @@ -0,0 +1,29 @@ + + + + + The main SessionConfigurator Class that handles the execution options and providers for the inference process. + + + + + Creates a SessionOptions with all available execution providers. + + SessionOptions with all available execution providers. + + + + Appends any execution provider available in the current system. + + + This function is mainly verbose for tracking implementation progress of different compute APIs. + + + + + Checks for available GPUs. + + An integer identifier for each compute platform. + + + \ No newline at end of file diff --git a/examples/2DTerrainChallenge/addons/godot_rl_agents/onnx/wrapper/ONNX_wrapper.gd b/examples/2DTerrainChallenge/addons/godot_rl_agents/onnx/wrapper/ONNX_wrapper.gd new file mode 100644 index 0000000..d201f0c --- /dev/null +++ b/examples/2DTerrainChallenge/addons/godot_rl_agents/onnx/wrapper/ONNX_wrapper.gd @@ -0,0 +1,54 @@ +extends Resource +class_name ONNXModel +var inferencer_script = load("res://addons/godot_rl_agents/onnx/csharp/ONNXInference.cs") + +var inferencer = null + +## How many action values the model outputs +var action_output_size: int + +## Used to differentiate models +## that only output continuous action mean (e.g. sb3, cleanrl export) +## versus models that output mean and logstd (e.g. rllib export) +var action_means_only: bool + +## Whether action_means_value has been set already for this model +var action_means_only_set: bool + + +# Must provide the path to the model and the batch size +func _init(model_path, batch_size): + inferencer = inferencer_script.new() + action_output_size = inferencer.Initialize(model_path, batch_size) + + +# This function is the one that will be called from the game, +# requires the observation as an array and the state_ins as an int +# returns an Array containing the action the model takes. +func run_inference(obs: Array, state_ins: int) -> Dictionary: + if inferencer == null: + printerr("Inferencer not initialized") + return {} + return inferencer.RunInference(obs, state_ins) + + +func _notification(what): + if what == NOTIFICATION_PREDELETE: + inferencer.FreeDisposables() + inferencer.free() + + +# Check whether agent uses a continuous actions model with only action means or not +func set_action_means_only(agent_action_space): + action_means_only_set = true + var continuous_only: bool = true + var continuous_actions: int + for action in agent_action_space: + if not agent_action_space[action]["action_type"] == "continuous": + continuous_only = false + break + else: + continuous_actions += agent_action_space[action]["size"] + if continuous_only: + if continuous_actions == action_output_size: + action_means_only = true diff --git a/examples/2DTerrainChallenge/addons/godot_rl_agents/plugin.cfg b/examples/2DTerrainChallenge/addons/godot_rl_agents/plugin.cfg new file mode 100644 index 0000000..b1bc988 --- /dev/null +++ b/examples/2DTerrainChallenge/addons/godot_rl_agents/plugin.cfg @@ -0,0 +1,7 @@ +[plugin] + +name="GodotRLAgents" +description="Custom nodes for the godot rl agents toolkit " +author="Edward Beeching" +version="0.1" +script="godot_rl_agents.gd" diff --git a/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_2d/ExampleRaycastSensor2D.tscn b/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_2d/ExampleRaycastSensor2D.tscn new file mode 100644 index 0000000..5edb6c7 --- /dev/null +++ b/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_2d/ExampleRaycastSensor2D.tscn @@ -0,0 +1,48 @@ +[gd_scene load_steps=5 format=3 uid="uid://ddeq7mn1ealyc"] + +[ext_resource type="Script" path="res://addons/godot_rl_agents/sensors/sensors_2d/RaycastSensor2D.gd" id="1"] + +[sub_resource type="GDScript" id="2"] +script/source = "extends Node2D + + + +func _physics_process(delta: float) -> void: + print(\"step start\") + +" + +[sub_resource type="GDScript" id="1"] +script/source = "extends RayCast2D + +var steps = 1 + +func _physics_process(delta: float) -> void: + print(\"processing raycast\") + steps += 1 + if steps % 2: + force_raycast_update() + + print(is_colliding()) +" + +[sub_resource type="CircleShape2D" id="3"] + +[node name="ExampleRaycastSensor2D" type="Node2D"] +script = SubResource("2") + +[node name="ExampleAgent" type="Node2D" parent="."] +position = Vector2(573, 314) +rotation = 0.286234 + +[node name="RaycastSensor2D" type="Node2D" parent="ExampleAgent"] +script = ExtResource("1") + +[node name="TestRayCast2D" type="RayCast2D" parent="."] +script = SubResource("1") + +[node name="StaticBody2D" type="StaticBody2D" parent="."] +position = Vector2(1, 52) + +[node name="CollisionShape2D" type="CollisionShape2D" parent="StaticBody2D"] +shape = SubResource("3") diff --git a/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_2d/GridSensor2D.gd b/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_2d/GridSensor2D.gd new file mode 100644 index 0000000..48b132e --- /dev/null +++ b/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_2d/GridSensor2D.gd @@ -0,0 +1,235 @@ +@tool +extends ISensor2D +class_name GridSensor2D + +@export var debug_view := false: + get: + return debug_view + set(value): + debug_view = value + _update() + +@export_flags_2d_physics var detection_mask := 0: + get: + return detection_mask + set(value): + detection_mask = value + _update() + +@export var collide_with_areas := false: + get: + return collide_with_areas + set(value): + collide_with_areas = value + _update() + +@export var collide_with_bodies := true: + get: + return collide_with_bodies + set(value): + collide_with_bodies = value + _update() + +@export_range(1, 200, 0.1) var cell_width := 20.0: + get: + return cell_width + set(value): + cell_width = value + _update() + +@export_range(1, 200, 0.1) var cell_height := 20.0: + get: + return cell_height + set(value): + cell_height = value + _update() + +@export_range(1, 21, 2, "or_greater") var grid_size_x := 3: + get: + return grid_size_x + set(value): + grid_size_x = value + _update() + +@export_range(1, 21, 2, "or_greater") var grid_size_y := 3: + get: + return grid_size_y + set(value): + grid_size_y = value + _update() + +var _obs_buffer: PackedFloat64Array +var _rectangle_shape: RectangleShape2D +var _collision_mapping: Dictionary +var _n_layers_per_cell: int + +var _highlighted_cell_color: Color +var _standard_cell_color: Color + + +func get_observation(): + return _obs_buffer + + +func _update(): + if Engine.is_editor_hint(): + if is_node_ready(): + _spawn_nodes() + + +func _ready() -> void: + _set_colors() + + if Engine.is_editor_hint(): + if get_child_count() == 0: + _spawn_nodes() + else: + _spawn_nodes() + + +func _set_colors() -> void: + _standard_cell_color = Color(100.0 / 255.0, 100.0 / 255.0, 100.0 / 255.0, 100.0 / 255.0) + _highlighted_cell_color = Color(255.0 / 255.0, 100.0 / 255.0, 100.0 / 255.0, 100.0 / 255.0) + + +func _get_collision_mapping() -> Dictionary: + # defines which layer is mapped to which cell obs index + var total_bits = 0 + var collision_mapping = {} + for i in 32: + var bit_mask = 2 ** i + if (detection_mask & bit_mask) > 0: + collision_mapping[i] = total_bits + total_bits += 1 + + return collision_mapping + + +func _spawn_nodes(): + for cell in get_children(): + cell.name = "_%s" % cell.name # Otherwise naming below will fail + cell.queue_free() + + _collision_mapping = _get_collision_mapping() + #prints("collision_mapping", _collision_mapping, len(_collision_mapping)) + # allocate memory for the observations + _n_layers_per_cell = len(_collision_mapping) + _obs_buffer = PackedFloat64Array() + _obs_buffer.resize(grid_size_x * grid_size_y * _n_layers_per_cell) + _obs_buffer.fill(0) + #prints(len(_obs_buffer), _obs_buffer ) + + _rectangle_shape = RectangleShape2D.new() + _rectangle_shape.set_size(Vector2(cell_width, cell_height)) + + var shift := Vector2( + -(grid_size_x / 2) * cell_width, + -(grid_size_y / 2) * cell_height, + ) + + for i in grid_size_x: + for j in grid_size_y: + var cell_position = Vector2(i * cell_width, j * cell_height) + shift + _create_cell(i, j, cell_position) + + +func _create_cell(i: int, j: int, position: Vector2): + var cell := Area2D.new() + cell.position = position + cell.name = "GridCell %s %s" % [i, j] + cell.modulate = _standard_cell_color + + if collide_with_areas: + cell.area_entered.connect(_on_cell_area_entered.bind(i, j)) + cell.area_exited.connect(_on_cell_area_exited.bind(i, j)) + + if collide_with_bodies: + cell.body_entered.connect(_on_cell_body_entered.bind(i, j)) + cell.body_exited.connect(_on_cell_body_exited.bind(i, j)) + + cell.collision_layer = 0 + cell.collision_mask = detection_mask + cell.monitorable = true + add_child(cell) + cell.set_owner(get_tree().edited_scene_root) + + var col_shape := CollisionShape2D.new() + col_shape.shape = _rectangle_shape + col_shape.name = "CollisionShape2D" + cell.add_child(col_shape) + col_shape.set_owner(get_tree().edited_scene_root) + + if debug_view: + var quad = MeshInstance2D.new() + quad.name = "MeshInstance2D" + var quad_mesh = QuadMesh.new() + + quad_mesh.set_size(Vector2(cell_width, cell_height)) + + quad.mesh = quad_mesh + cell.add_child(quad) + quad.set_owner(get_tree().edited_scene_root) + + +func _update_obs(cell_i: int, cell_j: int, collision_layer: int, entered: bool): + for key in _collision_mapping: + var bit_mask = 2 ** key + if (collision_layer & bit_mask) > 0: + var collison_map_index = _collision_mapping[key] + + var obs_index = ( + (cell_i * grid_size_y * _n_layers_per_cell) + + (cell_j * _n_layers_per_cell) + + collison_map_index + ) + #prints(obs_index, cell_i, cell_j) + if entered: + _obs_buffer[obs_index] += 1 + else: + _obs_buffer[obs_index] -= 1 + + +func _toggle_cell(cell_i: int, cell_j: int): + var cell = get_node_or_null("GridCell %s %s" % [cell_i, cell_j]) + + if cell == null: + print("cell not found, returning") + + var n_hits = 0 + var start_index = (cell_i * grid_size_y * _n_layers_per_cell) + (cell_j * _n_layers_per_cell) + for i in _n_layers_per_cell: + n_hits += _obs_buffer[start_index + i] + + if n_hits > 0: + cell.modulate = _highlighted_cell_color + else: + cell.modulate = _standard_cell_color + + +func _on_cell_area_entered(area: Area2D, cell_i: int, cell_j: int): + #prints("_on_cell_area_entered", cell_i, cell_j) + _update_obs(cell_i, cell_j, area.collision_layer, true) + if debug_view: + _toggle_cell(cell_i, cell_j) + #print(_obs_buffer) + + +func _on_cell_area_exited(area: Area2D, cell_i: int, cell_j: int): + #prints("_on_cell_area_exited", cell_i, cell_j) + _update_obs(cell_i, cell_j, area.collision_layer, false) + if debug_view: + _toggle_cell(cell_i, cell_j) + + +func _on_cell_body_entered(body: Node2D, cell_i: int, cell_j: int): + #prints("_on_cell_body_entered", cell_i, cell_j) + _update_obs(cell_i, cell_j, body.collision_layer, true) + if debug_view: + _toggle_cell(cell_i, cell_j) + + +func _on_cell_body_exited(body: Node2D, cell_i: int, cell_j: int): + #prints("_on_cell_body_exited", cell_i, cell_j) + _update_obs(cell_i, cell_j, body.collision_layer, false) + if debug_view: + _toggle_cell(cell_i, cell_j) diff --git a/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_2d/ISensor2D.gd b/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_2d/ISensor2D.gd new file mode 100644 index 0000000..67669a1 --- /dev/null +++ b/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_2d/ISensor2D.gd @@ -0,0 +1,25 @@ +extends Node2D +class_name ISensor2D + +var _obs: Array = [] +var _active := false + + +func get_observation(): + pass + + +func activate(): + _active = true + + +func deactivate(): + _active = false + + +func _update_observation(): + pass + + +func reset(): + pass diff --git a/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_2d/RGBCameraSensor2D.gd b/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_2d/RGBCameraSensor2D.gd new file mode 100644 index 0000000..3159c3a --- /dev/null +++ b/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_2d/RGBCameraSensor2D.gd @@ -0,0 +1,77 @@ +extends Node2D +class_name RGBCameraSensor2D +var camera_pixels = null + +@export var camera_zoom_factor := Vector2(0.1, 0.1) +@onready var camera := $SubViewport/Camera +@onready var preview_window := $Control +@onready var camera_texture := $Control/CameraTexture as Sprite2D +@onready var processed_texture := $Control/ProcessedTexture as Sprite2D +@onready var sub_viewport := $SubViewport as SubViewport +@onready var displayed_image: ImageTexture + +@export var render_image_resolution := Vector2(36, 36) +## Display size does not affect rendered or sent image resolution. +## Scale is relative to either render image or downscale image resolution +## depending on which mode is set. +@export var displayed_image_scale_factor := Vector2(8, 8) + +@export_group("Downscale image options") +## Enable to downscale the rendered image before sending the obs. +@export var downscale_image: bool = false +## If downscale_image is true, will display the downscaled image instead of rendered image. +@export var display_downscaled_image: bool = true +## This is the resolution of the image that will be sent after downscaling +@export var resized_image_resolution := Vector2(36, 36) + + +func _ready(): + DisplayServer.register_additional_output(self) + + camera.zoom = camera_zoom_factor + + var preview_size: Vector2 + + sub_viewport.world_2d = get_tree().get_root().get_world_2d() + sub_viewport.size = render_image_resolution + camera_texture.scale = displayed_image_scale_factor + + if downscale_image and display_downscaled_image: + camera_texture.visible = false + processed_texture.scale = displayed_image_scale_factor + preview_size = displayed_image_scale_factor * resized_image_resolution + else: + processed_texture.visible = false + preview_size = displayed_image_scale_factor * render_image_resolution + + preview_window.size = preview_size + + +func get_camera_pixel_encoding(): + var image := camera_texture.get_texture().get_image() as Image + + if downscale_image: + image.resize( + resized_image_resolution.x, resized_image_resolution.y, Image.INTERPOLATE_NEAREST + ) + if display_downscaled_image: + if not processed_texture.texture: + displayed_image = ImageTexture.create_from_image(image) + processed_texture.texture = displayed_image + else: + displayed_image.update(image) + + return image.get_data().hex_encode() + + +func get_camera_shape() -> Array: + var size = resized_image_resolution if downscale_image else render_image_resolution + + assert( + size.x >= 36 and size.y >= 36, + "Camera sensor sent image resolution must be 36x36 or larger." + ) + if sub_viewport.transparent_bg: + return [4, size.y, size.x] + else: + return [3, size.y, size.x] diff --git a/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_2d/RGBCameraSensor2D.tscn b/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_2d/RGBCameraSensor2D.tscn new file mode 100644 index 0000000..94ab778 --- /dev/null +++ b/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_2d/RGBCameraSensor2D.tscn @@ -0,0 +1,36 @@ +[gd_scene load_steps=3 format=3 uid="uid://bav1cl8uwc45c"] + +[ext_resource type="Script" path="res://addons/godot_rl_agents/sensors/sensors_2d/RGBCameraSensor2D.gd" id="1_txpo2"] + +[sub_resource type="ViewportTexture" id="ViewportTexture_jks1s"] +viewport_path = NodePath("SubViewport") + +[node name="RGBCameraSensor2D" type="Node2D"] +script = ExtResource("1_txpo2") +displayed_image_scale_factor = Vector2(3, 3) + +[node name="RemoteTransform" type="RemoteTransform2D" parent="."] +remote_path = NodePath("../SubViewport/Camera") + +[node name="SubViewport" type="SubViewport" parent="."] +canvas_item_default_texture_filter = 0 +size = Vector2i(36, 36) +render_target_update_mode = 4 + +[node name="Camera" type="Camera2D" parent="SubViewport"] +position_smoothing_speed = 2.0 + +[node name="Control" type="Window" parent="."] +canvas_item_default_texture_filter = 0 +title = "CameraSensor" +position = Vector2i(20, 40) +size = Vector2i(64, 64) +theme_override_font_sizes/title_font_size = 12 +metadata/_edit_use_anchors_ = true + +[node name="CameraTexture" type="Sprite2D" parent="Control"] +texture = SubResource("ViewportTexture_jks1s") +centered = false + +[node name="ProcessedTexture" type="Sprite2D" parent="Control"] +centered = false diff --git a/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_2d/RaycastSensor2D.gd b/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_2d/RaycastSensor2D.gd new file mode 100644 index 0000000..9bb54ed --- /dev/null +++ b/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_2d/RaycastSensor2D.gd @@ -0,0 +1,118 @@ +@tool +extends ISensor2D +class_name RaycastSensor2D + +@export_flags_2d_physics var collision_mask := 1: + get: + return collision_mask + set(value): + collision_mask = value + _update() + +@export var collide_with_areas := false: + get: + return collide_with_areas + set(value): + collide_with_areas = value + _update() + +@export var collide_with_bodies := true: + get: + return collide_with_bodies + set(value): + collide_with_bodies = value + _update() + +@export var n_rays := 16.0: + get: + return n_rays + set(value): + n_rays = value + _update() + +@export_range(5, 3000, 5.0) var ray_length := 200: + get: + return ray_length + set(value): + ray_length = value + _update() +@export_range(5, 360, 5.0) var cone_width := 360.0: + get: + return cone_width + set(value): + cone_width = value + _update() + +@export var debug_draw := true: + get: + return debug_draw + set(value): + debug_draw = value + _update() + +var _angles = [] +var rays := [] + + +func _update(): + if Engine.is_editor_hint(): + if debug_draw: + _spawn_nodes() + else: + for ray in get_children(): + if ray is RayCast2D: + remove_child(ray) + + +func _ready() -> void: + _spawn_nodes() + + +func _spawn_nodes(): + for ray in rays: + ray.queue_free() + rays = [] + + _angles = [] + var step = cone_width / (n_rays) + var start = step / 2 - cone_width / 2 + + for i in n_rays: + var angle = start + i * step + var ray = RayCast2D.new() + ray.set_target_position( + Vector2(ray_length * cos(deg_to_rad(angle)), ray_length * sin(deg_to_rad(angle))) + ) + ray.set_name("node_" + str(i)) + ray.enabled = false + ray.collide_with_areas = collide_with_areas + ray.collide_with_bodies = collide_with_bodies + ray.collision_mask = collision_mask + add_child(ray) + rays.append(ray) + + _angles.append(start + i * step) + + +func get_observation() -> Array: + return self.calculate_raycasts() + + +func calculate_raycasts() -> Array: + var result = [] + for ray in rays: + ray.enabled = true + ray.force_raycast_update() + var distance = _get_raycast_distance(ray) + result.append(distance) + ray.enabled = false + return result + + +func _get_raycast_distance(ray: RayCast2D) -> float: + if !ray.is_colliding(): + return 0.0 + + var distance = (global_position - ray.get_collision_point()).length() + distance = clamp(distance, 0.0, ray_length) + return (ray_length - distance) / ray_length diff --git a/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_2d/RaycastSensor2D.tscn b/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_2d/RaycastSensor2D.tscn new file mode 100644 index 0000000..5ca402c --- /dev/null +++ b/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_2d/RaycastSensor2D.tscn @@ -0,0 +1,7 @@ +[gd_scene load_steps=2 format=3 uid="uid://drvfihk5esgmv"] + +[ext_resource type="Script" path="res://addons/godot_rl_agents/sensors/sensors_2d/RaycastSensor2D.gd" id="1"] + +[node name="RaycastSensor2D" type="Node2D"] +script = ExtResource("1") +n_rays = 17.0 diff --git a/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_3d/ExampleRaycastSensor3D.tscn b/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_3d/ExampleRaycastSensor3D.tscn new file mode 100644 index 0000000..a8057c7 --- /dev/null +++ b/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_3d/ExampleRaycastSensor3D.tscn @@ -0,0 +1,6 @@ +[gd_scene format=3 uid="uid://biu787qh4woik"] + +[node name="ExampleRaycastSensor3D" type="Node3D"] + +[node name="Camera3D" type="Camera3D" parent="."] +transform = Transform3D(1, 0, 0, 0, 1, 0, 0, 0, 1, 0.804183, 0, 2.70146) diff --git a/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_3d/GridSensor3D.gd b/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_3d/GridSensor3D.gd new file mode 100644 index 0000000..24de9a4 --- /dev/null +++ b/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_3d/GridSensor3D.gd @@ -0,0 +1,258 @@ +@tool +extends ISensor3D +class_name GridSensor3D + +@export var debug_view := false: + get: + return debug_view + set(value): + debug_view = value + _update() + +@export_flags_3d_physics var detection_mask := 0: + get: + return detection_mask + set(value): + detection_mask = value + _update() + +@export var collide_with_areas := false: + get: + return collide_with_areas + set(value): + collide_with_areas = value + _update() + +@export var collide_with_bodies := false: + # NOTE! The sensor will not detect StaticBody3D, add an area to static bodies to detect them + get: + return collide_with_bodies + set(value): + collide_with_bodies = value + _update() + +@export_range(0.1, 2, 0.1) var cell_width := 1.0: + get: + return cell_width + set(value): + cell_width = value + _update() + +@export_range(0.1, 2, 0.1) var cell_height := 1.0: + get: + return cell_height + set(value): + cell_height = value + _update() + +@export_range(1, 21, 1, "or_greater") var grid_size_x := 3: + get: + return grid_size_x + set(value): + grid_size_x = value + _update() + +@export_range(1, 21, 1, "or_greater") var grid_size_z := 3: + get: + return grid_size_z + set(value): + grid_size_z = value + _update() + +var _obs_buffer: PackedFloat64Array +var _box_shape: BoxShape3D +var _collision_mapping: Dictionary +var _n_layers_per_cell: int + +var _highlighted_box_material: StandardMaterial3D +var _standard_box_material: StandardMaterial3D + + +func get_observation(): + return _obs_buffer + + +func reset(): + _obs_buffer.fill(0) + + +func _update(): + if Engine.is_editor_hint(): + if is_node_ready(): + _spawn_nodes() + + +func _ready() -> void: + _make_materials() + + if Engine.is_editor_hint(): + if get_child_count() == 0: + _spawn_nodes() + else: + _spawn_nodes() + + +func _make_materials() -> void: + if _highlighted_box_material != null and _standard_box_material != null: + return + + _standard_box_material = StandardMaterial3D.new() + _standard_box_material.set_transparency(1) # ALPHA + _standard_box_material.albedo_color = Color( + 100.0 / 255.0, 100.0 / 255.0, 100.0 / 255.0, 100.0 / 255.0 + ) + + _highlighted_box_material = StandardMaterial3D.new() + _highlighted_box_material.set_transparency(1) # ALPHA + _highlighted_box_material.albedo_color = Color( + 255.0 / 255.0, 100.0 / 255.0, 100.0 / 255.0, 100.0 / 255.0 + ) + + +func _get_collision_mapping() -> Dictionary: + # defines which layer is mapped to which cell obs index + var total_bits = 0 + var collision_mapping = {} + for i in 32: + var bit_mask = 2 ** i + if (detection_mask & bit_mask) > 0: + collision_mapping[i] = total_bits + total_bits += 1 + + return collision_mapping + + +func _spawn_nodes(): + for cell in get_children(): + cell.name = "_%s" % cell.name # Otherwise naming below will fail + cell.queue_free() + + _collision_mapping = _get_collision_mapping() + #prints("collision_mapping", _collision_mapping, len(_collision_mapping)) + # allocate memory for the observations + _n_layers_per_cell = len(_collision_mapping) + _obs_buffer = PackedFloat64Array() + _obs_buffer.resize(grid_size_x * grid_size_z * _n_layers_per_cell) + _obs_buffer.fill(0) + #prints(len(_obs_buffer), _obs_buffer ) + + _box_shape = BoxShape3D.new() + _box_shape.set_size(Vector3(cell_width, cell_height, cell_width)) + + var shift := Vector3( + -(grid_size_x / 2) * cell_width, + 0, + -(grid_size_z / 2) * cell_width, + ) + + for i in grid_size_x: + for j in grid_size_z: + var cell_position = Vector3(i * cell_width, 0.0, j * cell_width) + shift + _create_cell(i, j, cell_position) + + +func _create_cell(i: int, j: int, position: Vector3): + var cell := Area3D.new() + cell.position = position + cell.name = "GridCell %s %s" % [i, j] + + if collide_with_areas: + cell.area_entered.connect(_on_cell_area_entered.bind(i, j)) + cell.area_exited.connect(_on_cell_area_exited.bind(i, j)) + + if collide_with_bodies: + cell.body_entered.connect(_on_cell_body_entered.bind(i, j)) + cell.body_exited.connect(_on_cell_body_exited.bind(i, j)) + +# cell.body_shape_entered.connect(_on_cell_body_shape_entered.bind(i, j)) +# cell.body_shape_exited.connect(_on_cell_body_shape_exited.bind(i, j)) + + cell.collision_layer = 0 + cell.collision_mask = detection_mask + cell.monitorable = true + cell.input_ray_pickable = false + add_child(cell) + cell.set_owner(get_tree().edited_scene_root) + + var col_shape := CollisionShape3D.new() + col_shape.shape = _box_shape + col_shape.name = "CollisionShape3D" + cell.add_child(col_shape) + col_shape.set_owner(get_tree().edited_scene_root) + + if debug_view: + var box = MeshInstance3D.new() + box.name = "MeshInstance3D" + var box_mesh = BoxMesh.new() + + box_mesh.set_size(Vector3(cell_width, cell_height, cell_width)) + box_mesh.material = _standard_box_material + + box.mesh = box_mesh + cell.add_child(box) + box.set_owner(get_tree().edited_scene_root) + + +func _update_obs(cell_i: int, cell_j: int, collision_layer: int, entered: bool): + for key in _collision_mapping: + var bit_mask = 2 ** key + if (collision_layer & bit_mask) > 0: + var collison_map_index = _collision_mapping[key] + + var obs_index = ( + (cell_i * grid_size_z * _n_layers_per_cell) + + (cell_j * _n_layers_per_cell) + + collison_map_index + ) + #prints(obs_index, cell_i, cell_j) + if entered: + _obs_buffer[obs_index] += 1 + else: + _obs_buffer[obs_index] -= 1 + + +func _toggle_cell(cell_i: int, cell_j: int): + var cell = get_node_or_null("GridCell %s %s" % [cell_i, cell_j]) + + if cell == null: + print("cell not found, returning") + + var n_hits = 0 + var start_index = (cell_i * grid_size_z * _n_layers_per_cell) + (cell_j * _n_layers_per_cell) + for i in _n_layers_per_cell: + n_hits += _obs_buffer[start_index + i] + + var cell_mesh = cell.get_node_or_null("MeshInstance3D") + if n_hits > 0: + cell_mesh.mesh.material = _highlighted_box_material + else: + cell_mesh.mesh.material = _standard_box_material + + +func _on_cell_area_entered(area: Area3D, cell_i: int, cell_j: int): + #prints("_on_cell_area_entered", cell_i, cell_j) + _update_obs(cell_i, cell_j, area.collision_layer, true) + if debug_view: + _toggle_cell(cell_i, cell_j) + #print(_obs_buffer) + + +func _on_cell_area_exited(area: Area3D, cell_i: int, cell_j: int): + #prints("_on_cell_area_exited", cell_i, cell_j) + _update_obs(cell_i, cell_j, area.collision_layer, false) + if debug_view: + _toggle_cell(cell_i, cell_j) + + +func _on_cell_body_entered(body: Node3D, cell_i: int, cell_j: int): + #prints("_on_cell_body_entered", cell_i, cell_j) + _update_obs(cell_i, cell_j, body.collision_layer, true) + if debug_view: + _toggle_cell(cell_i, cell_j) + + +func _on_cell_body_exited(body: Node3D, cell_i: int, cell_j: int): + #prints("_on_cell_body_exited", cell_i, cell_j) + _update_obs(cell_i, cell_j, body.collision_layer, false) + if debug_view: + _toggle_cell(cell_i, cell_j) diff --git a/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_3d/ISensor3D.gd b/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_3d/ISensor3D.gd new file mode 100644 index 0000000..aca3c2d --- /dev/null +++ b/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_3d/ISensor3D.gd @@ -0,0 +1,25 @@ +extends Node3D +class_name ISensor3D + +var _obs: Array = [] +var _active := false + + +func get_observation(): + pass + + +func activate(): + _active = true + + +func deactivate(): + _active = false + + +func _update_observation(): + pass + + +func reset(): + pass diff --git a/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_3d/RGBCameraSensor3D.gd b/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_3d/RGBCameraSensor3D.gd new file mode 100644 index 0000000..96dfb6a --- /dev/null +++ b/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_3d/RGBCameraSensor3D.gd @@ -0,0 +1,63 @@ +extends Node3D +class_name RGBCameraSensor3D +var camera_pixels = null + +@onready var camera_texture := $Control/CameraTexture as Sprite2D +@onready var processed_texture := $Control/ProcessedTexture as Sprite2D +@onready var sub_viewport := $SubViewport as SubViewport +@onready var displayed_image: ImageTexture + +@export var render_image_resolution := Vector2(36, 36) +## Display size does not affect rendered or sent image resolution. +## Scale is relative to either render image or downscale image resolution +## depending on which mode is set. +@export var displayed_image_scale_factor := Vector2(8, 8) + +@export_group("Downscale image options") +## Enable to downscale the rendered image before sending the obs. +@export var downscale_image: bool = false +## If downscale_image is true, will display the downscaled image instead of rendered image. +@export var display_downscaled_image: bool = true +## This is the resolution of the image that will be sent after downscaling +@export var resized_image_resolution := Vector2(36, 36) + + +func _ready(): + sub_viewport.size = render_image_resolution + camera_texture.scale = displayed_image_scale_factor + + if downscale_image and display_downscaled_image: + camera_texture.visible = false + processed_texture.scale = displayed_image_scale_factor + else: + processed_texture.visible = false + + +func get_camera_pixel_encoding(): + var image := camera_texture.get_texture().get_image() as Image + + if downscale_image: + image.resize( + resized_image_resolution.x, resized_image_resolution.y, Image.INTERPOLATE_NEAREST + ) + if display_downscaled_image: + if not processed_texture.texture: + displayed_image = ImageTexture.create_from_image(image) + processed_texture.texture = displayed_image + else: + displayed_image.update(image) + + return image.get_data().hex_encode() + + +func get_camera_shape() -> Array: + var size = resized_image_resolution if downscale_image else render_image_resolution + + assert( + size.x >= 36 and size.y >= 36, + "Camera sensor sent image resolution must be 36x36 or larger." + ) + if sub_viewport.transparent_bg: + return [4, size.y, size.x] + else: + return [3, size.y, size.x] diff --git a/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_3d/RGBCameraSensor3D.tscn b/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_3d/RGBCameraSensor3D.tscn new file mode 100644 index 0000000..d58649c --- /dev/null +++ b/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_3d/RGBCameraSensor3D.tscn @@ -0,0 +1,35 @@ +[gd_scene load_steps=3 format=3 uid="uid://baaywi3arsl2m"] + +[ext_resource type="Script" path="res://addons/godot_rl_agents/sensors/sensors_3d/RGBCameraSensor3D.gd" id="1"] + +[sub_resource type="ViewportTexture" id="ViewportTexture_y72s3"] +viewport_path = NodePath("SubViewport") + +[node name="RGBCameraSensor3D" type="Node3D"] +script = ExtResource("1") + +[node name="RemoteTransform" type="RemoteTransform3D" parent="."] +remote_path = NodePath("../SubViewport/Camera") + +[node name="SubViewport" type="SubViewport" parent="."] +size = Vector2i(36, 36) +render_target_update_mode = 3 + +[node name="Camera" type="Camera3D" parent="SubViewport"] +near = 0.5 + +[node name="Control" type="Control" parent="."] +layout_mode = 3 +anchors_preset = 15 +anchor_right = 1.0 +anchor_bottom = 1.0 +grow_horizontal = 2 +grow_vertical = 2 +metadata/_edit_use_anchors_ = true + +[node name="CameraTexture" type="Sprite2D" parent="Control"] +texture = SubResource("ViewportTexture_y72s3") +centered = false + +[node name="ProcessedTexture" type="Sprite2D" parent="Control"] +centered = false diff --git a/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_3d/RaycastSensor3D.gd b/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_3d/RaycastSensor3D.gd new file mode 100644 index 0000000..1357529 --- /dev/null +++ b/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_3d/RaycastSensor3D.gd @@ -0,0 +1,185 @@ +@tool +extends ISensor3D +class_name RayCastSensor3D +@export_flags_3d_physics var collision_mask = 1: + get: + return collision_mask + set(value): + collision_mask = value + _update() +@export_flags_3d_physics var boolean_class_mask = 1: + get: + return boolean_class_mask + set(value): + boolean_class_mask = value + _update() + +@export var n_rays_width := 6.0: + get: + return n_rays_width + set(value): + n_rays_width = value + _update() + +@export var n_rays_height := 6.0: + get: + return n_rays_height + set(value): + n_rays_height = value + _update() + +@export var ray_length := 10.0: + get: + return ray_length + set(value): + ray_length = value + _update() + +@export var cone_width := 60.0: + get: + return cone_width + set(value): + cone_width = value + _update() + +@export var cone_height := 60.0: + get: + return cone_height + set(value): + cone_height = value + _update() + +@export var collide_with_areas := false: + get: + return collide_with_areas + set(value): + collide_with_areas = value + _update() + +@export var collide_with_bodies := true: + get: + return collide_with_bodies + set(value): + collide_with_bodies = value + _update() + +@export var class_sensor := false + +var rays := [] +var geo = null + + +func _update(): + if Engine.is_editor_hint(): + if is_node_ready(): + _spawn_nodes() + + +func _ready() -> void: + if Engine.is_editor_hint(): + if get_child_count() == 0: + _spawn_nodes() + else: + _spawn_nodes() + + +func _spawn_nodes(): + print("spawning nodes") + for ray in get_children(): + ray.queue_free() + if geo: + geo.clear() + #$Lines.remove_points() + rays = [] + + var horizontal_step = cone_width / (n_rays_width) + var vertical_step = cone_height / (n_rays_height) + + var horizontal_start = horizontal_step / 2 - cone_width / 2 + var vertical_start = vertical_step / 2 - cone_height / 2 + + var points = [] + + for i in n_rays_width: + for j in n_rays_height: + var angle_w = horizontal_start + i * horizontal_step + var angle_h = vertical_start + j * vertical_step + #angle_h = 0.0 + var ray = RayCast3D.new() + var cast_to = to_spherical_coords(ray_length, angle_w, angle_h) + ray.set_target_position(cast_to) + + points.append(cast_to) + + ray.set_name("node_" + str(i) + " " + str(j)) + ray.enabled = true + ray.collide_with_bodies = collide_with_bodies + ray.collide_with_areas = collide_with_areas + ray.collision_mask = collision_mask + add_child(ray) + ray.set_owner(get_tree().edited_scene_root) + rays.append(ray) + ray.force_raycast_update() + + +# if Engine.editor_hint: +# _create_debug_lines(points) + + +func _create_debug_lines(points): + if not geo: + geo = ImmediateMesh.new() + add_child(geo) + + geo.clear() + geo.begin(Mesh.PRIMITIVE_LINES) + for point in points: + geo.set_color(Color.AQUA) + geo.add_vertex(Vector3.ZERO) + geo.add_vertex(point) + geo.end() + + +func display(): + if geo: + geo.display() + + +func to_spherical_coords(r, inc, azimuth) -> Vector3: + return Vector3( + r * sin(deg_to_rad(inc)) * cos(deg_to_rad(azimuth)), + r * sin(deg_to_rad(azimuth)), + r * cos(deg_to_rad(inc)) * cos(deg_to_rad(azimuth)) + ) + + +func get_observation() -> Array: + return self.calculate_raycasts() + + +func calculate_raycasts() -> Array: + var result = [] + for ray in rays: + ray.set_enabled(true) + ray.force_raycast_update() + var distance = _get_raycast_distance(ray) + + result.append(distance) + if class_sensor: + var hit_class: float = 0 + if ray.get_collider(): + var hit_collision_layer = ray.get_collider().collision_layer + hit_collision_layer = hit_collision_layer & collision_mask + hit_class = (hit_collision_layer & boolean_class_mask) > 0 + result.append(float(hit_class)) + ray.set_enabled(false) + return result + + +func _get_raycast_distance(ray: RayCast3D) -> float: + if !ray.is_colliding(): + return 0.0 + + var distance = (global_transform.origin - ray.get_collision_point()).length() + distance = clamp(distance, 0.0, ray_length) + return (ray_length - distance) / ray_length diff --git a/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_3d/RaycastSensor3D.tscn b/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_3d/RaycastSensor3D.tscn new file mode 100644 index 0000000..35f9796 --- /dev/null +++ b/examples/2DTerrainChallenge/addons/godot_rl_agents/sensors/sensors_3d/RaycastSensor3D.tscn @@ -0,0 +1,27 @@ +[gd_scene load_steps=2 format=3 uid="uid://b803cbh1fmy66"] + +[ext_resource type="Script" path="res://addons/godot_rl_agents/sensors/sensors_3d/RaycastSensor3D.gd" id="1"] + +[node name="RaycastSensor3D" type="Node3D"] +script = ExtResource("1") +n_rays_width = 4.0 +n_rays_height = 2.0 +ray_length = 11.0 + +[node name="node_1 0" type="RayCast3D" parent="."] +target_position = Vector3(-1.38686, -2.84701, 10.5343) + +[node name="node_1 1" type="RayCast3D" parent="."] +target_position = Vector3(-1.38686, 2.84701, 10.5343) + +[node name="node_2 0" type="RayCast3D" parent="."] +target_position = Vector3(1.38686, -2.84701, 10.5343) + +[node name="node_2 1" type="RayCast3D" parent="."] +target_position = Vector3(1.38686, 2.84701, 10.5343) + +[node name="node_3 0" type="RayCast3D" parent="."] +target_position = Vector3(4.06608, -2.84701, 9.81639) + +[node name="node_3 1" type="RayCast3D" parent="."] +target_position = Vector3(4.06608, 2.84701, 9.81639) diff --git a/examples/2DTerrainChallenge/addons/godot_rl_agents/sync.gd b/examples/2DTerrainChallenge/addons/godot_rl_agents/sync.gd new file mode 100644 index 0000000..68c6581 --- /dev/null +++ b/examples/2DTerrainChallenge/addons/godot_rl_agents/sync.gd @@ -0,0 +1,594 @@ +extends Node +class_name Sync + +# --fixed-fps 2000 --disable-render-loop + +enum ControlModes { HUMAN, TRAINING, ONNX_INFERENCE } ## Test the environment manually ## Train a model ## Load a pretrained model using an .onnx file +@export var control_mode: ControlModes = ControlModes.TRAINING +## Action will be repeated for n frames (Godot physics steps). +@export_range(1, 10, 1, "or_greater") var action_repeat := 8 +## Speeds up the physics in the environment to enable faster training. +@export_range(0, 10, 0.1, "or_greater") var speed_up := 1.0 +## The path to a trained .onnx model file to use for inference (only needed for the 'Onnx Inference' control mode). +@export var onnx_model_path := "" + +# Onnx model stored for each requested path +var onnx_models: Dictionary + +@onready var start_time = Time.get_ticks_msec() + +const MAJOR_VERSION := "0" +const MINOR_VERSION := "7" +const DEFAULT_PORT := "11008" +const DEFAULT_SEED := "1" +var stream: StreamPeerTCP = null +var connected = false +var message_center +var should_connect = true + +var all_agents: Array +var agents_training: Array +## Policy name of each agent, for use with multi-policy multi-agent RL cases +var agents_training_policy_names: Array[String] = ["shared_policy"] +var agents_inference: Array +var agents_heuristic: Array + +## For recording expert demos +var agent_demo_record: Node +## File path for writing recorded trajectories +var expert_demo_save_path: String +## Stores recorded trajectories +var demo_trajectories: Array +## A trajectory includes obs: Array, acts: Array, terminal (set in Python env instead) +var current_demo_trajectory: Array + +var need_to_send_obs = false +var args = null +var initialized = false +var just_reset = false +var onnx_model = null +var n_action_steps = 0 + +var _action_space_training: Array[Dictionary] = [] +var _action_space_inference: Array[Dictionary] = [] +var _obs_space_training: Array[Dictionary] = [] + + +# Called when the node enters the scene tree for the first time. +func _ready(): + await get_parent().ready + get_tree().set_pause(true) + _initialize() + await get_tree().create_timer(1.0).timeout + get_tree().set_pause(false) + + +func _initialize(): + _get_agents() + args = _get_args() + Engine.physics_ticks_per_second = _get_speedup() * 60 # Replace with function body. + Engine.time_scale = _get_speedup() * 1.0 + prints( + "physics ticks", + Engine.physics_ticks_per_second, + Engine.time_scale, + _get_speedup(), + speed_up + ) + + _set_heuristic("human", all_agents) + + _initialize_training_agents() + _initialize_inference_agents() + _initialize_demo_recording() + + _set_seed() + _set_action_repeat() + initialized = true + + +func _initialize_training_agents(): + if agents_training.size() > 0: + _obs_space_training.resize(agents_training.size()) + _action_space_training.resize(agents_training.size()) + for agent_idx in range(0, agents_training.size()): + _obs_space_training[agent_idx] = agents_training[agent_idx].get_obs_space() + _action_space_training[agent_idx] = agents_training[agent_idx].get_action_space() + connected = connect_to_server() + if connected: + _set_heuristic("model", agents_training) + _handshake() + _send_env_info() + else: + push_warning( + "Couldn't connect to Python server, using human controls instead. ", + "Did you start the training server using e.g. `gdrl` from the console?" + ) + + +func _initialize_inference_agents(): + if agents_inference.size() > 0: + if control_mode == ControlModes.ONNX_INFERENCE: + assert( + FileAccess.file_exists(onnx_model_path), + "Onnx Model Path set on Sync node does not exist: %s" % onnx_model_path + ) + onnx_models[onnx_model_path] = ONNXModel.new(onnx_model_path, 1) + + for agent in agents_inference: + var action_space = agent.get_action_space() + _action_space_inference.append(action_space) + + var agent_onnx_model: ONNXModel + if agent.onnx_model_path.is_empty(): + assert( + onnx_models.has(onnx_model_path), + ( + "Node %s has no onnx model path set " % agent.get_path() + + "and sync node's control mode is not set to OnnxInference. " + + "Either add the path to the AIController, " + + "or if you want to use the path set on sync node instead, " + + "set control mode to OnnxInference." + ) + ) + prints( + "Info: AIController %s" % agent.get_path(), + "has no onnx model path set.", + "Using path set on the sync node instead." + ) + agent_onnx_model = onnx_models[onnx_model_path] + else: + if not onnx_models.has(agent.onnx_model_path): + assert( + FileAccess.file_exists(agent.onnx_model_path), + ( + "Onnx Model Path set on %s node does not exist: %s" + % [agent.get_path(), agent.onnx_model_path] + ) + ) + onnx_models[agent.onnx_model_path] = ONNXModel.new(agent.onnx_model_path, 1) + agent_onnx_model = onnx_models[agent.onnx_model_path] + + agent.onnx_model = agent_onnx_model + if not agent_onnx_model.action_means_only_set: + agent_onnx_model.set_action_means_only(action_space) + + _set_heuristic("model", agents_inference) + + +func _initialize_demo_recording(): + if agent_demo_record: + expert_demo_save_path = agent_demo_record.expert_demo_save_path + assert( + not expert_demo_save_path.is_empty(), + "Expert demo save path set in %s is empty." % agent_demo_record.get_path() + ) + + InputMap.add_action("RemoveLastDemoEpisode") + InputMap.action_add_event( + "RemoveLastDemoEpisode", agent_demo_record.remove_last_episode_key + ) + current_demo_trajectory.resize(2) + current_demo_trajectory[0] = [] + current_demo_trajectory[1] = [] + agent_demo_record.heuristic = "demo_record" + + +func _physics_process(_delta): + # two modes, human control, agent control + # pause tree, send obs, get actions, set actions, unpause tree + + _demo_record_process() + + if n_action_steps % action_repeat != 0: + n_action_steps += 1 + return + + n_action_steps += 1 + + _training_process() + _inference_process() + _heuristic_process() + + +func _training_process(): + if connected: + get_tree().set_pause(true) + + var obs = _get_obs_from_agents(agents_training) + var info = _get_info_from_agents(agents_training) + + if just_reset: + just_reset = false + + var reply = {"type": "reset", "obs": obs, "info": info} + _send_dict_as_json_message(reply) + # this should go straight to getting the action and setting it checked the agent, no need to perform one phyics tick + get_tree().set_pause(false) + return + + if need_to_send_obs: + need_to_send_obs = false + var reward = _get_reward_from_agents() + var done = _get_done_from_agents() + #_reset_agents_if_done() # this ensures the new observation is from the next env instance : NEEDS REFACTOR + + var reply = {"type": "step", "obs": obs, "reward": reward, "done": done, "info": info} + _send_dict_as_json_message(reply) + + var handled = handle_message() + + +func _inference_process(): + if agents_inference.size() > 0: + var obs: Array = _get_obs_from_agents(agents_inference) + var actions = [] + + for agent_id in range(0, agents_inference.size()): + var model: ONNXModel = agents_inference[agent_id].onnx_model + var action = model.run_inference(obs[agent_id]["obs"], 1.0) + var action_dict = _extract_action_dict( + action["output"], _action_space_inference[agent_id], model.action_means_only + ) + actions.append(action_dict) + + _set_agent_actions(actions, agents_inference) + _reset_agents_if_done(agents_inference) + get_tree().set_pause(false) + + +func _demo_record_process(): + if not agent_demo_record: + return + + if Input.is_action_just_pressed("RemoveLastDemoEpisode"): + print("[Sync script][Demo recorder] Removing last recorded episode.") + demo_trajectories.remove_at(demo_trajectories.size() - 1) + print("Remaining episode count: %d" % demo_trajectories.size()) + + if n_action_steps % agent_demo_record.action_repeat != 0: + return + + var obs_dict: Dictionary = agent_demo_record.get_obs() + + # Get the current obs from the agent + assert( + obs_dict.has("obs"), + "Demo recorder needs an 'obs' key in get_obs() returned dictionary to record obs from." + ) + current_demo_trajectory[0].append(obs_dict.obs) + + # Get the action applied for the current obs from the agent + agent_demo_record.set_action() + var acts = agent_demo_record.get_action() + + var terminal = agent_demo_record.get_done() + # Record actions only for non-terminal states + if terminal: + agent_demo_record.set_done_false() + else: + current_demo_trajectory[1].append(acts) + + if terminal: + #current_demo_trajectory[2].append(true) + demo_trajectories.append(current_demo_trajectory.duplicate(true)) + print("[Sync script][Demo recorder] Recorded episode count: %d" % demo_trajectories.size()) + current_demo_trajectory[0].clear() + current_demo_trajectory[1].clear() + + +func _heuristic_process(): + for agent in agents_heuristic: + _reset_agents_if_done(agents_heuristic) + + +func _extract_action_dict(action_array: Array, action_space: Dictionary, action_means_only: bool): + var index = 0 + var result = {} + for key in action_space.keys(): + var size = action_space[key]["size"] + var action_type = action_space[key]["action_type"] + if action_type == "discrete": + var largest_logit: float # Value of the largest logit for this action in the actions array + var largest_logit_idx: int # Index of the largest logit for this action in the actions array + for logit_idx in range(0, size): + var logit_value = action_array[index + logit_idx] + if logit_value > largest_logit: + largest_logit = logit_value + largest_logit_idx = logit_idx + result[key] = largest_logit_idx # Index of the largest logit is the discrete action value + index += size + elif action_type == "continuous": + # For continous actions, we only take the action mean values + result[key] = clamp_array(action_array.slice(index, index + size), -1.0, 1.0) + if action_means_only: + index += size # model only outputs action means, so we move index by size + else: + index += size * 2 # model outputs logstd after action mean, we skip the logstd part + + else: + assert( + false, + ( + 'Only "discrete" and "continuous" action types supported. Found: %s action type set.' + % action_type + ) + ) + + return result + + +## For AIControllers that inherit mode from sync, sets the correct mode. +func _set_agent_mode(agent: Node): + var agent_inherits_mode: bool = agent.control_mode == agent.ControlModes.INHERIT_FROM_SYNC + + if agent_inherits_mode: + match control_mode: + ControlModes.HUMAN: + agent.control_mode = agent.ControlModes.HUMAN + ControlModes.TRAINING: + agent.control_mode = agent.ControlModes.TRAINING + ControlModes.ONNX_INFERENCE: + agent.control_mode = agent.ControlModes.ONNX_INFERENCE + + +func _get_agents(): + all_agents = get_tree().get_nodes_in_group("AGENT") + for agent in all_agents: + _set_agent_mode(agent) + + if agent.control_mode == agent.ControlModes.TRAINING: + agents_training.append(agent) + elif agent.control_mode == agent.ControlModes.ONNX_INFERENCE: + agents_inference.append(agent) + elif agent.control_mode == agent.ControlModes.HUMAN: + agents_heuristic.append(agent) + elif agent.control_mode == agent.ControlModes.RECORD_EXPERT_DEMOS: + assert( + not agent_demo_record, + "Currently only a single AIController can be used for recording expert demos." + ) + agent_demo_record = agent + + var training_agent_count = agents_training.size() + agents_training_policy_names.resize(training_agent_count) + for i in range(0, training_agent_count): + agents_training_policy_names[i] = agents_training[i].policy_name + + +func _set_heuristic(heuristic, agents: Array): + for agent in agents: + agent.set_heuristic(heuristic) + + +func _handshake(): + print("performing handshake") + + var json_dict = _get_dict_json_message() + assert(json_dict["type"] == "handshake") + var major_version = json_dict["major_version"] + var minor_version = json_dict["minor_version"] + if major_version != MAJOR_VERSION: + print("WARNING: major verison mismatch ", major_version, " ", MAJOR_VERSION) + if minor_version != MINOR_VERSION: + print("WARNING: minor verison mismatch ", minor_version, " ", MINOR_VERSION) + + print("handshake complete") + + +func _get_dict_json_message(): + # returns a dictionary from of the most recent message + # this is not waiting + while stream.get_available_bytes() == 0: + stream.poll() + if stream.get_status() != 2: + print("server disconnected status, closing") + get_tree().quit() + return null + + OS.delay_usec(10) + + var message = stream.get_string() + var json_data = JSON.parse_string(message) + + return json_data + + +func _send_dict_as_json_message(dict): + stream.put_string(JSON.stringify(dict, "", false)) + + +func _send_env_info(): + var json_dict = _get_dict_json_message() + assert(json_dict["type"] == "env_info") + + var message = { + "type": "env_info", + "observation_space": _obs_space_training, + "action_space": _action_space_training, + "n_agents": len(agents_training), + "agent_policy_names": agents_training_policy_names + } + _send_dict_as_json_message(message) + + +func connect_to_server(): + print("Waiting for one second to allow server to start") + OS.delay_msec(1000) + print("trying to connect to server") + stream = StreamPeerTCP.new() + + # "localhost" was not working on windows VM, had to use the IP + var ip = "127.0.0.1" + var port = _get_port() + var connect = stream.connect_to_host(ip, port) + stream.set_no_delay(true) # TODO check if this improves performance or not + stream.poll() + # Fetch the status until it is either connected (2) or failed to connect (3) + while stream.get_status() < 2: + stream.poll() + return stream.get_status() == 2 + + +func _get_args(): + print("getting command line arguments") + var arguments = {} + for argument in OS.get_cmdline_args(): + print(argument) + if argument.find("=") > -1: + var key_value = argument.split("=") + arguments[key_value[0].lstrip("--")] = key_value[1] + else: + # Options without an argument will be present in the dictionary, + # with the value set to an empty string. + arguments[argument.lstrip("--")] = "" + + return arguments + + +func _get_speedup(): + print(args) + return args.get("speedup", str(speed_up)).to_float() + + +func _get_port(): + return args.get("port", DEFAULT_PORT).to_int() + + +func _set_seed(): + var _seed = args.get("env_seed", DEFAULT_SEED).to_int() + seed(_seed) + + +func _set_action_repeat(): + action_repeat = args.get("action_repeat", str(action_repeat)).to_int() + + +func disconnect_from_server(): + stream.disconnect_from_host() + + +func handle_message() -> bool: + # get json message: reset, step, close + var message = _get_dict_json_message() + if message["type"] == "close": + print("received close message, closing game") + get_tree().quit() + get_tree().set_pause(false) + return true + + if message["type"] == "reset": + print("resetting all agents") + _reset_agents() + just_reset = true + get_tree().set_pause(false) + #print("resetting forcing draw") +# RenderingServer.force_draw() +# var obs = _get_obs_from_agents() +# print("obs ", obs) +# var reply = { +# "type": "reset", +# "obs": obs +# } +# _send_dict_as_json_message(reply) + return true + + if message["type"] == "call": + var method = message["method"] + var returns = _call_method_on_agents(method) + var reply = {"type": "call", "returns": returns} + print("calling method from Python") + _send_dict_as_json_message(reply) + return handle_message() + + if message["type"] == "action": + var action = message["action"] + _set_agent_actions(action, agents_training) + need_to_send_obs = true + get_tree().set_pause(false) + return true + + print("message was not handled") + return false + + +func _call_method_on_agents(method): + var returns = [] + for agent in all_agents: + returns.append(agent.call(method)) + + return returns + + +func _reset_agents_if_done(agents = all_agents): + for agent in agents: + if agent.get_done(): + agent.set_done_false() + + +func _reset_agents(agents = all_agents): + for agent in agents: + agent.needs_reset = true + #agent.reset() + + +func _get_obs_from_agents(agents: Array = all_agents): + var obs = [] + for agent in agents: + obs.append(agent.get_obs()) + return obs + + +func _get_reward_from_agents(agents: Array = agents_training): + var rewards = [] + for agent in agents: + rewards.append(agent.get_reward()) + agent.zero_reward() + return rewards + + +func _get_info_from_agents(agents: Array = all_agents): + var info = [] + for agent in agents: + info.append(agent.get_info()) + return info + + +func _get_done_from_agents(agents: Array = agents_training): + var dones = [] + for agent in agents: + var done = agent.get_done() + if done: + agent.set_done_false() + dones.append(done) + return dones + + +func _set_agent_actions(actions, agents: Array = all_agents): + for i in range(len(actions)): + agents[i].set_action(actions[i]) + + +func clamp_array(arr: Array, min: float, max: float): + var output: Array = [] + for a in arr: + output.append(clamp(a, min, max)) + return output + + +## Save recorded export demos on window exit (Close game window instead of "Stop" button in Godot Editor) +func _notification(what): + if demo_trajectories.size() == 0 or expert_demo_save_path.is_empty(): + return + + if what == NOTIFICATION_PREDELETE: + var json_string = JSON.stringify(demo_trajectories, "", false) + var file = FileAccess.open(expert_demo_save_path, FileAccess.WRITE) + + if not file: + var error: Error = FileAccess.get_open_error() + assert(not error, "There was an error opening the file: %d" % error) + + file.store_line(json_string) + var error = file.get_error() + assert(not error, "There was an error after trying to write to the file: %d" % error) diff --git a/examples/2DTerrainChallenge/assets/car_body.png b/examples/2DTerrainChallenge/assets/car_body.png new file mode 100644 index 0000000..fb1362d Binary files /dev/null and b/examples/2DTerrainChallenge/assets/car_body.png differ diff --git a/examples/2DTerrainChallenge/assets/car_wheel.png b/examples/2DTerrainChallenge/assets/car_wheel.png new file mode 100644 index 0000000..4096e47 Binary files /dev/null and b/examples/2DTerrainChallenge/assets/car_wheel.png differ diff --git a/examples/2DTerrainChallenge/assets/license.md b/examples/2DTerrainChallenge/assets/license.md new file mode 100644 index 0000000..dd81700 --- /dev/null +++ b/examples/2DTerrainChallenge/assets/license.md @@ -0,0 +1,3 @@ +The following license is only for the graphical assets in the folder "assets", specifically .png files: +Author: Ivan Dodic (https://github.com/Ivan-267), +License: https://creativecommons.org/licenses/by/4.0/ \ No newline at end of file diff --git a/examples/2DTerrainChallenge/assets/sky.png b/examples/2DTerrainChallenge/assets/sky.png new file mode 100644 index 0000000..cd5ffbb Binary files /dev/null and b/examples/2DTerrainChallenge/assets/sky.png differ diff --git a/examples/2DTerrainChallenge/icon.svg b/examples/2DTerrainChallenge/icon.svg new file mode 100644 index 0000000..adc26df --- /dev/null +++ b/examples/2DTerrainChallenge/icon.svg @@ -0,0 +1 @@ + diff --git a/examples/2DTerrainChallenge/model.onnx b/examples/2DTerrainChallenge/model.onnx new file mode 100644 index 0000000..ef0b3b1 Binary files /dev/null and b/examples/2DTerrainChallenge/model.onnx differ diff --git a/examples/2DTerrainChallenge/project.godot b/examples/2DTerrainChallenge/project.godot new file mode 100644 index 0000000..fb5f527 --- /dev/null +++ b/examples/2DTerrainChallenge/project.godot @@ -0,0 +1,52 @@ +; Engine configuration file. +; It's best edited using the editor UI and not directly, +; since the parameters that go here are not all obvious. +; +; Format: +; [section] ; section goes between [] +; param=value ; assign values to parameters + +config_version=5 + +[application] + +config/name="2DTerrainChallenge" +run/main_scene="res://scenes/train_scene.tscn" +config/features=PackedStringArray("4.3", "C#", "Forward Plus") +config/icon="res://icon.svg" + +[display] + +window/stretch/aspect="ignore" + +[dotnet] + +project/assembly_name="2DTerrainChallenge" + +[editor_plugins] + +enabled=PackedStringArray() + +[input] + +move_left={ +"deadzone": 0.5, +"events": [Object(InputEventKey,"resource_local_to_scene":false,"resource_name":"","device":-1,"window_id":0,"alt_pressed":false,"shift_pressed":false,"ctrl_pressed":false,"meta_pressed":false,"pressed":false,"keycode":0,"physical_keycode":65,"key_label":0,"unicode":97,"location":0,"echo":false,"script":null) +] +} +move_right={ +"deadzone": 0.5, +"events": [Object(InputEventKey,"resource_local_to_scene":false,"resource_name":"","device":-1,"window_id":0,"alt_pressed":false,"shift_pressed":false,"ctrl_pressed":false,"meta_pressed":false,"pressed":false,"keycode":0,"physical_keycode":68,"key_label":0,"unicode":100,"location":0,"echo":false,"script":null) +] +} +reset={ +"deadzone": 0.5, +"events": [Object(InputEventKey,"resource_local_to_scene":false,"resource_name":"","device":-1,"window_id":0,"alt_pressed":false,"shift_pressed":false,"ctrl_pressed":false,"meta_pressed":false,"pressed":false,"keycode":0,"physical_keycode":82,"key_label":0,"unicode":114,"location":0,"echo":false,"script":null) +] +} + +[physics] + +common/max_physics_steps_per_frame=16 +common/physics_jitter_fix=0.0 +2d/solver/solver_iterations=8 diff --git a/examples/2DTerrainChallenge/readme.md b/examples/2DTerrainChallenge/readme.md new file mode 100644 index 0000000..d9c4e5d --- /dev/null +++ b/examples/2DTerrainChallenge/readme.md @@ -0,0 +1,113 @@ +# 2D Terrain Challenge + +https://github.com/user-attachments/assets/e083faf7-0afe-41af-9424-96d0d13afe56 + +# Goal: +The car needs to reach the goal near the end of the procedurally generated terrain +without flipping over or falling off. + +# Observations: +```gdscript +func get_obs() -> Dictionary: + var observations: Array + + var terrain_length: float = terrain_manager.get_terrain_length() + var velocity_scale_factor: float = 0.000116667 + + var car_base: RigidBody2D = car.base + var goal_position: Vector2 = car_base.to_local(goal_area.global_position) / terrain_length + var relative_velocity: Vector2 = ( + car_base.global_transform.basis_xform_inv(car_base.linear_velocity) * velocity_scale_factor + ) + + var angular_velocity := clampf(car_base.angular_velocity / 2.0, 0, 1.0) + + var car_orientation := car.base.global_transform.x + ( + observations + . append_array( + [ + car_orientation.x, + car_orientation.y, + clampf(goal_position.x, -1.0, 1.0), + clampf(goal_position.y, -1.0, 1.0), + clampf(relative_velocity.x, -1.0, 1.0), + clampf(relative_velocity.y, -1.0, 1.0), + angular_velocity, + ] + ) + ) + + for sensor in raycast_sensors: + observations.append_array(sensor.get_observation()) + + return {"obs": observations} +``` + +# Action space: + +There are 2 continuous actions used by the agent, for moving and turning: + +```gdscript +func get_action_space() -> Dictionary: + return { + "wheel_torque": {"size": 1, "action_type": "continuous"}, + } +``` + + +# Rewards and episode end condition: +A reward is given when: +- The car approaches the goal (best distance based) +- The car reaches the goal (ends episode) +- The car falls down (negative reward, ends episode) +- The car flips over (negative reward, ends episode) + +# Training: + +These are the training settings used to train the included onnx file: + +[SB3 example script](https://github.com/edbeeching/godot_rl_agents/blob/main/examples/stable_baselines3_example.py) was used for training, with the following changes: + +```python + learning_rate = 0.0003 if not args.linear_lr_schedule else linear_schedule(0.0003) + model: PPO = PPO( + "MlpPolicy", + env, + n_steps=512, + batch_size=512 * env.num_envs, + vf_coef=0.9, + gae_lambda=0.995, + gamma=0.995, + learning_rate=learning_rate, + n_epochs=100, + tensorboard_log=args.experiment_dir, + verbose=2, + ) +``` + +Note: Additional changes not detailed above are needed to use the MlpPolicy (`SBGSingleObsEnv` should be used), but you should be able to use the `MultiInput` policy instead. + +SB3 example script cmd arguments: + +```python +--n_parallel=4 +--onnx_export_path=model.onnx +--timesteps=10_000_000 +--save_model_path=model.zip +--speedup=20 +--linear_lr_schedule +--viz +``` + +![training session success rate](https://github.com/user-attachments/assets/037b685d-81d2-45a0-917f-9688053a92c8) + +Notes: +- If you don't need to observe the behavior during training, you can remove viz to decrease resource usage (mostly GPU). +- `--env_path` also needs to be set to the exported executable. + +This environment was made by [Ivan267.](https://github.com/Ivan-267) + + +# Testing the trained onnx: +A trained onnx file is included. To test it, open the environment, then open the `res://scenes/test_scene.tscn` scene, then press `F6`. diff --git a/examples/2DTerrainChallenge/scenes/car/car.gd b/examples/2DTerrainChallenge/scenes/car/car.gd new file mode 100644 index 0000000..1a23789 --- /dev/null +++ b/examples/2DTerrainChallenge/scenes/car/car.gd @@ -0,0 +1,61 @@ +extends Node2D +class_name Car + +@export var ai_controller: CarAIController +@export var wheel_torque_multiplier = 1500000 + +@onready var wheels = find_children("Wheel*") +@onready var wheel_initial_transforms = wheels.map(func(wheel): return wheel.transform) + +@onready var base: RigidBody2D = $Base +@onready var base_initial_transform = base.global_transform + +var requested_movement: float + +var terrain_manager: TerrainManager +var car_reached_goal: bool + +## Set by AI controller +var requested_torque: float + +var goal_position: float + + +func _physics_process(delta): + var torque = requested_torque * wheel_torque_multiplier + for wheel_idx in range(0, wheels.size()): + var wheel_state: PhysicsDirectBodyState2D = PhysicsServer2D.body_get_direct_state( + wheels[wheel_idx].get_rid() + ) + wheel_state.apply_torque(torque) + + process_car_fell_down() + + +func apply_wheel_torque(torque): + if is_zero_approx(torque): + return + for wheel in wheels: + wheel.apply_torque(torque) + + +func process_car_fell_down(): + if not ai_controller.done and base.position.y > terrain_manager.y_offset_multiplier * 1.01: + ai_controller.game_over(-1) + + +func reset(position_global: Vector2): + var base_state: PhysicsDirectBodyState2D = PhysicsServer2D.body_get_direct_state(base.get_rid()) + base_state.transform = Transform2D(0.0, position_global) + base_state.linear_velocity = Vector2.ZERO + base_state.angular_velocity = 0 + base.global_transform = base_state.transform + + for wheel_idx in range(0, wheels.size()): + var wheel_state: PhysicsDirectBodyState2D = PhysicsServer2D.body_get_direct_state( + wheels[wheel_idx].get_rid() + ) + wheel_state.transform = base_state.transform * wheel_initial_transforms[wheel_idx] + wheel_state.angular_velocity = 0 + wheel_state.linear_velocity = Vector2.ZERO + wheels[wheel_idx].global_transform = wheel_state.transform diff --git a/examples/2DTerrainChallenge/scenes/car/car.tscn b/examples/2DTerrainChallenge/scenes/car/car.tscn new file mode 100644 index 0000000..788e889 --- /dev/null +++ b/examples/2DTerrainChallenge/scenes/car/car.tscn @@ -0,0 +1,121 @@ +[gd_scene load_steps=11 format=3 uid="uid://d1uxkcttm306m"] + +[ext_resource type="Script" path="res://scenes/car/car.gd" id="1_6qjtc"] +[ext_resource type="Texture2D" uid="uid://vhrtedqiiu67" path="res://assets/car_wheel.png" id="2_462h5"] +[ext_resource type="Texture2D" uid="uid://cwtnpunccloy4" path="res://assets/car_body.png" id="3_06xwe"] +[ext_resource type="Script" path="res://scenes/car/car_ai_controller.gd" id="4_xblv3"] +[ext_resource type="Script" path="res://addons/godot_rl_agents/sensors/sensors_2d/RaycastSensor2D.gd" id="5_6pyw6"] + +[sub_resource type="InputEventKey" id="InputEventKey_8y7sl"] +device = -1 +keycode = 82 +unicode = 114 + +[sub_resource type="PhysicsMaterial" id="PhysicsMaterial_h17rd"] + +[sub_resource type="CircleShape2D" id="CircleShape2D_fs5fg"] +radius = 55.0 + +[sub_resource type="PhysicsMaterial" id="PhysicsMaterial_lhxix"] +friction = 0.85 +bounce = 0.05 + +[sub_resource type="RectangleShape2D" id="RectangleShape2D_e8ifa"] +size = Vector2(186.88, 7.29) + +[node name="CarAIController2D" type="Node2D" node_paths=PackedStringArray("raycast_sensors")] +script = ExtResource("4_xblv3") +raycast_sensors = [NodePath("Car/Base/RaycastSensorLocalTerrain")] +reset_after = 5000 +expert_demo_save_path = "works.json" +remove_last_episode_key = SubResource("InputEventKey_8y7sl") +action_repeat = 2 + +[node name="Car" type="Node2D" parent="." node_paths=PackedStringArray("ai_controller")] +script = ExtResource("1_6qjtc") +ai_controller = NodePath("..") + +[node name="Wheel" type="RigidBody2D" parent="Car"] +position = Vector2(-98, 104.015) +collision_layer = 2 +mass = 2.0 +physics_material_override = SubResource("PhysicsMaterial_h17rd") +gravity_scale = 2.0 +contact_monitor = true +max_contacts_reported = 1 + +[node name="CollisionShape2D" type="CollisionShape2D" parent="Car/Wheel"] +shape = SubResource("CircleShape2D_fs5fg") + +[node name="PinJoint2D" type="PinJoint2D" parent="Car/Wheel"] +node_a = NodePath("../../Base") +node_b = NodePath("..") +softness = 0.2 + +[node name="Sprite2D" type="Sprite2D" parent="Car/Wheel"] +scale = Vector2(1.1, 1.1) +texture = ExtResource("2_462h5") + +[node name="Wheel2" type="RigidBody2D" parent="Car"] +position = Vector2(98, 104.015) +collision_layer = 2 +mass = 2.0 +physics_material_override = SubResource("PhysicsMaterial_h17rd") +gravity_scale = 2.0 +contact_monitor = true +max_contacts_reported = 1 + +[node name="CollisionShape2D" type="CollisionShape2D" parent="Car/Wheel2"] +shape = SubResource("CircleShape2D_fs5fg") + +[node name="PinJoint2D" type="PinJoint2D" parent="Car/Wheel2"] +node_a = NodePath("../../Base") +node_b = NodePath("..") +softness = 0.2 + +[node name="Sprite2D" type="Sprite2D" parent="Car/Wheel2"] +scale = Vector2(1.1, 1.1) +texture = ExtResource("2_462h5") + +[node name="Base" type="RigidBody2D" parent="Car"] +collision_layer = 2 +collision_mask = 3 +mass = 20.0 +physics_material_override = SubResource("PhysicsMaterial_lhxix") +gravity_scale = 2.0 +center_of_mass_mode = 1 +center_of_mass = Vector2(0, 10) +contact_monitor = true +max_contacts_reported = 1 + +[node name="Camera2D" type="Camera2D" parent="Car/Base"] +zoom = Vector2(0.22, 0.22) +limit_bottom = 1200 +limit_smoothed = true +position_smoothing_speed = 15.0 + +[node name="Sprite2D" type="Sprite2D" parent="Car/Base"] +rotation = 3.14159 +scale = Vector2(0.3, -0.3) +texture = ExtResource("3_06xwe") + +[node name="CollisionPolygon2D" type="CollisionPolygon2D" parent="Car/Base"] +rotation = -3.14159 +scale = Vector2(0.3, -0.3) +polygon = PackedVector2Array(303.334, -263.333, 460, 40.0012, 506.666, 140.001, 553.333, 270.001, -543.334, 269.999, -517.1, 165.2, -533.334, 89.9987, -456.667, 36.6655, -306.666, -260.001) + +[node name="RoofCollisionDetector" type="Area2D" parent="Car/Base"] +position = Vector2(0, -94.09) +collision_layer = 0 +collision_mask = 3 + +[node name="CollisionShape2D" type="CollisionShape2D" parent="Car/Base/RoofCollisionDetector"] +shape = SubResource("RectangleShape2D_e8ifa") + +[node name="RaycastSensorLocalTerrain" type="Node2D" parent="Car/Base"] +rotation = 1.5708 +script = ExtResource("5_6pyw6") +n_rays = 8.0 +ray_length = 3000 + +[connection signal="body_entered" from="Car/Base/RoofCollisionDetector" to="." method="_on_roof_collision_detector_body_entered"] diff --git a/examples/2DTerrainChallenge/scenes/car/car_ai_controller.gd b/examples/2DTerrainChallenge/scenes/car/car_ai_controller.gd new file mode 100644 index 0000000..b746f46 --- /dev/null +++ b/examples/2DTerrainChallenge/scenes/car/car_ai_controller.gd @@ -0,0 +1,134 @@ +extends AIController2D +class_name CarAIController + +@export var raycast_sensors: Array[Node2D] +@export var terrain_manager: TerrainManager +@export var goal_area: GoalArea + +## Minimum car spawn offset (from the left side of the terrain) +@export var car_spawn_min_x_offset = 5500 +## Goal offset (from the right side of the terrain) +@export var goal_x_offset = 2500 + +@onready var car: Car = $Car + +var is_success: bool = false +var previous_distance_to_goal + + +func _ready(): + super._ready() + car.terrain_manager = terrain_manager + heuristic_init() + + +func get_info() -> Dictionary: + return {"is_success": is_success} + + +func get_obs() -> Dictionary: + var observations: Array + + var terrain_length: float = terrain_manager.get_terrain_length() + var velocity_scale_factor: float = 0.000116667 + + var car_base: RigidBody2D = car.base + var goal_position: Vector2 = car_base.to_local(goal_area.global_position) / terrain_length + var relative_velocity: Vector2 = ( + car_base.global_transform.basis_xform_inv(car_base.linear_velocity) * velocity_scale_factor + ) + + var angular_velocity := clampf(car_base.angular_velocity / 2.0, 0, 1.0) + + var car_orientation := car.base.global_transform.x + ( + observations + . append_array( + [ + car_orientation.x, + car_orientation.y, + clampf(goal_position.x, -1.0, 1.0), + clampf(goal_position.y, -1.0, 1.0), + clampf(relative_velocity.x, -1.0, 1.0), + clampf(relative_velocity.y, -1.0, 1.0), + angular_velocity, + ] + ) + ) + + for sensor in raycast_sensors: + observations.append_array(sensor.get_observation()) + + return {"obs": observations} + + +func get_reward() -> float: + var distance_to_goal = abs(goal_area.global_position.x - car.base.global_position.x) + if not previous_distance_to_goal: + previous_distance_to_goal = distance_to_goal +# + if distance_to_goal < previous_distance_to_goal: + reward += ((previous_distance_to_goal - distance_to_goal) / 225_000.0) + previous_distance_to_goal = distance_to_goal + return reward + + +func get_action_space() -> Dictionary: + return { + "wheel_torque": {"size": 1, "action_type": "continuous"}, + } + + +func _physics_process(_delta): + n_steps += 1 + if n_steps > reset_after: + game_over() + + if needs_reset: + reset() + + heuristic_process() + + +func reset(): + super.reset() + previous_distance_to_goal = null + terrain_manager.generate_terrain() + var new_car_pos = terrain_manager.get_terrain_position( + car_spawn_min_x_offset + randf_range(0, 200), -600 + ) + goal_area.global_position = terrain_manager.get_goal_position(goal_x_offset, 0) + car.reset(new_car_pos) + + +func set_action(action = null) -> void: + if action: + car.requested_torque = clampf(action.wheel_torque[0], -1.0, 1.0) + + +func heuristic_process() -> void: + if heuristic == "human": + car.requested_torque = ( + float(Input.is_action_pressed("move_right")) + - float(Input.is_action_pressed("move_left")) + ) + + +func heuristic_init() -> void: + if heuristic == "human": + reset() + + +func _on_roof_collision_detector_body_entered(_body): + game_over(-1) + + +func game_over(final_reward = 0, success := false): + is_success = success + reward += final_reward + done = true + reset() + + +func _on_goal_area_body_entered(body: Node2D) -> void: + game_over(+1, true) diff --git a/examples/2DTerrainChallenge/scenes/goal_area/goal_area.tscn b/examples/2DTerrainChallenge/scenes/goal_area/goal_area.tscn new file mode 100644 index 0000000..27ccb70 --- /dev/null +++ b/examples/2DTerrainChallenge/scenes/goal_area/goal_area.tscn @@ -0,0 +1,19 @@ +[gd_scene load_steps=3 format=3 uid="uid://bf3rfk5tvqlio"] + +[ext_resource type="Script" path="res://scenes/playing_area/goal_area.gd" id="1_0rw06"] + +[sub_resource type="RectangleShape2D" id="RectangleShape2D_c8fd7"] +size = Vector2(2000, 5000) + +[node name="GoalArea" type="Area2D"] +collision_layer = 4 +collision_mask = 2 +script = ExtResource("1_0rw06") + +[node name="Polygon2D" type="Polygon2D" parent="."] +position = Vector2(-1000, -2500) +color = Color(0, 1, 0, 0.4) +polygon = PackedVector2Array(0, 0, 2000, 0, 2000, 5000, 0, 5000) + +[node name="CollisionShape2D" type="CollisionShape2D" parent="."] +shape = SubResource("RectangleShape2D_c8fd7") diff --git a/examples/2DTerrainChallenge/scenes/playing_area/goal_area.gd b/examples/2DTerrainChallenge/scenes/playing_area/goal_area.gd new file mode 100644 index 0000000..4278763 --- /dev/null +++ b/examples/2DTerrainChallenge/scenes/playing_area/goal_area.gd @@ -0,0 +1,2 @@ +extends Area2D +class_name GoalArea diff --git a/examples/2DTerrainChallenge/scenes/playing_area/playing_area.tscn b/examples/2DTerrainChallenge/scenes/playing_area/playing_area.tscn new file mode 100644 index 0000000..49f79de --- /dev/null +++ b/examples/2DTerrainChallenge/scenes/playing_area/playing_area.tscn @@ -0,0 +1,42 @@ +[gd_scene load_steps=5 format=3 uid="uid://ca6csk8y5nvuu"] + +[ext_resource type="PackedScene" uid="uid://dmbj3jo7rdhj3" path="res://scenes/terrain_manager/terrain_manager.tscn" id="2_tyk7f"] +[ext_resource type="PackedScene" uid="uid://d1uxkcttm306m" path="res://scenes/car/car.tscn" id="3_g3hiu"] +[ext_resource type="PackedScene" uid="uid://bf3rfk5tvqlio" path="res://scenes/goal_area/goal_area.tscn" id="4_pgvws"] +[ext_resource type="Texture2D" uid="uid://c4200ixpcvf0x" path="res://assets/sky.png" id="5_apd32"] + +[node name="playing_area" type="Node2D"] + +[node name="CanvasLayer" type="CanvasLayer" parent="."] +layer = -1 + +[node name="Control" type="Control" parent="CanvasLayer"] +layout_mode = 3 +anchors_preset = 15 +anchor_right = 1.0 +anchor_bottom = 1.0 +grow_horizontal = 2 +grow_vertical = 2 +size_flags_horizontal = 3 +size_flags_vertical = 3 + +[node name="TextureRect" type="TextureRect" parent="CanvasLayer/Control"] +layout_mode = 1 +anchors_preset = 15 +anchor_right = 1.0 +anchor_bottom = 1.0 +grow_horizontal = 2 +grow_vertical = 2 +texture = ExtResource("5_apd32") +stretch_mode = 5 + +[node name="Terrain" parent="." instance=ExtResource("2_tyk7f")] + +[node name="GoalArea" parent="." instance=ExtResource("4_pgvws")] +position = Vector2(16423, 0) + +[node name="CarAIController2D" parent="." node_paths=PackedStringArray("terrain_manager", "goal_area") instance=ExtResource("3_g3hiu")] +terrain_manager = NodePath("../Terrain") +goal_area = NodePath("../GoalArea") + +[connection signal="body_entered" from="GoalArea" to="CarAIController2D" method="_on_goal_area_body_entered"] diff --git a/examples/2DTerrainChallenge/scenes/terrain_manager/terrain_manager.gd b/examples/2DTerrainChallenge/scenes/terrain_manager/terrain_manager.gd new file mode 100644 index 0000000..07281bf --- /dev/null +++ b/examples/2DTerrainChallenge/scenes/terrain_manager/terrain_manager.gd @@ -0,0 +1,83 @@ +extends StaticBody2D +class_name TerrainManager + +@onready var terrain_polygon: Polygon2D = $Polygon2D +@onready var collision_polygon: CollisionPolygon2D = $CollisionPolygon2D + +@export_group("Terrain settings") +@export var fixed_seed: bool = false +@export var x_point_count: float = 300 +@export var x_offset_between_points: int = 200 +@export var y_offset_multiplier: float = 1400 +@export var octaves: int = 3 +@export var frequency: float = 0.05 + +var noise: FastNoiseLite = FastNoiseLite.new() +var _point_heights: Array[float] + + +func _init() -> void: + noise.frequency = frequency + noise.fractal_octaves = octaves + noise.fractal_lacunarity = 1.5 + + +func generate_terrain(): + _point_heights.clear() + + noise.seed = randi() if not fixed_seed else 0 + var x_points = x_point_count + var x_offset = x_offset_between_points + var max_y_offset = y_offset_multiplier + + var new_poly := PackedVector2Array() + + var point_height: float + + var start_end_area_point_size: float = 150 + + for point_idx in range(0, x_points): + point_height = noise.get_noise_1d(point_idx) * max_y_offset + + if point_idx < start_end_area_point_size: + point_height *= 1 - (start_end_area_point_size - point_idx) / start_end_area_point_size + elif point_idx >= (x_points - start_end_area_point_size): + point_height *= (x_points - point_idx - 1) / (start_end_area_point_size) + + var point := Vector2(point_idx * x_offset, point_height) + new_poly.append(point) + _point_heights.append(point_height) + + point_height = noise.get_noise_1d(x_points - 1) * max_y_offset + ( + new_poly + . append_array( + [ + Vector2((x_points - 1) * x_offset, max_y_offset + 1000), + Vector2(0, max_y_offset + 1000), + ] + ) + ) + + collision_polygon.call_deferred("set_polygon", new_poly) + terrain_polygon.polygon = new_poly + + +func get_terrain_length() -> float: + return x_point_count * x_offset_between_points + + +func get_goal_position(x_offset_from_end, y_offset) -> Vector2: + return get_terrain_position( + (x_point_count * x_offset_between_points) - x_offset_from_end, y_offset + ) + + +func get_terrain_position(x_offset, y_offset) -> Vector2: + var scaled_x_offset = int(x_offset) / int(x_offset_between_points) + if scaled_x_offset < 0 or scaled_x_offset >= _point_heights.size(): + return Vector2(0.0, 0.0) + + return Vector2( + global_position.x + x_offset, global_position.y + _point_heights[scaled_x_offset] + y_offset + ) diff --git a/examples/2DTerrainChallenge/scenes/terrain_manager/terrain_manager.tscn b/examples/2DTerrainChallenge/scenes/terrain_manager/terrain_manager.tscn new file mode 100644 index 0000000..f1022ee --- /dev/null +++ b/examples/2DTerrainChallenge/scenes/terrain_manager/terrain_manager.tscn @@ -0,0 +1,42 @@ +[gd_scene load_steps=5 format=3 uid="uid://dmbj3jo7rdhj3"] + +[ext_resource type="Script" path="res://scenes/terrain_manager/terrain_manager.gd" id="1_tb7f1"] + +[sub_resource type="Gradient" id="Gradient_ii32c"] +offsets = PackedFloat32Array(0, 0.317136, 1) +colors = PackedColorArray(0.320313, 0.165997, 0.0702561, 1, 0.492188, 0.315034, 0.107524, 1, 0.578125, 0.425366, 0.125901, 1) + +[sub_resource type="FastNoiseLite" id="FastNoiseLite_cgubt"] +noise_type = 3 +seed = 420 +frequency = 0.0022 +fractal_octaves = 6 +fractal_gain = 1.185 +fractal_weighted_strength = 0.4 +domain_warp_enabled = true + +[sub_resource type="NoiseTexture2D" id="NoiseTexture2D_b6hcp"] +width = 1024 +height = 1024 +seamless = true +seamless_blend_skirt = 0.475 +color_ramp = SubResource("Gradient_ii32c") +noise = SubResource("FastNoiseLite_cgubt") + +[node name="Terrain" type="StaticBody2D"] +script = ExtResource("1_tb7f1") +x_point_count = 300.0 +x_offset_between_points = 200 +y_offset_multiplier = 1400.0 +octaves = 3 +frequency = 0.05 +terrain_start_lunacrity = 1.0 +terrain_end_lunacrity = 1.0 + +[node name="CollisionPolygon2D" type="CollisionPolygon2D" parent="."] +polygon = PackedVector2Array(0, 0, 100, 0, 100, 100, 0, 100) + +[node name="Polygon2D" type="Polygon2D" parent="."] +texture_repeat = 2 +texture = SubResource("NoiseTexture2D_b6hcp") +polygon = PackedVector2Array(0, 0, 100, 0, 100, 100, 0, 100) diff --git a/examples/2DTerrainChallenge/scenes/test_scene.tscn b/examples/2DTerrainChallenge/scenes/test_scene.tscn new file mode 100644 index 0000000..09ee1dc --- /dev/null +++ b/examples/2DTerrainChallenge/scenes/test_scene.tscn @@ -0,0 +1,13 @@ +[gd_scene load_steps=3 format=3 uid="uid://dbac4xoyhwxni"] + +[ext_resource type="Script" path="res://addons/godot_rl_agents/sync.gd" id="1_3wxae"] +[ext_resource type="PackedScene" uid="uid://ca6csk8y5nvuu" path="res://scenes/playing_area/playing_area.tscn" id="2_dp3en"] + +[node name="testing_scene" type="Node2D"] + +[node name="playing_area" parent="." instance=ExtResource("2_dp3en")] + +[node name="Sync" type="Node" parent="."] +script = ExtResource("1_3wxae") +control_mode = 2 +onnx_model_path = "model.onnx" diff --git a/examples/2DTerrainChallenge/scenes/train_scene.tscn b/examples/2DTerrainChallenge/scenes/train_scene.tscn new file mode 100644 index 0000000..b4142cd --- /dev/null +++ b/examples/2DTerrainChallenge/scenes/train_scene.tscn @@ -0,0 +1,102 @@ +[gd_scene load_steps=3 format=3 uid="uid://dt3fadynxnb2t"] + +[ext_resource type="Script" path="res://addons/godot_rl_agents/sync.gd" id="1_ge04o"] +[ext_resource type="PackedScene" uid="uid://ca6csk8y5nvuu" path="res://scenes/playing_area/playing_area.tscn" id="2_ogfjs"] + +[node name="training_scene" type="Node2D"] + +[node name="Sync" type="Node" parent="."] +script = ExtResource("1_ge04o") +speed_up = 16.0 + +[node name="playing_area" parent="." instance=ExtResource("2_ogfjs")] + +[node name="SubViewport" type="SubViewport" parent="."] +disable_3d = true +canvas_cull_mask = 4294967294 + +[node name="playing_area" parent="SubViewport" instance=ExtResource("2_ogfjs")] + +[node name="SubViewport2" type="SubViewport" parent="."] +disable_3d = true +canvas_cull_mask = 4294967294 + +[node name="playing_area" parent="SubViewport2" instance=ExtResource("2_ogfjs")] + +[node name="SubViewport3" type="SubViewport" parent="."] +disable_3d = true +canvas_cull_mask = 4294967294 + +[node name="playing_area" parent="SubViewport3" instance=ExtResource("2_ogfjs")] + +[node name="SubViewport4" type="SubViewport" parent="."] +disable_3d = true +canvas_cull_mask = 4294967294 + +[node name="playing_area" parent="SubViewport4" instance=ExtResource("2_ogfjs")] + +[node name="SubViewport5" type="SubViewport" parent="."] +disable_3d = true +canvas_cull_mask = 4294967294 + +[node name="playing_area" parent="SubViewport5" instance=ExtResource("2_ogfjs")] + +[node name="SubViewport6" type="SubViewport" parent="."] +disable_3d = true +canvas_cull_mask = 4294967294 + +[node name="playing_area" parent="SubViewport6" instance=ExtResource("2_ogfjs")] + +[node name="SubViewport7" type="SubViewport" parent="."] +disable_3d = true +canvas_cull_mask = 4294967294 + +[node name="playing_area" parent="SubViewport7" instance=ExtResource("2_ogfjs")] + +[node name="SubViewport8" type="SubViewport" parent="."] +disable_3d = true +canvas_cull_mask = 4294967294 + +[node name="playing_area" parent="SubViewport8" instance=ExtResource("2_ogfjs")] + +[node name="SubViewport9" type="SubViewport" parent="."] +disable_3d = true +canvas_cull_mask = 4294967294 + +[node name="playing_area" parent="SubViewport9" instance=ExtResource("2_ogfjs")] + +[node name="SubViewport10" type="SubViewport" parent="."] +disable_3d = true +canvas_cull_mask = 4294967294 + +[node name="playing_area" parent="SubViewport10" instance=ExtResource("2_ogfjs")] + +[node name="SubViewport11" type="SubViewport" parent="."] +disable_3d = true +canvas_cull_mask = 4294967294 + +[node name="playing_area" parent="SubViewport11" instance=ExtResource("2_ogfjs")] + +[node name="SubViewport12" type="SubViewport" parent="."] +disable_3d = true +canvas_cull_mask = 4294967294 + +[node name="playing_area" parent="SubViewport12" instance=ExtResource("2_ogfjs")] + +[node name="SubViewport13" type="SubViewport" parent="."] +disable_3d = true +canvas_cull_mask = 4294967294 + +[node name="playing_area" parent="SubViewport13" instance=ExtResource("2_ogfjs")] + +[node name="SubViewport14" type="SubViewport" parent="."] +disable_3d = true +canvas_cull_mask = 4294967294 + +[node name="playing_area" parent="SubViewport14" instance=ExtResource("2_ogfjs")] + +[node name="SubViewport15" type="SubViewport" parent="."] +disable_3d = true +canvas_cull_mask = 4294967294 + +[node name="playing_area" parent="SubViewport15" instance=ExtResource("2_ogfjs")]