Add pixel observations

CMakeLists.txt

@@ -44,6 +44,9 @@ message(STATUS "Building nle backend version: ${CMAKE_NLE_VERSION}")
 
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
+add_compile_options(-Wno-deprecated-non-prototype)
+add_compile_options(-Wno-unused-variable)
+
 set(HACKDIR
     "$ENV{HOME}/nethackdir.nle"
     CACHE STRING "Configuration files for nethack")
@@ -78,8 +81,7 @@ add_compile_definitions(
   HACKDIR="${HACKDIR}"
   DEFAULT_WINDOW_SYS="rl"
   DLB
-  NOCWD_ASSUMPTIONS
-  NLE_USE_TILES)
+  NOCWD_ASSUMPTIONS)
 
 set(NLE_SRC ${nle_SOURCE_DIR}/src)
 set(NLE_INC ${nle_SOURCE_DIR}/include)
@@ -140,6 +142,10 @@ set_target_properties(tmt PROPERTIES C_STANDARD 11)
 add_library(nethack SHARED ${NETHACK_SRC})
 add_dependencies(nethack util dat)
 set_target_properties(nethack PROPERTIES CXX_STANDARD 14 SUFFIX ".so")
+target_compile_options(
+  nethack
+  PRIVATE -Wno-deprecated-non-prototype
+  PRIVATE -Wno-unused-variable)
 target_include_directories(
   nethack
   PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${NLE_INC_GEN} /usr/local/include
@@ -176,8 +182,8 @@ pybind11_add_module(
   $<TARGET_OBJECTS:tile>)
 target_link_libraries(_pynethack PUBLIC nethackdl)
 set_target_properties(_pynethack PROPERTIES CXX_STANDARD 14)
-target_include_directories(_pynethack PUBLIC ${NLE_INC_GEN})
-add_dependencies(_pynethack util) # For pm.h.
+target_include_directories(_pynethack PUBLIC ${NLE_INC_GEN} ${NLE_WIN}/share)
+# add_dependencies(_pynethack util tile) # For pm.h.
 
 # ttyrec converter library
 add_library(
@@ -204,6 +210,10 @@ set_target_properties(_pyconverter PROPERTIES CXX_STANDARD 14)
 target_include_directories(
   _pyconverter PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/third_party/converter)
 
+set(TILE_FILES "win/share/monsters.txt" "win/share/objects.txt"
+               "win/share/other.txt")
+
+install(FILES ${TILE_FILES} DESTINATION ${INSTDIR}/tiles)
 # Only install if we are building as part of a Python project.
 if(DEFINED SKBUILD_PROJECT_VERSION)
   install(

doc/nle/source/index.rst

@@ -16,6 +16,7 @@ resembles the one used by people when playing the game.
    :caption: Getting Started
 
    getting_started
+   tiles
 
 
 .. toctree::

doc/nle/source/tiles.rst

@@ -0,0 +1,109 @@
+Using NetHack's tiles for image observations
+============================================
+
+Tiles
+*****
+
+NetHack as we know and love is a text based game with characters representing the dungeon 
+levels and objects. The RL Environment represents these in an observation numpy array:
+
+.. code-block:: python
+
+    obs[0]["chars"]
+
+Each character is also associated with a unique glyph id, which is represented in the "glyph" observation:
+
+.. code-block:: python
+
+    obs[0]["glyph"]
+
+NetHack also contains a set of tile descriptor files which can be used to generate
+the equivalent RGB values so that the game can be rendered as an image-based display.
+
+The source for the descriptor files are here:
+
+.. code-block:: console
+
+    win/share/monsters.txt
+    win/share/objects.txt
+    win/share/other.txt
+
+When converted to RGB, the full set of tiles looks like this:
+
+.. image:: https://github.com/NetHack-LE/nle/raw/main/dat/nle/tileset.png
+    :alt: NetHack tileset
+    :align: center
+\
+
+Installation
+************
+
+The tile descriptor files are included in the distribution and are installed in the 
+`nethackdir/tiles` directory.
+
+
+Initialisation
+**************
+
+To get NLE to render the tiles as an observation set, you must set the render_mode to
+"pixel" when the environment is created. For example:
+
+.. code-block:: python
+
+    env = gym.make("NetHack-v0", render_mode="pixel")
+
+The next step is to add the Gymnasium RenderObservationWrapper to the environment. This 
+ensures that every time the envrionment is rendered, the observations will include the
+RGB tile observations automatically.
+
+.. code-block:: python
+
+    env = gym.wrappers.AddRenderObservation(
+        env, render_only=False, render_key="pixel", obs_key="glyphs")
+
+RGB observations
+****************
+
+The RGB tiles representing the underlying dungeon can be accessed using the "pixel"
+key in the observations dictionary.
+
+.. code-block:: python
+
+    rgb_frame = obs[0]["pixel"]
+
+This frame is a 3D numpy array, each 2D slice represents all the pixels in the
+rendered game screen, and the 3rd dimension represents the RGB values for each pixel.
+
+Example
+*******
+
+Here's a short example of how to set up the environment to use the tile-based RGB observations:
+
+(Note that you need to install the Pillow library to run this example, which can be done with `pip install Pillow`)
+
+.. code-block:: python
+
+    import gymnasium as gym
+    from PIL import Image
+    import nle
+
+    env = gym.make("NetHack-v0", render_mode="pixel")
+    env = gym.wrappers.AddRenderObservation(
+        env, render_only=False, render_key="pixel", obs_key="glyphs")
+    env.unwrapped.seed(1234, 5678, False, 1)
+
+    obs, _ = env.reset()  # each reset generates a new dungeon
+    rgb_frame = obs["pixel"]
+
+    # Convert the RGB frame to an image and display it
+    img = Image.fromarray(rgb_frame)
+    img.show()
+
+
+Here's an animated example of what the tile-based rendering looks like
+after a few steps in the environment:
+
+.. image:: https://github.com/NetHack-LE/nle/raw/main/dat/nle/nethack_tiles_animation.gif
+    :alt: NetHack tileset
+    :align: center
+\

include/nletypes.h

@@ -46,8 +46,6 @@
 #define NLE_BL_CONDITION 25 /* condition bit mask */
 #define NLE_BL_ALIGN 26
 
-/* #define NLE_USE_TILES 1 */ /* Set in CMakeLists.txt. */
-
 /* NetHack defines boolean as follows:
     typedef xchar boolean;      (global.h:80)
     typedef schar xchar;        (global.h:73)

nle/env/base.py

@@ -151,7 +151,7 @@ class NLE(gym.Env):
     # but NetHack doesn't have any. Set it to 42, because
     # that is always the answer to life, the universe and
     # everything.
-    metadata = {"render_modes": ["human", "ansi", "full"], "render_fps": 42}
+    metadata = {"render_modes": ["human", "ansi", "full", "pixel"], "render_fps": 42}
 
     class StepStatus(enum.IntEnum):
         """Specifies the status of the terminal state.
@@ -332,6 +332,12 @@ def __init__(
 
         self.action_space = gym.spaces.Discrete(len(self.actions))
 
+        if render_mode == "pixel":
+            # Pre-load reference tilemap for pixel rendering
+            if not self.nethack.setup_tiles():
+                raise RuntimeError("Failed to setup tilemap for pixel rendering.")
+            self.rendered_frame = np.zeros(nethack.TILE_RENDER_SHAPE, dtype=np.uint8)
+
     def _get_observation(self, observation):
         return {
             key: observation[i]
@@ -549,6 +555,10 @@ def render(self):
             # TODO: Why return a string here but print in the other branches?
             return "\n".join([line.tobytes().decode("utf-8") for line in chars])
 
+        if mode == "pixel":
+            self.nethack.draw_frame(buffer=self.rendered_frame)
+            return self.rendered_frame
+
         return "\nInvalid render mode: " + mode
 
     def __repr__(self):

nle/nethack/__init__.py

@@ -9,6 +9,8 @@
 from nle.nethack.nethack import NETHACKOPTIONS
 from nle.nethack.nethack import OBSERVATION_DESC
 from nle.nethack.nethack import PROGRAM_STATE_SHAPE
+from nle.nethack.nethack import TILE_RENDER_SHAPE
+from nle.nethack.nethack import TILE_SHAPE
 from nle.nethack.nethack import TTYREC_VERSION
 from nle.nethack.nethack import Nethack
 from nle.nethack.nethack import tty_render

nle/nethack/nethack.py

@@ -14,6 +14,16 @@
 HACKDIR = os.path.join(os.path.dirname(_pynethack.__file__), "nethackdir")
 
 DUNGEON_SHAPE = (_pynethack.nethack.ROWNO, _pynethack.nethack.COLNO - 1)
+TILE_SHAPE = (
+    _pynethack.nethack.TILE_Y,
+    _pynethack.nethack.TILE_X,
+    _pynethack.nethack.TILE_Z,
+)
+TILE_RENDER_SHAPE = (
+    DUNGEON_SHAPE[0] * TILE_SHAPE[0],
+    DUNGEON_SHAPE[1] * TILE_SHAPE[1],
+    TILE_SHAPE[2],
+)
 BLSTATS_SHAPE = (_pynethack.nethack.NLE_BLSTATS_SIZE,)
 MESSAGE_SHAPE = (_pynethack.nethack.NLE_MESSAGE_SIZE,)
 PROGRAM_STATE_SHAPE = (_pynethack.nethack.NLE_PROGRAM_STATE_SIZE,)
@@ -318,3 +328,18 @@ def in_normal_game(self):
 
     def how_done(self):
         return self._pynethack.how_done()
+
+    def setup_tiles(self, tile_paths=None):
+        if tile_paths is None:
+            tile_paths = [
+                os.path.join(HACKDIR, "tiles", "monsters.txt"),
+                os.path.join(HACKDIR, "tiles", "objects.txt"),
+                os.path.join(HACKDIR, "tiles", "other.txt"),
+            ]
+        return self._pynethack.setup_tiles(tile_paths)
+
+    def get_tileset(self, buffer):
+        return self._pynethack.get_tileset(buffer)
+
+    def draw_frame(self, buffer):
+        return self._pynethack.draw_frame(buffer)

nle/scripts/pixels.py

@@ -0,0 +1,113 @@
+import gymnasium as gym
+from PIL import Image
+
+import nle  # noqa: F401
+
+env = gym.make("NetHack-v0", render_mode="pixel")
+env = gym.wrappers.AddRenderObservation(
+    env, render_only=False, render_key="pixel", obs_key="glyphs"
+)
+env.unwrapped.seed(1234, 5678, False, 1)
+
+
+frames = []
+
+obs = env.reset()
+frame = obs[0]["pixel"]
+img = Image.fromarray(frame, "RGB")
+frames.append(img)
+
+NORTH = 0
+WEST = 3
+SOUTH = 2
+EAST = 1
+
+# Get out of starting room
+steps = [EAST, EAST, NORTH, NORTH, WEST, WEST, WEST, WEST, WEST]
+# Go to room two
+steps += [
+    SOUTH,
+    SOUTH,
+    SOUTH,
+    SOUTH,
+    SOUTH,
+    WEST,
+    SOUTH,
+    WEST,
+    SOUTH,
+    SOUTH,
+    SOUTH,
+    SOUTH,
+    SOUTH,
+    WEST,
+    SOUTH,
+    WEST,
+    WEST,
+]
+# Traverse room two
+steps += [WEST, WEST, WEST, WEST, NORTH, NORTH, NORTH, NORTH]
+# Go to room three
+steps += [
+    WEST,
+    NORTH,
+    NORTH,
+    WEST,
+    WEST,
+    NORTH,
+    NORTH,
+    WEST,
+    WEST,
+    NORTH,
+    NORTH,
+    WEST,
+    WEST,
+    WEST,
+    WEST,
+    WEST,
+    WEST,
+    WEST,
+    WEST,
+    NORTH,
+]
+# Traverse room three
+steps += [NORTH, NORTH]
+# Go back towards room two and kill the monster
+steps += [SOUTH, SOUTH, SOUTH, EAST, EAST, EAST, EAST, EAST]
+# Continue to room two
+steps += [
+    EAST,
+    EAST,
+    EAST,
+    SOUTH,
+    SOUTH,
+    EAST,
+    EAST,
+    SOUTH,
+    SOUTH,
+    EAST,
+    EAST,
+    SOUTH,
+    SOUTH,
+    EAST,
+    SOUTH,
+    SOUTH,
+]
+# Traverse room two to other exit
+steps += [WEST, WEST, WEST, WEST, WEST, WEST, WEST, WEST, WEST, NORTH]
+# Go to room four
+steps += [NORTH, NORTH, NORTH, NORTH]
+
+for action in range(len(steps)):
+    obs = env.step(steps[action])
+    env.unwrapped.nethack.draw_frame(frame)
+    img = Image.fromarray(frame, "RGB")
+    frames.append(img)
+
+print(f"Saving animation with {len(frames)} frames...")
+frames[0].save(
+    "nethack_tiles_animation.gif",
+    save_all=True,
+    append_images=frames[1:],
+    duration=400,
+    loop=0,
+)