Run gfxstream end to end on macOS, guest to host over kumquat

.github/workflows/presubmit.yaml

@@ -79,10 +79,43 @@ jobs:
         //host:gfxstream_backend_static
         //host:gfxstream_backend_shared
         //host:gfxstream_backend
-      # Nothing is run yet: this job exists to keep the macOS host build
-      # compiling. Tests need a graphics driver environment that macOS does not
-      # have here yet.
-      run-tests: false
+      # Use the default host driver environment: SwiftShader can not be built on
+      # macOS. Vulkan-backed tests instead run against MoltenVK (installed via
+      # brew), pointed at by the ANDROID_EMU_VK_LOADER_PATH / VK_*_FILES env
+      # vars below.
+      graphics-drivers: host
+      run-tests: true
+      # The GLES framebuffer test needs the ANGLE GLES driver and runs in a
+      # separate job below, as do the end2end tests (which additionally need the
+      # mesa guest driver and the kumquat server built on top of it).
+      test-targets: >-
+        //host:gfxstream_vsyncthread_tests
+        //host/vulkan:vk_format_utils_tests
+        //host/vulkan:gfxstream_emulatedphysicalmemory_tests
+        //host/vulkan:gfxstream_emulatedphysicalqueue_tests
+        //host/vulkan:vk_common_operations_tests
+        //host/vulkan:gfxstream_displayvk_tests
+        //host/vulkan:gfxstream_compositorvk_tests
+      test-bazel-args: >-
+        --test_env=ANDROID_EMU_VK_LOADER_PATH=/opt/homebrew/lib/libvulkan.dylib
+        --test_env=VK_DRIVER_FILES=/opt/homebrew/etc/vulkan/icd.d/MoltenVK_icd.json
+        --test_env=VK_ICD_FILENAMES=/opt/homebrew/etc/vulkan/icd.d/MoltenVK_icd.json
+
+  # GLES tests use ANGLE (built on macOS) for the EGL/GLES driver, running on top
+  # of MoltenVK for Vulkan. This uses the gles_angle_vulkan_moltenvk test
+  # environment, which stages the ANGLE libraries and the system Vulkan loader
+  # together and points them at MoltenVK.
+  run-gfxstream-bazel-gles-build-macos:
+    uses: ./.github/workflows/presubmit_bazel.yml
+    with:
+      runner: macos-26
+      build-targets: //host:gfxstream_framebuffer_tests
+      graphics-drivers: gles_angle_vulkan_moltenvk
+      run-tests: true
+      test-targets: //host:gfxstream_framebuffer_tests
+      test-bazel-args: >-
+        --test_env=GFXSTREAM_TESTING_VULKAN_LOADER=/opt/homebrew/lib/libvulkan.1.dylib
+        --test_env=GFXSTREAM_TESTING_VULKAN_ICD=/opt/homebrew/etc/vulkan/icd.d/MoltenVK_icd.json
 
   run-gfxstream-cmake-build:
     runs-on: ubuntu-22.04

.github/workflows/presubmit_bazel.yml

@@ -25,6 +25,31 @@ on:
         type: boolean
         default: true
 
+      # Which graphics drivers/test environment to use. Defaults to the ANGLE +
+      # SwiftShader environment. Platforms without a buildable SwiftShader (e.g.
+      # macOS) can override this with `host`.
+      graphics-drivers:
+        type: string
+        default: 'gles_angle_vulkan_swiftshader'
+
+      # Extra flags passed only to the `bazel test` step (e.g. --test_env for
+      # pointing the tests at a Vulkan driver). Not passed to `bazel build`.
+      test-bazel-args:
+        type: string
+        default: ''
+
+      # Which test targets to run when run-tests is true. Platforms that can only
+      # run a subset (e.g. macOS) can override this with a curated list.
+      test-targets:
+        type: string
+        default: >-
+          host:gfxstream_framebuffer_tests
+          host/vulkan:gfxstream_compositorvk_tests
+          host/vulkan:gfxstream_emulatedphysicalmemory_tests
+          host/vulkan:vk_common_operations_tests
+          host/vulkan:vk_format_utils_tests
+          tests/end2end:gfxstream_end2end_tests
+
 permissions:
   contents: read
 
@@ -57,9 +82,12 @@ jobs:
       if: runner.os == 'Linux'
       run: sudo apt-get install -y libvulkan1
 
-    - name: Install bazel (macOS)
+    - name: Install dependencies (macOS)
       if: runner.os == 'macOS'
-      run: brew install bazelisk
+      # molten-vk provides a Vulkan-on-Metal driver and vulkan-loader provides
+      # libvulkan, which together give the host-driver tests a Vulkan
+      # implementation to run against.
+      run: brew install bazelisk molten-vk vulkan-loader
 
     - name: Load cache config
       run: cat .config/cache-config.env >> $GITHUB_ENV
@@ -82,27 +110,32 @@ jobs:
       env:
         BUILD_TARGETS: ${{ inputs.build-targets }}
         ADDITIONAL_BAZEL_ARGS: ${{ inputs.additional-bazel-args }}
+        GRAPHICS_DRIVERS: ${{ inputs.graphics-drivers }}
       run: |
         bazel build $BUILD_TARGETS \
           --disk_cache=$HOME/bazel-disk-cache \
-          --graphics_drivers=gles_angle_vulkan_swiftshader \
+          --graphics_drivers=$GRAPHICS_DRIVERS \
           --verbose_failures \
           $ADDITIONAL_BAZEL_ARGS
 
     - name: Test
       if: ${{ inputs.run-tests }}
       env:
         ADDITIONAL_BAZEL_ARGS: ${{ inputs.additional-bazel-args }}
+        GRAPHICS_DRIVERS: ${{ inputs.graphics-drivers }}
+        TEST_TARGETS: ${{ inputs.test-targets }}
+        TEST_BAZEL_ARGS: ${{ inputs.test-bazel-args }}
       run: |
+        # The variables below are deliberately unquoted so that the shell splits
+        # them into separate arguments. Disable globbing so that a value such as
+        # --test_arg=--gtest_filter=*Foo* is passed through literally rather than
+        # being expanded against the files in the workspace.
+        set -f
         bazel test \
           --disk_cache=$HOME/bazel-disk-cache \
-          --graphics_drivers=gles_angle_vulkan_swiftshader \
+          --graphics_drivers=$GRAPHICS_DRIVERS \
           --test_output=streamed \
           --verbose_failures \
-          host:gfxstream_framebuffer_tests \
-          host/vulkan:gfxstream_compositorvk_tests \
-          host/vulkan:gfxstream_emulatedphysicalmemory_tests \
-          host/vulkan:vk_common_operations_tests \
-          host/vulkan:vk_format_utils_tests \
-          tests/end2end:gfxstream_end2end_tests \
+          $TEST_TARGETS \
+          $TEST_BAZEL_ARGS \
           $ADDITIONAL_BAZEL_ARGS

common/testenv/BUILD.bazel

@@ -14,6 +14,7 @@ string_flag(
         "host",
         "gles_angle_vulkan_lavapipe",
         "gles_angle_vulkan_swiftshader",
+        "gles_angle_vulkan_moltenvk",
     ],
 )
 
@@ -38,6 +39,13 @@ config_setting(
     },
 )
 
+config_setting(
+    name = "graphics_test_environment_gles_angle_vulkan_moltenvk",
+    flag_values = {
+        ":graphics_test_environment": "gles_angle_vulkan_moltenvk",
+    },
+)
+
 copy_to_directory(
     name = "graphics_test_environment_drivers",
     srcs = select({
@@ -55,6 +63,12 @@ copy_to_directory(
             "@angle//:libGLESv2",
             "@swiftshader//:vk_swiftshader",
         ],
+        # macOS: ANGLE for GLES, running on top of the system MoltenVK Vulkan
+        # driver (provided at runtime, not as a Bazel artifact).
+        ":graphics_test_environment_gles_angle_vulkan_moltenvk": [
+            "@angle//:libEGL",
+            "@angle//:libGLESv2",
+        ],
         "//conditions:default": [
             # Default does not change anything and uses host drivers.
         ],
@@ -85,6 +99,9 @@ cc_library(
         ":graphics_test_environment_gles_angle_vulkan_swiftshader": [
             ":graphics_test_environment_drivers",
         ],
+        ":graphics_test_environment_gles_angle_vulkan_moltenvk": [
+            ":graphics_test_environment_drivers",
+        ],
         "//conditions:default": [],
     }),
     defines = select({
@@ -96,6 +113,10 @@ cc_library(
             "GFXSTREAM_TESTING_USE_GLES_ANGLE",
             "GFXSTREAM_TESTING_USE_VULKAN_SWIFTSHADER",
         ],
+        ":graphics_test_environment_gles_angle_vulkan_moltenvk": [
+            "GFXSTREAM_TESTING_USE_GLES_ANGLE",
+            "GFXSTREAM_TESTING_USE_VULKAN_MOLTENVK",
+        ],
         "//conditions:default": [
             # Default does not change anything and uses host drivers.
         ],
@@ -108,6 +129,9 @@ cc_library(
         ":graphics_test_environment_gles_angle_vulkan_swiftshader": [
             "@rules_cc//cc/runfiles",
         ],
+        ":graphics_test_environment_gles_angle_vulkan_moltenvk": [
+            "@rules_cc//cc/runfiles",
+        ],
         "//conditions:default": [],
     }) + [
         "//common/base:gfxstream_common_base",

common/testenv/graphics_test_environment.cpp

@@ -19,6 +19,10 @@
 #include <optional>
 #include <string>
 
+#if defined(__APPLE__)
+#include <unistd.h>
+#endif
+
 #ifdef BAZEL_CURRENT_REPOSITORY
 #include <rules_cc/cc/runfiles/runfiles.h>
 #endif
@@ -95,14 +99,21 @@ bool SetupGraphicsTestEnvironment() {
     //     gfxstream::base::setEnvironmentVariable("__EGL_VENDOR_LIBRARY_FILENAMES", driverEglIcd);
     //
     // For now, assume the ANGLE libs are directly used:
-    const auto driverGlesOpt = GetGraphicsDriverPath("libGLESv2.so");
+#if defined(__APPLE__)
+    static constexpr const char* kGlesDriverBasename = "libGLESv2.dylib";
+    static constexpr const char* kEglDriverBasename = "libEGL.dylib";
+#else
+    static constexpr const char* kGlesDriverBasename = "libGLESv2.so";
+    static constexpr const char* kEglDriverBasename = "libEGL.so";
+#endif
+    const auto driverGlesOpt = GetGraphicsDriverPath(kGlesDriverBasename);
     if (!driverGlesOpt) {
-        GFXSTREAM_ERROR("Failed to find libGLESv2.so.");
+        GFXSTREAM_ERROR("Failed to find %s.", kGlesDriverBasename);
         return false;
     }
-    const auto driverEglOpt = GetGraphicsDriverPath("libEGL.so");
+    const auto driverEglOpt = GetGraphicsDriverPath(kEglDriverBasename);
     if (!driverEglOpt) {
-        GFXSTREAM_ERROR("Failed to find libEGL.so");
+        GFXSTREAM_ERROR("Failed to find %s", kEglDriverBasename);
         return false;
     }
     const std::filesystem::path driverEgl = *driverEglOpt;
@@ -111,6 +122,67 @@ bool SetupGraphicsTestEnvironment() {
     const std::string currentLdLibraryPath = gfxstream::base::getEnvironmentVariable("LD_LIBRARY_PATH");
     const std::string updatedLdLibraryPath = driverDirectory.string() + ":" + currentLdLibraryPath;
     gfxstream::base::setEnvironmentVariable("LD_LIBRARY_PATH", updatedLdLibraryPath);
+
+#if defined(GFXSTREAM_TESTING_USE_VULKAN_MOLTENVK)
+    // ANGLE searches its own module directory for the Vulkan loader, and
+    // DYLD_LIBRARY_PATH is unavailable under macOS System Integrity Protection.
+    // The Bazel output directory holding the ANGLE libraries is read-only, so
+    // stage the ANGLE libraries and the system Vulkan loader together in a
+    // writable directory and load ANGLE from there. The loader path and the
+    // MoltenVK ICD are provided by the caller (see CI).
+    const std::string vulkanLoaderSource =
+        gfxstream::base::getEnvironmentVariable("GFXSTREAM_TESTING_VULKAN_LOADER");
+    if (vulkanLoaderSource.empty()) {
+        GFXSTREAM_ERROR("GFXSTREAM_TESTING_VULKAN_LOADER is not set for the MoltenVK environment.");
+        return false;
+    }
+    std::error_code stagingError;
+    const std::filesystem::path temporaryDirectory =
+        std::filesystem::temp_directory_path(stagingError);
+    if (stagingError) {
+        GFXSTREAM_ERROR("Failed to find a temporary directory: %s.",
+                        stagingError.message().c_str());
+        return false;
+    }
+    const std::filesystem::path stagingDirectory =
+        temporaryDirectory / ("gfxstream_moltenvk_drivers_" + std::to_string(getpid()));
+    std::filesystem::create_directories(stagingDirectory, stagingError);
+    if (stagingError) {
+        GFXSTREAM_ERROR("Failed to create staging directory %s: %s.",
+                        stagingDirectory.string().c_str(), stagingError.message().c_str());
+        return false;
+    }
+    // The staged copies inherit the read-only permissions of their Bazel source,
+    // so a later call (SetupGraphicsTestEnvironment runs once per test) cannot
+    // truncate them to overwrite. Remove any existing destination first, which
+    // also keeps staging idempotent across tests in the same process.
+    const auto stageInto = [&](const std::filesystem::path& source,
+                               const std::filesystem::path& destination) -> bool {
+        std::error_code removeError;
+        std::filesystem::remove(destination, removeError);
+        std::error_code copyError;
+        std::filesystem::copy_file(source, destination,
+                                   std::filesystem::copy_options::overwrite_existing, copyError);
+        if (copyError) {
+            GFXSTREAM_ERROR("Failed to stage %s into %s: %s.", source.string().c_str(),
+                            destination.string().c_str(), copyError.message().c_str());
+            return false;
+        }
+        return true;
+    };
+    const auto stageFile = [&](const std::filesystem::path& source) -> bool {
+        return stageInto(source, stagingDirectory / source.filename());
+    };
+    if (!stageFile(*driverGlesOpt) || !stageFile(driverEgl)) {
+        return false;
+    }
+    // Stage the loader under the name ANGLE looks for.
+    if (!stageInto(vulkanLoaderSource, stagingDirectory / "libvulkan.dylib")) {
+        return false;
+    }
+    gfxstream::base::setEnvironmentVariable(
+        "LD_LIBRARY_PATH", stagingDirectory.string() + ":" + updatedLdLibraryPath);
+#endif  // defined(GFXSTREAM_TESTING_USE_VULKAN_MOLTENVK)
 #else
     GFXSTREAM_INFO("GraphicsTestEnvironment: not changing host EGL/GLES driver configuration.");
 #endif  // defined(GFXSTREAM_TESTING_USE_GLES_ANGLE)
@@ -149,6 +221,19 @@ bool SetupGraphicsTestEnvironment() {
     const std::string driverLavapipeIcd = driverLavapipeIcdOpt->string();
     gfxstream::base::setEnvironmentVariable("VK_DRIVER_FILES", driverLavapipeIcd);
     gfxstream::base::setEnvironmentVariable("VK_ICD_FILENAMES", driverLavapipeIcd);
+#elif defined(GFXSTREAM_TESTING_USE_VULKAN_MOLTENVK)
+    GFXSTREAM_INFO("GraphicsTestEnvironment: configuring MoltenVK as the Vulkan driver.");
+
+    // MoltenVK is provided by the system (e.g. Homebrew), not as a build
+    // artifact. The caller provides the ICD via GFXSTREAM_TESTING_VULKAN_ICD.
+    const std::string moltenVkIcd =
+        gfxstream::base::getEnvironmentVariable("GFXSTREAM_TESTING_VULKAN_ICD");
+    if (moltenVkIcd.empty()) {
+        GFXSTREAM_ERROR("GFXSTREAM_TESTING_VULKAN_ICD is not set for the MoltenVK environment.");
+        return false;
+    }
+    gfxstream::base::setEnvironmentVariable("VK_DRIVER_FILES", moltenVkIcd);
+    gfxstream::base::setEnvironmentVariable("VK_ICD_FILENAMES", moltenVkIcd);
 #else
     GFXSTREAM_INFO("GraphicsTestEnvironment: not changing host Vulkan driver configuration.");
 #endif  // defined(GFXSTREAM_TESTING_USE_VULKAN_LAVAPIPE) || defined(GFXSTREAM_TESTING_USE_VULKAN_SWIFTSHADER)
@@ -161,6 +246,8 @@ bool IsGraphicsTestEnvironmentProvidingVulkanDriver() {
     return true;
 #elif defined(GFXSTREAM_TESTING_USE_VULKAN_SWIFTSHADER)
     return true;
+#elif defined(GFXSTREAM_TESTING_USE_VULKAN_MOLTENVK)
+    return true;
 #else
     return false;
 #endif

host/library/shared_library.cpp

@@ -185,6 +185,25 @@ SharedLibrary* SharedLibrary::do_open(const char* libraryName,
             libraryName, libPath);
         lib = dlopen(libPath, RTLD_NOW);
     }
+    if (lib == NULL) {
+        // DYLD_LIBRARY_PATH is stripped from the environment in some contexts
+        // (e.g. macOS System Integrity Protection), so a bare dlopen can not find
+        // libraries there. Search LD_LIBRARY_PATH manually with full paths, which
+        // is not subject to that stripping.
+        const std::vector<std::string> libraryPaths =
+            gfxstream::Split(gfxstream::base::getEnvironmentVariable("LD_LIBRARY_PATH"), ":");
+        for (const std::string& libraryPath : libraryPaths) {
+            if (libraryPath.empty()) {
+                continue;
+            }
+            const std::string fullpath = PathUtils::join(libraryPath, libPath);
+            GFXSTREAM_VERBOSE("Calling dlopen on %s.", fullpath.c_str());
+            lib = dlopen(fullpath.c_str(), RTLD_NOW);
+            if (lib != nullptr) {
+                break;
+            }
+        }
+    }
 #else
     GFXSTREAM_VERBOSE("SharedLibrary::open for [%s] (posix,linux): call dlopen on [%s]", libraryName, libPath);
     void* lib = nullptr;