Turbopack: Allow turbopack-node backend to be swapped at runtime using an experimental config option

crates/next-api/Cargo.toml

@@ -9,6 +9,11 @@ autobenches = false
 [lib]
 bench = false
 
+[features]
+default = ["process_pool"]
+process_pool = ["next-core/process_pool", "turbopack-node/process_pool"]
+worker_pool = ["next-core/worker_pool", "turbopack-node/worker_pool"]
+
 [lints]
 workspace = true
 

crates/next-api/src/project.rs

@@ -14,7 +14,9 @@ use next_core::{
     next_client::{
         ClientChunkingContextOptions, get_client_chunking_context, get_client_compile_time_info,
     },
-    next_config::{ModuleIds as ModuleIdStrategyConfig, NextConfig},
+    next_config::{
+        ModuleIds as ModuleIdStrategyConfig, NextConfig, TurbopackPluginRuntimeStrategy,
+    },
     next_edge::context::EdgeChunkingContextOptions,
     next_server::{
         ServerChunkingContextOptions, ServerContextType, get_server_chunking_context,
@@ -79,7 +81,11 @@ use turbopack_core::{
         NotFoundVersion, OptionVersionedContent, Update, Version, VersionState, VersionedContent,
     },
 };
+#[cfg(feature = "process_pool")]
+use turbopack_node::child_process_backend;
 use turbopack_node::execution_context::ExecutionContext;
+#[cfg(feature = "worker_pool")]
+use turbopack_node::worker_threads_backend;
 use turbopack_nodejs::NodeJsChunkingContext;
 
 use crate::{
@@ -1217,6 +1223,16 @@ impl Project {
     pub(super) async fn execution_context(self: Vc<Self>) -> Result<Vc<ExecutionContext>> {
         let node_root = self.node_root().owned().await?;
         let next_mode = self.next_mode().await?;
+        let strategy = *self
+            .next_config()
+            .turbopack_plugin_runtime_strategy()
+            .await?;
+        let node_backend = match strategy {
+            #[cfg(feature = "worker_pool")]
+            TurbopackPluginRuntimeStrategy::WorkerThreads => worker_threads_backend(),
+            #[cfg(feature = "process_pool")]
+            TurbopackPluginRuntimeStrategy::ChildProcesses => child_process_backend(),
+        };
 
         let node_execution_chunking_context = Vc::upcast(
             NodeJsChunkingContext::builder(
@@ -1237,6 +1253,7 @@ impl Project {
             self.project_path().owned().await?,
             node_execution_chunking_context,
             self.env(),
+            node_backend,
         ))
     }
 
@@ -1475,10 +1492,12 @@ impl Project {
 
             // At this point all modules have been computed and we can get rid of the node.js
             // process pools
+            let execution_context = self.execution_context().await?;
+            let node_backend = execution_context.node_backend.into_trait_ref().await?;
             if *self.is_watch_enabled().await? {
-                turbopack_node::evaluate::scale_down();
+                node_backend.scale_down()?;
             } else {
-                turbopack_node::evaluate::scale_zero();
+                node_backend.scale_zero()?;
             }
 
             Ok(module_graphs_vc)

crates/next-core/Cargo.toml

@@ -87,6 +87,9 @@ turbopack-static = { workspace = true }
 turbopack-trace-utils = { workspace = true }
 
 [features]
+default = ["process_pool"]
+process_pool = ["turbopack-node/process_pool"]
+worker_pool = ["turbopack-node/worker_pool"]
 next-font-local = []
 plugin = [
   "swc_core/plugin_transform_host_native",

crates/next-core/src/next_config.rs

@@ -920,6 +920,16 @@ pub enum ModuleIds {
 #[turbo_tasks::value(transparent)]
 pub struct OptionModuleIds(pub Option<ModuleIds>);
 
+#[turbo_tasks::value(operation)]
+#[derive(Copy, Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub enum TurbopackPluginRuntimeStrategy {
+    #[cfg(feature = "worker_pool")]
+    WorkerThreads,
+    #[cfg(feature = "process_pool")]
+    ChildProcesses,
+}
+
 #[derive(
     Clone, Debug, PartialEq, Deserialize, TraceRawVcs, NonLocalValue, OperationValue, Encode, Decode,
 )]
@@ -1150,6 +1160,7 @@ pub struct ExperimentalConfig {
 
     turbopack_minify: Option<bool>,
     turbopack_module_ids: Option<ModuleIds>,
+    turbopack_plugin_runtime_strategy: Option<TurbopackPluginRuntimeStrategy>,
     turbopack_source_maps: Option<bool>,
     turbopack_input_source_maps: Option<bool>,
     turbopack_tree_shaking: Option<bool>,
@@ -2046,6 +2057,19 @@ impl NextConfig {
         )
     }
 
+    #[turbo_tasks::function]
+    pub fn turbopack_plugin_runtime_strategy(&self) -> Vc<TurbopackPluginRuntimeStrategy> {
+        #[cfg(feature = "worker_pool")]
+        let default = TurbopackPluginRuntimeStrategy::WorkerThreads;
+        #[cfg(all(not(feature = "worker_pool"), feature = "process_pool"))]
+        let default = TurbopackPluginRuntimeStrategy::ChildProcesses;
+
+        self.experimental
+            .turbopack_plugin_runtime_strategy
+            .unwrap_or(default)
+            .cell()
+    }
+
     #[turbo_tasks::function]
     pub async fn module_ids(&self, mode: Vc<NextMode>) -> Result<Vc<ModuleIds>> {
         Ok(match *mode.await? {

crates/next-core/src/next_font/google/mod.rs

@@ -741,6 +741,7 @@ async fn get_mock_stylesheet(
         env,
         project_path: _,
         chunking_context,
+        node_backend,
     } = *execution_context.await?;
     let asset_context = node_evaluate_asset_context(
         execution_context,
@@ -770,7 +771,7 @@ async fn get_mock_stylesheet(
         )
         .module();
 
-    let entries = get_evaluate_entries(mocked_response_asset, asset_context, None);
+    let entries = get_evaluate_entries(mocked_response_asset, asset_context, *node_backend, None);
     let module_graph = ModuleGraph::from_single_graph(SingleModuleGraph::new_with_entries(
         entries.graph_entries().to_resolved().await?,
         false,
@@ -783,6 +784,7 @@ async fn get_mock_stylesheet(
         entries,
         root,
         *env,
+        *node_backend,
         loader_source,
         *chunking_context,
         module_graph,

crates/next-napi-bindings/Cargo.toml

@@ -110,7 +110,7 @@ turbo-tasks = { workspace = true }
 turbo-tasks-backend = { workspace = true }
 turbo-tasks-fs = { workspace = true }
 turbo-unix-path = { workspace = true }
-next-api = { workspace = true }
+next-api = { workspace = true, features = ["worker_pool"] }
 next-build = { workspace = true }
 next-core = { workspace = true }
 

eslint.config.mjs

@@ -308,6 +308,7 @@ export default defineConfig([
             'itCI',
             'itHeaded',
             'itTurbopackDev',
+            'itOnlyTurbopack',
           ],
         },
       ],

packages/next/src/server/config-schema.ts

@@ -324,6 +324,9 @@ export const experimentalSchema = {
   webpackBuildWorker: z.boolean().optional(),
   webpackMemoryOptimizations: z.boolean().optional(),
   turbopackMemoryLimit: z.number().optional(),
+  turbopackPluginRuntimeStrategy: z
+    .enum(['workerThreads', 'childProcesses'])
+    .optional(),
   turbopackMinify: z.boolean().optional(),
   turbopackFileSystemCacheForDev: z.boolean().optional(),
   turbopackFileSystemCacheForBuild: z.boolean().optional(),

packages/next/src/server/config-shared.ts

@@ -497,6 +497,11 @@ export interface ExperimentalConfig {
    */
   turbopackMemoryLimit?: number
 
+  /**
+   * Selects the runtime backend used by Turbopack for Node.js evaluation.
+   */
+  turbopackPluginRuntimeStrategy?: 'workerThreads' | 'childProcesses'
+
   /**
    * Enable minification. Defaults to true in build mode and false in dev mode.
    */
@@ -1717,6 +1722,7 @@ export const defaultConfig = Object.freeze({
     turbopackFileSystemCacheForDev: true,
     turbopackFileSystemCacheForBuild: false,
     turbopackInferModuleSideEffects: true,
+    turbopackPluginRuntimeStrategy: 'workerThreads',
     devCacheControlNoCache: false,
   },
   htmlLimitedBots: undefined,

test/production/turbopack-node-backend/input.pid

@@ -0,0 +1 @@
+pid

test/production/turbopack-node-backend/next.config.js

@@ -0,0 +1,21 @@
+const runtimeStrategy =
+  process.env.TEST_TURBOPACK_PLUGIN_RUNTIME_STRATEGY || 'workerThreads'
+
+module.exports = {
+  experimental: {
+    turbopackPluginRuntimeStrategy: runtimeStrategy,
+  },
+  compiler: {
+    defineServer: {
+      'process.env.__TEST_BUILD_PID': String(process.pid),
+    },
+  },
+  turbopack: {
+    rules: {
+      '*.pid': {
+        as: '*.js',
+        loaders: ['./pid-loader.js'],
+      },
+    },
+  },
+}

test/production/turbopack-node-backend/pages/api/pid.js

@@ -0,0 +1,8 @@
+const loaderData = require('../../input.pid')
+
+export default function handler(_req, res) {
+  res.status(200).json({
+    loaderPid: String(loaderData.loaderPid),
+    buildPid: String(process.env.__TEST_BUILD_PID),
+  })
+}

test/production/turbopack-node-backend/pages/index.js

@@ -0,0 +1,3 @@
+export default function Page() {
+  return 'ok'
+}

test/production/turbopack-node-backend/pid-loader.js

@@ -0,0 +1,3 @@
+module.exports = function loader() {
+  return `module.exports = ${JSON.stringify({ loaderPid: String(process.pid) })}`
+}

test/production/turbopack-node-backend/turbopack-node-backend.test.ts

@@ -0,0 +1,33 @@
+import { nextTestSetup } from 'e2e-utils'
+
+describe.each([
+  ['workerThreads', true],
+  ['childProcesses', false],
+] as const)(
+  'turbopack-node-backend (%s)',
+  (turbopackPluginRuntimeStrategy, expectSamePid) => {
+    const { next, isTurbopack } = nextTestSetup({
+      files: __dirname,
+      env: {
+        TEST_TURBOPACK_PLUGIN_RUNTIME_STRATEGY: turbopackPluginRuntimeStrategy,
+      },
+    })
+
+    const itOnlyTurbopack = isTurbopack ? it : it.skip
+
+    itOnlyTurbopack('should match expected loader pid behavior', async () => {
+      const response = await next.fetch('/api/pid')
+      expect(response.status).toBe(200)
+
+      const data = await response.json()
+      expect(data.buildPid).toBeDefined()
+      expect(data.loaderPid).toBeDefined()
+
+      if (expectSamePid) {
+        expect(data.loaderPid).toBe(data.buildPid)
+      } else {
+        expect(data.loaderPid).not.toBe(data.buildPid)
+      }
+    })
+  }
+)

turbopack/crates/turbopack-cli/src/build/mod.rs

@@ -44,7 +44,7 @@ use turbopack_css::chunk::CssChunkType;
 use turbopack_ecmascript::chunk::EcmascriptChunkType;
 use turbopack_ecmascript_runtime::RuntimeType;
 use turbopack_env::dotenv::load_env;
-use turbopack_node::execution_context::ExecutionContext;
+use turbopack_node::{child_process_backend, execution_context::ExecutionContext};
 use turbopack_nodejs::NodeJsChunkingContext;
 
 use crate::{
@@ -225,6 +225,7 @@ async fn build_internal(
     };
 
     let compile_time_info = get_client_compile_time_info(browserslist_query.clone(), node_env);
+    let node_backend = child_process_backend();
     let execution_context = ExecutionContext::new(
         root_path.clone(),
         Vc::upcast(
@@ -245,6 +246,7 @@ async fn build_internal(
             .build(),
         ),
         load_env(root_path.clone()),
+        node_backend,
     );
 
     let asset_context = get_client_asset_context(

turbopack/crates/turbopack-cli/src/dev/mod.rs

@@ -40,7 +40,7 @@ use turbopack_dev_server::{
 };
 use turbopack_ecmascript_runtime::RuntimeType;
 use turbopack_env::dotenv::load_env;
-use turbopack_node::execution_context::ExecutionContext;
+use turbopack_node::{child_process_backend, execution_context::ExecutionContext};
 use turbopack_nodejs::NodeJsChunkingContext;
 
 use self::web_entry_source::create_web_entry_source;
@@ -298,8 +298,13 @@ async fn source(
     )
     .build();
 
-    let execution_context =
-        ExecutionContext::new(root_path.clone(), Vc::upcast(build_chunking_context), env);
+    let node_backend = child_process_backend();
+    let execution_context = ExecutionContext::new(
+        root_path.clone(),
+        Vc::upcast(build_chunking_context),
+        env,
+        node_backend,
+    );
 
     let server_fs = Vc::upcast::<Box<dyn FileSystem>>(ServerFileSystem::new());
     let server_root = server_fs.root().owned().await?;

turbopack/crates/turbopack-node/src/backend.rs

@@ -0,0 +1,48 @@
+use std::{future::Future, path::PathBuf, pin::Pin};
+
+use anyhow::Result;
+use rustc_hash::FxHashMap;
+use turbo_rcstr::RcStr;
+use turbo_tasks::ResolvedVc;
+use turbo_tasks_fs::FileSystemPath;
+
+use crate::{AssetsForSourceMapping, evaluate::EvaluatePool};
+
+pub struct CreatePoolOptions {
+    pub cwd: PathBuf,
+    pub entrypoint: PathBuf,
+    pub env: FxHashMap<RcStr, RcStr>,
+    pub assets_for_source_mapping: ResolvedVc<AssetsForSourceMapping>,
+    pub assets_root: FileSystemPath,
+    pub project_dir: FileSystemPath,
+    pub concurrency: usize,
+    pub debug: bool,
+}
+
+pub type CreatePoolFuture = Pin<Box<dyn Future<Output = Result<EvaluatePool>> + Send + 'static>>;
+
+mod sealed {
+    #[turbo_tasks::value_trait]
+    pub(crate) trait Sealed {}
+}
+
+#[cfg(feature = "worker_pool")]
+#[turbo_tasks::value_impl]
+impl sealed::Sealed for crate::worker_pool::WorkerThreadsBackend {}
+
+#[cfg(feature = "process_pool")]
+#[turbo_tasks::value_impl]
+impl sealed::Sealed for crate::process_pool::ChildProcessesBackend {}
+
+#[turbo_tasks::value_trait]
+pub trait NodeBackend: sealed::Sealed {
+    fn runtime_module_path(&self) -> RcStr;
+
+    fn globals_module_path(&self) -> RcStr;
+
+    fn create_pool(&self, options: CreatePoolOptions) -> CreatePoolFuture;
+
+    fn scale_down(&self) -> Result<()>;
+
+    fn scale_zero(&self) -> Result<()>;
+}