diff --git a/crates/next-api/Cargo.toml b/crates/next-api/Cargo.toml index 6068bd198d2aac..e7240d1d63023c 100644 --- a/crates/next-api/Cargo.toml +++ b/crates/next-api/Cargo.toml @@ -9,6 +9,11 @@ autobenches = false [lib] bench = false +[features] +default = ["process_pool"] +process_pool = ["next-core/process_pool", "turbopack-node/process_pool"] +worker_pool = ["next-core/worker_pool", "turbopack-node/worker_pool"] + [lints] workspace = true diff --git a/crates/next-api/src/project.rs b/crates/next-api/src/project.rs index 7e206cafad96f7..5f34accd7c6607 100644 --- a/crates/next-api/src/project.rs +++ b/crates/next-api/src/project.rs @@ -14,7 +14,9 @@ use next_core::{ next_client::{ ClientChunkingContextOptions, get_client_chunking_context, get_client_compile_time_info, }, - next_config::{ModuleIds as ModuleIdStrategyConfig, NextConfig}, + next_config::{ + ModuleIds as ModuleIdStrategyConfig, NextConfig, TurbopackPluginRuntimeStrategy, + }, next_edge::context::EdgeChunkingContextOptions, next_server::{ ServerChunkingContextOptions, ServerContextType, get_server_chunking_context, @@ -79,7 +81,11 @@ use turbopack_core::{ NotFoundVersion, OptionVersionedContent, Update, Version, VersionState, VersionedContent, }, }; +#[cfg(feature = "process_pool")] +use turbopack_node::child_process_backend; use turbopack_node::execution_context::ExecutionContext; +#[cfg(feature = "worker_pool")] +use turbopack_node::worker_threads_backend; use turbopack_nodejs::NodeJsChunkingContext; use crate::{ @@ -1217,6 +1223,16 @@ impl Project { pub(super) async fn execution_context(self: Vc) -> Result> { let node_root = self.node_root().owned().await?; let next_mode = self.next_mode().await?; + let strategy = *self + .next_config() + .turbopack_plugin_runtime_strategy() + .await?; + let node_backend = match strategy { + #[cfg(feature = "worker_pool")] + TurbopackPluginRuntimeStrategy::WorkerThreads => worker_threads_backend(), + #[cfg(feature = "process_pool")] + TurbopackPluginRuntimeStrategy::ChildProcesses => child_process_backend(), + }; let node_execution_chunking_context = Vc::upcast( NodeJsChunkingContext::builder( @@ -1237,6 +1253,7 @@ impl Project { self.project_path().owned().await?, node_execution_chunking_context, self.env(), + node_backend, )) } @@ -1475,10 +1492,12 @@ impl Project { // At this point all modules have been computed and we can get rid of the node.js // process pools + let execution_context = self.execution_context().await?; + let node_backend = execution_context.node_backend.into_trait_ref().await?; if *self.is_watch_enabled().await? { - turbopack_node::evaluate::scale_down(); + node_backend.scale_down()?; } else { - turbopack_node::evaluate::scale_zero(); + node_backend.scale_zero()?; } Ok(module_graphs_vc) diff --git a/crates/next-core/Cargo.toml b/crates/next-core/Cargo.toml index db2eaacd9e05a0..c4736692bf1c04 100644 --- a/crates/next-core/Cargo.toml +++ b/crates/next-core/Cargo.toml @@ -87,6 +87,9 @@ turbopack-static = { workspace = true } turbopack-trace-utils = { workspace = true } [features] +default = ["process_pool"] +process_pool = ["turbopack-node/process_pool"] +worker_pool = ["turbopack-node/worker_pool"] next-font-local = [] plugin = [ "swc_core/plugin_transform_host_native", diff --git a/crates/next-core/src/next_config.rs b/crates/next-core/src/next_config.rs index d4b6015adeddc6..4961606e63d5f3 100644 --- a/crates/next-core/src/next_config.rs +++ b/crates/next-core/src/next_config.rs @@ -920,6 +920,16 @@ pub enum ModuleIds { #[turbo_tasks::value(transparent)] pub struct OptionModuleIds(pub Option); +#[turbo_tasks::value(operation)] +#[derive(Copy, Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum TurbopackPluginRuntimeStrategy { + #[cfg(feature = "worker_pool")] + WorkerThreads, + #[cfg(feature = "process_pool")] + ChildProcesses, +} + #[derive( Clone, Debug, PartialEq, Deserialize, TraceRawVcs, NonLocalValue, OperationValue, Encode, Decode, )] @@ -1150,6 +1160,7 @@ pub struct ExperimentalConfig { turbopack_minify: Option, turbopack_module_ids: Option, + turbopack_plugin_runtime_strategy: Option, turbopack_source_maps: Option, turbopack_input_source_maps: Option, turbopack_tree_shaking: Option, @@ -2046,6 +2057,19 @@ impl NextConfig { ) } + #[turbo_tasks::function] + pub fn turbopack_plugin_runtime_strategy(&self) -> Vc { + #[cfg(feature = "worker_pool")] + let default = TurbopackPluginRuntimeStrategy::WorkerThreads; + #[cfg(all(not(feature = "worker_pool"), feature = "process_pool"))] + let default = TurbopackPluginRuntimeStrategy::ChildProcesses; + + self.experimental + .turbopack_plugin_runtime_strategy + .unwrap_or(default) + .cell() + } + #[turbo_tasks::function] pub async fn module_ids(&self, mode: Vc) -> Result> { Ok(match *mode.await? { diff --git a/crates/next-core/src/next_font/google/mod.rs b/crates/next-core/src/next_font/google/mod.rs index 12f574605b67d6..43bc857c4d28d7 100644 --- a/crates/next-core/src/next_font/google/mod.rs +++ b/crates/next-core/src/next_font/google/mod.rs @@ -741,6 +741,7 @@ async fn get_mock_stylesheet( env, project_path: _, chunking_context, + node_backend, } = *execution_context.await?; let asset_context = node_evaluate_asset_context( execution_context, @@ -770,7 +771,7 @@ async fn get_mock_stylesheet( ) .module(); - let entries = get_evaluate_entries(mocked_response_asset, asset_context, None); + let entries = get_evaluate_entries(mocked_response_asset, asset_context, *node_backend, None); let module_graph = ModuleGraph::from_single_graph(SingleModuleGraph::new_with_entries( entries.graph_entries().to_resolved().await?, false, @@ -783,6 +784,7 @@ async fn get_mock_stylesheet( entries, root, *env, + *node_backend, loader_source, *chunking_context, module_graph, diff --git a/crates/next-napi-bindings/Cargo.toml b/crates/next-napi-bindings/Cargo.toml index a044a787c8ffab..0e134e7d145b42 100644 --- a/crates/next-napi-bindings/Cargo.toml +++ b/crates/next-napi-bindings/Cargo.toml @@ -110,7 +110,7 @@ turbo-tasks = { workspace = true } turbo-tasks-backend = { workspace = true } turbo-tasks-fs = { workspace = true } turbo-unix-path = { workspace = true } -next-api = { workspace = true } +next-api = { workspace = true, features = ["worker_pool"] } next-build = { workspace = true } next-core = { workspace = true } diff --git a/eslint.config.mjs b/eslint.config.mjs index 61183ef1e304e8..d0cee73dcac520 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -308,6 +308,7 @@ export default defineConfig([ 'itCI', 'itHeaded', 'itTurbopackDev', + 'itOnlyTurbopack', ], }, ], diff --git a/packages/next/src/server/config-schema.ts b/packages/next/src/server/config-schema.ts index e3d9e2e9fec137..d21b3623a60f0a 100644 --- a/packages/next/src/server/config-schema.ts +++ b/packages/next/src/server/config-schema.ts @@ -324,6 +324,9 @@ export const experimentalSchema = { webpackBuildWorker: z.boolean().optional(), webpackMemoryOptimizations: z.boolean().optional(), turbopackMemoryLimit: z.number().optional(), + turbopackPluginRuntimeStrategy: z + .enum(['workerThreads', 'childProcesses']) + .optional(), turbopackMinify: z.boolean().optional(), turbopackFileSystemCacheForDev: z.boolean().optional(), turbopackFileSystemCacheForBuild: z.boolean().optional(), diff --git a/packages/next/src/server/config-shared.ts b/packages/next/src/server/config-shared.ts index 892054946bdcd6..e3c1a69cc305d2 100644 --- a/packages/next/src/server/config-shared.ts +++ b/packages/next/src/server/config-shared.ts @@ -497,6 +497,11 @@ export interface ExperimentalConfig { */ turbopackMemoryLimit?: number + /** + * Selects the runtime backend used by Turbopack for Node.js evaluation. + */ + turbopackPluginRuntimeStrategy?: 'workerThreads' | 'childProcesses' + /** * Enable minification. Defaults to true in build mode and false in dev mode. */ @@ -1717,6 +1722,7 @@ export const defaultConfig = Object.freeze({ turbopackFileSystemCacheForDev: true, turbopackFileSystemCacheForBuild: false, turbopackInferModuleSideEffects: true, + turbopackPluginRuntimeStrategy: 'workerThreads', devCacheControlNoCache: false, }, htmlLimitedBots: undefined, diff --git a/test/production/turbopack-node-backend/input.pid b/test/production/turbopack-node-backend/input.pid new file mode 100644 index 00000000000000..7fe681d26773ab --- /dev/null +++ b/test/production/turbopack-node-backend/input.pid @@ -0,0 +1 @@ +pid diff --git a/test/production/turbopack-node-backend/next.config.js b/test/production/turbopack-node-backend/next.config.js new file mode 100644 index 00000000000000..d359dc20168d4d --- /dev/null +++ b/test/production/turbopack-node-backend/next.config.js @@ -0,0 +1,21 @@ +const runtimeStrategy = + process.env.TEST_TURBOPACK_PLUGIN_RUNTIME_STRATEGY || 'workerThreads' + +module.exports = { + experimental: { + turbopackPluginRuntimeStrategy: runtimeStrategy, + }, + compiler: { + defineServer: { + 'process.env.__TEST_BUILD_PID': String(process.pid), + }, + }, + turbopack: { + rules: { + '*.pid': { + as: '*.js', + loaders: ['./pid-loader.js'], + }, + }, + }, +} diff --git a/test/production/turbopack-node-backend/pages/api/pid.js b/test/production/turbopack-node-backend/pages/api/pid.js new file mode 100644 index 00000000000000..9e8d524c89201a --- /dev/null +++ b/test/production/turbopack-node-backend/pages/api/pid.js @@ -0,0 +1,8 @@ +const loaderData = require('../../input.pid') + +export default function handler(_req, res) { + res.status(200).json({ + loaderPid: String(loaderData.loaderPid), + buildPid: String(process.env.__TEST_BUILD_PID), + }) +} diff --git a/test/production/turbopack-node-backend/pages/index.js b/test/production/turbopack-node-backend/pages/index.js new file mode 100644 index 00000000000000..11f5e989753c47 --- /dev/null +++ b/test/production/turbopack-node-backend/pages/index.js @@ -0,0 +1,3 @@ +export default function Page() { + return 'ok' +} diff --git a/test/production/turbopack-node-backend/pid-loader.js b/test/production/turbopack-node-backend/pid-loader.js new file mode 100644 index 00000000000000..dab2c436819e3b --- /dev/null +++ b/test/production/turbopack-node-backend/pid-loader.js @@ -0,0 +1,3 @@ +module.exports = function loader() { + return `module.exports = ${JSON.stringify({ loaderPid: String(process.pid) })}` +} diff --git a/test/production/turbopack-node-backend/turbopack-node-backend.test.ts b/test/production/turbopack-node-backend/turbopack-node-backend.test.ts new file mode 100644 index 00000000000000..f480a5ec072690 --- /dev/null +++ b/test/production/turbopack-node-backend/turbopack-node-backend.test.ts @@ -0,0 +1,33 @@ +import { nextTestSetup } from 'e2e-utils' + +describe.each([ + ['workerThreads', true], + ['childProcesses', false], +] as const)( + 'turbopack-node-backend (%s)', + (turbopackPluginRuntimeStrategy, expectSamePid) => { + const { next, isTurbopack } = nextTestSetup({ + files: __dirname, + env: { + TEST_TURBOPACK_PLUGIN_RUNTIME_STRATEGY: turbopackPluginRuntimeStrategy, + }, + }) + + const itOnlyTurbopack = isTurbopack ? it : it.skip + + itOnlyTurbopack('should match expected loader pid behavior', async () => { + const response = await next.fetch('/api/pid') + expect(response.status).toBe(200) + + const data = await response.json() + expect(data.buildPid).toBeDefined() + expect(data.loaderPid).toBeDefined() + + if (expectSamePid) { + expect(data.loaderPid).toBe(data.buildPid) + } else { + expect(data.loaderPid).not.toBe(data.buildPid) + } + }) + } +) diff --git a/turbopack/crates/turbopack-cli/src/build/mod.rs b/turbopack/crates/turbopack-cli/src/build/mod.rs index 99403b4975af74..822c53ceb0b5ca 100644 --- a/turbopack/crates/turbopack-cli/src/build/mod.rs +++ b/turbopack/crates/turbopack-cli/src/build/mod.rs @@ -44,7 +44,7 @@ use turbopack_css::chunk::CssChunkType; use turbopack_ecmascript::chunk::EcmascriptChunkType; use turbopack_ecmascript_runtime::RuntimeType; use turbopack_env::dotenv::load_env; -use turbopack_node::execution_context::ExecutionContext; +use turbopack_node::{child_process_backend, execution_context::ExecutionContext}; use turbopack_nodejs::NodeJsChunkingContext; use crate::{ @@ -225,6 +225,7 @@ async fn build_internal( }; let compile_time_info = get_client_compile_time_info(browserslist_query.clone(), node_env); + let node_backend = child_process_backend(); let execution_context = ExecutionContext::new( root_path.clone(), Vc::upcast( @@ -245,6 +246,7 @@ async fn build_internal( .build(), ), load_env(root_path.clone()), + node_backend, ); let asset_context = get_client_asset_context( diff --git a/turbopack/crates/turbopack-cli/src/dev/mod.rs b/turbopack/crates/turbopack-cli/src/dev/mod.rs index 143f520913c2ba..ff0dff606762c2 100644 --- a/turbopack/crates/turbopack-cli/src/dev/mod.rs +++ b/turbopack/crates/turbopack-cli/src/dev/mod.rs @@ -40,7 +40,7 @@ use turbopack_dev_server::{ }; use turbopack_ecmascript_runtime::RuntimeType; use turbopack_env::dotenv::load_env; -use turbopack_node::execution_context::ExecutionContext; +use turbopack_node::{child_process_backend, execution_context::ExecutionContext}; use turbopack_nodejs::NodeJsChunkingContext; use self::web_entry_source::create_web_entry_source; @@ -298,8 +298,13 @@ async fn source( ) .build(); - let execution_context = - ExecutionContext::new(root_path.clone(), Vc::upcast(build_chunking_context), env); + let node_backend = child_process_backend(); + let execution_context = ExecutionContext::new( + root_path.clone(), + Vc::upcast(build_chunking_context), + env, + node_backend, + ); let server_fs = Vc::upcast::>(ServerFileSystem::new()); let server_root = server_fs.root().owned().await?; diff --git a/turbopack/crates/turbopack-node/src/backend.rs b/turbopack/crates/turbopack-node/src/backend.rs new file mode 100644 index 00000000000000..7da3f47a7ad356 --- /dev/null +++ b/turbopack/crates/turbopack-node/src/backend.rs @@ -0,0 +1,48 @@ +use std::{future::Future, path::PathBuf, pin::Pin}; + +use anyhow::Result; +use rustc_hash::FxHashMap; +use turbo_rcstr::RcStr; +use turbo_tasks::ResolvedVc; +use turbo_tasks_fs::FileSystemPath; + +use crate::{AssetsForSourceMapping, evaluate::EvaluatePool}; + +pub struct CreatePoolOptions { + pub cwd: PathBuf, + pub entrypoint: PathBuf, + pub env: FxHashMap, + pub assets_for_source_mapping: ResolvedVc, + pub assets_root: FileSystemPath, + pub project_dir: FileSystemPath, + pub concurrency: usize, + pub debug: bool, +} + +pub type CreatePoolFuture = Pin> + Send + 'static>>; + +mod sealed { + #[turbo_tasks::value_trait] + pub(crate) trait Sealed {} +} + +#[cfg(feature = "worker_pool")] +#[turbo_tasks::value_impl] +impl sealed::Sealed for crate::worker_pool::WorkerThreadsBackend {} + +#[cfg(feature = "process_pool")] +#[turbo_tasks::value_impl] +impl sealed::Sealed for crate::process_pool::ChildProcessesBackend {} + +#[turbo_tasks::value_trait] +pub trait NodeBackend: sealed::Sealed { + fn runtime_module_path(&self) -> RcStr; + + fn globals_module_path(&self) -> RcStr; + + fn create_pool(&self, options: CreatePoolOptions) -> CreatePoolFuture; + + fn scale_down(&self) -> Result<()>; + + fn scale_zero(&self) -> Result<()>; +} diff --git a/turbopack/crates/turbopack-node/src/evaluate.rs b/turbopack/crates/turbopack-node/src/evaluate.rs index e6d9b01778c619..b991e171f13f39 100644 --- a/turbopack/crates/turbopack-node/src/evaluate.rs +++ b/turbopack/crates/turbopack-node/src/evaluate.rs @@ -10,8 +10,8 @@ use serde::{Deserialize, Serialize, de::DeserializeOwned}; use serde_json::Value as JsonValue; use turbo_rcstr::{RcStr, rcstr}; use turbo_tasks::{ - Completion, Effects, FxIndexMap, NonLocalValue, OperationVc, PrettyPrintError, ReadRef, - ResolvedVc, TaskInput, TryJoinIterExt, Vc, duration_span, fxindexmap, get_effects, + Completion, Effects, FxIndexMap, IntoTraitRef, NonLocalValue, OperationVc, PrettyPrintError, + ReadRef, ResolvedVc, TaskInput, TryJoinIterExt, Vc, duration_span, fxindexmap, get_effects, trace::TraceRawVcs, }; use turbo_tasks_env::{EnvMap, ProcessEnv}; @@ -35,13 +35,14 @@ use turbopack_core::{ virtual_source::VirtualSource, }; -#[cfg(all(feature = "process_pool", not(feature = "worker_pool")))] -use crate::process_pool::ChildProcessPool; -#[cfg(feature = "worker_pool")] -use crate::worker_pool::WorkerThreadPool; use crate::{ - AssetsForSourceMapping, embed_js::embed_file_path, emit, emit_package_json, - format::FormattingMode, internal_assets_for_source_mapping, pool_stats::PoolStatsSnapshot, + AssetsForSourceMapping, + backend::{CreatePoolOptions, NodeBackend}, + embed_js::embed_file_path, + emit, emit_package_json, + format::FormattingMode, + internal_assets_for_source_mapping, + pool_stats::PoolStatsSnapshot, source_map::StructuredError, }; @@ -98,7 +99,6 @@ impl EvaluatePool { self.pool.stats() } - #[cfg(all(feature = "process_pool", not(feature = "worker_pool")))] pub fn pre_warm(&self) { self.pool.pre_warm() } @@ -108,7 +108,11 @@ impl EvaluatePool { pub trait EvaluateOperation: Send + Sync { async fn operation(&self) -> Result>; fn stats(&self) -> PoolStatsSnapshot; - #[cfg(all(feature = "process_pool", not(feature = "worker_pool")))] + /// Eagerly spawn a Node.js worker so it's ready when the first [`Self::operation`] is called. + /// The worker should go into the idle queue. + /// + /// If a worker request comes in while this is still initializing, it should wait on the bootup + /// semaphore and will resume when the worker is ready. fn pre_warm(&self); } @@ -209,6 +213,7 @@ pub async fn get_evaluate_pool( entries: ResolvedVc, cwd: FileSystemPath, env: ResolvedVc>, + node_backend: ResolvedVc>, chunking_context: ResolvedVc>, module_graph: ResolvedVc, additional_invalidation: ResolvedVc, @@ -253,31 +258,19 @@ pub async fn get_evaluate_pool( } }; - #[cfg(all(feature = "process_pool", not(feature = "worker_pool")))] - #[allow(unused_variables)] - let pool = ChildProcessPool::create( - cwd.clone(), - entrypoint.clone(), - env.iter().map(|(k, v)| (k.clone(), v.clone())).collect(), - assets_for_source_mapping, - output_root.clone(), - chunking_context.root_path().owned().await?, - available_parallelism().map_or(1, |v| v.get()), - debug, - ); - #[cfg(feature = "worker_pool")] - let pool = WorkerThreadPool::create( - cwd, - entrypoint, - env.iter().map(|(k, v)| (k.clone(), v.clone())).collect(), - assets_for_source_mapping, - output_root.clone(), - chunking_context.root_path().owned().await?, - available_parallelism().map_or(1, |v| v.get()), - debug, - ) - .await; - #[cfg(all(feature = "process_pool", not(feature = "worker_pool")))] + let node_backend = node_backend.into_trait_ref().await?; + let pool = node_backend + .create_pool(CreatePoolOptions { + cwd, + entrypoint, + env: env.iter().map(|(k, v)| (k.clone(), v.clone())).collect(), + assets_for_source_mapping, + assets_root: output_root.clone(), + project_dir: chunking_context.root_path().owned().await?, + concurrency: available_parallelism().map_or(1, |v| v.get()), + debug, + }) + .await?; pool.pre_warm(); additional_invalidation.await?; Ok(pool.cell()) @@ -430,13 +423,11 @@ impl EvaluateEntries { pub async fn get_evaluate_entries( module_asset: ResolvedVc>, asset_context: ResolvedVc>, + node_backend: ResolvedVc>, runtime_entries: Option>, ) -> Result> { - let runtime_module_path = if cfg!(all(feature = "process_pool", not(feature = "worker_pool"))) { - rcstr!("child_process/evaluate.ts") - } else { - rcstr!("worker_thread/evaluate.ts") - }; + let node_backend = node_backend.into_trait_ref().await?; + let runtime_module_path = node_backend.runtime_module_path(); let runtime_asset = asset_context .process( @@ -471,12 +462,7 @@ pub async fn get_evaluate_entries( let runtime_entries = { let mut entries = vec![]; - let global_module_path = - if cfg!(all(feature = "process_pool", not(feature = "worker_pool"))) { - rcstr!("child_process/globals.ts") - } else { - rcstr!("worker_thread/globals.ts") - }; + let global_module_path = node_backend.globals_module_path(); let globals_module = asset_context .process( @@ -521,6 +507,7 @@ pub async fn evaluate( entries: ResolvedVc, cwd: FileSystemPath, env: ResolvedVc>, + node_backend: ResolvedVc>, context_source_for_issue: ResolvedVc>, chunking_context: ResolvedVc>, module_graph: ResolvedVc, @@ -532,6 +519,7 @@ pub async fn evaluate( entries, cwd, env, + node_backend, context_source_for_issue, chunking_context, module_graph, @@ -606,6 +594,7 @@ struct BasicEvaluateContext { entries: ResolvedVc, cwd: FileSystemPath, env: ResolvedVc>, + node_backend: ResolvedVc>, context_source_for_issue: ResolvedVc>, chunking_context: ResolvedVc>, module_graph: ResolvedVc, @@ -625,6 +614,7 @@ impl EvaluateContext for BasicEvaluateContext { self.entries, self.cwd.clone(), self.env, + self.node_backend, self.chunking_context, self.module_graph, self.additional_invalidation, @@ -731,25 +721,3 @@ impl Issue for EvaluationIssue { Vc::cell(Some(self.source)) } } - -pub fn scale_down() { - #[cfg(all(feature = "process_pool", not(feature = "worker_pool")))] - { - ChildProcessPool::scale_down(); - } - #[cfg(feature = "worker_pool")] - { - WorkerThreadPool::scale_down(); - } -} - -pub fn scale_zero() { - #[cfg(all(feature = "process_pool", not(feature = "worker_pool")))] - { - ChildProcessPool::scale_zero(); - } - #[cfg(feature = "worker_pool")] - { - WorkerThreadPool::scale_zero(); - } -} diff --git a/turbopack/crates/turbopack-node/src/execution_context.rs b/turbopack/crates/turbopack-node/src/execution_context.rs index b6fe5a2a737f66..af98a71a611393 100644 --- a/turbopack/crates/turbopack-node/src/execution_context.rs +++ b/turbopack/crates/turbopack-node/src/execution_context.rs @@ -3,11 +3,14 @@ use turbo_tasks_env::ProcessEnv; use turbo_tasks_fs::FileSystemPath; use turbopack_core::chunk::ChunkingContext; +use crate::backend::NodeBackend; + #[turbo_tasks::value] pub struct ExecutionContext { pub project_path: FileSystemPath, pub chunking_context: ResolvedVc>, pub env: ResolvedVc>, + pub node_backend: ResolvedVc>, } #[turbo_tasks::value_impl] @@ -17,11 +20,13 @@ impl ExecutionContext { project_path: FileSystemPath, chunking_context: ResolvedVc>, env: ResolvedVc>, + node_backend: ResolvedVc>, ) -> Vc { ExecutionContext { project_path, chunking_context, env, + node_backend, } .cell() } @@ -40,4 +45,9 @@ impl ExecutionContext { pub fn env(&self) -> Vc> { *self.env } + + #[turbo_tasks::function] + pub fn node_backend(&self) -> Vc> { + *self.node_backend + } } diff --git a/turbopack/crates/turbopack-node/src/lib.rs b/turbopack/crates/turbopack-node/src/lib.rs index 011d7d993463e4..7bd8fb1219e1f0 100644 --- a/turbopack/crates/turbopack-node/src/lib.rs +++ b/turbopack/crates/turbopack-node/src/lib.rs @@ -13,20 +13,30 @@ use turbopack_core::{ virtual_output::VirtualOutputAsset, }; +mod backend; pub mod debug; pub mod embed_js; pub mod evaluate; pub mod execution_context; mod format; mod pool_stats; -// When both features are enabled, worker_pool takes priority -#[cfg(all(feature = "process_pool", not(feature = "worker_pool")))] +#[cfg(feature = "process_pool")] pub mod process_pool; pub mod source_map; pub mod transforms; #[cfg(feature = "worker_pool")] pub mod worker_pool; +pub use backend::{CreatePoolFuture, CreatePoolOptions, NodeBackend}; +#[cfg(feature = "process_pool")] +pub fn child_process_backend() -> Vc> { + Vc::upcast(process_pool::ChildProcessesBackend.cell()) +} +#[cfg(feature = "worker_pool")] +pub fn worker_threads_backend() -> Vc> { + Vc::upcast(worker_pool::WorkerThreadsBackend.cell()) +} + #[turbo_tasks::function] async fn emit( intermediate_asset: Vc>, diff --git a/turbopack/crates/turbopack-node/src/process_pool/mod.rs b/turbopack/crates/turbopack-node/src/process_pool/mod.rs index 7b722878c99fbd..e3d584dffff7b0 100644 --- a/turbopack/crates/turbopack-node/src/process_pool/mod.rs +++ b/turbopack/crates/turbopack-node/src/process_pool/mod.rs @@ -24,13 +24,14 @@ use tokio::{ sync::Semaphore, time::{sleep, timeout}, }; -use turbo_rcstr::RcStr; +use turbo_rcstr::{RcStr, rcstr}; use turbo_tasks::{FxIndexSet, ResolvedVc, Vc, duration_span}; use turbo_tasks_fs::FileSystemPath; use turbopack_ecmascript::magic_identifier::unmangle_identifiers; use crate::{ AssetsForSourceMapping, + backend::{CreatePoolFuture, CreatePoolOptions, NodeBackend}, evaluate::{EvaluateOperation, EvaluatePool, Operation}, format::FormattingMode, pool_stats::{AcquiredPermits, NodeJsPoolStats, PoolStatsSnapshot}, @@ -597,6 +598,56 @@ impl ChildProcessPool { } } +#[turbo_tasks::value(shared)] +pub(crate) struct ChildProcessesBackend; + +#[turbo_tasks::value_impl] +impl NodeBackend for ChildProcessesBackend { + fn runtime_module_path(&self) -> RcStr { + rcstr!("child_process/evaluate.ts") + } + + fn globals_module_path(&self) -> RcStr { + rcstr!("child_process/globals.ts") + } + + fn create_pool(&self, options: CreatePoolOptions) -> CreatePoolFuture { + Box::pin(async move { + let CreatePoolOptions { + cwd, + entrypoint, + env, + assets_for_source_mapping, + assets_root, + project_dir, + concurrency, + debug, + } = options; + + Ok(ChildProcessPool::create( + cwd, + entrypoint, + env, + assets_for_source_mapping, + assets_root, + project_dir, + concurrency, + debug, + )) + }) + } + + fn scale_down(&self) -> Result<()> { + ChildProcessPool::scale_down(); + Ok(()) + } + + fn scale_zero(&self) -> Result<()> { + ChildProcessPool::scale_zero(); + Ok(()) + } +} + #[async_trait::async_trait] impl EvaluateOperation for ChildProcessPool { async fn operation(&self) -> Result> { diff --git a/turbopack/crates/turbopack-node/src/transforms/postcss.rs b/turbopack/crates/turbopack-node/src/transforms/postcss.rs index 1bb5a31c836ab5..7445f9c85e2483 100644 --- a/turbopack/crates/turbopack-node/src/transforms/postcss.rs +++ b/turbopack/crates/turbopack-node/src/transforms/postcss.rs @@ -461,6 +461,7 @@ impl PostCssTransformedAsset { project_path, chunking_context, env, + node_backend, } = &*self.execution_context.await?; // For this postcss transform, there is no guarantee that looking up for the @@ -508,9 +509,10 @@ impl PostCssTransformedAsset { let postcss_executor = postcss_executor(*evaluate_context, project_path.clone(), config_path).module(); - let entries = get_evaluate_entries(postcss_executor, *evaluate_context, None) - .to_resolved() - .await?; + let entries = + get_evaluate_entries(postcss_executor, *evaluate_context, **node_backend, None) + .to_resolved() + .await?; let module_graph = ModuleGraph::from_single_graph(SingleModuleGraph::new_with_entries( entries.graph_entries().to_resolved().await?, @@ -537,6 +539,7 @@ impl PostCssTransformedAsset { entries, cwd: project_path.clone(), env: *env, + node_backend: *node_backend, context_source_for_issue: self.source, chunking_context: *chunking_context, module_graph, diff --git a/turbopack/crates/turbopack-node/src/transforms/webpack.rs b/turbopack/crates/turbopack-node/src/transforms/webpack.rs index c4538ee061af42..74b3830c2b3e4a 100644 --- a/turbopack/crates/turbopack-node/src/transforms/webpack.rs +++ b/turbopack/crates/turbopack-node/src/transforms/webpack.rs @@ -51,6 +51,7 @@ use turbopack_resolve::{ use crate::{ AssetsForSourceMapping, + backend::NodeBackend, debug::should_debug, embed_js::embed_file_path, evaluate::{ @@ -209,6 +210,7 @@ impl WebpackLoadersProcessedAsset { project_path, chunking_context, env, + node_backend, } = &*transform.execution_context.await?; let source_content = self.source.content(); let AssetContent::File(file) = *source_content.await? else { @@ -240,9 +242,14 @@ impl WebpackLoadersProcessedAsset { let webpack_loaders_executor = webpack_loaders_executor(*evaluate_context).module(); - let entries = get_evaluate_entries(webpack_loaders_executor, *evaluate_context, None) - .to_resolved() - .await?; + let entries = get_evaluate_entries( + webpack_loaders_executor, + *evaluate_context, + **node_backend, + None, + ) + .to_resolved() + .await?; let module_graph = ModuleGraph::from_single_graph(SingleModuleGraph::new_with_entries( entries.graph_entries().to_resolved().await?, @@ -264,6 +271,7 @@ impl WebpackLoadersProcessedAsset { entries, cwd: project_path.clone(), env: *env, + node_backend: *node_backend, context_source_for_issue: self.source, chunking_context: *chunking_context, module_graph, @@ -434,6 +442,7 @@ pub struct WebpackLoaderContext { pub entries: ResolvedVc, pub cwd: FileSystemPath, pub env: ResolvedVc>, + pub node_backend: ResolvedVc>, pub context_source_for_issue: ResolvedVc>, pub module_graph: ResolvedVc, pub chunking_context: ResolvedVc>, @@ -453,6 +462,7 @@ impl EvaluateContext for WebpackLoaderContext { self.entries, self.cwd.clone(), self.env, + self.node_backend, self.chunking_context, self.module_graph, self.additional_invalidation, diff --git a/turbopack/crates/turbopack-node/src/worker_pool/mod.rs b/turbopack/crates/turbopack-node/src/worker_pool/mod.rs index fb6b2d11141fdb..717d89aa2bfda1 100644 --- a/turbopack/crates/turbopack-node/src/worker_pool/mod.rs +++ b/turbopack/crates/turbopack-node/src/worker_pool/mod.rs @@ -13,12 +13,13 @@ use tokio::{ sync::{Semaphore, oneshot}, time::sleep, }; -use turbo_rcstr::RcStr; +use turbo_rcstr::{RcStr, rcstr}; use turbo_tasks::{ResolvedVc, duration_span}; use turbo_tasks_fs::FileSystemPath; use crate::{ AssetsForSourceMapping, + backend::{CreatePoolFuture, CreatePoolOptions, NodeBackend}, evaluate::{EvaluateOperation, EvaluatePool, Operation}, pool_stats::{AcquiredPermits, PoolStatsSnapshot}, worker_pool::{ @@ -148,6 +149,57 @@ impl WorkerThreadPool { } } +#[turbo_tasks::value(shared)] +pub(crate) struct WorkerThreadsBackend; + +#[turbo_tasks::value_impl] +impl NodeBackend for WorkerThreadsBackend { + fn runtime_module_path(&self) -> RcStr { + rcstr!("worker_thread/evaluate.ts") + } + + fn globals_module_path(&self) -> RcStr { + rcstr!("worker_thread/globals.ts") + } + + fn create_pool(&self, options: CreatePoolOptions) -> CreatePoolFuture { + Box::pin(async move { + let CreatePoolOptions { + cwd, + entrypoint, + env, + assets_for_source_mapping, + assets_root, + project_dir, + concurrency, + debug, + } = options; + + Ok(WorkerThreadPool::create( + cwd, + entrypoint, + env, + assets_for_source_mapping, + assets_root, + project_dir, + concurrency, + debug, + ) + .await) + }) + } + + fn scale_down(&self) -> Result<()> { + WorkerThreadPool::scale_down(); + Ok(()) + } + + fn scale_zero(&self) -> Result<()> { + WorkerThreadPool::scale_zero(); + Ok(()) + } +} + impl WorkerThreadPool { pub fn scale_down() { let _ = WORKER_POOL_OPERATION.scale_down(); @@ -207,4 +259,8 @@ impl EvaluateOperation for WorkerThreadPool { fn stats(&self) -> PoolStatsSnapshot { self.state.stats.lock().snapshot() } + + fn pre_warm(&self) { + // TODO: This is a no-op for worker_pool right now, only process_pool implements it + } } diff --git a/turbopack/crates/turbopack-tests/tests/execution.rs b/turbopack/crates/turbopack-tests/tests/execution.rs index 37e31d5ad0795e..ea6b5cd2204c35 100644 --- a/turbopack/crates/turbopack-tests/tests/execution.rs +++ b/turbopack/crates/turbopack-tests/tests/execution.rs @@ -51,6 +51,7 @@ use turbopack_css::chunk::CssChunkType; use turbopack_ecmascript::{TreeShakingMode, chunk::EcmascriptChunkType}; use turbopack_ecmascript_runtime::RuntimeType; use turbopack_node::{ + child_process_backend, debug::should_debug, evaluate::{evaluate, get_evaluate_entries}, }; @@ -482,7 +483,9 @@ async fn run_test_operation(prepared_test: ResolvedVc) -> Result) -> Result