@@ -93,6 +93,13 @@ const decideStdout = (cmd: RecordedCommand): string => {
9393const nvidiaRuntimeFailure =
9494 "Error response from daemon: failed to create task for container: nvidia-container-cli: initialization error: load library failed: libnvidia-ml.so.1"
9595
96+ const hasNvidiaFallbackWarning = ( logs : ReadonlyArray < string > ) : boolean =>
97+ logs . some ( ( entry ) =>
98+ entry . includes ( "NVIDIA runtime failed" ) &&
99+ entry . includes ( "libnvidia-ml.so.1" ) &&
100+ entry . includes ( "GPU access disabled" )
101+ )
102+
96103const shouldFailNvidiaGpuComposeUp = ( cmd : RecordedCommand ) : boolean =>
97104 isDockerComposeUpWithBuild ( cmd ) || isDockerComposeUpReuse ( cmd )
98105
@@ -292,11 +299,56 @@ describe("runDockerComposeUpWithPortCheck", () => {
292299 const configAfter = yield * _ ( fs . readFileString ( path . join ( outDir , "docker-git.json" ) ) )
293300 expect ( configAfter ) . toContain ( '"gpu": "none"' )
294301 expect ( recorded . filter ( ( entry ) => isDockerComposeUpWithBuild ( entry ) ) . length ) . toBe ( 2 )
295- expect ( logs . some ( ( entry ) =>
296- entry . includes ( "NVIDIA runtime failed" ) &&
297- entry . includes ( "libnvidia-ml.so.1" ) &&
298- entry . includes ( "GPU access disabled" )
299- ) ) . toBe ( true )
302+ expect ( hasNvidiaFallbackWarning ( logs ) ) . toBe ( true )
303+ } )
304+ ) . pipe ( Effect . provide ( NodeContext . layer ) ) )
305+
306+ it . effect ( "falls back to GPU none before retrying reuse mode when the host NVIDIA runtime is unavailable" , ( ) =>
307+ withTempDir ( ( root ) =>
308+ Effect . gen ( function * ( _ ) {
309+ const fs = yield * _ ( FileSystem . FileSystem )
310+ const path = yield * _ ( Path . Path )
311+ const outDir = path . join ( root , "project" )
312+ const targetDir = "/home/dev/workspaces/org/repo"
313+ const globalConfig = makeTemplateConfig ( root , outDir , path , targetDir )
314+ const projectConfig : TemplateConfig = {
315+ ...makeTemplateConfig ( root , outDir , path , targetDir ) ,
316+ gpu : "all"
317+ }
318+ const recorded : Array < RecordedCommand > = [ ]
319+ const logs : Array < string > = [ ]
320+ const executor = makeFakeExecutor ( recorded , { failGpuComposeUp : true } )
321+ const logger = Logger . make ( ( { message } ) => {
322+ logs . push ( String ( message ) )
323+ } )
324+
325+ yield * _ (
326+ prepareProjectFiles ( outDir , root , globalConfig , projectConfig , {
327+ force : false ,
328+ forceEnv : false
329+ } )
330+ )
331+
332+ const started = yield * _ (
333+ runDockerComposeUpWithPortCheck ( outDir , {
334+ buildMode : "reuse" ,
335+ waitForPostStart : false
336+ } ) . pipe (
337+ Effect . provideService ( CommandExecutor . CommandExecutor , executor ) ,
338+ Effect . provide ( Logger . replace ( Logger . defaultLogger , logger ) )
339+ )
340+ )
341+
342+ expect ( started . gpu ) . toBe ( "none" )
343+
344+ const composeAfter = yield * _ ( fs . readFileString ( path . join ( outDir , "docker-compose.yml" ) ) )
345+ expect ( composeAfter ) . not . toContain ( " gpus: all\n" )
346+
347+ const configAfter = yield * _ ( fs . readFileString ( path . join ( outDir , "docker-git.json" ) ) )
348+ expect ( configAfter ) . toContain ( '"gpu": "none"' )
349+ expect ( recorded . filter ( ( entry ) => isDockerComposeUpReuse ( entry ) ) . length ) . toBe ( 2 )
350+ expect ( recorded . filter ( ( entry ) => isDockerComposeUpWithBuild ( entry ) ) . length ) . toBe ( 0 )
351+ expect ( hasNvidiaFallbackWarning ( logs ) ) . toBe ( true )
300352 } )
301353 ) . pipe ( Effect . provide ( NodeContext . layer ) ) )
302354
0 commit comments