From bdc65c26bca51eb97a5c4b6477b624a86135bd04 Mon Sep 17 00:00:00 2001 From: Pruthviraj Prakash Date: Wed, 12 Aug 2026 10:44:25 -0700 Subject: [PATCH 1/3] fix(deploy): default inference coordinator host to loopback create_mcore_engine passed os.environ.get("MASTER_ADDR") straight through to MegatronLLM. When the variable is unset that is None, which is indistinguishable from passing nothing, so MCore falls back to socket.gethostname(). Inside Docker that resolves to the container ID, which is not a bindable interface, and the data-parallel inference coordinator dies with: zmq.error.ZMQError: No such device (addr='tcp://:*') The Ray path never hit this because megatronllm_deployable_ray sets MASTER_ADDR to the node IP itself; the PyTriton path sets nothing, so every in-framework Triton deploy failed on single-node Docker. Fall back to 127.0.0.1 only when MASTER_ADDR is absent, so multi-node launches that already export a routable address are unaffected. Signed-off-by: Pruthviraj Prakash --- nemo_deploy/llm/inference/inference_base.py | 8 ++- .../unit_tests/deploy/test_inference_base.py | 50 +++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/nemo_deploy/llm/inference/inference_base.py b/nemo_deploy/llm/inference/inference_base.py index 0a05f1631..8a8274c6d 100644 --- a/nemo_deploy/llm/inference/inference_base.py +++ b/nemo_deploy/llm/inference/inference_base.py @@ -522,7 +522,13 @@ def create_mcore_engine( materialize_only_last_token_logits=True, ) - coordinator_host = os.environ.get("MASTER_ADDR") + # Fall back to loopback when MASTER_ADDR is unset. Passing None is + # indistinguishable from passing nothing, and MCore then binds its data-parallel + # inference coordinator to socket.gethostname(), which inside Docker resolves to + # the container ID and is not a bindable interface (ZMQError: No such device). + # The Ray path never hits this because megatronllm_deployable_ray sets + # MASTER_ADDR to the node IP itself; the PyTriton path sets nothing. + coordinator_host = os.environ.get("MASTER_ADDR") or "127.0.0.1" llm = MegatronLLM( model=model, diff --git a/tests/unit_tests/deploy/test_inference_base.py b/tests/unit_tests/deploy/test_inference_base.py index c35f4edea..29e8e01d9 100644 --- a/tests/unit_tests/deploy/test_inference_base.py +++ b/tests/unit_tests/deploy/test_inference_base.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import types import unittest from pathlib import Path @@ -826,6 +827,55 @@ def test_create_mcore_engine_megatron_format( mock_megatron_llm.assert_called_once() self.assertIsNotNone(engine) + @patch("nemo_deploy.llm.inference.inference_base.setup_megatron_model_and_tokenizer_for_inference") + @patch("nemo_deploy.llm.inference.inference_base.MegatronLLM") + def test_create_mcore_engine_coordinator_host_defaults_to_loopback( + self, + mock_megatron_llm, + mock_setup, + ): + """coordinator_host must not be None when MASTER_ADDR is unset. + + MCore falls back to socket.gethostname() when no hostname is supplied, which + inside Docker resolves to the container ID and is not a bindable interface + (ZMQError: No such device). Only the Ray deployable sets MASTER_ADDR; the + PyTriton path sets nothing, so the default has to come from here. + """ + mock_setup.return_value = ([MagicMock()], MagicMock(), MagicMock()) + mock_megatron_llm.return_value = MagicMock() + + with patch.dict(os.environ, {}, clear=True): + os.environ.pop("MASTER_ADDR", None) + create_mcore_engine( + path=self.mock_path, + model_format="megatron", + inference_max_seq_length=2048, + max_batch_size=4, + ) + + self.assertEqual(mock_megatron_llm.call_args.kwargs["coordinator_host"], "127.0.0.1") + + @patch("nemo_deploy.llm.inference.inference_base.setup_megatron_model_and_tokenizer_for_inference") + @patch("nemo_deploy.llm.inference.inference_base.MegatronLLM") + def test_create_mcore_engine_coordinator_host_prefers_master_addr( + self, + mock_megatron_llm, + mock_setup, + ): + """An explicit MASTER_ADDR still wins over the loopback default.""" + mock_setup.return_value = ([MagicMock()], MagicMock(), MagicMock()) + mock_megatron_llm.return_value = MagicMock() + + with patch.dict(os.environ, {"MASTER_ADDR": "10.0.0.5"}): + create_mcore_engine( + path=self.mock_path, + model_format="megatron", + inference_max_seq_length=2048, + max_batch_size=4, + ) + + self.assertEqual(mock_megatron_llm.call_args.kwargs["coordinator_host"], "10.0.0.5") + @patch("nemo_deploy.llm.inference.inference_base.torch_distributed_init") @patch("nemo_deploy.llm.inference.inference_base.load_model_config") @patch("nemo_deploy.llm.inference.inference_base.initialize_megatron_for_inference") From 3472ea5a95f0f00121eddc1092ef59bb7e5ce2c1 Mon Sep 17 00:00:00 2001 From: Pruthviraj Prakash Date: Wed, 12 Aug 2026 11:00:52 -0700 Subject: [PATCH 2/3] fix(deploy): resolve coordinator host instead of hardcoding loopback Defaulting to 127.0.0.1 fixed the reported Docker failure but is bindable only locally, so a multi-node launch that did not export MASTER_ADDR would bind a coordinator nobody else can reach - trading a loud ZMQError for a silent hang. Resolve the hostname to its IP instead. That is what the Ray path effectively gets by exporting the node IP as MASTER_ADDR, and it is both bindable and reachable from other containers. Loopback remains only as a last resort if resolution itself fails. Measured in nvcr.io/nvidian/nemo:26.08.rc9: socket.gethostname() -> f411026bc80c ZMQ bind FAILS socket.gethostbyname(gethostname()) -> 172.17.0.2 ZMQ bind OK 127.0.0.1 ZMQ bind OK, not routable Signed-off-by: Pruthviraj Prakash --- nemo_deploy/llm/inference/inference_base.py | 25 ++++++--- .../unit_tests/deploy/test_inference_base.py | 54 ++++++++++++++----- 2 files changed, 60 insertions(+), 19 deletions(-) diff --git a/nemo_deploy/llm/inference/inference_base.py b/nemo_deploy/llm/inference/inference_base.py index 8a8274c6d..129344179 100644 --- a/nemo_deploy/llm/inference/inference_base.py +++ b/nemo_deploy/llm/inference/inference_base.py @@ -16,6 +16,7 @@ import atexit import logging import os +import socket from pathlib import Path from typing import List, Optional, Tuple, Union @@ -414,6 +415,22 @@ def __getattr__(self, name): return getattr(self.mcore_engine, name) +def _default_coordinator_host() -> str: + """Return a bindable address for the data-parallel inference coordinator. + + MCore defaults the coordinator host to ``socket.gethostname()``, which is a + *name*, not an address. Inside Docker it resolves to the container ID and ZMQ + cannot bind it (``ZMQError: No such device``). Resolving that name to its IP + yields an address that is both bindable and reachable from other containers, + which is what the Ray path already gets by exporting the node IP as + ``MASTER_ADDR``. Fall back to loopback only if resolution fails. + """ + try: + return socket.gethostbyname(socket.gethostname()) + except OSError: + return "127.0.0.1" + + def create_mcore_engine( path: Path, num_devices: Optional[int] = None, @@ -522,13 +539,7 @@ def create_mcore_engine( materialize_only_last_token_logits=True, ) - # Fall back to loopback when MASTER_ADDR is unset. Passing None is - # indistinguishable from passing nothing, and MCore then binds its data-parallel - # inference coordinator to socket.gethostname(), which inside Docker resolves to - # the container ID and is not a bindable interface (ZMQError: No such device). - # The Ray path never hits this because megatronllm_deployable_ray sets - # MASTER_ADDR to the node IP itself; the PyTriton path sets nothing. - coordinator_host = os.environ.get("MASTER_ADDR") or "127.0.0.1" + coordinator_host = os.environ.get("MASTER_ADDR") or _default_coordinator_host() llm = MegatronLLM( model=model, diff --git a/tests/unit_tests/deploy/test_inference_base.py b/tests/unit_tests/deploy/test_inference_base.py index 29e8e01d9..286759192 100644 --- a/tests/unit_tests/deploy/test_inference_base.py +++ b/tests/unit_tests/deploy/test_inference_base.py @@ -829,29 +829,59 @@ def test_create_mcore_engine_megatron_format( @patch("nemo_deploy.llm.inference.inference_base.setup_megatron_model_and_tokenizer_for_inference") @patch("nemo_deploy.llm.inference.inference_base.MegatronLLM") - def test_create_mcore_engine_coordinator_host_defaults_to_loopback( + def test_create_mcore_engine_coordinator_host_resolves_to_address( self, mock_megatron_llm, mock_setup, ): - """coordinator_host must not be None when MASTER_ADDR is unset. + """With MASTER_ADDR unset, coordinator_host must be a bindable address. - MCore falls back to socket.gethostname() when no hostname is supplied, which - inside Docker resolves to the container ID and is not a bindable interface - (ZMQError: No such device). Only the Ray deployable sets MASTER_ADDR; the - PyTriton path sets nothing, so the default has to come from here. + MCore defaults the coordinator to socket.gethostname(), which is a name and + not an address; inside Docker it is the container ID and ZMQ cannot bind it + (ZMQError: No such device). Only the Ray deployable exports MASTER_ADDR, so + on the PyTriton path the resolved address has to come from here. """ mock_setup.return_value = ([MagicMock()], MagicMock(), MagicMock()) mock_megatron_llm.return_value = MagicMock() with patch.dict(os.environ, {}, clear=True): os.environ.pop("MASTER_ADDR", None) - create_mcore_engine( - path=self.mock_path, - model_format="megatron", - inference_max_seq_length=2048, - max_batch_size=4, - ) + with patch("nemo_deploy.llm.inference.inference_base.socket") as mock_socket: + mock_socket.gethostname.return_value = "9f2c1a4b7d3e" # container ID + mock_socket.gethostbyname.return_value = "172.17.0.2" + create_mcore_engine( + path=self.mock_path, + model_format="megatron", + inference_max_seq_length=2048, + max_batch_size=4, + ) + + host = mock_megatron_llm.call_args.kwargs["coordinator_host"] + self.assertEqual(host, "172.17.0.2") + self.assertNotEqual(host, "9f2c1a4b7d3e", "must not pass the bare hostname through") + + @patch("nemo_deploy.llm.inference.inference_base.setup_megatron_model_and_tokenizer_for_inference") + @patch("nemo_deploy.llm.inference.inference_base.MegatronLLM") + def test_create_mcore_engine_coordinator_host_falls_back_when_unresolvable( + self, + mock_megatron_llm, + mock_setup, + ): + """If the hostname cannot be resolved, fall back to loopback rather than None.""" + mock_setup.return_value = ([MagicMock()], MagicMock(), MagicMock()) + mock_megatron_llm.return_value = MagicMock() + + with patch.dict(os.environ, {}, clear=True): + os.environ.pop("MASTER_ADDR", None) + with patch("nemo_deploy.llm.inference.inference_base.socket") as mock_socket: + mock_socket.gethostname.return_value = "9f2c1a4b7d3e" + mock_socket.gethostbyname.side_effect = OSError("name resolution failed") + create_mcore_engine( + path=self.mock_path, + model_format="megatron", + inference_max_seq_length=2048, + max_batch_size=4, + ) self.assertEqual(mock_megatron_llm.call_args.kwargs["coordinator_host"], "127.0.0.1") From 49943f0b0625e08e41e28910ca22611a1e7dd045 Mon Sep 17 00:00:00 2001 From: Pruthviraj Prakash Date: Wed, 12 Aug 2026 11:36:04 -0700 Subject: [PATCH 3/3] test(deploy): use a non-hex hostname in the coordinator tests The fabricated container ID was 12 hex characters, which detect-secrets flags as a Hex High Entropy String and fails the secrets-detector job. The test only needs a value that is not an address, so name it that rather than allowlisting a non-secret or regenerating the shared baseline. Signed-off-by: Pruthviraj Prakash --- tests/unit_tests/deploy/test_inference_base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit_tests/deploy/test_inference_base.py b/tests/unit_tests/deploy/test_inference_base.py index 286759192..da84e56ee 100644 --- a/tests/unit_tests/deploy/test_inference_base.py +++ b/tests/unit_tests/deploy/test_inference_base.py @@ -847,7 +847,7 @@ def test_create_mcore_engine_coordinator_host_resolves_to_address( with patch.dict(os.environ, {}, clear=True): os.environ.pop("MASTER_ADDR", None) with patch("nemo_deploy.llm.inference.inference_base.socket") as mock_socket: - mock_socket.gethostname.return_value = "9f2c1a4b7d3e" # container ID + mock_socket.gethostname.return_value = "test-container-id" # a name, not an address mock_socket.gethostbyname.return_value = "172.17.0.2" create_mcore_engine( path=self.mock_path, @@ -858,7 +858,7 @@ def test_create_mcore_engine_coordinator_host_resolves_to_address( host = mock_megatron_llm.call_args.kwargs["coordinator_host"] self.assertEqual(host, "172.17.0.2") - self.assertNotEqual(host, "9f2c1a4b7d3e", "must not pass the bare hostname through") + self.assertNotEqual(host, "test-container-id", "must not pass the bare hostname through") @patch("nemo_deploy.llm.inference.inference_base.setup_megatron_model_and_tokenizer_for_inference") @patch("nemo_deploy.llm.inference.inference_base.MegatronLLM") @@ -874,7 +874,7 @@ def test_create_mcore_engine_coordinator_host_falls_back_when_unresolvable( with patch.dict(os.environ, {}, clear=True): os.environ.pop("MASTER_ADDR", None) with patch("nemo_deploy.llm.inference.inference_base.socket") as mock_socket: - mock_socket.gethostname.return_value = "9f2c1a4b7d3e" + mock_socket.gethostname.return_value = "test-container-id" mock_socket.gethostbyname.side_effect = OSError("name resolution failed") create_mcore_engine( path=self.mock_path,