Added code to handle masterless sync of salt-minion, tests work-in-progress

Author: David Murphy

salt/fileclient.py

@@ -45,12 +45,15 @@
 MAX_FILENAME_LENGTH = 255
 
 
-def get_file_client(opts, pillar=False):
+def get_file_client(opts, pillar=False, force_local=False):
     """
     Read in the ``file_client`` option and return the correct type of file
     server
     """
-    client = opts.get("file_client", "remote")
+    if force_local:
+        client = "local"
+    else:
+        client = opts.get("file_client", "remote")
 
     if pillar and client == "local":
         client = "pillar"

salt/minion.py

@@ -939,6 +939,7 @@ def __init__(self, opts, context=None):
         # Late setup of the opts grains, so we can log from the grains module
         import salt.loader
 
+        _sync_grains(opts)
         opts["grains"] = salt.loader.grains(opts)
         super().__init__(opts)
 

salt/utils/extmods.py

@@ -39,6 +39,7 @@ def sync(
     saltenv=None,
     extmod_whitelist=None,
     extmod_blacklist=None,
+    force_local=False,
 ):
     """
     Sync custom modules into the extension_modules directory
@@ -82,7 +83,10 @@ def sync(
                         "Cannot create cache module directory %s. Check permissions.",
                         mod_dir,
                     )
-            with salt.fileclient.get_file_client(opts) as fileclient:
+            ## DGM with salt.fileclient.get_file_client(opts) as fileclient:
+            with salt.fileclient.get_file_client(
+                opts, pillar=False, force_local=force_local
+            ) as fileclient:
                 for sub_env in saltenv:
                     log.info("Syncing %s for environment '%s'", form, sub_env)
                     cache = []

tests/pytests/integration/cli/test_salt_call.py

@@ -429,3 +429,134 @@ def test_local_salt_call_no_function_no_retcode(salt_call_cli):
     assert "test" in ret.data
     assert ret.data["test"] == "'test' is not available."
     assert "test.echo" in ret.data
+
+
+def test_state_highstate_custom_grains_masterless_mode(
+    salt_master, salt_minion_factory
+):
+    """
+    This test ensure that custom grains in salt://_grains are loaded before pillar compilation
+    to ensure that any use of custom grains in pillar files are available when in masterless mode,
+    this implies that a sync of grains occurs before loading the regular
+    /etc/salt/grains or configuration file grains, as well as the usual grains.
+    Note: cannot use salt_minion and salt_call_cli, since these will be loaded before
+    the pillar and custom_grains files are written, hence using salt_minion_factory.
+    """
+    pillar_top_sls = """
+    base:
+      '*':
+        - defaults
+        """
+
+    pillar_defaults_sls = """
+    mypillar: "{{ grains['custom_grain'] }}"
+    """
+
+    salt_top_sls = """
+    base:
+      '*':
+        - test
+        """
+
+    salt_test_sls = """
+    "donothing":
+      test.nop: []
+    """
+
+    salt_custom_grains_py = """
+    def main():
+        return {'custom_grain': 'test_value'}
+    """
+
+    ## DGM TBD need to get masterless mode
+    assert salt_master.is_running()
+    with salt_minion_factory.started():
+        salt_minion = salt_minion_factory
+        salt_call_cli = salt_minion_factory.salt_call_cli()
+        with salt_minion.pillar_tree.base.temp_file(
+            "top.sls", pillar_top_sls
+        ), salt_minion.pillar_tree.base.temp_file(
+            "defaults.sls", pillar_defaults_sls
+        ), salt_minion.state_tree.base.temp_file(
+            "top.sls", salt_top_sls
+        ), salt_minion.state_tree.base.temp_file(
+            "test.sls", salt_test_sls
+        ), salt_minion.state_tree.base.temp_file(
+            "_grains/custom_grain.py", salt_custom_grains_py
+        ):
+            ret = salt_call_cli.run("state.highstate")
+            assert ret.returncode == 0
+            ret = salt_call_cli.run("pillar.items")
+            assert ret.returncode == 0
+            assert ret.data
+            pillar_items = ret.data
+            assert "mypillar" in pillar_items
+            assert pillar_items["mypillar"] == "test_value"
+
+
+def test_state_highstate_custom_grains_master_mode(salt_master, salt_minion_factory):
+    """
+    This test ensure that custom grains in salt://_grains are loaded before pillar compilation
+    to ensure that any use of custom grains in pillar files are unaffected by changes for
+    sync custom grains for the salt-minion in masterless mode,
+    """
+    pillar_top_sls = """
+    base:
+      '*':
+        - defaults
+        """
+
+    pillar_defaults_sls = """
+    mypillar: "{{ grains['custom_grain'] }}"
+    """
+
+    salt_top_sls = """
+    base:
+      '*':
+        - test
+        """
+
+    salt_test_sls = """
+    "donothing":
+      test.nop: []
+    """
+
+    salt_custom_grains_py = """
+    def main():
+        return {'custom_grain': 'test_value'}
+    """
+
+    ## DGM TBD need to ensure master mode
+    assert salt_master.is_running()
+    with salt_minion_factory.started():
+        salt_minion = salt_minion_factory
+        salt_call_cli = salt_minion_factory.salt_call_cli()
+        with salt_minion.pillar_tree.base.temp_file(
+            "top.sls", pillar_top_sls
+        ), salt_minion.pillar_tree.base.temp_file(
+            "defaults.sls", pillar_defaults_sls
+        ), salt_minion.state_tree.base.temp_file(
+            "top.sls", salt_top_sls
+        ), salt_minion.state_tree.base.temp_file(
+            "test.sls", salt_test_sls
+        ), salt_minion.state_tree.base.temp_file(
+            "_grains/custom_grain.py", salt_custom_grains_py
+        ):
+            ret = salt_call_cli.run("state.highstate")
+            assert ret.returncode == 0
+            ret = salt_call_cli.run("pillar.items")
+            assert ret.returncode == 0
+            assert ret.data
+            pillar_items = ret.data
+            assert "mypillar" not in pillar_items
+
+
+def test_salt_call_versions(salt_call_cli, caplog):
+    """
+    Call test.versions without '--local' to test grains
+    are sync'd without any missing keys in opts
+    """
+    with caplog.at_level(logging.DEBUG):
+        ret = salt_call_cli.run("test.versions")
+        assert ret.returncode == 0
+        assert "Failed to sync grains module: 'master_uri'" not in caplog.messages