Add SafeDict/SafeList pillar wrapping with Pydantic secrets, output redaction, and no_log

changelog/68907.added.md

@@ -0,0 +1 @@
+Pillar data is now wrapped in SafeDict/SafeList with Pydantic SecretStr/SecretBytes for safer logging and output; optional state `no_log` and automatic redaction of pillar literals in state returns and minion job logs.

doc/topics/tutorials/pillar.rst

@@ -40,6 +40,23 @@ Arbitrary Data:
     so a key/value store can be defined making it easy to iterate over a group
     of values in sls formulas.
 
+.. versionadded:: 3008
+
+On the minion, compiled pillar is stored using :mod:`salt.utils.safepillar`
+containers: string and bytes leaves are wrapped with Pydantic ``SecretStr`` and
+``SecretBytes``, and mappings/lists use ``SafeDict`` / ``SafeList`` so later
+in-place updates stay wrapped. State returns and minion logs redact known pillar
+literals where possible. States may set ``no_log: true`` (runtime keyword, not
+passed to the state function) to mask that state's ``comment`` and ``changes``
+in output. Custom code that assumes ``isinstance(value, str)`` for pillar
+values may need to use ``get_secret_value()`` on secret types or compare with
+``SecretStr``.
+
+The execution functions ``pillar.items`` and ``pillar.get`` are exceptions: they
+return plain Python strings, bytes, dicts, and lists (unwrapped) because the
+operator explicitly asked for pillar data. Other code paths use wrapped
+in-memory pillar unless they call those functions.
+
 Pillar is therefore one of the most important systems when using Salt. This
 walkthrough is designed to get a simple Pillar up and running in a few minutes
 and then to dive into the capabilities of Pillar and where the data is

requirements/base.txt

@@ -40,6 +40,7 @@ psutil>=5.0.0; python_version >= '3.10'
 pymssql>=2.2.1; sys_platform == 'win32' and python_version < '3.11'
 pymssql==2.3.11; sys_platform == 'win32' and python_version >= '3.11'
 pymysql>=1.0.2; sys_platform == 'win32'
+pydantic>=2.4.0
 pyopenssl>=25.0.0
 python-dateutil>=2.8.1
 python-gnupg>=0.4.7

requirements/static/ci/py3.10/cloud.txt

@@ -17,6 +17,11 @@ aiosignal==1.4.0
     #   -c requirements/static/ci/py3.10/linux.txt
     #   -c requirements/static/pkg/py3.10/linux.txt
     #   aiohttp
+annotated-types==0.7.0
+    # via
+    #   -c requirements/static/ci/py3.10/linux.txt
+    #   -c requirements/static/pkg/py3.10/linux.txt
+    #   pydantic
 apache-libcloud==3.9.0
     # via
     #   -c requirements/static/ci/py3.10/linux.txt
@@ -439,6 +444,16 @@ pycryptodomex==3.19.1
     #   -c requirements/static/ci/py3.10/linux.txt
     #   -c requirements/static/pkg/py3.10/linux.txt
     #   -r requirements/crypto.txt
+pydantic==2.12.5
+    # via
+    #   -c requirements/static/ci/py3.10/linux.txt
+    #   -c requirements/static/pkg/py3.10/linux.txt
+    #   -r requirements/base.txt
+pydantic-core==2.41.5
+    # via
+    #   -c requirements/static/ci/py3.10/linux.txt
+    #   -c requirements/static/pkg/py3.10/linux.txt
+    #   pydantic
 pyfakefs==5.3.1
     # via
     #   -c requirements/static/ci/py3.10/linux.txt
@@ -725,9 +740,17 @@ typing-extensions==4.15.0
     #   aiosignal
     #   cryptography
     #   multidict
+    #   pydantic
+    #   pydantic-core
     #   pyopenssl
     #   pytest-system-statistics
+    #   typing-inspection
     #   virtualenv
+typing-inspection==0.4.2
+    # via
+    #   -c requirements/static/ci/py3.10/linux.txt
+    #   -c requirements/static/pkg/py3.10/linux.txt
+    #   pydantic
 urllib3==2.6.3
     # via
     #   -c requirements/static/ci/py3.10/linux.txt

requirements/static/ci/py3.10/darwin.txt

@@ -14,6 +14,10 @@ aiosignal==1.4.0
     # via
     #   -c requirements/static/pkg/py3.10/darwin.txt
     #   aiohttp
+annotated-types==0.7.0
+    # via
+    #   -c requirements/static/pkg/py3.10/darwin.txt
+    #   pydantic
 apache-libcloud==3.9.0
     # via
     #   -c requirements/static/pkg/py3.10/darwin.txt
@@ -323,6 +327,14 @@ pycryptodomex==3.19.1
     # via
     #   -c requirements/static/pkg/py3.10/darwin.txt
     #   -r requirements/crypto.txt
+pydantic==2.12.5
+    # via
+    #   -c requirements/static/pkg/py3.10/darwin.txt
+    #   -r requirements/base.txt
+pydantic-core==2.41.5
+    # via
+    #   -c requirements/static/pkg/py3.10/darwin.txt
+    #   pydantic
 pyfakefs==5.3.1
     # via -r requirements/pytest.txt
 pygit2==1.13.1
@@ -504,9 +516,16 @@ typing-extensions==4.15.0
     #   aiosignal
     #   cryptography
     #   multidict
+    #   pydantic
+    #   pydantic-core
     #   pyopenssl
     #   pytest-system-statistics
+    #   typing-inspection
     #   virtualenv
+typing-inspection==0.4.2
+    # via
+    #   -c requirements/static/pkg/py3.10/darwin.txt
+    #   pydantic
 urllib3==2.6.3
     # via
     #   -c requirements/static/pkg/py3.10/darwin.txt

requirements/static/ci/py3.10/docs.txt

@@ -14,6 +14,10 @@ aiosignal==1.4.0
     #   aiohttp
 alabaster==0.7.13
     # via sphinx
+annotated-types==0.7.0
+    # via
+    #   -c requirements/static/ci/py3.10/linux.txt
+    #   pydantic
 apache-libcloud==3.9.0
     # via
     #   -c requirements/static/ci/py3.10/linux.txt
@@ -216,6 +220,14 @@ pycryptodomex==3.19.1
     # via
     #   -c requirements/static/ci/py3.10/linux.txt
     #   -r requirements/crypto.txt
+pydantic==2.12.5
+    # via
+    #   -c requirements/static/ci/py3.10/linux.txt
+    #   -r requirements/base.txt
+pydantic-core==2.41.5
+    # via
+    #   -c requirements/static/ci/py3.10/linux.txt
+    #   pydantic
 pyenchant==3.2.2
     # via sphinxcontrib-spelling
 pygments==2.17.2
@@ -316,8 +328,15 @@ typing-extensions==4.15.0
     #   aiosignal
     #   cryptography
     #   multidict
+    #   pydantic
+    #   pydantic-core
     #   pyopenssl
+    #   typing-inspection
     #   virtualenv
+typing-inspection==0.4.2
+    # via
+    #   -c requirements/static/ci/py3.10/linux.txt
+    #   pydantic
 uc-micro-py==1.0.2
     # via linkify-it-py
 urllib3==2.6.3

requirements/static/ci/py3.10/freebsd.txt

@@ -14,6 +14,10 @@ aiosignal==1.4.0
     # via
     #   -c requirements/static/pkg/py3.10/freebsd.txt
     #   aiohttp
+annotated-types==0.7.0
+    # via
+    #   -c requirements/static/pkg/py3.10/freebsd.txt
+    #   pydantic
 apache-libcloud==3.9.0
     # via
     #   -c requirements/static/pkg/py3.10/freebsd.txt
@@ -341,6 +345,14 @@ pycryptodomex==3.19.1
     # via
     #   -c requirements/static/pkg/py3.10/freebsd.txt
     #   -r requirements/crypto.txt
+pydantic==2.12.5
+    # via
+    #   -c requirements/static/pkg/py3.10/freebsd.txt
+    #   -r requirements/base.txt
+pydantic-core==2.41.5
+    # via
+    #   -c requirements/static/pkg/py3.10/freebsd.txt
+    #   pydantic
 pyfakefs==5.3.1
     # via -r requirements/pytest.txt
 pyinotify==0.9.6 ; platform_system != 'openbsd' and sys_platform != 'darwin' and sys_platform != 'win32'
@@ -568,9 +580,16 @@ typing-extensions==4.15.0
     #   aiosignal
     #   cryptography
     #   multidict
+    #   pydantic
+    #   pydantic-core
     #   pyopenssl
     #   pytest-system-statistics
+    #   typing-inspection
     #   virtualenv
+typing-inspection==0.4.2
+    # via
+    #   -c requirements/static/pkg/py3.10/freebsd.txt
+    #   pydantic
 urllib3==2.6.3
     # via
     #   -c requirements/static/pkg/py3.10/freebsd.txt

requirements/static/ci/py3.10/lint.txt

@@ -23,6 +23,11 @@ aiosignal==1.4.0
     #   -c requirements/static/ci/py3.10/linux.txt
     #   -c requirements/static/pkg/py3.10/linux.txt
     #   aiohttp
+annotated-types==0.7.0
+    # via
+    #   -c requirements/static/ci/py3.10/linux.txt
+    #   -c requirements/static/pkg/py3.10/linux.txt
+    #   pydantic
 ansible==10.7.0
     # via
     #   -c requirements/static/ci/py3.10/linux.txt
@@ -461,6 +466,16 @@ pycryptodomex==3.19.1
     #   -c requirements/static/ci/py3.10/linux.txt
     #   -c requirements/static/pkg/py3.10/linux.txt
     #   -r requirements/crypto.txt
+pydantic==2.12.5
+    # via
+    #   -c requirements/static/ci/py3.10/linux.txt
+    #   -c requirements/static/pkg/py3.10/linux.txt
+    #   -r requirements/base.txt
+pydantic-core==2.41.5
+    # via
+    #   -c requirements/static/ci/py3.10/linux.txt
+    #   -c requirements/static/pkg/py3.10/linux.txt
+    #   pydantic
 pygit2==1.13.1
     # via
     #   -c requirements/static/ci/py3.10/linux.txt
@@ -733,8 +748,16 @@ typing-extensions==4.15.0
     #   astroid
     #   cryptography
     #   multidict
+    #   pydantic
+    #   pydantic-core
     #   pyopenssl
+    #   typing-inspection
     #   virtualenv
+typing-inspection==0.4.2
+    # via
+    #   -c requirements/static/ci/py3.10/linux.txt
+    #   -c requirements/static/pkg/py3.10/linux.txt
+    #   pydantic
 urllib3==2.6.3
     # via
     #   -c requirements/static/ci/py3.10/linux.txt

requirements/static/ci/py3.10/linux.txt

@@ -18,6 +18,10 @@ aiosignal==1.4.0
     # via
     #   -c requirements/static/pkg/py3.10/linux.txt
     #   aiohttp
+annotated-types==0.7.0
+    # via
+    #   -c requirements/static/pkg/py3.10/linux.txt
+    #   pydantic
 ansible==10.7.0
     # via -r requirements/static/ci/linux.in
 ansible-core==2.17.14
@@ -351,6 +355,14 @@ pycryptodomex==3.19.1
     # via
     #   -c requirements/static/pkg/py3.10/linux.txt
     #   -r requirements/crypto.txt
+pydantic==2.12.5
+    # via
+    #   -c requirements/static/pkg/py3.10/linux.txt
+    #   -r requirements/base.txt
+pydantic-core==2.41.5
+    # via
+    #   -c requirements/static/pkg/py3.10/linux.txt
+    #   pydantic
 pyfakefs==5.3.1
     # via -r requirements/pytest.txt
 pygit2==1.13.1
@@ -571,9 +583,16 @@ typing-extensions==4.15.0
     #   aiosignal
     #   cryptography
     #   multidict
+    #   pydantic
+    #   pydantic-core
     #   pyopenssl
     #   pytest-system-statistics
+    #   typing-inspection
     #   virtualenv
+typing-inspection==0.4.2
+    # via
+    #   -c requirements/static/pkg/py3.10/linux.txt
+    #   pydantic
 urllib3==2.6.3
     # via
     #   -c requirements/static/pkg/py3.10/linux.txt

requirements/static/ci/py3.10/windows.txt

@@ -18,6 +18,10 @@ annotated-doc==0.0.4
     # via
     #   -c requirements/static/pkg/py3.10/windows.txt
     #   typer
+annotated-types==0.7.0
+    # via
+    #   -c requirements/static/pkg/py3.10/windows.txt
+    #   pydantic
 apache-libcloud==3.9.0
     # via
     #   -c requirements/static/pkg/py3.10/windows.txt
@@ -314,15 +318,23 @@ pycryptodomex==3.23.0
     # via
     #   -c requirements/static/pkg/py3.10/windows.txt
     #   -r requirements/crypto.txt
+pydantic==2.12.5
+    # via
+    #   -c requirements/static/pkg/py3.10/windows.txt
+    #   -r requirements/base.txt
+pydantic-core==2.41.5
+    # via
+    #   -c requirements/static/pkg/py3.10/windows.txt
+    #   pydantic
 pyfakefs==5.3.1
     # via -r requirements/pytest.txt
 pygit2==1.13.1
     # via -r requirements/static/ci/windows.in
-pygments==2.19.2
+pygments==2.20.0
     # via
     #   -c requirements/static/pkg/py3.10/windows.txt
     #   rich
-pymssql==2.3.11
+pymssql==2.3.1
     # via
     #   -c requirements/static/pkg/py3.10/windows.txt
     #   -r requirements/base.txt
@@ -522,9 +534,16 @@ typing-extensions==4.15.0
     #   aiosignal
     #   cryptography
     #   multidict
+    #   pydantic
+    #   pydantic-core
     #   pyopenssl
     #   pytest-system-statistics
+    #   typing-inspection
     #   virtualenv
+typing-inspection==0.4.2
+    # via
+    #   -c requirements/static/pkg/py3.10/windows.txt
+    #   pydantic
 urllib3==2.6.3
     # via
     #   -c requirements/static/pkg/py3.10/windows.txt

requirements/static/ci/py3.11/cloud.txt

@@ -17,6 +17,11 @@ aiosignal==1.4.0
     #   -c requirements/static/ci/py3.11/linux.txt
     #   -c requirements/static/pkg/py3.11/linux.txt
     #   aiohttp
+annotated-types==0.7.0
+    # via
+    #   -c requirements/static/ci/py3.11/linux.txt
+    #   -c requirements/static/pkg/py3.11/linux.txt
+    #   pydantic
 apache-libcloud==3.9.0
     # via
     #   -c requirements/static/ci/py3.11/linux.txt
@@ -429,6 +434,16 @@ pycryptodomex==3.19.1
     #   -c requirements/static/ci/py3.11/linux.txt
     #   -c requirements/static/pkg/py3.11/linux.txt
     #   -r requirements/crypto.txt
+pydantic==2.12.5
+    # via
+    #   -c requirements/static/ci/py3.11/linux.txt
+    #   -c requirements/static/pkg/py3.11/linux.txt
+    #   -r requirements/base.txt
+pydantic-core==2.41.5
+    # via
+    #   -c requirements/static/ci/py3.11/linux.txt
+    #   -c requirements/static/pkg/py3.11/linux.txt
+    #   pydantic
 pyfakefs==5.3.1
     # via
     #   -c requirements/static/ci/py3.11/linux.txt
@@ -709,8 +724,16 @@ typing-extensions==4.14.1
     #   -c requirements/static/ci/py3.11/linux.txt
     #   -c requirements/static/pkg/py3.11/linux.txt
     #   aiosignal
+    #   pydantic
+    #   pydantic-core
     #   pyopenssl
     #   pytest-system-statistics
+    #   typing-inspection
+typing-inspection==0.4.2
+    # via
+    #   -c requirements/static/ci/py3.11/linux.txt
+    #   -c requirements/static/pkg/py3.11/linux.txt
+    #   pydantic
 urllib3==2.6.3
     # via
     #   -c requirements/static/ci/py3.11/linux.txt