[master] Access pillar data while rendering pillars

changelog/64043.added.md

@@ -0,0 +1 @@
+Added the ability to access already compiled pillar data during the pillar rendering process via the `__pillar__` global in templates and matchers.

changelog/65724.added.md

@@ -0,0 +1 @@
+Pillar dunder is now available in extension modules during pillar render.

salt/loader/__init__.py

@@ -263,6 +263,7 @@ def minion_mods(
     notify=False,
     static_modules=None,
     proxy=None,
+    pillar=None,
     file_client=None,
 ):
     """
@@ -306,17 +307,20 @@ def minion_mods(
     # TODO Publish documentation for module whitelisting
     if not whitelist:
         whitelist = opts.get("whitelist_modules", None)
+    pack = {
+        "__context__": context,
+        "__utils__": utils,
+        "__proxy__": proxy,
+        "__opts__": opts,
+        "__file_client__": file_client,
+    }
+    if pillar is not None:
+        pack["__pillar__"] = pillar
     ret = LazyLoader(
         _module_dirs(opts, "modules", "module"),
         opts,
         tag="module",
-        pack={
-            "__context__": context,
-            "__utils__": utils,
-            "__proxy__": proxy,
-            "__opts__": opts,
-            "__file_client__": file_client,
-        },
+        pack=pack,
         whitelist=whitelist,
         loaded_base_name=loaded_base_name,
         static_modules=static_modules,
@@ -413,19 +417,35 @@ def metaproxy(opts, loaded_base_name=None):
     )
 
 
-def matchers(opts, loaded_base_name=None):
+def matchers(opts, loaded_base_name=None, context=None, pillar=None):
     """
     Return the matcher services plugins
 
     :param dict opts: The Salt options dictionary
     :param str loaded_base_name: The imported modules namespace when imported
                                  by the salt loader.
+    :param dict context: The Salt context dictionary
+    :param dict pillar: The Salt pillar dictionary
     """
+    if context is None:
+        context = {}
+
+    pack = {
+        "__salt__": {},
+        "__runners__": {},
+        "__grains__": opts.get("grains", {}),
+        "__context__": context,
+        "__file_client__": None,
+    }
+    if pillar is not None:
+        pack["__pillar__"] = pillar
+
     return LazyLoader(
         _module_dirs(opts, "matchers"),
         opts,
         tag="matchers",
         loaded_base_name=loaded_base_name,
+        pack=pack,
     )
 
 
@@ -526,6 +546,7 @@ def utils(
     context=None,
     proxy=None,
     file_client=None,
+    pillar=None,
     pack_self=None,
     loaded_base_name=None,
 ):
@@ -540,6 +561,9 @@ def utils(
     :param str loaded_base_name: The imported modules namespace when imported
                                  by the salt loader.
     """
+    pack = {"__context__": context, "__proxy__": proxy or {}}
+    if pillar is not None:
+        pack["__pillar__"] = pillar
     return LazyLoader(
         _module_dirs(opts, "utils", ext_type_dirs="utils_dirs", load_extensions=False),
         opts,
@@ -556,7 +580,7 @@ def utils(
     )
 
 
-def pillars(opts, functions, context=None, loaded_base_name=None):
+def pillars(opts, functions, context=None, pillar=None, loaded_base_name=None):
     """
     Returns the pillars modules
 
@@ -568,11 +592,14 @@ def pillars(opts, functions, context=None, loaded_base_name=None):
                                  by the salt loader.
     """
     _utils = utils(opts)
+    pack = {"__salt__": functions, "__context__": context, "__utils__": _utils}
+    if pillar is not None:
+        pack["__pillar__"] = pillar
     ret = LazyLoader(
         _module_dirs(opts, "pillar"),
         opts,
         tag="pillar",
-        pack={"__salt__": functions, "__context__": context, "__utils__": _utils},
+        pack=pack,
         extra_module_dirs=_utils.module_dirs,
         pack_self="__ext_pillar__",
         loaded_base_name=loaded_base_name,
@@ -916,6 +943,7 @@ def render(
     proxy=None,
     context=None,
     file_client=None,
+    pillar=None,
     loaded_base_name=None,
 ):
     """
@@ -939,6 +967,8 @@ def render(
         "__context__": context,
         "__file_client__": file_client,
     }
+    if pillar is not None:
+        pack["__pillar__"] = pillar
 
     if states:
         pack["__states__"] = states

salt/matchers/confirm_top.py

@@ -25,7 +25,11 @@ def confirm_top(match, data, nodegroups=None):
     if "matchers" in __context__:
         matchers = __context__["matchers"]
     else:
-        matchers = salt.loader.matchers(__opts__)
+        # Matchers need pillar data if available
+        pillar = __pillar__ if "__pillar__" in globals() else None
+        if hasattr(pillar, "value"):
+            pillar = pillar.value()
+        matchers = salt.loader.matchers(__opts__, context=__context__, pillar=pillar)
         __context__["matchers"] = matchers
     funcname = matcher + "_match.match"
     if matcher == "nodegroup":

salt/matchers/pillar_exact_match.py

@@ -20,11 +20,16 @@ def match(tgt, delimiter=":", opts=None, minion_id=None):
         log.error("Got insufficient arguments for pillar match statement from master")
         return False
 
-    if "pillar" in opts:
+    if opts.get("pillar"):
         pillar = opts["pillar"]
-    elif "ext_pillar" in opts:
-        log.info("No pillar found, fallback to ext_pillar")
+    elif "__pillar__" in globals():
+        pillar = __pillar__
+        if hasattr(pillar, "value"):
+            pillar = pillar.value()
+    elif opts.get("ext_pillar"):
         pillar = opts["ext_pillar"]
+    else:
+        pillar = {}
 
     return salt.utils.data.subdict_match(
         pillar, tgt, delimiter=delimiter, exact_match=True

salt/matchers/pillar_match.py

@@ -21,10 +21,15 @@ def match(tgt, delimiter=DEFAULT_TARGET_DELIM, opts=None, minion_id=None):
         log.error("Got insufficient arguments for pillar match statement from master")
         return False
 
-    if "pillar" in opts:
+    if opts.get("pillar"):
         pillar = opts["pillar"]
-    elif "ext_pillar" in opts:
-        log.info("No pillar found, fallback to ext_pillar")
+    elif "__pillar__" in globals():
+        pillar = __pillar__
+        if hasattr(pillar, "value"):
+            pillar = pillar.value()
+    elif opts.get("ext_pillar"):
         pillar = opts["ext_pillar"]
+    else:
+        pillar = {}
 
     return salt.utils.data.subdict_match(pillar, tgt, delimiter=delimiter)

salt/matchers/pillar_pcre_match.py

@@ -23,11 +23,16 @@ def match(tgt, delimiter=DEFAULT_TARGET_DELIM, opts=None, minion_id=None):
         )
         return False
 
-    if "pillar" in opts:
+    if opts.get("pillar"):
         pillar = opts["pillar"]
-    elif "ext_pillar" in opts:
-        log.info("No pillar found, fallback to ext_pillar")
+    elif "__pillar__" in globals():
+        pillar = __pillar__
+        if hasattr(pillar, "value"):
+            pillar = pillar.value()
+    elif opts.get("ext_pillar"):
         pillar = opts["ext_pillar"]
+    else:
+        pillar = {}
 
     return salt.utils.data.subdict_match(
         pillar, tgt, delimiter=delimiter, regex_match=True

salt/pillar/__init__.py

@@ -418,6 +418,12 @@ def __init__(
         self.client = salt.fileclient.get_file_client(self.opts, True)
         self.fileclient = salt.fileclient.get_file_client(self.opts, False)
         self.avail = self.__gather_avail()
+        self.pillar_data = self.opts.get("pillar", {})
+        if not isinstance(self.pillar_data, dict):
+            self.pillar_data = {}
+        else:
+            # Ensure we have a plain dict and not a proxy into opts
+            self.pillar_data = dict(self.pillar_data)
 
         if opts.get("file_client", "") == "local" and not opts.get(
             "use_master_when_local", False
@@ -432,20 +438,30 @@ def __init__(
                     opts,
                     utils=utils,
                     file_client=salt.fileclient.ContextlessFileClient(self.fileclient),
+                    pillar=self.pillar_data,
                 )
             else:
                 self.functions = salt.loader.minion_mods(
                     self.opts,
                     utils=utils,
                     file_client=salt.fileclient.ContextlessFileClient(self.fileclient),
+                    pillar=self.pillar_data,
                 )
         else:
             self.functions = functions
+            if hasattr(self.functions, "pack"):
+                self.functions.pack["__pillar__"] = self.pillar_data
 
         self.opts["minion_id"] = minion_id
-        self.matchers = salt.loader.matchers(self.opts)
+        self.matchers = salt.loader.matchers(self.opts, pillar=self.pillar_data)
+        if hasattr(self.matchers, "pack"):
+            self.matchers.pack["__pillar__"] = self.pillar_data
         self.rend = salt.loader.render(
-            self.opts, self.functions, self.client, file_client=self.client
+            self.opts,
+            self.functions,
+            self.client,
+            file_client=self.client,
+            pillar=self.pillar_data,
         )
         ext_pillar_opts = copy.deepcopy(self.opts)
         # Keep the incoming opts ID intact, ie, the master id
@@ -455,7 +471,9 @@ def __init__(
         if opts.get("pillar_source_merging_strategy"):
             self.merge_strategy = opts["pillar_source_merging_strategy"]
 
-        self.ext_pillars = salt.loader.pillars(ext_pillar_opts, self.functions)
+        self.ext_pillars = salt.loader.pillars(
+            ext_pillar_opts, self.functions, pillar=self.pillar_data
+        )
         self.ignored_pillars = {}
         self.pillar_override = pillar_override or {}
         if not isinstance(self.pillar_override, dict):
@@ -737,7 +755,8 @@ def top_matches(self, top, reload=False):
         """
         matches = {}
         if reload:
-            self.matchers = salt.loader.matchers(self.opts)
+            self.matchers = salt.loader.matchers(self.opts, pillar=self.pillar_data)
+            self._update_loader_packs()
         for saltenv, body in top.items():
             if self.opts["pillarenv"]:
                 if saltenv != self.opts["pillarenv"]:
@@ -1110,29 +1129,49 @@ def compile_pillar(self, ext=True):
         top, top_errors = self.get_top()
         if ext:
             if self.opts.get("ext_pillar_first", False):
-                self.opts["pillar"], errors = self.ext_pillar(self.pillar_override)
-                self.rend = salt.loader.render(self.opts, self.functions)
+                pillar, errors = self.ext_pillar(self.pillar_override)
+                self.pillar_data.update(pillar)
+                self._update_loader_packs()
+                self.rend = salt.loader.render(
+                    self.opts,
+                    self.functions,
+                    self.client,
+                    file_client=self.client,
+                    pillar=self.pillar_data,
+                )
                 matches = self.top_matches(top, reload=True)
                 pillar, errors = self.render_pillar(matches, errors=errors)
                 pillar = merge(
-                    self.opts["pillar"],
+                    self.pillar_data,
                     pillar,
                     self.merge_strategy,
                     self.opts.get("renderer", "yaml"),
                     self.opts.get("pillar_merge_lists", False),
                 )
+                self.pillar_data.update(pillar)
+                self._update_loader_packs()
             else:
                 matches = self.top_matches(top)
                 pillar, errors = self.render_pillar(matches)
-                pillar, errors = self.ext_pillar(pillar, errors=errors)
+                self.pillar_data.update(pillar)
+                self._update_loader_packs()
+                pillar, errors = self.ext_pillar(self.pillar_data, errors=errors)
+                self.pillar_data.update(pillar)
+                self._update_loader_packs()
         else:
             matches = self.top_matches(top)
             pillar, errors = self.render_pillar(matches)
+            self.pillar_data.update(pillar)
+            self._update_loader_packs()
+
+        pillar = self.pillar_data
         errors.extend(top_errors)
         if self.opts.get("pillar_opts", False):
-            mopts = dict(self.opts)
-            if "grains" in mopts:
-                mopts.pop("grains")
+            mopts = {}
+            for key, val in self.opts.items():
+                if key in ("pillar", "__context__", "functions", "matchers", "rend"):
+                    continue
+                mopts[key] = val
             mopts["saltversion"] = __version__
             pillar["master"] = mopts
         if "pillar" in self.opts and self.opts.get("ssh_merge_pillar", False):
@@ -1160,8 +1199,61 @@ def compile_pillar(self, ext=True):
         decrypt_errors = self.decrypt_pillar(pillar)
         if decrypt_errors:
             pillar.setdefault("_errors", []).extend(decrypt_errors)
+        self.pillar_data.update(pillar)
+        self._update_loader_packs()
         return pillar
 
+    def _update_loader_packs(self):
+        """
+        Update the loader packs with the current pillar data
+        """
+        for loader in (self.functions, self.matchers):
+            if hasattr(loader, "pack"):
+                if "__pillar__" in loader.pack:
+                    if loader.pack["__pillar__"] is self.pillar_data:
+                        continue
+                    if isinstance(loader.pack["__pillar__"], dict):
+                        loader.pack["__pillar__"].clear()
+                        loader.pack["__pillar__"].update(self.pillar_data)
+                    else:
+                        loader.pack["__pillar__"] = self.pillar_data
+                else:
+                    loader.pack["__pillar__"] = self.pillar_data
+
+        # Also update matchers in context if they exist
+        context = {}
+        if hasattr(self.functions, "pack"):
+            context = self.functions.pack.get("__context__", {})
+        if not context and hasattr(self.matchers, "pack"):
+            context = self.matchers.pack.get("__context__", {})
+
+        if hasattr(context, "value"):
+            context = context.value()
+
+        if isinstance(context, dict) and "matchers" in context:
+            m = context["matchers"]
+            if hasattr(m, "pack"):
+                if "__pillar__" in m.pack:
+                    if m.pack["__pillar__"] is not self.pillar_data:
+                        if isinstance(m.pack["__pillar__"], dict):
+                            m.pack["__pillar__"].clear()
+                            m.pack["__pillar__"].update(self.pillar_data)
+                        else:
+                            m.pack["__pillar__"] = self.pillar_data
+                else:
+                    m.pack["__pillar__"] = self.pillar_data
+
+        if hasattr(self.rend, "pack"):
+            if "__pillar__" in self.rend.pack:
+                if self.rend.pack["__pillar__"] is not self.pillar_data:
+                    if isinstance(self.rend.pack["__pillar__"], dict):
+                        self.rend.pack["__pillar__"].clear()
+                        self.rend.pack["__pillar__"].update(self.pillar_data)
+                    else:
+                        self.rend.pack["__pillar__"] = self.pillar_data
+            else:
+                self.rend.pack["__pillar__"] = self.pillar_data
+
     def decrypt_pillar(self, pillar):
         """
         Decrypt the specified pillar dictionary items, if configured to do so