From 047eef4163ae3b18f266ffea9501fc3d6c2f62ef Mon Sep 17 00:00:00 2001 From: wk333 <13474090681@163.com> Date: Tue, 1 Apr 2025 17:16:43 +0800 Subject: [PATCH] Fix CVE-2023-5764 (cherry picked from commit f831272e2fb56bd7e1a0f5f752ef4191d095047d) --- CVE-2023-5764.patch | 2010 +++++++++++++++++++++++++++++++++++++++++++ CVE-2024-0690.patch | 968 ++++++++++++++++++++- ansible.spec | 6 +- 3 files changed, 2968 insertions(+), 16 deletions(-) create mode 100644 CVE-2023-5764.patch diff --git a/CVE-2023-5764.patch b/CVE-2023-5764.patch new file mode 100644 index 0000000..3ef4a99 --- /dev/null +++ b/CVE-2023-5764.patch @@ -0,0 +1,2010 @@ +From f374897940ad8f388c273af91db4863c51799c6d Mon Sep 17 00:00:00 2001 +From: Abhijeet Kasurde +Date: Tue, 7 Sep 2021 10:21:47 +0530 +Subject: [PATCH 1/2] Ensure that unsafe is more difficult to lose + [stable-2.14] (#82295) + +Origin: https://build.opensuse.org/projects/SUSE:SLE-15-SP3:Update/packages/ansible/files/0001-Ensure-that-unsafe-is-more-difficult-to-lose-stable-.patch?expand=1 + +* Ensure that unsafe is more difficult to lose + +* Add Task.untemplated_args, and switch assert over to use it +* Don't use re in first_found, switch to using native string methods +* If nested templating results in unsafe, just error, don't continue + +(cherry picked from commit 586f1924512b01305f896d9ae4732773023013a3) + +* ci_complete + +yaml dumper: Add YAML respresenter for AnsibleUndefined (#75078) + +Fixes: #75072 + +Signed-off-by: Abhijeet Kasurde + +Ensure single vaulted values aren't counted as sequences. Fixes #70784 (#70786) + +AnsibleVaultEncryptedUnicode should be considered a string (#71609) + +* AnsibleVaultEncryptedUnicode should be considered a string + +* linting fix + +* clog frag + +2.12: Add YAML representers for NativeJinjaUnsafeText and NativeJinjaText (#77299) + +* Add a YAML representer for NativeJinjaUnsafeText (#76186) + +(cherry picked from commit dd220ddc2faf9510bdfedacf8b755798038591d9) + +* Add a YAML representer for NativeJinjaText (#77282) + +Fixes #77280 + +(cherry picked from commit c9db73f04e7a5fae7bbbdff8efbd585d15971d31) +--- + .../fragments/70784-vault-is-string.yml | 3 + + .../fragments/71609-is_string-vault.yml | 3 + + changelogs/fragments/75072_undefined_yaml.yml | 3 + + changelogs/fragments/cve-2023-5764.yml | 6 + + .../nativejinjatext-yaml-representer.yml | 2 + + ...nativejinjaunsafetext-yaml-representer.yml | 2 + + .../module_utils/common/collections.py | 3 +- + lib/ansible/module_utils/common/json.py | 14 +- + lib/ansible/parsing/yaml/dumper.py | 50 +++- + lib/ansible/playbook/conditional.py | 9 +- + lib/ansible/playbook/task.py | 24 ++ + lib/ansible/plugins/action/assert.py | 23 +- + lib/ansible/plugins/callback/__init__.py | 90 +++++- + lib/ansible/plugins/filter/core.py | 25 +- + lib/ansible/plugins/lookup/first_found.py | 25 ++ + lib/ansible/template/__init__.py | 23 +- + lib/ansible/utils/unsafe_proxy.py | 263 +++++++++++++++++- + .../targets/apt_repository/tasks/apt.yml | 11 +- + .../assert/assert.out.nested_tmpl.stderr | 4 + + .../assert/assert.out.nested_tmpl.stdout | 12 + + ...t.quiet.stderr => assert.out.quiet.stderr} | 0 + ...t.quiet.stdout => assert.out.quiet.stdout} | 0 + .../targets/assert/nested_tmpl.yml | 9 + + test/integration/targets/assert/quiet.yml | 4 +- + test/integration/targets/assert/runme.sh | 3 +- + .../targets/command_shell/tasks/main.yml | 2 +- + test/integration/targets/copy/tasks/tests.yml | 42 +-- + test/integration/targets/debug/runme.sh | 2 + + test/integration/targets/debug/unsafe.yml | 13 + + .../integration/targets/expect/tasks/main.yml | 7 +- + .../targets/file/tasks/state_link.yml | 2 +- + test/integration/targets/find/tasks/main.yml | 88 +++++- + .../gathering_facts/test_gathering_facts.yml | 4 +- + test/integration/targets/git/tasks/depth.yml | 2 +- + .../targets/git/tasks/localmods.yml | 4 +- + .../targets/git/tasks/submodules.yml | 25 +- + .../targets/include_vars/tasks/main.yml | 20 +- + .../tests/cli/merged.yaml | 9 +- + .../tests/cli/replaced.yaml | 10 +- + .../test_lookup_properties.yml | 2 +- + .../modules_test_multiple_roles.yml | 2 +- + ...ules_test_multiple_roles_reverse_order.yml | 2 +- + .../multiple_roles/bar/tasks/main.yml | 2 +- + .../multiple_roles/foo/tasks/main.yml | 2 +- + .../integration/targets/script/tasks/main.yml | 4 +- + test/integration/targets/slurp/tasks/main.yml | 2 +- + .../targets/template/tasks/main.yml | 2 +- + .../targets/unarchive/tasks/test_mode.yml | 8 +- + .../tasks/test_unprivileged_user.yml | 2 +- + .../targets/unarchive/tasks/test_zip.yml | 2 +- + .../roles/test_vault_embedded/tasks/main.yml | 2 +- + .../tasks/main.yml | 2 +- + .../vyos_config/tests/cli/check_config.yaml | 4 +- + .../vyos_interfaces/tests/cli/deleted.yaml | 9 +- + .../vyos_interfaces/tests/cli/overridden.yaml | 10 +- + .../targets/wait_for/tasks/main.yml | 20 +- + .../module_utils/common/test_collections.py | 15 +- + test/units/parsing/test_ajson.py | 1 + + test/units/parsing/yaml/test_dumper.py | 20 +- + 59 files changed, 828 insertions(+), 126 deletions(-) + create mode 100644 changelogs/fragments/70784-vault-is-string.yml + create mode 100644 changelogs/fragments/71609-is_string-vault.yml + create mode 100644 changelogs/fragments/75072_undefined_yaml.yml + create mode 100644 changelogs/fragments/cve-2023-5764.yml + create mode 100644 changelogs/fragments/nativejinjatext-yaml-representer.yml + create mode 100644 changelogs/fragments/nativejinjaunsafetext-yaml-representer.yml + create mode 100644 test/integration/targets/assert/assert.out.nested_tmpl.stderr + create mode 100644 test/integration/targets/assert/assert.out.nested_tmpl.stdout + rename test/integration/targets/assert/{assert_quiet.out.quiet.stderr => assert.out.quiet.stderr} (100%) + rename test/integration/targets/assert/{assert_quiet.out.quiet.stdout => assert.out.quiet.stdout} (100%) + create mode 100644 test/integration/targets/assert/nested_tmpl.yml + create mode 100644 test/integration/targets/debug/unsafe.yml + +diff --git a/changelogs/fragments/70784-vault-is-string.yml b/changelogs/fragments/70784-vault-is-string.yml +new file mode 100644 +index 0000000000..8dc1164a85 +--- /dev/null ++++ b/changelogs/fragments/70784-vault-is-string.yml +@@ -0,0 +1,3 @@ ++bugfixes: ++- JSON Encoder - Ensure we treat single vault encrypted values as strings ++ (https://github.com/ansible/ansible/issues/70784) +diff --git a/changelogs/fragments/71609-is_string-vault.yml b/changelogs/fragments/71609-is_string-vault.yml +new file mode 100644 +index 0000000000..89ddd91913 +--- /dev/null ++++ b/changelogs/fragments/71609-is_string-vault.yml +@@ -0,0 +1,3 @@ ++bugfixes: ++- is_string/vault - Ensure the is_string helper properly identifies AnsibleVaultEncryptedUnicode ++ as a string (https://github.com/ansible/ansible/pull/71609) +diff --git a/changelogs/fragments/75072_undefined_yaml.yml b/changelogs/fragments/75072_undefined_yaml.yml +new file mode 100644 +index 0000000000..227c24de1b +--- /dev/null ++++ b/changelogs/fragments/75072_undefined_yaml.yml +@@ -0,0 +1,3 @@ ++--- ++minor_changes: ++- yaml dumper - YAML representer for AnsibleUndefined (https://github.com/ansible/ansible/issues/75072). +diff --git a/changelogs/fragments/cve-2023-5764.yml b/changelogs/fragments/cve-2023-5764.yml +new file mode 100644 +index 0000000000..c37127dac1 +--- /dev/null ++++ b/changelogs/fragments/cve-2023-5764.yml +@@ -0,0 +1,6 @@ ++security_fixes: ++- templating - Address issues where internal templating can cause unsafe ++ variables to lose their unsafe designation (CVE-2023-5764) ++breaking_changes: ++- assert - Nested templating may result in an inability for the conditional ++ to be evaluated. See the porting guide for more information. +diff --git a/changelogs/fragments/nativejinjatext-yaml-representer.yml b/changelogs/fragments/nativejinjatext-yaml-representer.yml +new file mode 100644 +index 0000000000..ef2f460a09 +--- /dev/null ++++ b/changelogs/fragments/nativejinjatext-yaml-representer.yml +@@ -0,0 +1,2 @@ ++bugfixes: ++ - Add a YAML representer for ``NativeJinjaText`` +diff --git a/changelogs/fragments/nativejinjaunsafetext-yaml-representer.yml b/changelogs/fragments/nativejinjaunsafetext-yaml-representer.yml +new file mode 100644 +index 0000000000..e13486fb30 +--- /dev/null ++++ b/changelogs/fragments/nativejinjaunsafetext-yaml-representer.yml +@@ -0,0 +1,2 @@ ++bugfixes: ++ - Add a YAML representer for ``NativeJinjaUnsafeText`` +diff --git a/lib/ansible/module_utils/common/collections.py b/lib/ansible/module_utils/common/collections.py +index 0a166cd4cf..123ee8354b 100644 +--- a/lib/ansible/module_utils/common/collections.py ++++ b/lib/ansible/module_utils/common/collections.py +@@ -58,7 +58,8 @@ class ImmutableDict(Hashable, Mapping): + + def is_string(seq): + """Identify whether the input has a string-like type (inclding bytes).""" +- return isinstance(seq, (text_type, binary_type)) ++ # AnsibleVaultEncryptedUnicode inherits from Sequence, but is expected to be a string like object ++ return isinstance(seq, (text_type, binary_type)) or getattr(seq, '__ENCRYPTED__', False) + + + def is_iterable(seq, include_strings=False): +diff --git a/lib/ansible/module_utils/common/json.py b/lib/ansible/module_utils/common/json.py +index 3018e9e238..41f9b85fb7 100644 +--- a/lib/ansible/module_utils/common/json.py ++++ b/lib/ansible/module_utils/common/json.py +@@ -15,14 +15,22 @@ from ansible.module_utils.common._collections_compat import Mapping + from ansible.module_utils.common.collections import is_sequence + + ++def _is_unsafe(value): ++ return getattr(value, '__UNSAFE__', False) and not getattr(value, '__ENCRYPTED__', False) ++ ++ ++def _is_vault(value): ++ return getattr(value, '__ENCRYPTED__', False) ++ ++ + def _preprocess_unsafe_encode(value): + """Recursively preprocess a data structure converting instances of ``AnsibleUnsafe`` + into their JSON dict representations + + Used in ``AnsibleJSONEncoder.iterencode`` + """ +- if getattr(value, '__UNSAFE__', False) and not getattr(value, '__ENCRYPTED__', False): +- value = {'__ansible_unsafe': to_text(value, errors='surrogate_or_strict', nonstring='strict')} ++ if _is_unsafe(value): ++ value = {'__ansible_unsafe': to_text(value._strip_unsafe(), errors='surrogate_or_strict', nonstring='strict')} + elif is_sequence(value): + value = [_preprocess_unsafe_encode(v) for v in value] + elif isinstance(value, Mapping): +@@ -51,7 +59,7 @@ class AnsibleJSONEncoder(json.JSONEncoder): + value = {'__ansible_vault': to_text(o._ciphertext, errors='surrogate_or_strict', nonstring='strict')} + elif getattr(o, '__UNSAFE__', False): + # unsafe object, this will never be triggered, see ``AnsibleJSONEncoder.iterencode`` +- value = {'__ansible_unsafe': to_text(o, errors='surrogate_or_strict', nonstring='strict')} ++ value = {'__ansible_unsafe': to_text(o._strip_unsafe(), errors='surrogate_or_strict', nonstring='strict')} + elif isinstance(o, Mapping): + # hostvars and other objects + value = dict(o) +diff --git a/lib/ansible/parsing/yaml/dumper.py b/lib/ansible/parsing/yaml/dumper.py +index 67a2efb36d..ddb1363a9e 100644 +--- a/lib/ansible/parsing/yaml/dumper.py ++++ b/lib/ansible/parsing/yaml/dumper.py +@@ -21,9 +21,10 @@ __metaclass__ = type + + import yaml + +-from ansible.module_utils.six import PY3 ++from ansible.module_utils.six import PY3, text_type + from ansible.parsing.yaml.objects import AnsibleUnicode, AnsibleSequence, AnsibleMapping, AnsibleVaultEncryptedUnicode +-from ansible.utils.unsafe_proxy import AnsibleUnsafeText, AnsibleUnsafeBytes ++from ansible.utils.unsafe_proxy import AnsibleUnsafeText, AnsibleUnsafeBytes, NativeJinjaUnsafeText, NativeJinjaText, _is_unsafe ++from ansible.template import AnsibleUndefined + from ansible.vars.hostvars import HostVars, HostVarsVars + + +@@ -44,12 +45,30 @@ def represent_vault_encrypted_unicode(self, data): + return self.represent_scalar(u'!vault', data._ciphertext.decode(), style='|') + + +-if PY3: +- represent_unicode = yaml.representer.SafeRepresenter.represent_str +- represent_binary = yaml.representer.SafeRepresenter.represent_binary +-else: +- represent_unicode = yaml.representer.SafeRepresenter.represent_unicode +- represent_binary = yaml.representer.SafeRepresenter.represent_str ++def represent_unicode(self, data): ++ if _is_unsafe(data): ++ data = data._strip_unsafe() ++ if PY3: ++ return yaml.representer.SafeRepresenter.represent_str(self, text_type(data)) ++ else: ++ return yaml.representer.SafeRepresenter.represent_unicode(self, text_type(data)) ++ ++ ++def represent_binary(self, data): ++ if _is_unsafe(data): ++ data = data._strip_unsafe() ++ if PY3: ++ return yaml.representer.SafeRepresenter.represent_binary(self, binary_type(data)) ++ else: ++ return yaml.representer.SafeRepresenter.represent_str(self, binary_type(data)) ++ ++ ++def represent_undefined(self, data): ++ # Here bool will ensure _fail_with_undefined_error happens ++ # if the value is Undefined. ++ # This happens because Jinja sets __bool__ on StrictUndefined ++ return bool(data) ++ + + AnsibleDumper.add_representer( + AnsibleUnicode, +@@ -90,3 +109,18 @@ AnsibleDumper.add_representer( + AnsibleVaultEncryptedUnicode, + represent_vault_encrypted_unicode, + ) ++ ++AnsibleDumper.add_representer( ++ AnsibleUndefined, ++ represent_undefined, ++) ++ ++AnsibleDumper.add_representer( ++ NativeJinjaUnsafeText, ++ represent_unicode, ++) ++ ++AnsibleDumper.add_representer( ++ NativeJinjaText, ++ represent_unicode, ++) +diff --git a/lib/ansible/playbook/conditional.py b/lib/ansible/playbook/conditional.py +index 2fadf77487..ac4fc0c568 100644 +--- a/lib/ansible/playbook/conditional.py ++++ b/lib/ansible/playbook/conditional.py +@@ -26,7 +26,7 @@ from jinja2.compiler import generate + from jinja2.exceptions import UndefinedError + + from ansible import constants as C +-from ansible.errors import AnsibleError, AnsibleUndefinedVariable ++from ansible.errors import AnsibleError, AnsibleUndefinedVariable, AnsibleTemplateError + from ansible.module_utils.six import text_type + from ansible.module_utils._text import to_native + from ansible.playbook.attribute import FieldAttribute +@@ -138,9 +138,10 @@ class Conditional: + if not isinstance(conditional, text_type) or conditional == "": + return conditional + +- # update the lookups flag, as the string returned above may now be unsafe +- # and we don't want future templating calls to do unsafe things +- disable_lookups |= hasattr(conditional, '__UNSAFE__') ++ # If the result of the first-pass template render (to resolve inline templates) is marked unsafe, ++ # explicitly fail since the next templating operation would never evaluate ++ if hasattr(conditional, '__UNSAFE__'): ++ raise AnsibleTemplateError('Conditional is marked as unsafe, and cannot be evaluated.') + + # First, we do some low-level jinja2 parsing involving the AST format of the + # statement to ensure we don't do anything unsafe (using the disable_lookup flag above) +diff --git a/lib/ansible/playbook/task.py b/lib/ansible/playbook/task.py +index 7fd480c895..e24d27ba8a 100644 +--- a/lib/ansible/playbook/task.py ++++ b/lib/ansible/playbook/task.py +@@ -298,6 +298,30 @@ class Task(Base, Conditional, Taggable, CollectionSearch): + + super(Task, self).post_validate(templar) + ++ def _post_validate_args(self, attr, value, templar): ++ # smuggle an untemplated copy of the task args for actions that need more control over the templating of their ++ # input (eg, debug's var/msg, assert's "that" conditional expressions) ++ self.untemplated_args = value ++ ++ # now recursively template the args dict ++ args = templar.template(value) ++ ++ # FIXME: could we just nuke this entirely and/or wrap it up in ModuleArgsParser or something? ++ if '_variable_params' in args: ++ variable_params = args.pop('_variable_params') ++ if isinstance(variable_params, dict): ++ if C.INJECT_FACTS_AS_VARS: ++ display.warning("Using a variable for a task's 'args' is unsafe in some situations " ++ "(see https://docs.ansible.com/ansible/devel/reference_appendices/faq.html#argsplat-unsafe)") ++ variable_params.update(args) ++ args = variable_params ++ else: ++ # if we didn't get a dict, it means there's garbage remaining after k=v parsing, just give up ++ # see https://github.com/ansible/ansible/issues/79862 ++ raise AnsibleError(f"invalid or malformed argument: '{variable_params}'") ++ ++ return args ++ + def _post_validate_loop(self, attr, value, templar): + ''' + Override post validation for the loop field, which is templated +diff --git a/lib/ansible/plugins/action/assert.py b/lib/ansible/plugins/action/assert.py +index 7721a6b47c..e8ab6a9a4f 100644 +--- a/lib/ansible/plugins/action/assert.py ++++ b/lib/ansible/plugins/action/assert.py +@@ -63,8 +63,29 @@ class ActionModule(ActionBase): + + quiet = boolean(self._task.args.get('quiet', False), strict=False) + ++ # directly access 'that' via untemplated args from the task so we can intelligently trust embedded ++ # templates and preserve the original inputs/locations for better messaging on assert failures and ++ # errors. ++ # FIXME: even in devel, things like `that: item` don't always work properly (truthy string value ++ # is not really an embedded expression) ++ # we could fix that by doing direct var lookups on the inputs ++ # FIXME: some form of this code should probably be shared between debug, assert, and ++ # Task.post_validate, since they ++ # have a lot of overlapping needs ++ try: ++ thats = self._task.untemplated_args['that'] ++ except KeyError: ++ # in the case of "we got our entire args dict from a template", we can just consult the ++ # post-templated dict (the damage has likely already been done for embedded templates anyway) ++ thats = self._task.args['that'] ++ ++ # FIXME: this is a case where we only want to resolve indirections, NOT recurse containers ++ # (and even then, the leaf-most expression being wrapped is at least suboptimal ++ # (since its expression will be "eaten"). ++ if isinstance(thats, str): ++ thats = self._templar.template(thats) ++ + # make sure the 'that' items are a list +- thats = self._task.args['that'] + if not isinstance(thats, list): + thats = [thats] + +diff --git a/lib/ansible/plugins/callback/__init__.py b/lib/ansible/plugins/callback/__init__.py +index 71287f8b5e..08b3b4cb29 100644 +--- a/lib/ansible/plugins/callback/__init__.py ++++ b/lib/ansible/plugins/callback/__init__.py +@@ -22,6 +22,7 @@ __metaclass__ = type + import difflib + import json + import os ++import re + import sys + import warnings + +@@ -29,12 +30,15 @@ from copy import deepcopy + + from ansible import constants as C + from ansible.module_utils.common._collections_compat import MutableMapping +-from ansible.module_utils.six import PY3 ++from ansible.module_utils.six import PY3, text_type + from ansible.module_utils._text import to_text + from ansible.parsing.ajson import AnsibleJSONEncoder ++from ansible.parsing.yaml.dumper import AnsibleDumper ++from ansible.parsing.yaml.objects import AnsibleUnicode + from ansible.plugins import AnsiblePlugin, get_plugin_class + from ansible.utils.color import stringc + from ansible.utils.display import Display ++from ansible.utils.unsafe_proxy import AnsibleUnsafeText, NativeJinjaUnsafeText, _is_unsafe + from ansible.vars.clean import strip_internal_keys, module_response_deepcopy + + if PY3: +@@ -51,6 +55,90 @@ __all__ = ["CallbackBase"] + + + _DEBUG_ALLOWED_KEYS = frozenset(('msg', 'exception', 'warnings', 'deprecations')) ++_YAML_TEXT_TYPES = (text_type, AnsibleUnicode, AnsibleUnsafeText, NativeJinjaUnsafeText) ++# Characters that libyaml/pyyaml consider breaks ++_YAML_BREAK_CHARS = '\n\x85\u2028\u2029' # NL, NEL, LS, PS ++# regex representation of libyaml/pyyaml of a space followed by a break character ++_SPACE_BREAK_RE = re.compile(fr' +([{_YAML_BREAK_CHARS}])') ++ ++ ++class _AnsibleCallbackDumper(AnsibleDumper): ++ def __init__(self, lossy=False): ++ self._lossy = lossy ++ ++ def __call__(self, *args, **kwargs): ++ # pyyaml expects that we are passing an object that can be instantiated, but to ++ # smuggle the ``lossy`` configuration, we do that in ``__init__`` and then ++ # define this ``__call__`` that will mimic the ability for pyyaml to instantiate class ++ super().__init__(*args, **kwargs) ++ return self ++ ++ ++def _should_use_block(scalar): ++ """Returns true if string should be in block format based on the existence of various newline separators""" ++ # This method of searching is faster than using a regex ++ for ch in _YAML_BREAK_CHARS: ++ if ch in scalar: ++ return True ++ return False ++ ++ ++class _SpecialCharacterTranslator: ++ def __getitem__(self, ch): ++ # "special character" logic from pyyaml yaml.emitter.Emitter.analyze_scalar, translated to decimal ++ # for perf w/ str.translate ++ if (ch == 10 or ++ 32 <= ch <= 126 or ++ ch == 133 or ++ 160 <= ch <= 55295 or ++ 57344 <= ch <= 65533 or ++ 65536 <= ch < 1114111)\ ++ and ch != 65279: ++ return ch ++ return None ++ ++ ++def _filter_yaml_special(scalar): ++ """Filter a string removing any character that libyaml/pyyaml declare as special""" ++ return scalar.translate(_SpecialCharacterTranslator()) ++ ++ ++def _munge_data_for_lossy_yaml(scalar): ++ """Modify a string so that analyze_scalar in libyaml/pyyaml will allow block formatting""" ++ # we care more about readability than accuracy, so... ++ # ...libyaml/pyyaml does not permit trailing spaces for block scalars ++ scalar = scalar.rstrip() ++ # ...libyaml/pyyaml does not permit tabs for block scalars ++ scalar = scalar.expandtabs() ++ # ...libyaml/pyyaml only permits special characters for double quoted scalars ++ scalar = _filter_yaml_special(scalar) ++ # ...libyaml/pyyaml only permits spaces followed by breaks for double quoted scalars ++ return _SPACE_BREAK_RE.sub(r'\1', scalar) ++ ++ ++def _pretty_represent_str(self, data): ++ """Uses block style for multi-line strings""" ++ if _is_unsafe(data): ++ data = data._strip_unsafe() ++ data = text_type(data) ++ if _should_use_block(data): ++ style = '|' ++ if self._lossy: ++ data = _munge_data_for_lossy_yaml(data) ++ else: ++ style = self.default_style ++ ++ node = yaml.representer.ScalarNode('tag:yaml.org,2002:str', data, style=style) ++ if self.alias_key is not None: ++ self.represented_objects[self.alias_key] = node ++ return node ++ ++ ++for data_type in _YAML_TEXT_TYPES: ++ _AnsibleCallbackDumper.add_representer( ++ data_type, ++ _pretty_represent_str ++ ) + + + class CallbackBase(AnsiblePlugin): +diff --git a/lib/ansible/plugins/filter/core.py b/lib/ansible/plugins/filter/core.py +index 1fc6790a4d..d39d8426bc 100644 +--- a/lib/ansible/plugins/filter/core.py ++++ b/lib/ansible/plugins/filter/core.py +@@ -53,6 +53,7 @@ from ansible.utils.display import Display + from ansible.utils.encrypt import passlib_or_crypt + from ansible.utils.hashing import md5s, checksum_s + from ansible.utils.unicode import unicode_wrap ++from ansible.utils.unsafe_proxy import _is_unsafe + from ansible.utils.vars import merge_hash + + display = Display() +@@ -63,13 +64,19 @@ UUID_NAMESPACE_ANSIBLE = uuid.UUID('361E6D51-FAEC-444A-9079-341386DA8E2E') + def to_yaml(a, *args, **kw): + '''Make verbose, human readable yaml''' + default_flow_style = kw.pop('default_flow_style', None) +- transformed = yaml.dump(a, Dumper=AnsibleDumper, allow_unicode=True, default_flow_style=default_flow_style, **kw) ++ try: ++ transformed = yaml.dump(a, Dumper=AnsibleDumper, allow_unicode=True, default_flow_style=default_flow_style, **kw) ++ except Exception as e: ++ raise AnsibleFilterError("to_yaml - %s" % to_native(e), orig_exc=e) + return to_text(transformed) + + + def to_nice_yaml(a, indent=4, *args, **kw): + '''Make verbose, human readable yaml''' +- transformed = yaml.dump(a, Dumper=AnsibleDumper, indent=indent, allow_unicode=True, default_flow_style=False, **kw) ++ try: ++ transformed = yaml.dump(a, Dumper=AnsibleDumper, indent=indent, allow_unicode=True, default_flow_style=False, **kw) ++ except Exception as e: ++ raise AnsibleFilterError("to_nice_yaml - %s" % to_native(e), orig_exc=e) + return to_text(transformed) + + +@@ -207,13 +214,23 @@ def regex_escape(string, re_type='python'): + + def from_yaml(data): + if isinstance(data, string_types): +- return yaml.safe_load(data) ++ # The ``text_type`` call here strips any custom ++ # string wrapper class, so that CSafeLoader can ++ # read the data ++ if _is_unsafe(data): ++ data = data._strip_unsafe() ++ return yaml_load(text_type(to_text(data, errors='surrogate_or_strict'))) + return data + + + def from_yaml_all(data): + if isinstance(data, string_types): +- return yaml.safe_load_all(data) ++ # The ``text_type`` call here strips any custom ++ # string wrapper class, so that CSafeLoader can ++ # read the data ++ if _is_unsafe(data): ++ data = data._strip_unsafe() ++ return yaml_load_all(text_type(to_text(data, errors='surrogate_or_strict'))) + return data + + +diff --git a/lib/ansible/plugins/lookup/first_found.py b/lib/ansible/plugins/lookup/first_found.py +index a1828e6b9c..39a2f1aac3 100644 +--- a/lib/ansible/plugins/lookup/first_found.py ++++ b/lib/ansible/plugins/lookup/first_found.py +@@ -102,6 +102,8 @@ RETURN = """ + """ + import os + ++from collections.abc import Mapping, Sequence ++ + from jinja2.exceptions import UndefinedError + + from ansible.errors import AnsibleFileNotFound, AnsibleLookupError, AnsibleUndefinedVariable +@@ -110,6 +112,29 @@ from ansible.module_utils.parsing.convert_bool import boolean + from ansible.plugins.lookup import LookupBase + + ++def _splitter(value, chars): ++ chars = set(chars) ++ v = '' ++ for c in value: ++ if c in chars: ++ yield v ++ v = '' ++ continue ++ v += c ++ yield v ++ ++ ++def _split_on(terms, spliters=','): ++ termlist = [] ++ if isinstance(terms, string_types): ++ termlist = list(_splitter(terms, spliters)) ++ else: ++ # added since options will already listify ++ for t in terms: ++ termlist.extend(_split_on(t, spliters)) ++ return termlist ++ ++ + class LookupModule(LookupBase): + + def run(self, terms, variables, **kwargs): +diff --git a/lib/ansible/template/__init__.py b/lib/ansible/template/__init__.py +index a20b1bae68..94ab31e58d 100644 +--- a/lib/ansible/template/__init__.py ++++ b/lib/ansible/template/__init__.py +@@ -35,7 +35,7 @@ try: + except ImportError: + from sha import sha as sha1 + +-from jinja2.exceptions import TemplateSyntaxError, UndefinedError ++from jinja2.exceptions import TemplateSyntaxError, UndefinedError, SecurityError + from jinja2.loaders import FileSystemLoader + from jinja2.runtime import Context, StrictUndefined + +@@ -53,6 +53,9 @@ from ansible.template.vars import AnsibleJ2Vars + from ansible.utils.collection_loader import AnsibleCollectionRef + from ansible.utils.display import Display + from ansible.utils.unsafe_proxy import wrap_var ++from ansible.utils.listify import listify_lookup_plugin_terms ++from ansible.utils.native_jinja import NativeJinjaText ++from ansible.utils.unsafe_proxy import wrap_var, AnsibleUnsafeText, AnsibleUnsafeBytes, NativeJinjaUnsafeText + + display = Display() + +@@ -259,10 +262,21 @@ class AnsibleContext(Context): + flag is checked post-templating, and (when set) will result in the + final templated result being wrapped in AnsibleUnsafe. + ''' ++ _disallowed_callables = frozenset({ ++ AnsibleUnsafeText._strip_unsafe.__qualname__, ++ AnsibleUnsafeBytes._strip_unsafe.__qualname__, ++ NativeJinjaUnsafeText._strip_unsafe.__qualname__, ++ }) ++ + def __init__(self, *args, **kwargs): + super(AnsibleContext, self).__init__(*args, **kwargs) + self.unsafe = False + ++ def call(self, obj, *args, **kwargs): ++ if getattr(obj, '__qualname__', None) in self._disallowed_callables or obj in self._disallowed_callables: ++ raise SecurityError(f"{obj!r} is not safely callable") ++ return super().call(obj, *args, **kwargs) ++ + def _is_unsafe(self, val): + ''' + Our helper function, which will also recursively check dict and +@@ -874,8 +888,11 @@ class Templar: + rf = t.root_render_func(new_context) + + try: +- res = j2_concat(rf) +- unsafe = getattr(new_context, 'unsafe', False) ++ if self.jinja2_native: ++ res = ansible_native_concat(rf) ++ else: ++ res = j2_concat(rf) ++ unsafe = getattr(self.cur_context, 'unsafe', False) + if unsafe: + res = wrap_var(res) + except TypeError as te: +diff --git a/lib/ansible/utils/unsafe_proxy.py b/lib/ansible/utils/unsafe_proxy.py +index ffb5f37503..0bfd6340ff 100644 +--- a/lib/ansible/utils/unsafe_proxy.py ++++ b/lib/ansible/utils/unsafe_proxy.py +@@ -67,10 +67,267 @@ class AnsibleUnsafe(object): + + + class AnsibleUnsafeBytes(binary_type, AnsibleUnsafe): +- pass ++ def _strip_unsafe(self): ++ return super().__bytes__() ++ ++ def __str__(self): # pylint: disable=invalid-str-returned ++ return self.encode() ++ ++ def __bytes__(self): # pylint: disable=invalid-bytes-returned ++ return self ++ ++ def __repr__(self): # pylint: disable=invalid-repr-returned ++ return AnsibleUnsafeText(super().__repr__()) ++ ++ def __format__(self, format_spec): # pylint: disable=invalid-format-returned ++ return self.__class__(super().__format__(format_spec)) ++ ++ def __getitem__(self, key): ++ return self.__class__(super().__getitem__(key)) ++ ++ def __iter__(self): ++ cls = self.__class__ ++ return (cls(c) for c in super().__iter__()) ++ ++ def __reversed__(self): ++ return self[::-1] ++ ++ def __add__(self, value): ++ return self.__class__(super().__add__(value)) ++ ++ def __radd__(self, value): ++ return self.__class__(value.__add__(self)) ++ ++ def __mul__(self, value): ++ return self.__class__(super().__mul__(value)) ++ ++ __rmul__ = __mul__ ++ ++ def __mod__(self, value): ++ return self.__class__(super().__mod__(value)) ++ ++ def __rmod__(self, value): ++ return self.__class__(super().__rmod__(value)) ++ ++ def capitalize(self): ++ return self.__class__(super().capitalize()) ++ ++ def casefold(self): ++ return self.__class__(super().casefold()) ++ ++ def center(self, width, fillchar=b' '): ++ return self.__class__(super().center(width, fillchar)) ++ ++ def decode(self, encoding='utf-8', errors='strict'): ++ return AnsibleUnsafeText(super().decode(encoding=encoding, errors=errors)) ++ ++ def removeprefix(self, prefix): ++ return self.__class__(super().removeprefix(prefix)) ++ ++ def removesuffix(self, suffix): ++ return self.__class__(super().removesuffix(suffix)) ++ ++ def expandtabs(self, tabsize=8): ++ return self.__class__(super().expandtabs(tabsize)) ++ ++ def format(self, *args, **kwargs): ++ return self.__class__(super().format(*args, **kwargs)) ++ ++ def format_map(self, mapping): ++ return self.__class__(super().format_map(mapping)) ++ ++ def join(self, iterable_of_bytes): ++ return self.__class__(super().join(iterable_of_bytes)) ++ ++ def ljust(self, width, fillchar=b' '): ++ return self.__class__(super().ljust(width, fillchar)) ++ ++ def lower(self): ++ return self.__class__(super().lower()) ++ ++ def lstrip(self, bytes=None): ++ return self.__class__(super().lstrip(bytes)) ++ ++ def partition(self, sep): ++ cls = self.__class__ ++ return tuple(cls(e) for e in super().partition(sep)) ++ ++ def replace(self, old, new, count=-1): ++ return self.__class__(super().replace(old, new, count)) ++ ++ def rjust(self, width, fillchar=b' '): ++ return self.__class__(super().rjust(width, fillchar)) ++ ++ def rpartition(self, sep): ++ cls = self.__class__ ++ return tuple(cls(e) for e in super().rpartition(sep)) ++ ++ def rstrip(self, bytes=None): ++ return self.__class__(super().rstrip(bytes)) ++ ++ def split(self, sep=None, maxsplit=-1): ++ cls = self.__class__ ++ return [cls(e) for e in super().split(sep=sep, maxsplit=maxsplit)] ++ ++ def rsplit(self, sep=None, maxsplit=-1): ++ cls = self.__class__ ++ return [cls(e) for e in super().rsplit(sep=sep, maxsplit=maxsplit)] ++ ++ def splitlines(self, keepends=False): ++ cls = self.__class__ ++ return [cls(e) for e in super().splitlines(keepends=keepends)] ++ ++ def strip(self, bytes=None): ++ return self.__class__(super().strip(bytes)) ++ ++ def swapcase(self): ++ return self.__class__(super().swapcase()) ++ ++ def title(self): ++ return self.__class__(super().title()) ++ ++ def translate(self, table, delete=b''): ++ return self.__class__(super().translate(table, delete=delete)) ++ ++ def upper(self): ++ return self.__class__(super().upper()) ++ ++ def zfill(self, width): ++ return self.__class__(super().zfill(width)) + + + class AnsibleUnsafeText(text_type, AnsibleUnsafe): ++ # def __getattribute__(self, name): ++ # print(f'attr: {name}') ++ # return object.__getattribute__(self, name) ++ ++ def _strip_unsafe(self): ++ return super().__str__() ++ ++ def __str__(self): # pylint: disable=invalid-str-returned ++ return self ++ ++ def __repr__(self): # pylint: disable=invalid-repr-returned ++ return self.__class__(super().__repr__()) ++ ++ def __format__(self, format_spec): # pylint: disable=invalid-format-returned ++ return self.__class__(super().__format__(format_spec)) ++ ++ def __getitem__(self, key): ++ return self.__class__(super().__getitem__(key)) ++ ++ def __iter__(self): ++ cls = self.__class__ ++ return (cls(c) for c in super().__iter__()) ++ ++ def __reversed__(self): ++ return self[::-1] ++ ++ def __add__(self, value): ++ return self.__class__(super().__add__(value)) ++ ++ def __radd__(self, value): ++ return self.__class__(value.__add__(self)) ++ ++ def __mul__(self, value): ++ return self.__class__(super().__mul__(value)) ++ ++ __rmul__ = __mul__ ++ ++ def __mod__(self, value): ++ return self.__class__(super().__mod__(value)) ++ ++ def __rmod__(self, value): ++ return self.__class__(super().__rmod__(value)) ++ ++ def capitalize(self): ++ return self.__class__(super().capitalize()) ++ ++ def casefold(self): ++ return self.__class__(super().casefold()) ++ ++ def center(self, width, fillchar=' '): ++ return self.__class__(super().center(width, fillchar)) ++ ++ def encode(self, encoding='utf-8', errors='strict'): ++ return AnsibleUnsafeBytes(super().encode(encoding=encoding, errors=errors)) ++ ++ def removeprefix(self, prefix): ++ return self.__class__(super().removeprefix(prefix)) ++ ++ def removesuffix(self, suffix): ++ return self.__class__(super().removesuffix(suffix)) ++ ++ def expandtabs(self, tabsize=8): ++ return self.__class__(super().expandtabs(tabsize)) ++ ++ def format(self, *args, **kwargs): ++ return self.__class__(super().format(*args, **kwargs)) ++ ++ def format_map(self, mapping): ++ return self.__class__(super().format_map(mapping)) ++ ++ def join(self, iterable): ++ return self.__class__(super().join(iterable)) ++ ++ def ljust(self, width, fillchar=' '): ++ return self.__class__(super().ljust(width, fillchar)) ++ ++ def lower(self): ++ return self.__class__(super().lower()) ++ ++ def lstrip(self, chars=None): ++ return self.__class__(super().lstrip(chars)) ++ ++ def partition(self, sep): ++ cls = self.__class__ ++ return tuple(cls(e) for e in super().partition(sep)) ++ ++ def replace(self, old, new, count=-1): ++ return self.__class__(super().replace(old, new, count)) ++ ++ def rjust(self, width, fillchar=' '): ++ return self.__class__(super().rjust(width, fillchar)) ++ ++ def rpartition(self, sep): ++ cls = self.__class__ ++ return tuple(cls(e) for e in super().rpartition(sep)) ++ ++ def rstrip(self, chars=None): ++ return self.__class__(super().rstrip(chars)) ++ ++ def split(self, sep=None, maxsplit=-1): ++ cls = self.__class__ ++ return [cls(e) for e in super().split(sep=sep, maxsplit=maxsplit)] ++ ++ def rsplit(self, sep=None, maxsplit=-1): ++ cls = self.__class__ ++ return [cls(e) for e in super().rsplit(sep=sep, maxsplit=maxsplit)] ++ ++ def splitlines(self, keepends=False): ++ cls = self.__class__ ++ return [cls(e) for e in super().splitlines(keepends=keepends)] ++ ++ def strip(self, chars=None): ++ return self.__class__(super().strip(chars)) ++ ++ def swapcase(self): ++ return self.__class__(super().swapcase()) ++ ++ def title(self): ++ return self.__class__(super().title()) ++ ++ def translate(self, table): ++ return self.__class__(super().translate(table)) ++ ++ def upper(self): ++ return self.__class__(super().upper()) ++ ++ def zfill(self, width): ++ return self.__class__(super().zfill(width)) ++ ++ ++class NativeJinjaUnsafeText(NativeJinjaText, AnsibleUnsafeText): + pass + + +@@ -133,3 +390,7 @@ def to_unsafe_bytes(*args, **kwargs): + + def to_unsafe_text(*args, **kwargs): + return wrap_var(to_text(*args, **kwargs)) ++ ++ ++def _is_unsafe(obj): ++ return getattr(obj, '__UNSAFE__', False) is True +diff --git a/test/integration/targets/apt_repository/tasks/apt.yml b/test/integration/targets/apt_repository/tasks/apt.yml +index 941335f2c6..8d0f4ad896 100644 +--- a/test/integration/targets/apt_repository/tasks/apt.yml ++++ b/test/integration/targets/apt_repository/tasks/apt.yml +@@ -50,7 +50,7 @@ + that: + - 'result.changed' + - 'result.state == "present"' +- - 'result.repo == "{{test_ppa_name}}"' ++ - 'result.repo == test_ppa_name' + + - name: 'examine apt cache mtime' + stat: path='/var/cache/apt/pkgcache.bin' +@@ -81,7 +81,7 @@ + that: + - 'result.changed' + - 'result.state == "present"' +- - 'result.repo == "{{test_ppa_name}}"' ++ - 'result.repo == test_ppa_name' + + - name: 'examine apt cache mtime' + stat: path='/var/cache/apt/pkgcache.bin' +@@ -112,7 +112,7 @@ + that: + - 'result.changed' + - 'result.state == "present"' +- - 'result.repo == "{{test_ppa_name}}"' ++ - 'result.repo == test_ppa_name' + + - name: 'examine apt cache mtime' + stat: path='/var/cache/apt/pkgcache.bin' +@@ -146,7 +146,8 @@ + that: + - 'result.changed' + - 'result.state == "present"' +- - 'result.repo == "{{test_ppa_spec}}"' ++ - 'result.repo == test_ppa_spec' ++ - result_cache is not changed + + - name: 'examine apt cache mtime' + stat: path='/var/cache/apt/pkgcache.bin' +@@ -181,7 +182,7 @@ + that: + - 'result.changed' + - 'result.state == "present"' +- - 'result.repo == "{{test_ppa_spec}}"' ++ - 'result.repo == test_ppa_spec' + + - name: 'examine source file' + stat: path='/etc/apt/sources.list.d/{{test_ppa_filename}}.list' +diff --git a/test/integration/targets/assert/assert.out.nested_tmpl.stderr b/test/integration/targets/assert/assert.out.nested_tmpl.stderr +new file mode 100644 +index 0000000000..ea208a41c7 +--- /dev/null ++++ b/test/integration/targets/assert/assert.out.nested_tmpl.stderr +@@ -0,0 +1,4 @@ +++ ansible-playbook -i localhost, -c local nested_tmpl.yml ++++ set +x ++[WARNING]: conditional statements should not include jinja2 templating ++delimiters such as {{ }} or {% %}. Found: "{{ foo }}" == "bar" +diff --git a/test/integration/targets/assert/assert.out.nested_tmpl.stdout b/test/integration/targets/assert/assert.out.nested_tmpl.stdout +new file mode 100644 +index 0000000000..8ca3fb76d4 +--- /dev/null ++++ b/test/integration/targets/assert/assert.out.nested_tmpl.stdout +@@ -0,0 +1,12 @@ ++ ++PLAY [localhost] *************************************************************** ++ ++TASK [assert] ****************************************************************** ++ok: [localhost] => { ++ "changed": false, ++ "msg": "All assertions passed" ++} ++ ++PLAY RECAP ********************************************************************* ++localhost : ok=1 changed=0 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 ++ +diff --git a/test/integration/targets/assert/assert_quiet.out.quiet.stderr b/test/integration/targets/assert/assert.out.quiet.stderr +similarity index 100% +rename from test/integration/targets/assert/assert_quiet.out.quiet.stderr +rename to test/integration/targets/assert/assert.out.quiet.stderr +diff --git a/test/integration/targets/assert/assert_quiet.out.quiet.stdout b/test/integration/targets/assert/assert.out.quiet.stdout +similarity index 100% +rename from test/integration/targets/assert/assert_quiet.out.quiet.stdout +rename to test/integration/targets/assert/assert.out.quiet.stdout +diff --git a/test/integration/targets/assert/nested_tmpl.yml b/test/integration/targets/assert/nested_tmpl.yml +new file mode 100644 +index 0000000000..3da4b1d80e +--- /dev/null ++++ b/test/integration/targets/assert/nested_tmpl.yml +@@ -0,0 +1,9 @@ ++- hosts: localhost ++ gather_facts: False ++ tasks: ++ - assert: ++ that: ++ - '"{{ foo }}" == "bar"' ++ - foo == "bar" ++ vars: ++ foo: bar +diff --git a/test/integration/targets/assert/quiet.yml b/test/integration/targets/assert/quiet.yml +index 6834712c2c..1c425cb5ba 100644 +--- a/test/integration/targets/assert/quiet.yml ++++ b/test/integration/targets/assert/quiet.yml +@@ -5,12 +5,12 @@ + item_A: yes + tasks: + - assert: +- that: "{{ item }} is defined" ++ that: "item is defined" + quiet: True + with_items: + - item_A + - assert: +- that: "{{ item }} is defined" ++ that: "item is defined" + quiet: False + with_items: + - item_A +diff --git a/test/integration/targets/assert/runme.sh b/test/integration/targets/assert/runme.sh +index ca0a858726..b79072813d 100755 +--- a/test/integration/targets/assert/runme.sh ++++ b/test/integration/targets/assert/runme.sh +@@ -45,7 +45,7 @@ cleanup() { + fi + } + +-BASEFILE=assert_quiet.out ++BASEFILE=assert.out + + ORIGFILE="${BASEFILE}" + OUTFILE="${BASEFILE}.new" +@@ -69,3 +69,4 @@ export ANSIBLE_NOCOLOR=1 + export ANSIBLE_RETRY_FILES_ENABLED=0 + + run_test quiet ++run_test nested_tmpl +diff --git a/test/integration/targets/command_shell/tasks/main.yml b/test/integration/targets/command_shell/tasks/main.yml +index 9dc6c9beb6..d685443b55 100644 +--- a/test/integration/targets/command_shell/tasks/main.yml ++++ b/test/integration/targets/command_shell/tasks/main.yml +@@ -286,7 +286,7 @@ + assert: + that: + - shell_result0 is changed +- - shell_result0.cmd == '{{ output_dir_test }}/test.sh' ++ - shell_result0.cmd == output_dir_test ~ '/test.sh' + - shell_result0.rc == 0 + - shell_result0.stderr == '' + - shell_result0.stdout == 'win' +diff --git a/test/integration/targets/copy/tasks/tests.yml b/test/integration/targets/copy/tasks/tests.yml +index 900f86a66a..2a150c4217 100644 +--- a/test/integration/targets/copy/tasks/tests.yml ++++ b/test/integration/targets/copy/tasks/tests.yml +@@ -1176,7 +1176,7 @@ + assert: + that: + - "copy_result6.changed" +- - "copy_result6.dest == '{{remote_dir_expanded}}/multiline.txt'" ++ - "copy_result6.dest == remote_dir_expanded ~ '/multiline.txt'" + - "copy_result6.checksum == '9cd0697c6a9ff6689f0afb9136fa62e0b3fee903'" + + # test overwriting a file as an unprivileged user (pull request #8624) +@@ -2060,26 +2060,26 @@ + assert: + that: + - testcase5 is changed +- - "stat_new_dir_with_chown.stat.uid == {{ ansible_copy_test_user.uid }}" +- - "stat_new_dir_with_chown.stat.gid == {{ ansible_copy_test_group.gid }}" +- - "stat_new_dir_with_chown.stat.pw_name == '{{ ansible_copy_test_user_name }}'" +- - "stat_new_dir_with_chown.stat.gr_name == '{{ ansible_copy_test_user_name }}'" +- - "stat_new_dir_with_chown_file1.stat.uid == {{ ansible_copy_test_user.uid }}" +- - "stat_new_dir_with_chown_file1.stat.gid == {{ ansible_copy_test_group.gid }}" +- - "stat_new_dir_with_chown_file1.stat.pw_name == '{{ ansible_copy_test_user_name }}'" +- - "stat_new_dir_with_chown_file1.stat.gr_name == '{{ ansible_copy_test_user_name }}'" +- - "stat_new_dir_with_chown_subdir.stat.uid == {{ ansible_copy_test_user.uid }}" +- - "stat_new_dir_with_chown_subdir.stat.gid == {{ ansible_copy_test_group.gid }}" +- - "stat_new_dir_with_chown_subdir.stat.pw_name == '{{ ansible_copy_test_user_name }}'" +- - "stat_new_dir_with_chown_subdir.stat.gr_name == '{{ ansible_copy_test_user_name }}'" +- - "stat_new_dir_with_chown_subdir_file12.stat.uid == {{ ansible_copy_test_user.uid }}" +- - "stat_new_dir_with_chown_subdir_file12.stat.gid == {{ ansible_copy_test_group.gid }}" +- - "stat_new_dir_with_chown_subdir_file12.stat.pw_name == '{{ ansible_copy_test_user_name }}'" +- - "stat_new_dir_with_chown_subdir_file12.stat.gr_name == '{{ ansible_copy_test_user_name }}'" +- - "stat_new_dir_with_chown_link_file12.stat.uid == {{ ansible_copy_test_user.uid }}" +- - "stat_new_dir_with_chown_link_file12.stat.gid == {{ ansible_copy_test_group.gid }}" +- - "stat_new_dir_with_chown_link_file12.stat.pw_name == '{{ ansible_copy_test_user_name }}'" +- - "stat_new_dir_with_chown_link_file12.stat.gr_name == '{{ ansible_copy_test_user_name }}'" ++ - "stat_new_dir_with_chown.stat.uid == ansible_copy_test_user.uid" ++ - "stat_new_dir_with_chown.stat.gid == ansible_copy_test_group.gid" ++ - "stat_new_dir_with_chown.stat.pw_name == ansible_copy_test_user_name" ++ - "stat_new_dir_with_chown.stat.gr_name == ansible_copy_test_user_name" ++ - "stat_new_dir_with_chown_file1.stat.uid == ansible_copy_test_user.uid" ++ - "stat_new_dir_with_chown_file1.stat.gid == ansible_copy_test_group.gid" ++ - "stat_new_dir_with_chown_file1.stat.pw_name == ansible_copy_test_user_name" ++ - "stat_new_dir_with_chown_file1.stat.gr_name == ansible_copy_test_user_name" ++ - "stat_new_dir_with_chown_subdir.stat.uid == ansible_copy_test_user.uid" ++ - "stat_new_dir_with_chown_subdir.stat.gid == ansible_copy_test_group.gid" ++ - "stat_new_dir_with_chown_subdir.stat.pw_name == ansible_copy_test_user_name" ++ - "stat_new_dir_with_chown_subdir.stat.gr_name == ansible_copy_test_user_name" ++ - "stat_new_dir_with_chown_subdir_file12.stat.uid == ansible_copy_test_user.uid" ++ - "stat_new_dir_with_chown_subdir_file12.stat.gid == ansible_copy_test_group.gid" ++ - "stat_new_dir_with_chown_subdir_file12.stat.pw_name == ansible_copy_test_user_name" ++ - "stat_new_dir_with_chown_subdir_file12.stat.gr_name == ansible_copy_test_user_name" ++ - "stat_new_dir_with_chown_link_file12.stat.uid == ansible_copy_test_user.uid" ++ - "stat_new_dir_with_chown_link_file12.stat.gid == ansible_copy_test_group.gid" ++ - "stat_new_dir_with_chown_link_file12.stat.pw_name == ansible_copy_test_user_name" ++ - "stat_new_dir_with_chown_link_file12.stat.gr_name == ansible_copy_test_user_name" + + always: + - name: execute - remove the user for test +diff --git a/test/integration/targets/debug/runme.sh b/test/integration/targets/debug/runme.sh +index 5ccb1bfda6..05357c2a37 100755 +--- a/test/integration/targets/debug/runme.sh ++++ b/test/integration/targets/debug/runme.sh +@@ -15,3 +15,5 @@ for i in 1 2 3; do + grep "ok: \[localhost\] => (item=$i)" out + grep "\"item\": $i" out + done ++ ++ansible-playbook unsafe.yml "$@" +diff --git a/test/integration/targets/debug/unsafe.yml b/test/integration/targets/debug/unsafe.yml +new file mode 100644 +index 0000000000..6a78af1a69 +--- /dev/null ++++ b/test/integration/targets/debug/unsafe.yml +@@ -0,0 +1,13 @@ ++- hosts: localhost ++ gather_facts: false ++ vars: ++ unsafe_var: !unsafe undef()|mandatory ++ tasks: ++ - debug: ++ var: '{{ unsafe_var }}' ++ ignore_errors: true ++ register: result ++ ++ - assert: ++ that: ++ - result is successful +diff --git a/test/integration/targets/expect/tasks/main.yml b/test/integration/targets/expect/tasks/main.yml +index 0c408d282d..b46474367f 100644 +--- a/test/integration/targets/expect/tasks/main.yml ++++ b/test/integration/targets/expect/tasks/main.yml +@@ -109,10 +109,15 @@ + foo: bar + register: chdir_result + ++- name: get output_dir real path ++ raw: > ++ {{ ansible_python_interpreter }} -c 'import os; os.chdir("{{output_dir}}"); print(os.getcwd())' ++ register: output_dir_real_path ++ + - name: assert chdir works + assert: + that: +- - "'{{chdir_result.stdout |expanduser | realpath }}' == '{{output_dir | expanduser | realpath}}'" ++ - "chdir_result.stdout | trim == output_dir_real_path.stdout | trim" + + - name: test timeout option + expect: +diff --git a/test/integration/targets/file/tasks/state_link.yml b/test/integration/targets/file/tasks/state_link.yml +index d9ac96740c..9e13eccbd1 100644 +--- a/test/integration/targets/file/tasks/state_link.yml ++++ b/test/integration/targets/file/tasks/state_link.yml +@@ -196,7 +196,7 @@ + - "missing_dst_no_follow_enable_force_use_mode2 is changed" + - "missing_dst_no_follow_enable_force_use_mode3 is not changed" + - "soft3_result['stat'].islnk" +- - "soft3_result['stat'].lnk_target == '{{ user.home }}/nonexistent'" ++ - "soft3_result['stat'].lnk_target == user.home ~ '/nonexistent'" + + # + # Test creating a link to a directory https://github.com/ansible/ansible/issues/1369 +diff --git a/test/integration/targets/find/tasks/main.yml b/test/integration/targets/find/tasks/main.yml +index 028d3cb45c..f8ea1a45f6 100644 +--- a/test/integration/targets/find/tasks/main.yml ++++ b/test/integration/targets/find/tasks/main.yml +@@ -116,4 +116,90 @@ + - name: assert we skipped the ogg file + assert: + that: +- - '"{{ output_dir_test }}/e/f/g/h/8.ogg" not in find_test3_list' +\ No newline at end of file ++ - 'output_dir_test ~ "/e/f/g/h/8.ogg" not in find_test3_list' ++ ++- name: patterns with regex ++ find: ++ paths: "{{ output_dir_test }}" ++ recurse: yes ++ use_regex: true ++ patterns: .*\.ogg ++ register: find_test4 ++ ++- name: assert we matched the ogg file ++ assert: ++ that: ++ - output_dir_test ~ "/e/f/g/h/8.ogg" in find_test4.files|map(attribute="path") ++ ++- name: create our age/size testing sub-directory ++ file: ++ path: "{{ output_dir_test }}/astest" ++ state: directory ++ ++- name: create test file with old timestamps ++ file: ++ path: "{{ output_dir_test }}/astest/old.txt" ++ state: touch ++ modification_time: "202001011200.0" ++ ++- name: create test file with current timestamps ++ file: ++ path: "{{ output_dir_test }}/astest/new.txt" ++ state: touch ++ ++- name: create hidden test file with current timestamps ++ file: ++ path: "{{ output_dir_test }}/astest/.hidden.txt" ++ state: touch ++ ++- name: find files older than 1 week ++ find: ++ path: "{{ output_dir_test }}/astest" ++ age: 1w ++ hidden: true ++ register: result ++ ++- set_fact: ++ astest_list: "{{ result.files|map(attribute='path') }}" ++ ++- name: assert we only find the old file ++ assert: ++ that: ++ - result.matched == 1 ++ - 'output_dir_test ~ "/astest/old.txt" in astest_list' ++ ++- name: find files newer than 1 week ++ find: ++ path: "{{ output_dir_test }}/astest" ++ age: -1w ++ register: result ++ ++- set_fact: ++ astest_list: "{{ result.files|map(attribute='path') }}" ++ ++- name: assert we only find the current file ++ assert: ++ that: ++ - result.matched == 1 ++ - 'output_dir_test ~ "/astest/new.txt" in astest_list' ++ ++- name: add some content to the new file ++ shell: "echo hello world > {{ output_dir_test }}/astest/new.txt" ++ ++- name: find files with MORE than 5 bytes, also get checksums ++ find: ++ path: "{{ output_dir_test }}/astest" ++ size: 5 ++ hidden: true ++ get_checksum: true ++ register: result ++ ++- set_fact: ++ astest_list: "{{ result.files|map(attribute='path') }}" ++ ++- name: assert we only find the hello world file ++ assert: ++ that: ++ - result.matched == 1 ++ - 'output_dir_test ~ "/astest/new.txt" in astest_list' ++ - '"checksum" in result.files[0]' +diff --git a/test/integration/targets/gathering_facts/test_gathering_facts.yml b/test/integration/targets/gathering_facts/test_gathering_facts.yml +index 5924a15649..c32bb3c8b8 100644 +--- a/test/integration/targets/gathering_facts/test_gathering_facts.yml ++++ b/test/integration/targets/gathering_facts/test_gathering_facts.yml +@@ -371,7 +371,7 @@ + - name: Test reading facts from default fact_path + assert: + that: +- - '"{{ ansible_local.testfact.fact_dir }}" == "default"' ++ - 'ansible_local.testfact.fact_dir == "default"' + + - hosts: facthost9 + tags: [ 'fact_local'] +@@ -382,7 +382,7 @@ + - name: Test reading facts from custom fact_path + assert: + that: +- - '"{{ ansible_local.testfact.fact_dir }}" == "custom"' ++ - 'ansible_local.testfact.fact_dir == "custom"' + + - hosts: facthost20 + tags: [ 'fact_facter_ohai' ] +diff --git a/test/integration/targets/git/tasks/depth.yml b/test/integration/targets/git/tasks/depth.yml +index 547f84f7b5..e0585ca39b 100644 +--- a/test/integration/targets/git/tasks/depth.yml ++++ b/test/integration/targets/git/tasks/depth.yml +@@ -169,7 +169,7 @@ + - name: DEPTH | check update arrived + assert: + that: +- - "{{ a_file.content | b64decode | trim }} == 3" ++ - a_file.content | b64decode | trim == "3" + - git_fetch is changed + + - name: DEPTH | clear checkout_dir +diff --git a/test/integration/targets/git/tasks/localmods.yml b/test/integration/targets/git/tasks/localmods.yml +index 09a1326d58..0e0cf684ed 100644 +--- a/test/integration/targets/git/tasks/localmods.yml ++++ b/test/integration/targets/git/tasks/localmods.yml +@@ -47,7 +47,7 @@ + - name: LOCALMODS | check update arrived + assert: + that: +- - "{{ a_file.content | b64decode | trim }} == 2" ++ - a_file.content | b64decode | trim == "2" + - git_fetch_force is changed + + - name: LOCALMODS | clear checkout_dir +@@ -105,7 +105,7 @@ + - name: LOCALMODS | check update arrived + assert: + that: +- - "{{ a_file.content | b64decode | trim }} == 2" ++ - a_file.content | b64decode | trim == "2" + - git_fetch_force is changed + + - name: LOCALMODS | clear checkout_dir +diff --git a/test/integration/targets/git/tasks/submodules.yml b/test/integration/targets/git/tasks/submodules.yml +index 647d1e23b4..1ba84afbde 100644 +--- a/test/integration/targets/git/tasks/submodules.yml ++++ b/test/integration/targets/git/tasks/submodules.yml +@@ -32,7 +32,7 @@ + + - name: SUBMODULES | Ensure submodu1 is at the appropriate commit + assert: +- that: '{{ submodule1.stdout_lines | length }} == 2' ++ that: 'submodule1.stdout_lines | length == 2' + + - name: SUBMODULES | clear checkout_dir + file: +@@ -53,7 +53,7 @@ + + - name: SUBMODULES | Ensure submodule1 is at the appropriate commit + assert: +- that: '{{ submodule1.stdout_lines | length }} == 4' ++ that: 'submodule1.stdout_lines | length == 4' + + - name: SUBMODULES | Copy the checkout so we can run several different tests on it + command: 'cp -pr {{ checkout_dir }} {{ checkout_dir }}.bak' +@@ -84,8 +84,8 @@ + - name: SUBMODULES | Ensure both submodules are at the appropriate commit + assert: + that: +- - '{{ submodule1.stdout_lines|length }} == 4' +- - '{{ submodule2.stdout_lines|length }} == 2' ++ - 'submodule1.stdout_lines|length == 4' ++ - 'submodule2.stdout_lines|length == 2' + + + - name: SUBMODULES | Remove checkout dir +@@ -112,7 +112,7 @@ + + - name: SUBMODULES | Ensure submodule1 is at the appropriate commit + assert: +- that: '{{ submodule1.stdout_lines | length }} == 5' ++ that: 'submodule1.stdout_lines | length == 5' + + + - name: SUBMODULES | Test that update with recursive found new submodules +@@ -121,4 +121,17 @@ + + - name: SUBMODULES | Enusre submodule2 is at the appropriate commit + assert: +- that: '{{ submodule2.stdout_lines | length }} == 4' ++ that: 'submodule2.stdout_lines | length == 4' ++ ++- name: SUBMODULES | clear checkout_dir ++ file: ++ state: absent ++ path: "{{ checkout_dir }}" ++ ++ ++- name: SUBMODULES | Clone main submodule repository ++ git: ++ repo: "{{ repo_submodules }}" ++ dest: "{{ checkout_dir }}/test.gitdir" ++ version: 45c6c07ef10fd9e453d90207e63da1ce5bd3ae1e ++ recursive: yes +diff --git a/test/integration/targets/include_vars/tasks/main.yml b/test/integration/targets/include_vars/tasks/main.yml +index 799d7b26a6..3c5816b0d4 100644 +--- a/test/integration/targets/include_vars/tasks/main.yml ++++ b/test/integration/targets/include_vars/tasks/main.yml +@@ -15,7 +15,7 @@ + that: + - "testing == 789" + - "base_dir == 'environments/development'" +- - "{{ included_one_file.ansible_included_var_files | length }} == 1" ++ - "included_one_file.ansible_included_var_files | length == 1" + - "'vars/environments/development/all.yml' in included_one_file.ansible_included_var_files[0]" + + - name: include the vars/environments/development/all.yml and save results in all +@@ -51,7 +51,7 @@ + assert: + that: + - webapp_version is defined +- - "'file_without_extension' in '{{ include_without_file_extension.ansible_included_var_files | join(' ') }}'" ++ - "'file_without_extension' in include_without_file_extension.ansible_included_var_files | join(' ')" + + - name: include every directory in vars + include_vars: +@@ -65,7 +65,7 @@ + - "testing == 456" + - "base_dir == 'services'" + - "webapp_containers == 10" +- - "{{ include_every_dir.ansible_included_var_files | length }} == 7" ++ - "include_every_dir.ansible_included_var_files | length == 7" + - "'vars/all/all.yml' in include_every_dir.ansible_included_var_files[0]" + - "'vars/environments/development/all.yml' in include_every_dir.ansible_included_var_files[1]" + - "'vars/environments/development/services/webapp.yml' in include_every_dir.ansible_included_var_files[2]" +@@ -85,9 +85,9 @@ + that: + - "testing == 789" + - "base_dir == 'environments/development'" +- - "{{ include_without_webapp.ansible_included_var_files | length }} == 4" +- - "'webapp.yml' not in '{{ include_without_webapp.ansible_included_var_files | join(' ') }}'" +- - "'file_without_extension' not in '{{ include_without_webapp.ansible_included_var_files | join(' ') }}'" ++ - "include_without_webapp.ansible_included_var_files | length == 4" ++ - "'webapp.yml' not in include_without_webapp.ansible_included_var_files | join(' ')" ++ - "'file_without_extension' not in include_without_webapp.ansible_included_var_files | join(' ')" + + - name: include only files matching webapp.yml + include_vars: +@@ -101,9 +101,9 @@ + - "testing == 101112" + - "base_dir == 'development/services'" + - "webapp_containers == 20" +- - "{{ include_match_webapp.ansible_included_var_files | length }} == 1" ++ - "include_match_webapp.ansible_included_var_files | length == 1" + - "'vars/environments/development/services/webapp.yml' in include_match_webapp.ansible_included_var_files[0]" +- - "'all.yml' not in '{{ include_match_webapp.ansible_included_var_files | join(' ') }}'" ++ - "'all.yml' not in include_match_webapp.ansible_included_var_files | join(' ')" + + - name: include only files matching webapp.yml and store results in webapp + include_vars: +@@ -162,3 +162,7 @@ + that: + - "'my_custom_service' == service_name_fqcn" + - "'my_custom_service' == service_name_tmpl_fqcn" ++ ++- assert: ++ that: ++ - baz.ansible_facts.foo|type_debug != "AnsibleUnsafeText" +diff --git a/test/integration/targets/iosxr_lacp_interfaces/tests/cli/merged.yaml b/test/integration/targets/iosxr_lacp_interfaces/tests/cli/merged.yaml +index be5134091b..a2947d2959 100644 +--- a/test/integration/targets/iosxr_lacp_interfaces/tests/cli/merged.yaml ++++ b/test/integration/targets/iosxr_lacp_interfaces/tests/cli/merged.yaml +@@ -24,17 +24,17 @@ + + - name: Assert that before dicts were correctly generated + assert: +- that: "{{ merged['before'] | symmetric_difference(result['before']) |length == 0 }}" ++ that: "merged['before'] | symmetric_difference(result['before']) |length == 0" + + - name: Assert that correct set of commands were generated + assert: + that: +- - "{{ merged['commands'] | symmetric_difference(result['commands']) |length == 0 }}" ++ - "merged['commands'] | symmetric_difference(result['commands']) |length == 0" + + - name: Assert that after dicts was correctly generated + assert: + that: +- - "{{ merged['after'] | symmetric_difference(result['after']) |length == 0 }}" ++ - "merged['after'] | symmetric_difference(result['after']) |length == 0" + + - name: Merge the provided configuration with the existing running configuration (IDEMPOTENT) + iosxr_lacp_interfaces: *merged +@@ -49,6 +49,7 @@ + - name: Assert that before dicts were correctly generated + assert: + that: +- - "{{ merged['after'] | symmetric_difference(result['before']) |length == 0 }}" ++ - "merged['after'] | symmetric_difference(result['before']) |length == 0" ++ + always: + - include_tasks: _remove_config.yaml +diff --git a/test/integration/targets/iosxr_lacp_interfaces/tests/cli/replaced.yaml b/test/integration/targets/iosxr_lacp_interfaces/tests/cli/replaced.yaml +index 0dcb8505e0..ccf9d803b2 100644 +--- a/test/integration/targets/iosxr_lacp_interfaces/tests/cli/replaced.yaml ++++ b/test/integration/targets/iosxr_lacp_interfaces/tests/cli/replaced.yaml +@@ -21,17 +21,17 @@ + - name: Assert that correct set of commands were generated + assert: + that: +- - "{{ replaced['commands'] | symmetric_difference(result['commands']) |length == 0 }}" ++ - "replaced['commands'] | symmetric_difference(result['commands']) |length == 0" + + - name: Assert that before dicts are correctly generated + assert: + that: +- - "{{ populate | symmetric_difference(result['before']) |length == 0 }}" ++ - "populate | symmetric_difference(result['before']) |length == 0" + + - name: Assert that after dict is correctly generated + assert: + that: +- - "{{ replaced['after'] | symmetric_difference(result['after']) |length == 0 }}" ++ - "replaced['after'] | symmetric_difference(result['after']) |length == 0" + + - name: Replace device configurations of listed interfaces with provided configurarions (IDEMPOTENT) + iosxr_lacp_interfaces: *replaced +@@ -46,7 +46,7 @@ + - name: Assert that before dict is correctly generated + assert: + that: +- - "{{ replaced['after'] | symmetric_difference(result['before']) |length == 0 }}" +- ++ - "replaced['after'] | symmetric_difference(result['before']) |length == 0" ++ + always: + - include_tasks: _remove_config.yaml +diff --git a/test/integration/targets/lookup_properties/test_lookup_properties.yml b/test/integration/targets/lookup_properties/test_lookup_properties.yml +index a8cad9de48..7c33a70b0b 100644 +--- a/test/integration/targets/lookup_properties/test_lookup_properties.yml ++++ b/test/integration/targets/lookup_properties/test_lookup_properties.yml +@@ -10,7 +10,7 @@ + test_dot: "{{lookup('ini', 'value.dot type=properties file=lookup.properties')}}" + field_with_space: "{{lookup('ini', 'field.with.space type=properties file=lookup.properties')}}" + - assert: +- that: "{{item}} is defined" ++ that: "item is defined" + with_items: [ 'test1', 'test2', 'test_dot', 'field_with_space' ] + - name: "read ini value" + set_fact: +diff --git a/test/integration/targets/module_precedence/modules_test_multiple_roles.yml b/test/integration/targets/module_precedence/modules_test_multiple_roles.yml +index f4bd264957..182c2158e8 100644 +--- a/test/integration/targets/module_precedence/modules_test_multiple_roles.yml ++++ b/test/integration/targets/module_precedence/modules_test_multiple_roles.yml +@@ -14,4 +14,4 @@ + - assert: + that: + - '"location" in result' +- - 'result["location"] == "{{ expected_location}}"' ++ - 'result["location"] == expected_location' +diff --git a/test/integration/targets/module_precedence/modules_test_multiple_roles_reverse_order.yml b/test/integration/targets/module_precedence/modules_test_multiple_roles_reverse_order.yml +index 5403ae238c..ec5619f39e 100644 +--- a/test/integration/targets/module_precedence/modules_test_multiple_roles_reverse_order.yml ++++ b/test/integration/targets/module_precedence/modules_test_multiple_roles_reverse_order.yml +@@ -13,4 +13,4 @@ + - assert: + that: + - '"location" in result' +- - 'result["location"] == "{{ expected_location}}"' ++ - 'result["location"] == expected_location' +diff --git a/test/integration/targets/module_precedence/multiple_roles/bar/tasks/main.yml b/test/integration/targets/module_precedence/multiple_roles/bar/tasks/main.yml +index 52c3402013..62b38a7cb5 100644 +--- a/test/integration/targets/module_precedence/multiple_roles/bar/tasks/main.yml ++++ b/test/integration/targets/module_precedence/multiple_roles/bar/tasks/main.yml +@@ -7,4 +7,4 @@ + assert: + that: + - '"location" in result' +- - 'result["location"] == "{{ expected_location }}"' ++ - 'result["location"] == expected_location' +diff --git a/test/integration/targets/module_precedence/multiple_roles/foo/tasks/main.yml b/test/integration/targets/module_precedence/multiple_roles/foo/tasks/main.yml +index 52c3402013..62b38a7cb5 100644 +--- a/test/integration/targets/module_precedence/multiple_roles/foo/tasks/main.yml ++++ b/test/integration/targets/module_precedence/multiple_roles/foo/tasks/main.yml +@@ -7,4 +7,4 @@ + assert: + that: + - '"location" in result' +- - 'result["location"] == "{{ expected_location }}"' ++ - 'result["location"] == expected_location' +diff --git a/test/integration/targets/script/tasks/main.yml b/test/integration/targets/script/tasks/main.yml +index f1746f7c48..3a10a1fec4 100644 +--- a/test/integration/targets/script/tasks/main.yml ++++ b/test/integration/targets/script/tasks/main.yml +@@ -197,7 +197,7 @@ + assert: + that: + - _check_mode_test2 is skipped +- - '_check_mode_test2.msg == "{{ output_dir_test | expanduser }}/afile2.txt exists, matching creates option"' ++ - '_check_mode_test2.msg == output_dir_test | expanduser ~ "/afile2.txt exists, matching creates option"' + + - name: Remove afile2.txt + file: +@@ -219,7 +219,7 @@ + assert: + that: + - _check_mode_test3 is skipped +- - '_check_mode_test3.msg == "{{ output_dir_test | expanduser }}/afile2.txt does not exist, matching removes option"' ++ - '_check_mode_test3.msg == output_dir_test | expanduser ~ "/afile2.txt does not exist, matching removes option"' + + # executable + +diff --git a/test/integration/targets/slurp/tasks/main.yml b/test/integration/targets/slurp/tasks/main.yml +index 4f3556fad4..fd61b7f4bc 100644 +--- a/test/integration/targets/slurp/tasks/main.yml ++++ b/test/integration/targets/slurp/tasks/main.yml +@@ -33,7 +33,7 @@ + - 'slurp_existing.encoding == "base64"' + - 'slurp_existing is not changed' + - 'slurp_existing is not failed' +- - '"{{ slurp_existing.content | b64decode }}" == "We are at the café"' ++ - 'slurp_existing.content | b64decode == "We are at the café"' + + - name: Create a binary file to test with + copy: +diff --git a/test/integration/targets/template/tasks/main.yml b/test/integration/targets/template/tasks/main.yml +index da80343686..daed110855 100644 +--- a/test/integration/targets/template/tasks/main.yml ++++ b/test/integration/targets/template/tasks/main.yml +@@ -356,7 +356,7 @@ + - assert: + that: + - "\"foo t'e~m\\plated\" in unusual_results.stdout_lines" +- - "{{unusual_results.stdout_lines| length}} == 1" ++ - "unusual_results.stdout_lines| length == 1" + + - name: check that the unusual filename can be checked for changes + template: +diff --git a/test/integration/targets/unarchive/tasks/test_mode.yml b/test/integration/targets/unarchive/tasks/test_mode.yml +index c69e3bd2b2..06fbc7b8d9 100644 +--- a/test/integration/targets/unarchive/tasks/test_mode.yml ++++ b/test/integration/targets/unarchive/tasks/test_mode.yml +@@ -24,7 +24,7 @@ + - "unarchive06_stat.stat.mode == '0600'" + # Verify that file list is generated + - "'files' in unarchive06" +- - "{{unarchive06['files']| length}} == 1" ++ - "unarchive06['files']| length == 1" + - "'foo-unarchive.txt' in unarchive06['files']" + + - name: remove our tar.gz unarchive destination +@@ -74,7 +74,7 @@ + - "unarchive07.changed == false" + # Verify that file list is generated + - "'files' in unarchive07" +- - "{{unarchive07['files']| length}} == 1" ++ - "unarchive07['files']| length == 1" + - "'foo-unarchive.txt' in unarchive07['files']" + + - name: remove our tar.gz unarchive destination +@@ -108,7 +108,7 @@ + - "unarchive08_stat.stat.mode == '0601'" + # Verify that file list is generated + - "'files' in unarchive08" +- - "{{unarchive08['files']| length}} == 3" ++ - "unarchive08['files']| length == 3" + - "'foo-unarchive.txt' in unarchive08['files']" + - "'foo-unarchive-777.txt' in unarchive08['files']" + - "'FOO-UNAR.TXT' in unarchive08['files']" +@@ -140,7 +140,7 @@ + - "unarchive08_stat.stat.mode == '0601'" + # Verify that file list is generated + - "'files' in unarchive08" +- - "{{unarchive08['files']| length}} == 3" ++ - "unarchive08['files']| length == 3" + - "'foo-unarchive.txt' in unarchive08['files']" + - "'foo-unarchive-777.txt' in unarchive08['files']" + - "'FOO-UNAR.TXT' in unarchive08['files']" +diff --git a/test/integration/targets/unarchive/tasks/test_unprivileged_user.yml b/test/integration/targets/unarchive/tasks/test_unprivileged_user.yml +index 6181e3bd62..b3653c0872 100644 +--- a/test/integration/targets/unarchive/tasks/test_unprivileged_user.yml ++++ b/test/integration/targets/unarchive/tasks/test_unprivileged_user.yml +@@ -48,7 +48,7 @@ + - unarchive10 is changed + # Verify that file list is generated + - "'files' in unarchive10" +- - "{{unarchive10['files']| length}} == 1" ++ - "unarchive10['files']| length == 1" + - "'foo-unarchive.txt' in unarchive10['files']" + - archive_path.stat.exists + +diff --git a/test/integration/targets/unarchive/tasks/test_zip.yml b/test/integration/targets/unarchive/tasks/test_zip.yml +index aae57d8ec9..d11c5f7223 100644 +--- a/test/integration/targets/unarchive/tasks/test_zip.yml ++++ b/test/integration/targets/unarchive/tasks/test_zip.yml +@@ -17,7 +17,7 @@ + - "unarchive03.changed == true" + # Verify that file list is generated + - "'files' in unarchive03" +- - "{{unarchive03['files']| length}} == 3" ++ - "unarchive03['files']| length == 3" + - "'foo-unarchive.txt' in unarchive03['files']" + - "'foo-unarchive-777.txt' in unarchive03['files']" + - "'FOO-UNAR.TXT' in unarchive03['files']" +diff --git a/test/integration/targets/vault/roles/test_vault_embedded/tasks/main.yml b/test/integration/targets/vault/roles/test_vault_embedded/tasks/main.yml +index eba938966d..98ef751b86 100644 +--- a/test/integration/targets/vault/roles/test_vault_embedded/tasks/main.yml ++++ b/test/integration/targets/vault/roles/test_vault_embedded/tasks/main.yml +@@ -2,7 +2,7 @@ + - name: Assert that a embedded vault of a string with no newline works + assert: + that: +- - '"{{ vault_encrypted_one_line_var }}" == "Setec Astronomy"' ++ - 'vault_encrypted_one_line_var == "Setec Astronomy"' + + - name: Assert that a multi line embedded vault works, including new line + assert: +diff --git a/test/integration/targets/vault/roles/test_vault_file_encrypted_embedded/tasks/main.yml b/test/integration/targets/vault/roles/test_vault_file_encrypted_embedded/tasks/main.yml +index e09004a1d9..107e65cb11 100644 +--- a/test/integration/targets/vault/roles/test_vault_file_encrypted_embedded/tasks/main.yml ++++ b/test/integration/targets/vault/roles/test_vault_file_encrypted_embedded/tasks/main.yml +@@ -2,7 +2,7 @@ + - name: Assert that a vault encrypted file with embedded vault of a string with no newline works + assert: + that: +- - '"{{ vault_file_encrypted_with_encrypted_one_line_var }}" == "Setec Astronomy"' ++ - 'vault_file_encrypted_with_encrypted_one_line_var == "Setec Astronomy"' + + - name: Assert that a vault encrypted file with multi line embedded vault works, including new line + assert: +diff --git a/test/integration/targets/vyos_config/tests/cli/check_config.yaml b/test/integration/targets/vyos_config/tests/cli/check_config.yaml +index 65076b3c54..30c43599af 100644 +--- a/test/integration/targets/vyos_config/tests/cli/check_config.yaml ++++ b/test/integration/targets/vyos_config/tests/cli/check_config.yaml +@@ -22,7 +22,7 @@ + - name: Check that multiple duplicate lines collapse into a single commands + assert: + that: +- - "{{ result.commands|length }} == 1" ++ - "result.commands|length == 1" + + - name: Check that set is correctly prepended + assert: +@@ -58,6 +58,6 @@ + + - assert: + that: +- - "{{ result.filtered|length }} == 2" ++ - "result.filtered|length == 2" + + - debug: msg="END cli/config_check.yaml on connection={{ ansible_connection }}" +diff --git a/test/integration/targets/vyos_interfaces/tests/cli/deleted.yaml b/test/integration/targets/vyos_interfaces/tests/cli/deleted.yaml +index 5b08ea95f8..b2aa51fc77 100644 +--- a/test/integration/targets/vyos_interfaces/tests/cli/deleted.yaml ++++ b/test/integration/targets/vyos_interfaces/tests/cli/deleted.yaml +@@ -16,17 +16,17 @@ + - name: Assert that the before dicts were correctly generated + assert: + that: +- - "{{ populate | symmetric_difference(result['before']) |length == 0 }}" ++ - "populate | symmetric_difference(result['before']) |length == 0" + + - name: Assert that the correct set of commands were generated + assert: + that: +- - "{{ deleted['commands'] | symmetric_difference(result['commands']) |length == 0 }}" ++ - "deleted['commands'] | symmetric_difference(result['commands']) |length == 0" + + - name: Assert that the after dicts were correctly generated + assert: + that: +- - "{{ deleted['after'] | symmetric_difference(result['after']) |length == 0 }}" ++ - "deleted['after'] | symmetric_difference(result['after']) |length == 0" + + - name: Delete attributes of given interfaces (IDEMPOTENT) + vyos_interfaces: *deleted +@@ -40,7 +40,6 @@ + - name: Assert that the before dicts were correctly generated + assert: + that: +- - "{{ deleted['after'] | symmetric_difference(result['before']) |length == 0 }}" +- ++ - "deleted['after'] | symmetric_difference(result['before']) |length == 0" + always: + - include_tasks: _remove_config.yaml +diff --git a/test/integration/targets/vyos_interfaces/tests/cli/overridden.yaml b/test/integration/targets/vyos_interfaces/tests/cli/overridden.yaml +index 43040c1e67..4f2e323b36 100644 +--- a/test/integration/targets/vyos_interfaces/tests/cli/overridden.yaml ++++ b/test/integration/targets/vyos_interfaces/tests/cli/overridden.yaml +@@ -22,17 +22,17 @@ + - name: Assert that before dicts were correctly generated + assert: + that: +- - "{{ populate | symmetric_difference(result['before']) |length == 0 }}" ++ - "populate_intf | symmetric_difference(result['before']) |length == 0" + + - name: Assert that correct commands were generated + assert: + that: +- - "{{ overridden['commands'] | symmetric_difference(result['commands']) |length == 0 }}" ++ - "overridden['commands'] | symmetric_difference(result['commands']) |length == 0" + + - name: Assert that after dicts were correctly generated + assert: + that: +- - "{{ overridden['after'] | symmetric_difference(result['after']) |length == 0 }}" ++ - "overridden['after'] | symmetric_difference(result['after']) |length == 0" + + - name: Overrides all device configuration with provided configurations (IDEMPOTENT) + vyos_interfaces: *overridden +@@ -46,7 +46,7 @@ + - name: Assert that before dicts were correctly generated + assert: + that: +- - "{{ overridden['after'] | symmetric_difference(result['before']) |length == 0 }}" +- ++ - "overridden['after'] | symmetric_difference(result['before']) |length == 0" ++ + always: + - include_tasks: _remove_config.yaml +diff --git a/test/integration/targets/wait_for/tasks/main.yml b/test/integration/targets/wait_for/tasks/main.yml +index 1898fd1253..7bdbdd951a 100644 +--- a/test/integration/targets/wait_for/tasks/main.yml ++++ b/test/integration/targets/wait_for/tasks/main.yml +@@ -29,7 +29,7 @@ + assert: + that: + - waitfor is successful +- - waitfor.path == "{{ output_dir | expanduser }}/wait_for_file" ++ - waitfor.path == output_dir | expanduser ~ "/wait_for_file" + - waitfor.elapsed >= 2 + - waitfor.elapsed <= 15 + +@@ -47,7 +47,7 @@ + assert: + that: + - waitfor is successful +- - waitfor.path == "{{ output_dir | expanduser }}/wait_for_file" ++ - waitfor.path == output_dir | expanduser ~ "/wait_for_file" + - waitfor.elapsed >= 2 + - waitfor.elapsed <= 15 + +@@ -135,7 +135,7 @@ + that: + - waitfor is successful + - waitfor is not changed +- - "waitfor.port == {{ http_port }}" ++ - "waitfor.port == http_port" + + - name: install psutil using pip (non-Linux only) + pip: +@@ -163,4 +163,16 @@ + that: + - waitfor is successful + - waitfor is not changed +- - "waitfor.port == {{ http_port }}" ++ - "waitfor.port == http_port" ++ ++- name: test wait_for with delay ++ wait_for: ++ timeout: 2 ++ delay: 2 ++ register: waitfor ++ ++- name: verify test wait_for with delay ++ assert: ++ that: ++ - waitfor is successful ++ - waitfor.elapsed >= 4 +diff --git a/test/units/module_utils/common/test_collections.py b/test/units/module_utils/common/test_collections.py +index eb6e376a2c..95b2a402f2 100644 +--- a/test/units/module_utils/common/test_collections.py ++++ b/test/units/module_utils/common/test_collections.py +@@ -35,8 +35,21 @@ class IterableStub: + return IteratorStub() + + ++class FakeAnsibleVaultEncryptedUnicode(Sequence): ++ __ENCRYPTED__ = True ++ ++ def __init__(self, data): ++ self.data = data ++ ++ def __getitem__(self, index): ++ return self.data[index] ++ ++ def __len__(self): ++ return len(self.data) ++ ++ + TEST_STRINGS = u'he', u'Україна', u'Česká republika' +-TEST_STRINGS = TEST_STRINGS + tuple(s.encode('utf-8') for s in TEST_STRINGS) ++TEST_STRINGS = TEST_STRINGS + tuple(s.encode('utf-8') for s in TEST_STRINGS) + (FakeAnsibleVaultEncryptedUnicode(u'foo'),) + + TEST_ITEMS_NON_SEQUENCES = ( + {}, object(), frozenset(), +diff --git a/test/units/parsing/test_ajson.py b/test/units/parsing/test_ajson.py +index 929d19966d..c38f43ea57 100644 +--- a/test/units/parsing/test_ajson.py ++++ b/test/units/parsing/test_ajson.py +@@ -158,6 +158,7 @@ class TestAnsibleJSONEncoder: + Test for passing AnsibleVaultEncryptedUnicode to AnsibleJSONEncoder.default(). + """ + assert ansible_json_encoder.default(test_input) == {'__ansible_vault': expected} ++ assert json.dumps(test_input, cls=AnsibleJSONEncoder, preprocess_unsafe=True) == '{"__ansible_vault": "%s"}' % expected.replace('\n', '\\n') + + @pytest.mark.parametrize( + 'test_input,expected', +diff --git a/test/units/parsing/yaml/test_dumper.py b/test/units/parsing/yaml/test_dumper.py +index 8129ca3ab2..ee9ea8b07a 100644 +--- a/test/units/parsing/yaml/test_dumper.py ++++ b/test/units/parsing/yaml/test_dumper.py +@@ -19,13 +19,16 @@ from __future__ import (absolute_import, division, print_function) + __metaclass__ = type + + import io ++import yaml ++ ++from jinja2.exceptions import UndefinedError + + from units.compat import unittest + from ansible.parsing import vault + from ansible.parsing.yaml import dumper, objects + from ansible.parsing.yaml.loader import AnsibleLoader + from ansible.module_utils.six import PY2 +-from ansible.utils.unsafe_proxy import AnsibleUnsafeText, AnsibleUnsafeBytes ++from ansible.template import AnsibleUndefined + + from units.mock.yaml_helper import YamlTestUtils + from units.mock.vault_helper import TextVaultSecret +@@ -64,8 +67,7 @@ class TestAnsibleDumper(unittest.TestCase, YamlTestUtils): + + def test_bytes(self): + b_text = u'tréma'.encode('utf-8') +- unsafe_object = AnsibleUnsafeBytes(b_text) +- yaml_out = self._dump_string(unsafe_object, dumper=self.dumper) ++ yaml_out = self._dump_string(b_text, dumper=self.dumper) + + stream = self._build_stream(yaml_out) + loader = self._loader(stream) +@@ -92,8 +94,7 @@ class TestAnsibleDumper(unittest.TestCase, YamlTestUtils): + + def test_unicode(self): + u_text = u'nöel' +- unsafe_object = AnsibleUnsafeText(u_text) +- yaml_out = self._dump_string(unsafe_object, dumper=self.dumper) ++ yaml_out = self._dump_string(u_text, dumper=self.dumper) + + stream = self._build_stream(yaml_out) + loader = self._loader(stream) +@@ -101,3 +102,12 @@ class TestAnsibleDumper(unittest.TestCase, YamlTestUtils): + data_from_yaml = loader.get_single_data() + + self.assertEqual(u_text, data_from_yaml) ++ ++ def test_undefined(self): ++ undefined_object = AnsibleUndefined() ++ try: ++ yaml_out = self._dump_string(undefined_object, dumper=self.dumper) ++ except UndefinedError: ++ yaml_out = None ++ ++ self.assertIsNone(yaml_out) +-- +2.44.0 + diff --git a/CVE-2024-0690.patch b/CVE-2024-0690.patch index e96ab50..f250889 100644 --- a/CVE-2024-0690.patch +++ b/CVE-2024-0690.patch @@ -1,33 +1,162 @@ -From beb04bc2642c208447c5a936f94310528a1946b1 Mon Sep 17 00:00:00 2001 +From f5cb50f79af310b917e6932a0c0d8e9a73261b7f Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Thu, 18 Jan 2024 17:17:23 -0600 -Subject: [PATCH] [stable-2.14] Ensure ANSIBLE_NO_LOG is respected +Subject: [PATCH 2/2] [stable-2.14] Ensure ANSIBLE_NO_LOG is respected (CVE-2024-0690) (#82565) (#82568) -Origin: https://github.com/ansible/ansible/commit/beb04bc2642c208447c5a936f94310528a1946b1 +Origin:https://build.opensuse.org/projects/SUSE:SLE-15-SP3:Update/packages/ansible/files/0002-Ensure-ANSIBLE_NO_LOG-is-respected-CVE-2024-0690-825.patch?expand=1 +https://github.com/ansible/ansible/pull/82566 +https://github.com/ansible/ansible/pull/68560 +https://github.com/ansible/ansible/pull/68014 +https://github.com/ansible/ansible/pull/71313 +https://github.com/ansible/ansible/pull/60513 (cherry picked from commit 6935c8e) +Force template module to use non-native Jinja2 (#68560) + +Fixes #46169 + +Auto unroll generators produced by jinja filters (#68014) + +* Auto unroll generators produced by jinja filters + +* Unroll for native in finalize + +* Fix indentation + +Co-authored-by: Sam Doran + +* Add changelog fragment + +* ci_complete + +* Always unroll regardless of jinja2 + +* ci_complete + +Co-authored-by: Sam Doran + +Skip literal_eval for string filters results in native jinja. (#70988) (#71313) + +Fixes #70831 + +(cherry picked from commit b66d66027ece03f3f0a3fdb5fd6b8213965a2f1d) + +Introduce context manager for temporary templar context changes (#60513) + +* Introduce context manager for temporary templar context changes. Fixes #60106 + +* Rename and docstring + +* Make set_temporary_context more generic, don't hardcode each thing you can set, apply to template action too + +* not None + +* linting fix + +* Ignore invalid attrs + +* Catch the right things, loop the right things + +* Use set_temporary_context in a few extra action plugins --- - changelogs/fragments/cve-2024-0690.yml | 2 ++ - lib/ansible/playbook/base.py | 2 +- - lib/ansible/playbook/play_context.py | 4 ---- - test/integration/targets/no_log/no_log_config.yml | 13 +++++++++++++ - test/integration/targets/no_log/runme.sh | 5 +++++ - 5 files changed, 21 insertions(+), 5 deletions(-) + .../46169-non-native-template-module.yml | 2 + + .../60106-templar-contextmanager.yml | 4 + + .../68014-auto-unroll-jinja2-generators.yml | 3 + + ...iteral_eval-string-filter-native-jinja.yml | 2 + + changelogs/fragments/cve-2024-0690.yml | 2 + + lib/ansible/config/base.yml | 2 +- + lib/ansible/playbook/base.py | 2 +- + lib/ansible/playbook/conditional.py | 4 +- + lib/ansible/playbook/play_context.py | 4 - + lib/ansible/plugins/action/ce_template.py | 4 +- + lib/ansible/plugins/action/network.py | 4 +- + lib/ansible/plugins/action/template.py | 32 ++- + lib/ansible/plugins/lookup/template.py | 50 ++-- + lib/ansible/template/__init__.py | 227 ++++++++++++++++-- + lib/ansible/template/native_helpers.py | 39 +++ + lib/ansible/utils/native_jinja.py | 13 + + lib/ansible/utils/unsafe_proxy.py | 7 + + .../jinja2_native_types/test_casting.yml | 7 + + .../jinja2_native_types/test_dunder.yml | 2 +- + .../targets/no_log/no_log_config.yml | 13 + + test/integration/targets/no_log/runme.sh | 5 + + .../template_jinja2_non_native/46169.yml | 32 +++ + .../template_jinja2_non_native/aliases | 1 + + .../template_jinja2_non_native/runme.sh | 7 + + .../templates/46169.json.j2 | 3 + + 25 files changed, 398 insertions(+), 73 deletions(-) + create mode 100644 changelogs/fragments/46169-non-native-template-module.yml + create mode 100644 changelogs/fragments/60106-templar-contextmanager.yml + create mode 100644 changelogs/fragments/68014-auto-unroll-jinja2-generators.yml + create mode 100644 changelogs/fragments/70831-skip-literal_eval-string-filter-native-jinja.yml create mode 100644 changelogs/fragments/cve-2024-0690.yml + create mode 100644 lib/ansible/utils/native_jinja.py create mode 100644 test/integration/targets/no_log/no_log_config.yml + create mode 100644 test/integration/targets/template_jinja2_non_native/46169.yml + create mode 100644 test/integration/targets/template_jinja2_non_native/aliases + create mode 100755 test/integration/targets/template_jinja2_non_native/runme.sh + create mode 100644 test/integration/targets/template_jinja2_non_native/templates/46169.json.j2 +diff --git a/changelogs/fragments/46169-non-native-template-module.yml b/changelogs/fragments/46169-non-native-template-module.yml +new file mode 100644 +index 0000000000..7d004a6296 +--- /dev/null ++++ b/changelogs/fragments/46169-non-native-template-module.yml +@@ -0,0 +1,2 @@ ++minor_changes: ++ - Force the template module to use non-native Jinja2 (https://github.com/ansible/ansible/issues/46169) +diff --git a/changelogs/fragments/60106-templar-contextmanager.yml b/changelogs/fragments/60106-templar-contextmanager.yml +new file mode 100644 +index 0000000000..45afc1544a +--- /dev/null ++++ b/changelogs/fragments/60106-templar-contextmanager.yml +@@ -0,0 +1,4 @@ ++bugfixes: ++- template lookup - ensure changes to the templar in the lookup, do not ++ affect the templar context outside of the lookup ++ (https://github.com/ansible/ansible/issues/60106) +diff --git a/changelogs/fragments/68014-auto-unroll-jinja2-generators.yml b/changelogs/fragments/68014-auto-unroll-jinja2-generators.yml +new file mode 100644 +index 0000000000..211d2fd665 +--- /dev/null ++++ b/changelogs/fragments/68014-auto-unroll-jinja2-generators.yml +@@ -0,0 +1,3 @@ ++minor_changes: ++- Templating - Add support to auto unroll generators produced by jinja2 filters, to prevent the need of explicit use of ``|list`` ++ (https://github.com/ansible/ansible/pull/68014) +diff --git a/changelogs/fragments/70831-skip-literal_eval-string-filter-native-jinja.yml b/changelogs/fragments/70831-skip-literal_eval-string-filter-native-jinja.yml +new file mode 100644 +index 0000000000..40b426e50b +--- /dev/null ++++ b/changelogs/fragments/70831-skip-literal_eval-string-filter-native-jinja.yml +@@ -0,0 +1,2 @@ ++bugfixes: ++ - Skip literal_eval for string filters results in native jinja. (https://github.com/ansible/ansible/issues/70831) diff --git a/changelogs/fragments/cve-2024-0690.yml b/changelogs/fragments/cve-2024-0690.yml new file mode 100644 -index 00000000..0e030d88 +index 0000000000..0e030d8886 --- /dev/null +++ b/changelogs/fragments/cve-2024-0690.yml @@ -0,0 +1,2 @@ +security_fixes: +- ANSIBLE_NO_LOG - Address issue where ANSIBLE_NO_LOG was ignored (CVE-2024-0690) +diff --git a/lib/ansible/config/base.yml b/lib/ansible/config/base.yml +index 3d3916a7fc..96d38e7f51 100644 +--- a/lib/ansible/config/base.yml ++++ b/lib/ansible/config/base.yml +@@ -1757,7 +1757,7 @@ SHOW_CUSTOM_STATS: + type: bool + STRING_TYPE_FILTERS: + name: Filters to preserve strings +- default: [string, to_json, to_nice_json, to_yaml, ppretty, json] ++ default: [string, to_json, to_nice_json, to_yaml, to_nice_yaml, ppretty, json] + description: + - "This list of filters avoids 'type conversion' when templating variables" + - Useful when you want to avoid conversion into lists or dictionaries for JSON strings, for example. diff --git a/lib/ansible/playbook/base.py b/lib/ansible/playbook/base.py -index 0f4dc4e4..172963a2 100644 +index 0f4dc4e430..172963a218 100644 --- a/lib/ansible/playbook/base.py +++ b/lib/ansible/playbook/base.py @@ -613,7 +613,7 @@ class Base(FieldAttributeBase): @@ -39,8 +168,23 @@ index 0f4dc4e4..172963a2 100644 _run_once = FieldAttribute(isa='bool') _ignore_errors = FieldAttribute(isa='bool') _ignore_unreachable = FieldAttribute(isa='bool') +diff --git a/lib/ansible/playbook/conditional.py b/lib/ansible/playbook/conditional.py +index ac4fc0c568..be4b75986c 100644 +--- a/lib/ansible/playbook/conditional.py ++++ b/lib/ansible/playbook/conditional.py +@@ -173,8 +173,8 @@ class Conditional: + ) + try: + e = templar.environment.overlay() +- e.filters.update(templar._get_filters()) +- e.tests.update(templar._get_tests()) ++ e.filters.update(templar.environment.filters) ++ e.tests.update(templar.environment.tests) + + res = e._parse(conditional, None, None) + res = generate(res, e, None, None) diff --git a/lib/ansible/playbook/play_context.py b/lib/ansible/playbook/play_context.py -index 10dd57aa..5b8b2852 100644 +index 10dd57aa3f..5b8b28526c 100644 --- a/lib/ansible/playbook/play_context.py +++ b/lib/ansible/playbook/play_context.py @@ -318,10 +318,6 @@ class PlayContext(Base): @@ -54,9 +198,736 @@ index 10dd57aa..5b8b2852 100644 if task.check_mode is not None: new_info.check_mode = task.check_mode +diff --git a/lib/ansible/plugins/action/ce_template.py b/lib/ansible/plugins/action/ce_template.py +index 8d62b25647..4a72fbbfa8 100644 +--- a/lib/ansible/plugins/action/ce_template.py ++++ b/lib/ansible/plugins/action/ce_template.py +@@ -100,5 +100,5 @@ class ActionModule(_ActionModule): + for role in dep_chain: + searchpath.append(role._role_path) + searchpath.append(os.path.dirname(source)) +- self._templar.environment.loader.searchpath = searchpath +- self._task.args['src'] = self._templar.template(template_data) ++ with self._templar.set_temporary_context(searchpath=searchpath): ++ self._task.args['src'] = self._templar.template(template_data) +diff --git a/lib/ansible/plugins/action/network.py b/lib/ansible/plugins/action/network.py +index f0d0ca3ba7..d91c9b2af9 100644 +--- a/lib/ansible/plugins/action/network.py ++++ b/lib/ansible/plugins/action/network.py +@@ -160,8 +160,8 @@ class ActionModule(_ActionModule): + for role in dep_chain: + searchpath.append(role._role_path) + searchpath.append(os.path.dirname(source)) +- self._templar.environment.loader.searchpath = searchpath +- self._task.args['src'] = self._templar.template(template_data, convert_data=convert_data) ++ with self._templar.set_temporary_context(searchpath=searchpath): ++ self._task.args['src'] = self._templar.template(template_data, convert_data=convert_data) + + def _get_network_os(self, task_vars): + if 'network_os' in self._task.args and self._task.args['network_os']: +diff --git a/lib/ansible/plugins/action/template.py b/lib/ansible/plugins/action/template.py +index 8fb7393ff9..cede680ca6 100644 +--- a/lib/ansible/plugins/action/template.py ++++ b/lib/ansible/plugins/action/template.py +@@ -17,7 +17,7 @@ from ansible.module_utils._text import to_bytes, to_text, to_native + from ansible.module_utils.parsing.convert_bool import boolean + from ansible.module_utils.six import string_types + from ansible.plugins.action import ActionBase +-from ansible.template import generate_ansible_template_vars ++from ansible.template import generate_ansible_template_vars, AnsibleEnvironment + + + class ActionModule(ActionBase): +@@ -127,27 +127,23 @@ class ActionModule(ActionBase): + newsearchpath.append(p) + searchpath = newsearchpath + +- self._templar.environment.loader.searchpath = searchpath +- self._templar.environment.newline_sequence = newline_sequence +- if block_start_string is not None: +- self._templar.environment.block_start_string = block_start_string +- if block_end_string is not None: +- self._templar.environment.block_end_string = block_end_string +- if variable_start_string is not None: +- self._templar.environment.variable_start_string = variable_start_string +- if variable_end_string is not None: +- self._templar.environment.variable_end_string = variable_end_string +- self._templar.environment.trim_blocks = trim_blocks +- self._templar.environment.lstrip_blocks = lstrip_blocks +- + # add ansible 'template' vars + temp_vars = task_vars.copy() + temp_vars.update(generate_ansible_template_vars(source, dest)) + +- old_vars = self._templar.available_variables +- self._templar.available_variables = temp_vars +- resultant = self._templar.do_template(template_data, preserve_trailing_newlines=True, escape_backslashes=False) +- self._templar.available_variables = old_vars ++ # force templar to use AnsibleEnvironment to prevent issues with native types ++ # https://github.com/ansible/ansible/issues/46169 ++ templar = self._templar.copy_with_new_env(environment_class=AnsibleEnvironment, ++ searchpath=searchpath, ++ newline_sequence=newline_sequence, ++ block_start_string=block_start_string, ++ block_end_string=block_end_string, ++ variable_start_string=variable_start_string, ++ variable_end_string=variable_end_string, ++ trim_blocks=trim_blocks, ++ lstrip_blocks=lstrip_blocks, ++ available_variables=temp_vars) ++ resultant = templar.do_template(template_data, preserve_trailing_newlines=True, escape_backslashes=False) + except AnsibleAction: + raise + except Exception as e: +diff --git a/lib/ansible/plugins/lookup/template.py b/lib/ansible/plugins/lookup/template.py +index 4fd3584b65..c04b5e0d6a 100644 +--- a/lib/ansible/plugins/lookup/template.py ++++ b/lib/ansible/plugins/lookup/template.py +@@ -17,7 +17,9 @@ DOCUMENTATION = """ + description: list of files to template + convert_data: + type: bool +- description: whether to convert YAML into data. If False, strings that are YAML will be left untouched. ++ description: ++ - Whether to convert YAML into data. If False, strings that are YAML will be left untouched. ++ - Mutually exclusive with the jinja2_native option. + variable_start_string: + description: The string marking the beginning of a print statement. + default: '{{' +@@ -28,6 +30,16 @@ DOCUMENTATION = """ + default: '}}' + version_added: '2.8' + type: str ++ jinja2_native: ++ description: ++ - Controls whether to use Jinja2 native types. ++ - It is off by default even if global jinja2_native is True. ++ - Has no effect if global jinja2_native is False. ++ - This offers more flexibility than the template module which does not use Jinja2 native types at all. ++ - Mutually exclusive with the convert_data option. ++ default: False ++ version_added: '2.11' ++ type: bool + """ + + EXAMPLES = """ +@@ -51,24 +63,31 @@ import os + from ansible.errors import AnsibleError + from ansible.plugins.lookup import LookupBase + from ansible.module_utils._text import to_bytes, to_text +-from ansible.template import generate_ansible_template_vars ++from ansible.template import generate_ansible_template_vars, AnsibleEnvironment, USE_JINJA2_NATIVE + from ansible.utils.display import Display + ++if USE_JINJA2_NATIVE: ++ from ansible.utils.native_jinja import NativeJinjaText ++ ++ + display = Display() + + + class LookupModule(LookupBase): + + def run(self, terms, variables, **kwargs): +- + convert_data_p = kwargs.get('convert_data', True) + lookup_template_vars = kwargs.get('template_vars', {}) ++ jinja2_native = kwargs.get('jinja2_native', False) + ret = [] + + variable_start_string = kwargs.get('variable_start_string', None) + variable_end_string = kwargs.get('variable_end_string', None) + +- old_vars = self._templar.available_variables ++ if USE_JINJA2_NATIVE and not jinja2_native: ++ templar = self._templar.copy_with_new_env(environment_class=AnsibleEnvironment) ++ else: ++ templar = self._templar + + for term in terms: + display.debug("File lookup term: %s" % term) +@@ -92,12 +111,6 @@ class LookupModule(LookupBase): + searchpath = newsearchpath + searchpath.insert(0, os.path.dirname(lookupfile)) + +- self._templar.environment.loader.searchpath = searchpath +- if variable_start_string is not None: +- self._templar.environment.variable_start_string = variable_start_string +- if variable_end_string is not None: +- self._templar.environment.variable_end_string = variable_end_string +- + # The template will have access to all existing variables, + # plus some added by ansible (e.g., template_{path,mtime}), + # plus anything passed to the lookup with the template_vars= +@@ -105,17 +118,20 @@ class LookupModule(LookupBase): + vars = deepcopy(variables) + vars.update(generate_ansible_template_vars(lookupfile)) + vars.update(lookup_template_vars) +- self._templar.available_variables = vars + +- # do the templating +- res = self._templar.template(template_data, preserve_trailing_newlines=True, +- convert_data=convert_data_p, escape_backslashes=False) ++ with templar.set_temporary_context(variable_start_string=variable_start_string, ++ variable_end_string=variable_end_string, ++ available_variables=vars, searchpath=searchpath): ++ res = templar.template(template_data, preserve_trailing_newlines=True, ++ convert_data=convert_data_p, escape_backslashes=False) ++ ++ if USE_JINJA2_NATIVE and not jinja2_native: ++ # jinja2_native is true globally but off for the lookup, we need this text ++ # not to be processed by literal_eval anywhere in Ansible ++ res = NativeJinjaText(res) + + ret.append(res) + else: + raise AnsibleError("the template file %s could not be found for the lookup" % term) + +- # restore old variables +- self._templar.available_variables = old_vars +- + return ret +diff --git a/lib/ansible/template/__init__.py b/lib/ansible/template/__init__.py +index 94ab31e58d..35c9dac194 100644 +--- a/lib/ansible/template/__init__.py ++++ b/lib/ansible/template/__init__.py +@@ -28,6 +28,7 @@ import re + import time + + from distutils.version import LooseVersion ++from contextlib import contextmanager + from numbers import Number + + try: +@@ -42,8 +43,9 @@ from jinja2.runtime import Context, StrictUndefined + from ansible import constants as C + from ansible.errors import AnsibleError, AnsibleFilterError, AnsibleUndefinedVariable, AnsibleAssertionError + from ansible.module_utils.six import iteritems, string_types, text_type ++from ansible.module_utils.six.moves import range + from ansible.module_utils._text import to_native, to_text, to_bytes +-from ansible.module_utils.common._collections_compat import Sequence, Mapping, MutableMapping ++from ansible.module_utils.common._collections_compat import Iterator, Sequence, Mapping, MappingView, MutableMapping + from ansible.module_utils.common.collections import is_sequence + from ansible.module_utils.compat.importlib import import_module + from ansible.plugins.loader import filter_loader, lookup_loader, test_loader +@@ -71,12 +73,16 @@ NON_TEMPLATED_TYPES = (bool, Number) + JINJA2_OVERRIDE = '#jinja2:' + + from jinja2 import __version__ as j2_version ++from jinja2 import Environment ++from jinja2.utils import concat as j2_concat ++ + + USE_JINJA2_NATIVE = False + if C.DEFAULT_JINJA2_NATIVE: + try: +- from jinja2.nativetypes import NativeEnvironment as Environment +- from ansible.template.native_helpers import ansible_native_concat as j2_concat ++ from jinja2.nativetypes import NativeEnvironment ++ from ansible.template.native_helpers import ansible_native_concat ++ from ansible.utils.native_jinja import NativeJinjaText + USE_JINJA2_NATIVE = True + except ImportError: + from jinja2 import Environment +@@ -85,15 +91,15 @@ if C.DEFAULT_JINJA2_NATIVE: + 'jinja2_native requires Jinja 2.10 and above. ' + 'Version detected: %s. Falling back to default.' % j2_version + ) +-else: +- from jinja2 import Environment +- from jinja2.utils import concat as j2_concat + + + JINJA2_BEGIN_TOKENS = frozenset(('variable_begin', 'block_begin', 'comment_begin', 'raw_begin')) + JINJA2_END_TOKENS = frozenset(('variable_end', 'block_end', 'comment_end', 'raw_end')) + + ++RANGE_TYPE = type(range(0)) ++ ++ + def generate_ansible_template_vars(path, dest_path=None): + b_path = to_bytes(path) + try: +@@ -230,6 +236,60 @@ def recursive_check_defined(item): + raise AnsibleFilterError("{0} is undefined".format(item)) + + ++def _is_rolled(value): ++ """Helper method to determine if something is an unrolled generator, ++ iterator, or similar object ++ """ ++ return ( ++ isinstance(value, Iterator) or ++ isinstance(value, MappingView) or ++ isinstance(value, RANGE_TYPE) ++ ) ++ ++ ++def _unroll_iterator(func): ++ """Wrapper function, that intercepts the result of a filter ++ and auto unrolls a generator, so that users are not required to ++ explicitly use ``|list`` to unroll. ++ """ ++ def wrapper(*args, **kwargs): ++ ret = func(*args, **kwargs) ++ if _is_rolled(ret): ++ return list(ret) ++ return ret ++ ++ return _update_wrapper(wrapper, func) ++ ++ ++def _update_wrapper(wrapper, func): ++ # This code is duplicated from ``functools.update_wrapper`` from Py3.7. ++ # ``functools.update_wrapper`` was failing when the func was ``functools.partial`` ++ for attr in ('__module__', '__name__', '__qualname__', '__doc__', '__annotations__'): ++ try: ++ value = getattr(func, attr) ++ except AttributeError: ++ pass ++ else: ++ setattr(wrapper, attr, value) ++ for attr in ('__dict__',): ++ getattr(wrapper, attr).update(getattr(func, attr, {})) ++ wrapper.__wrapped__ = func ++ return wrapper ++ ++ ++def _wrap_native_text(func): ++ """Wrapper function, that intercepts the result of a filter ++ and wraps it into NativeJinjaText which is then used ++ in ``ansible_native_concat`` to indicate that it is a text ++ which should not be passed into ``literal_eval``. ++ """ ++ def wrapper(*args, **kwargs): ++ ret = func(*args, **kwargs) ++ return NativeJinjaText(ret) ++ ++ return _update_wrapper(wrapper, func) ++ ++ + class AnsibleUndefined(StrictUndefined): + ''' + A custom Undefined class, which returns further Undefined objects on access, +@@ -350,10 +410,11 @@ class AnsibleContext(Context): + + + class JinjaPluginIntercept(MutableMapping): +- def __init__(self, delegatee, pluginloader, *args, **kwargs): ++ def __init__(self, delegatee, pluginloader, jinja2_native, *args, **kwargs): + super(JinjaPluginIntercept, self).__init__(*args, **kwargs) + self._delegatee = delegatee + self._pluginloader = pluginloader ++ self._jinja2_native = jinja2_native + + if self._pluginloader.class_name == 'FilterModule': + self._method_map_name = 'filters' +@@ -406,10 +467,13 @@ class JinjaPluginIntercept(MutableMapping): + + method_map = getattr(plugin_impl, self._method_map_name) + +- for f in iteritems(method_map()): +- fq_name = '.'.join((parent_prefix, f[0])) ++ for func_name, func in iteritems(method_map()): ++ fq_name = '.'.join((parent_prefix, func_name)) + # FIXME: detect/warn on intra-collection function name collisions +- self._collection_jinja_func_cache[fq_name] = f[1] ++ if self._jinja2_native and func_name in C.STRING_TYPE_FILTERS: ++ self._collection_jinja_func_cache[fq_name] = _wrap_native_text(func) ++ else: ++ self._collection_jinja_func_cache[fq_name] = _unroll_iterator(func) + + function_impl = self._collection_jinja_func_cache[key] + return function_impl +@@ -433,6 +497,9 @@ class AnsibleEnvironment(Environment): + ''' + Our custom environment, which simply allows us to override the class-level + values for the Template and Context classes used by jinja2 internally. ++ ++ NOTE: Any changes to this class must be reflected in ++ :class:`AnsibleNativeEnvironment` as well. + ''' + context_class = AnsibleContext + template_class = AnsibleJ2Template +@@ -440,8 +507,27 @@ class AnsibleEnvironment(Environment): + def __init__(self, *args, **kwargs): + super(AnsibleEnvironment, self).__init__(*args, **kwargs) + +- self.filters = JinjaPluginIntercept(self.filters, filter_loader) +- self.tests = JinjaPluginIntercept(self.tests, test_loader) ++ self.filters = JinjaPluginIntercept(self.filters, filter_loader, jinja2_native=False) ++ self.tests = JinjaPluginIntercept(self.tests, test_loader, jinja2_native=False) ++ ++ ++if USE_JINJA2_NATIVE: ++ class AnsibleNativeEnvironment(NativeEnvironment): ++ ''' ++ Our custom environment, which simply allows us to override the class-level ++ values for the Template and Context classes used by jinja2 internally. ++ ++ NOTE: Any changes to this class must be reflected in ++ :class:`AnsibleEnvironment` as well. ++ ''' ++ context_class = AnsibleContext ++ template_class = AnsibleJ2Template ++ ++ def __init__(self, *args, **kwargs): ++ super(AnsibleNativeEnvironment, self).__init__(*args, **kwargs) ++ ++ self.filters = JinjaPluginIntercept(self.filters, filter_loader, jinja2_native=True) ++ self.tests = JinjaPluginIntercept(self.tests, test_loader, jinja2_native=True) + + + class Templar: +@@ -478,7 +564,9 @@ class Templar: + self._fail_on_filter_errors = True + self._fail_on_undefined_errors = C.DEFAULT_UNDEFINED_VAR_BEHAVIOR + +- self.environment = AnsibleEnvironment( ++ environment_class = AnsibleNativeEnvironment if USE_JINJA2_NATIVE else AnsibleEnvironment ++ ++ self.environment = environment_class( + trim_blocks=True, + undefined=AnsibleUndefined, + extensions=self._get_extensions(), +@@ -489,17 +577,50 @@ class Templar: + # the current rendering context under which the templar class is working + self.cur_context = None + ++ # FIXME these regular expressions should be re-compiled each time variable_start_string and variable_end_string are changed + self.SINGLE_VAR = re.compile(r"^%s\s*(\w*)\s*%s$" % (self.environment.variable_start_string, self.environment.variable_end_string)) +- +- self._clean_regex = re.compile(r'(?:%s|%s|%s|%s)' % ( +- self.environment.variable_start_string, +- self.environment.block_start_string, +- self.environment.block_end_string, +- self.environment.variable_end_string +- )) + self._no_type_regex = re.compile(r'.*?\|\s*(?:%s)(?:\([^\|]*\))?\s*\)?\s*(?:%s)' % + ('|'.join(C.STRING_TYPE_FILTERS), self.environment.variable_end_string)) + ++ @property ++ def jinja2_native(self): ++ return not isinstance(self.environment, AnsibleEnvironment) ++ ++ def copy_with_new_env(self, environment_class=AnsibleEnvironment, **kwargs): ++ r"""Creates a new copy of Templar with a new environment. The new environment is based on ++ given environment class and kwargs. ++ ++ :kwarg environment_class: Environment class used for creating a new environment. ++ :kwarg \*\*kwargs: Optional arguments for the new environment that override existing ++ environment attributes. ++ ++ :returns: Copy of Templar with updated environment. ++ """ ++ # We need to use __new__ to skip __init__, mainly not to create a new ++ # environment there only to override it below ++ new_env = object.__new__(environment_class) ++ new_env.__dict__.update(self.environment.__dict__) ++ ++ new_templar = object.__new__(Templar) ++ new_templar.__dict__.update(self.__dict__) ++ new_templar.environment = new_env ++ ++ mapping = { ++ 'available_variables': new_templar, ++ 'searchpath': new_env.loader, ++ } ++ ++ for key, value in kwargs.items(): ++ obj = mapping.get(key, new_env) ++ try: ++ if value is not None: ++ setattr(obj, key, value) ++ except AttributeError: ++ # Ignore invalid attrs, lstrip_blocks was added in jinja2==2.7 ++ pass ++ ++ return new_templar ++ + def _get_filters(self): + ''' + Returns filter plugins, after loading and caching them if need be +@@ -513,6 +634,17 @@ class Templar: + for fp in self._filter_loader.all(): + self._filters.update(fp.filters()) + ++ if self.jinja2_native: ++ for string_filter in C.STRING_TYPE_FILTERS: ++ try: ++ orig_filter = self._filters[string_filter] ++ except KeyError: ++ try: ++ orig_filter = self.environment.filters[string_filter] ++ except KeyError: ++ continue ++ self._filters[string_filter] = _wrap_native_text(orig_filter) ++ + return self._filters.copy() + + def _get_tests(self): +@@ -570,6 +702,36 @@ class Templar: + ) + self.available_variables = variables + ++ @contextmanager ++ def set_temporary_context(self, **kwargs): ++ """Context manager used to set temporary templating context, without having to worry about resetting ++ original values afterward ++ ++ Use a keyword that maps to the attr you are setting. Applies to ``self.environment`` by default, to ++ set context on another object, it must be in ``mapping``. ++ """ ++ mapping = { ++ 'available_variables': self, ++ 'searchpath': self.environment.loader, ++ } ++ original = {} ++ ++ for key, value in kwargs.items(): ++ obj = mapping.get(key, self.environment) ++ try: ++ original[key] = getattr(obj, key) ++ if value is not None: ++ setattr(obj, key, value) ++ except AttributeError: ++ # Ignore invalid attrs, lstrip_blocks was added in jinja2==2.7 ++ pass ++ ++ yield ++ ++ for key in original: ++ obj = mapping.get(key, self.environment) ++ setattr(obj, key, original[key]) ++ + def template(self, variable, convert_bare=False, preserve_trailing_newlines=True, escape_backslashes=True, fail_on_undefined=None, overrides=None, + convert_data=True, static_vars=None, cache=True, disable_lookups=False): + ''' +@@ -632,7 +794,7 @@ class Templar: + disable_lookups=disable_lookups, + ) + +- if not USE_JINJA2_NATIVE: ++ if not self.jinja2_native: + unsafe = hasattr(result, '__UNSAFE__') + if convert_data and not self._no_type_regex.match(variable): + # if this looks like a dictionary or list, convert it to such using the safe_eval method +@@ -746,8 +908,18 @@ class Templar: + + If using ANSIBLE_JINJA2_NATIVE we bypass this and return the actual value always + ''' +- if USE_JINJA2_NATIVE: ++ if _is_rolled(thing): ++ # Auto unroll a generator, so that users are not required to ++ # explicitly use ``|list`` to unroll ++ # This only affects the scenario where the final result of templating ++ # is a generator, and not where a filter creates a generator in the middle ++ # of a template. See ``_unroll_iterator`` for the other case. This is probably ++ # unncessary ++ return list(thing) ++ ++ if self.jinja2_native: + return thing ++ + return thing if thing is not None else '' + + def _fail_lookup(self, name, *args, **kwargs): +@@ -802,7 +974,10 @@ class Templar: + ran = wrap_var(ran) + else: + try: +- ran = wrap_var(",".join(ran)) ++ if self.jinja2_native and isinstance(ran[0], NativeJinjaText): ++ ran = wrap_var(NativeJinjaText(",".join(ran))) ++ else: ++ ran = wrap_var(",".join(ran)) + except TypeError: + # Lookup Plugins should always return lists. Throw an error if that's not + # the case: +@@ -824,7 +999,7 @@ class Templar: + raise AnsibleError("lookup plugin (%s) not found" % name) + + def do_template(self, data, preserve_trailing_newlines=True, escape_backslashes=True, fail_on_undefined=None, overrides=None, disable_lookups=False): +- if USE_JINJA2_NATIVE and not isinstance(data, string_types): ++ if self.jinja2_native and not isinstance(data, string_types): + return data + + # For preserving the number of input newlines in the output (used +@@ -853,6 +1028,8 @@ class Templar: + + # Adds Ansible custom filters and tests + myenv.filters.update(self._get_filters()) ++ for k in myenv.filters: ++ myenv.filters[k] = _unroll_iterator(myenv.filters[k]) + myenv.tests.update(self._get_tests()) + + if escape_backslashes: +@@ -904,7 +1081,7 @@ class Templar: + display.debug("failing because of a type error, template data is: %s" % to_text(data)) + raise AnsibleError("Unexpected templating type error occurred on (%s): %s" % (to_native(data), to_native(te))) + +- if USE_JINJA2_NATIVE and not isinstance(res, string_types): ++ if self.jinja2_native and not isinstance(res, string_types): + return res + + if preserve_trailing_newlines: +diff --git a/lib/ansible/template/native_helpers.py b/lib/ansible/template/native_helpers.py +index 11c14b7fa1..84296ad9b6 100644 +--- a/lib/ansible/template/native_helpers.py ++++ b/lib/ansible/template/native_helpers.py +@@ -14,6 +14,34 @@ from jinja2._compat import text_type + + from jinja2.runtime import StrictUndefined + from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode ++from ansible.utils.native_jinja import NativeJinjaText ++ ++ ++def _fail_on_undefined(data): ++ """Recursively find an undefined value in a nested data structure ++ and properly raise the undefined exception. ++ """ ++ if isinstance(data, Mapping): ++ for value in data.values(): ++ _fail_on_undefined(value) ++ elif is_sequence(data): ++ for item in data: ++ _fail_on_undefined(item) ++ else: ++ if isinstance(data, StrictUndefined): ++ # To actually raise the undefined exception we need to ++ # access the undefined object otherwise the exception would ++ # be raised on the next access which might not be properly ++ # handled. ++ # See https://github.com/ansible/ansible/issues/52158 ++ # and StrictUndefined implementation in upstream Jinja2. ++ str(data) ++ ++ return data ++ ++ ++class NativeJinjaText(text_type): ++ pass + + + def ansible_native_concat(nodes): +@@ -49,9 +77,20 @@ def ansible_native_concat(nodes): + # We do that only here because it is taken care of by text_type() in the else block below already. + str(out) + ++ if isinstance(out, NativeJinjaText): ++ # Sometimes (e.g. ``| string``) we need to mark variables ++ # in a special way so that they remain strings and are not ++ # passed into literal_eval. ++ # See: ++ # https://github.com/ansible/ansible/issues/70831 ++ # https://github.com/pallets/jinja/issues/1200 ++ # https://github.com/ansible/ansible/issues/70831#issuecomment-664190894 ++ return out ++ + # short circuit literal_eval when possible + if not isinstance(out, list): + return out ++ + else: + if isinstance(nodes, types.GeneratorType): + nodes = chain(head, nodes) +diff --git a/lib/ansible/utils/native_jinja.py b/lib/ansible/utils/native_jinja.py +new file mode 100644 +index 0000000000..53ef14004a +--- /dev/null ++++ b/lib/ansible/utils/native_jinja.py +@@ -0,0 +1,13 @@ ++# Copyright: (c) 2020, Ansible Project ++# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) ++ ++# Make coding more python3-ish ++from __future__ import (absolute_import, division, print_function) ++__metaclass__ = type ++ ++ ++from ansible.module_utils.six import text_type ++ ++ ++class NativeJinjaText(text_type): ++ pass +diff --git a/lib/ansible/utils/unsafe_proxy.py b/lib/ansible/utils/unsafe_proxy.py +index 0bfd6340ff..54bebd177a 100644 +--- a/lib/ansible/utils/unsafe_proxy.py ++++ b/lib/ansible/utils/unsafe_proxy.py +@@ -57,6 +57,7 @@ from ansible.module_utils._text import to_bytes, to_text + from ansible.module_utils.common._collections_compat import Mapping, Set + from ansible.module_utils.common.collections import is_sequence + from ansible.module_utils.six import string_types, binary_type, text_type ++from ansible.utils.native_jinja import NativeJinjaText + + + __all__ = ['AnsibleUnsafe', 'wrap_var'] +@@ -331,6 +332,10 @@ class NativeJinjaUnsafeText(NativeJinjaText, AnsibleUnsafeText): + pass + + ++class NativeJinjaUnsafeText(NativeJinjaText, AnsibleUnsafeText): ++ pass ++ ++ + class UnsafeProxy(object): + def __new__(cls, obj, *args, **kwargs): + from ansible.utils.display import Display +@@ -376,6 +381,8 @@ def wrap_var(v): + v = _wrap_set(v) + elif is_sequence(v): + v = _wrap_sequence(v) ++ elif isinstance(v, NativeJinjaText): ++ v = NativeJinjaUnsafeText(v) + elif isinstance(v, binary_type): + v = AnsibleUnsafeBytes(v) + elif isinstance(v, text_type): +diff --git a/test/integration/targets/jinja2_native_types/test_casting.yml b/test/integration/targets/jinja2_native_types/test_casting.yml +index 5b4fe3ac0e..da06ab2e28 100644 +--- a/test/integration/targets/jinja2_native_types/test_casting.yml ++++ b/test/integration/targets/jinja2_native_types/test_casting.yml +@@ -1,17 +1,22 @@ + - name: cast things to other things + set_fact: + int_to_str: "{{ i_two|to_text }}" ++ int_to_str2: "{{ i_two | string }}" + str_to_int: "{{ s_two|int }}" + dict_to_str: "{{ dict_one|to_text }}" + list_to_str: "{{ list_one|to_text }}" + int_to_bool: "{{ i_one|bool }}" + str_true_to_bool: "{{ s_true|bool }}" + str_false_to_bool: "{{ s_false|bool }}" ++ list_to_json_str: "{{ list_one | to_json }}" ++ list_to_yaml_str: "{{ list_one | to_yaml }}" + + - assert: + that: + - 'int_to_str == "2"' + - 'int_to_str|type_debug in ["str", "unicode"]' ++ - 'int_to_str2 == "2"' ++ - 'int_to_str2|type_debug in ["NativeJinjaText"]' + - 'str_to_int == 2' + - 'str_to_int|type_debug == "int"' + - 'dict_to_str|type_debug in ["str", "unicode"]' +@@ -22,3 +27,5 @@ + - 'str_true_to_bool|type_debug == "bool"' + - 'str_false_to_bool is sameas false' + - 'str_false_to_bool|type_debug == "bool"' ++ - 'list_to_json_str|type_debug in ["NativeJinjaText"]' ++ - 'list_to_yaml_str|type_debug in ["NativeJinjaText"]' +diff --git a/test/integration/targets/jinja2_native_types/test_dunder.yml b/test/integration/targets/jinja2_native_types/test_dunder.yml +index 46fd4d0a90..df5ea9276b 100644 +--- a/test/integration/targets/jinja2_native_types/test_dunder.yml ++++ b/test/integration/targets/jinja2_native_types/test_dunder.yml +@@ -20,4 +20,4 @@ + + - assert: + that: +- - 'const_dunder|type_debug in ["str", "unicode"]' ++ - 'const_dunder|type_debug in ["str", "unicode", "NativeJinjaText"]' diff --git a/test/integration/targets/no_log/no_log_config.yml b/test/integration/targets/no_log/no_log_config.yml new file mode 100644 -index 00000000..8a508805 +index 0000000000..8a5088059d --- /dev/null +++ b/test/integration/targets/no_log/no_log_config.yml @@ -0,0 +1,13 @@ @@ -74,7 +945,7 @@ index 00000000..8a508805 + - debug: + loop: '{{ range(3) }}' diff --git a/test/integration/targets/no_log/runme.sh b/test/integration/targets/no_log/runme.sh -index bb5c048f..8bfe019b 100755 +index bb5c048fc9..8bfe019bb9 100755 --- a/test/integration/targets/no_log/runme.sh +++ b/test/integration/targets/no_log/runme.sh @@ -19,3 +19,8 @@ set -eux @@ -86,6 +957,73 @@ index bb5c048f..8bfe019b 100755 +[ "$(ansible-playbook no_log_config.yml -i ../../inventory -vvvvv "$@" | grep -Ec 'the output has been hidden')" = "1" ] +[ "$(ANSIBLE_NO_LOG=0 ansible-playbook no_log_config.yml -i ../../inventory -vvvvv "$@" | grep -Ec 'the output has been hidden')" = "1" ] +[ "$(ANSIBLE_NO_LOG=1 ansible-playbook no_log_config.yml -i ../../inventory -vvvvv "$@" | grep -Ec 'the output has been hidden')" = "6" ] +diff --git a/test/integration/targets/template_jinja2_non_native/46169.yml b/test/integration/targets/template_jinja2_non_native/46169.yml +new file mode 100644 +index 0000000000..efb443eae0 +--- /dev/null ++++ b/test/integration/targets/template_jinja2_non_native/46169.yml +@@ -0,0 +1,32 @@ ++- hosts: localhost ++ gather_facts: no ++ tasks: ++ - set_fact: ++ output_dir: "{{ lookup('env', 'OUTPUT_DIR') }}" ++ ++ - template: ++ src: templates/46169.json.j2 ++ dest: "{{ output_dir }}/result.json" ++ ++ - command: "diff templates/46169.json.j2 {{ output_dir }}/result.json" ++ register: diff_result ++ ++ - assert: ++ that: ++ - diff_result.stdout == "" ++ ++ - block: ++ - set_fact: ++ non_native_lookup: "{{ lookup('template', 'templates/46169.json.j2') }}" ++ ++ - assert: ++ that: ++ - non_native_lookup | type_debug == 'NativeJinjaUnsafeText' ++ ++ - set_fact: ++ native_lookup: "{{ lookup('template', 'templates/46169.json.j2', jinja2_native=true) }}" ++ ++ - assert: ++ that: ++ - native_lookup | type_debug == 'dict' ++ when: lookup('pipe', ansible_python_interpreter ~ ' -c "import jinja2; print(jinja2.__version__)"') is version('2.10', '>=') +diff --git a/test/integration/targets/template_jinja2_non_native/aliases b/test/integration/targets/template_jinja2_non_native/aliases +new file mode 100644 +index 0000000000..b59832142f +--- /dev/null ++++ b/test/integration/targets/template_jinja2_non_native/aliases +@@ -0,0 +1 @@ ++shippable/posix/group3 +diff --git a/test/integration/targets/template_jinja2_non_native/runme.sh b/test/integration/targets/template_jinja2_non_native/runme.sh +new file mode 100755 +index 0000000000..fe9d495a3e +--- /dev/null ++++ b/test/integration/targets/template_jinja2_non_native/runme.sh +@@ -0,0 +1,7 @@ ++#!/usr/bin/env bash ++ ++set -eux ++ ++export ANSIBLE_JINJA2_NATIVE=1 ++ansible-playbook 46169.yml -v "$@" ++unset ANSIBLE_JINJA2_NATIVE +diff --git a/test/integration/targets/template_jinja2_non_native/templates/46169.json.j2 b/test/integration/targets/template_jinja2_non_native/templates/46169.json.j2 +new file mode 100644 +index 0000000000..a4fc3f6717 +--- /dev/null ++++ b/test/integration/targets/template_jinja2_non_native/templates/46169.json.j2 +@@ -0,0 +1,3 @@ ++{ ++ "key": "bar" ++} -- -2.33.0 +2.44.0 diff --git a/ansible.spec b/ansible.spec index 767b1ea..3d3902c 100644 --- a/ansible.spec +++ b/ansible.spec @@ -10,7 +10,7 @@ Name: ansible Summary: SSH-based configuration management, deployment, and task execution system Version: 2.9.27 -Release: 6 +Release: 7 License: Python-2.0 and MIT and GPL+ Url: http://ansible.com Source0: https://releases.ansible.com/ansible/%{name}-%{version}.tar.gz @@ -20,6 +20,7 @@ Patch2: CVE-2024-8775.patch Patch3: CVE-2024-9902.patch Patch4: CVE-2022-3697.patch Patch5: CVE-2023-5115.patch +Patch6: CVE-2023-5764.patch BuildArch: noarch Provides: ansible-fireball = %{version}-%{release} Obsoletes: ansible-fireball < 1.2.4 @@ -103,6 +104,9 @@ cp -pr docs/docsite/rst . %endif %changelog +* Tue Apr 01 2025 wangkai <13474090681@163.com> - 2.9.27-7 +- Fix CVE-2023-5764 + * Sat Feb 08 2025 wangkai <13474090681@163.com> - 2.9.27-6 - Fix CVE-2022-3697 CVE-2023-5115 -- Gitee