From f01cb885f7bc26495ef1f3a9b0df4da44e3e9952 Mon Sep 17 00:00:00 2001 From: xinsheng3 Date: Tue, 24 Sep 2024 20:07:22 +0800 Subject: [PATCH] fix CVE-2024-6232,CVE-2024-3219,CVE-2024-0450,CVE-2023-6597,CVE-2024-4032 --- ...pfile.TemporaryDirectory-fix-symlink.patch | 0 ...x-locking-in-cert_store_stats-and-ge.patch | 0 ...otect-zipfile-from-quoted-overlap-zi.patch | 16 +- ...thenticate-socket-connection-for-soc.patch | 218 ++++++++++ ...work-pure-Python-socketpair-tests-to.patch | 207 +++++++++ ...-65056-Fix-private-non-global-IP-add.patch | 402 ++++++++++++++++++ ...emove-backtracking-when-parsing-tarf.patch | 248 +++++++++++ ...code-newlines-in-headers-and-verify-.patch | 133 +++--- ...adratic-complexity-in-parsing-quoted.patch | 0 ...ed-SanitizedNames-with-a-more-surgic.patch | 0 python3.spec | 39 +- 11 files changed, 1187 insertions(+), 76 deletions(-) rename backport-3.9-gh-91133-tempfile.TemporaryDirectory-fix-symlink.patch => backport-CVE-2023-6597-gh-91133-tempfile.TemporaryDirectory-fix-symlink.patch (100%) rename backport-3.9-gh-114572-Fix-locking-in-cert_store_stats-and-ge.patch => backport-CVE-2024-0397-gh-114572-Fix-locking-in-cert_store_stats-and-ge.patch (100%) rename backport-3.9-gh-109858-Protect-zipfile-from-quoted-overlap-zi.patch => backport-CVE-2024-0450-gh-109858-Protect-zipfile-from-quoted-overlap-zi.patch (93%) create mode 100644 backport-CVE-2024-3219-1-gh-122133-Authenticate-socket-connection-for-soc.patch create mode 100644 backport-CVE-2024-3219-2-gh-122133-Rework-pure-Python-socketpair-tests-to.patch create mode 100644 backport-CVE-2024-4032-gh-113171-gh-65056-Fix-private-non-global-IP-add.patch create mode 100644 backport-CVE-2024-6232-gh-121285-Remove-backtracking-when-parsing-tarf.patch rename backport-gh-121650-Encode-newlines-in-headers-and-verify-head.patch => backport-CVE-2024-6923-gh-121650-Encode-newlines-in-headers-and-verify-.patch (85%) rename backport-gh-123067-Fix-quadratic-complexity-in-parsing-quoted.patch => backport-CVE-2024-7592-gh-123067-Fix-quadratic-complexity-in-parsing-quoted.patch (100%) rename backport-gh-123270-Replaced-SanitizedNames-with-a-more-surgic.patch => backport-CVE-2024-8088-gh-123270-Replaced-SanitizedNames-with-a-more-surgic.patch (100%) diff --git a/backport-3.9-gh-91133-tempfile.TemporaryDirectory-fix-symlink.patch b/backport-CVE-2023-6597-gh-91133-tempfile.TemporaryDirectory-fix-symlink.patch similarity index 100% rename from backport-3.9-gh-91133-tempfile.TemporaryDirectory-fix-symlink.patch rename to backport-CVE-2023-6597-gh-91133-tempfile.TemporaryDirectory-fix-symlink.patch diff --git a/backport-3.9-gh-114572-Fix-locking-in-cert_store_stats-and-ge.patch b/backport-CVE-2024-0397-gh-114572-Fix-locking-in-cert_store_stats-and-ge.patch similarity index 100% rename from backport-3.9-gh-114572-Fix-locking-in-cert_store_stats-and-ge.patch rename to backport-CVE-2024-0397-gh-114572-Fix-locking-in-cert_store_stats-and-ge.patch diff --git a/backport-3.9-gh-109858-Protect-zipfile-from-quoted-overlap-zi.patch b/backport-CVE-2024-0450-gh-109858-Protect-zipfile-from-quoted-overlap-zi.patch similarity index 93% rename from backport-3.9-gh-109858-Protect-zipfile-from-quoted-overlap-zi.patch rename to backport-CVE-2024-0450-gh-109858-Protect-zipfile-from-quoted-overlap-zi.patch index 91fcdb5..258b670 100644 --- a/backport-3.9-gh-109858-Protect-zipfile-from-quoted-overlap-zi.patch +++ b/backport-CVE-2024-0450-gh-109858-Protect-zipfile-from-quoted-overlap-zi.patch @@ -18,10 +18,10 @@ Co-authored-by: Serhiy Storchaka create mode 100644 Misc/NEWS.d/next/Library/2023-09-28-13-15-51.gh-issue-109858.43e2dg.rst diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py -index bd383d3f68..17e95eb862 100644 +index bd383d3f685..17e95eb8623 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py -@@ -2045,6 +2045,66 @@ def test_decompress_without_3rd_party_library(self): +@@ -2045,6 +2045,66 @@ class OtherTests(unittest.TestCase): with zipfile.ZipFile(zip_file) as zf: self.assertRaises(RuntimeError, zf.extract, 'a.txt') @@ -89,7 +89,7 @@ index bd383d3f68..17e95eb862 100644 unlink(TESTFN) unlink(TESTFN2) diff --git a/Lib/zipfile.py b/Lib/zipfile.py -index 1e942a503e..95f95ee112 100644 +index 1e942a503e8..95f95ee1126 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -338,6 +338,7 @@ class ZipInfo (object): @@ -100,7 +100,7 @@ index 1e942a503e..95f95ee112 100644 ) def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)): -@@ -379,6 +380,7 @@ def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)): +@@ -379,6 +380,7 @@ class ZipInfo (object): self.external_attr = 0 # External file attributes self.compress_size = 0 # Size of the compressed file self.file_size = 0 # Size of the uncompressed file @@ -108,7 +108,7 @@ index 1e942a503e..95f95ee112 100644 # Other attributes are set by class ZipFile: # header_offset Byte offset to the file header # CRC CRC-32 of the uncompressed file -@@ -1399,6 +1401,12 @@ def _RealGetContents(self): +@@ -1399,6 +1401,12 @@ class ZipFile: if self.debug > 2: print("total", total) @@ -121,7 +121,7 @@ index 1e942a503e..95f95ee112 100644 def namelist(self): """Return a list of file names in the archive.""" -@@ -1554,6 +1562,10 @@ def open(self, name, mode="r", pwd=None, *, force_zip64=False): +@@ -1554,6 +1562,10 @@ class ZipFile: 'File name in directory %r and header %r differ.' % (zinfo.orig_filename, fname)) @@ -134,7 +134,7 @@ index 1e942a503e..95f95ee112 100644 if is_encrypted: diff --git a/Misc/NEWS.d/next/Library/2023-09-28-13-15-51.gh-issue-109858.43e2dg.rst b/Misc/NEWS.d/next/Library/2023-09-28-13-15-51.gh-issue-109858.43e2dg.rst new file mode 100644 -index 0000000000..be279caffc +index 00000000000..be279caffc4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-28-13-15-51.gh-issue-109858.43e2dg.rst @@ -0,0 +1,3 @@ @@ -142,5 +142,5 @@ index 0000000000..be279caffc +BadZipFile when try to read an entry that overlaps with other entry or +central directory. -- -2.34.1.windows.1 +2.33.0 diff --git a/backport-CVE-2024-3219-1-gh-122133-Authenticate-socket-connection-for-soc.patch b/backport-CVE-2024-3219-1-gh-122133-Authenticate-socket-connection-for-soc.patch new file mode 100644 index 0000000..e375f5a --- /dev/null +++ b/backport-CVE-2024-3219-1-gh-122133-Authenticate-socket-connection-for-soc.patch @@ -0,0 +1,218 @@ +From 06fa244666ec6335a3b9bf2367e31b42b9a89b20 Mon Sep 17 00:00:00 2001 +From: "Miss Islington (bot)" + <31488909+miss-islington@users.noreply.github.com> +Date: Tue, 30 Jul 2024 14:44:26 +0200 +Subject: [PATCH] [3.9] gh-122133: Authenticate socket connection for + `socket.socketpair()` fallback (GH-122134) (#122428) + +Authenticate socket connection for `socket.socketpair()` fallback when the platform does not have a native `socketpair` C API. We authenticate in-process using `getsocketname` and `getpeername` (thanks to Nathaniel J Smith for that suggestion). + +(cherry picked from commit 78df1043dbdce5c989600616f9f87b4ee72944e5) + +Co-authored-by: Seth Michael Larson +Co-authored-by: Gregory P. Smith +--- + Lib/socket.py | 17 +++ + Lib/test/test_socket.py | 128 +++++++++++++++++- + ...-07-22-13-11-28.gh-issue-122133.0mPeta.rst | 5 + + 3 files changed, 147 insertions(+), 3 deletions(-) + create mode 100644 Misc/NEWS.d/next/Security/2024-07-22-13-11-28.gh-issue-122133.0mPeta.rst + +diff --git a/Lib/socket.py b/Lib/socket.py +index 46fc49ca323..643f218d2f7 100755 +--- a/Lib/socket.py ++++ b/Lib/socket.py +@@ -646,6 +646,23 @@ else: + raise + finally: + lsock.close() ++ ++ # Authenticating avoids using a connection from something else ++ # able to connect to {host}:{port} instead of us. ++ # We expect only AF_INET and AF_INET6 families. ++ try: ++ if ( ++ ssock.getsockname() != csock.getpeername() ++ or csock.getsockname() != ssock.getpeername() ++ ): ++ raise ConnectionError("Unexpected peer connection") ++ except: ++ # getsockname() and getpeername() can fail ++ # if either socket isn't connected. ++ ssock.close() ++ csock.close() ++ raise ++ + return (ssock, csock) + __all__.append("socketpair") + +diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py +index 043e5543889..ea812408042 100755 +--- a/Lib/test/test_socket.py ++++ b/Lib/test/test_socket.py +@@ -555,19 +555,27 @@ class SocketPairTest(unittest.TestCase, ThreadableTest): + def __init__(self, methodName='runTest'): + unittest.TestCase.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) ++ self.cli = None ++ self.serv = None ++ ++ def socketpair(self): ++ # To be overridden by some child classes. ++ return socket.socketpair() + + def setUp(self): +- self.serv, self.cli = socket.socketpair() ++ self.serv, self.cli = self.socketpair() + + def tearDown(self): +- self.serv.close() ++ if self.serv: ++ self.serv.close() + self.serv = None + + def clientSetUp(self): + pass + + def clientTearDown(self): +- self.cli.close() ++ if self.cli: ++ self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + +@@ -4613,6 +4621,120 @@ class BasicSocketPairTest(SocketPairTest): + self.assertEqual(msg, MSG) + + ++class PurePythonSocketPairTest(SocketPairTest): ++ ++ # Explicitly use socketpair AF_INET or AF_INET6 to ensure that is the ++ # code path we're using regardless platform is the pure python one where ++ # `_socket.socketpair` does not exist. (AF_INET does not work with ++ # _socket.socketpair on many platforms). ++ def socketpair(self): ++ # called by super().setUp(). ++ try: ++ return socket.socketpair(socket.AF_INET6) ++ except OSError: ++ return socket.socketpair(socket.AF_INET) ++ ++ # Local imports in this class make for easy security fix backporting. ++ ++ def setUp(self): ++ import _socket ++ self._orig_sp = getattr(_socket, 'socketpair', None) ++ if self._orig_sp is not None: ++ # This forces the version using the non-OS provided socketpair ++ # emulation via an AF_INET socket in Lib/socket.py. ++ del _socket.socketpair ++ import importlib ++ global socket ++ socket = importlib.reload(socket) ++ else: ++ pass # This platform already uses the non-OS provided version. ++ super().setUp() ++ ++ def tearDown(self): ++ super().tearDown() ++ import _socket ++ if self._orig_sp is not None: ++ # Restore the default socket.socketpair definition. ++ _socket.socketpair = self._orig_sp ++ import importlib ++ global socket ++ socket = importlib.reload(socket) ++ ++ def test_recv(self): ++ msg = self.serv.recv(1024) ++ self.assertEqual(msg, MSG) ++ ++ def _test_recv(self): ++ self.cli.send(MSG) ++ ++ def test_send(self): ++ self.serv.send(MSG) ++ ++ def _test_send(self): ++ msg = self.cli.recv(1024) ++ self.assertEqual(msg, MSG) ++ ++ def test_ipv4(self): ++ cli, srv = socket.socketpair(socket.AF_INET) ++ cli.close() ++ srv.close() ++ ++ def _test_ipv4(self): ++ pass ++ ++ @unittest.skipIf(not hasattr(_socket, 'IPPROTO_IPV6') or ++ not hasattr(_socket, 'IPV6_V6ONLY'), ++ "IPV6_V6ONLY option not supported") ++ @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'IPv6 required for this test') ++ def test_ipv6(self): ++ cli, srv = socket.socketpair(socket.AF_INET6) ++ cli.close() ++ srv.close() ++ ++ def _test_ipv6(self): ++ pass ++ ++ def test_injected_authentication_failure(self): ++ orig_getsockname = socket.socket.getsockname ++ inject_sock = None ++ ++ def inject_getsocketname(self): ++ nonlocal inject_sock ++ sockname = orig_getsockname(self) ++ # Connect to the listening socket ahead of the ++ # client socket. ++ if inject_sock is None: ++ inject_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) ++ inject_sock.setblocking(False) ++ try: ++ inject_sock.connect(sockname[:2]) ++ except (BlockingIOError, InterruptedError): ++ pass ++ inject_sock.setblocking(True) ++ return sockname ++ ++ sock1 = sock2 = None ++ try: ++ socket.socket.getsockname = inject_getsocketname ++ with self.assertRaises(OSError): ++ sock1, sock2 = socket.socketpair() ++ finally: ++ socket.socket.getsockname = orig_getsockname ++ if inject_sock: ++ inject_sock.close() ++ if sock1: # This cleanup isn't needed on a successful test. ++ sock1.close() ++ if sock2: ++ sock2.close() ++ ++ def _test_injected_authentication_failure(self): ++ # No-op. Exists for base class threading infrastructure to call. ++ # We could refactor this test into its own lesser class along with the ++ # setUp and tearDown code to construct an ideal; it is simpler to keep ++ # it here and live with extra overhead one this _one_ failure test. ++ pass ++ ++ + class NonBlockingTCPTests(ThreadedTCPSocketTest): + + def __init__(self, methodName='runTest'): +diff --git a/Misc/NEWS.d/next/Security/2024-07-22-13-11-28.gh-issue-122133.0mPeta.rst b/Misc/NEWS.d/next/Security/2024-07-22-13-11-28.gh-issue-122133.0mPeta.rst +new file mode 100644 +index 00000000000..3544eb3824d +--- /dev/null ++++ b/Misc/NEWS.d/next/Security/2024-07-22-13-11-28.gh-issue-122133.0mPeta.rst +@@ -0,0 +1,5 @@ ++Authenticate the socket connection for the ``socket.socketpair()`` fallback ++on platforms where ``AF_UNIX`` is not available like Windows. ++ ++Patch by Gregory P. Smith and Seth Larson . Reported by Ellie ++ +-- +2.33.0 + diff --git a/backport-CVE-2024-3219-2-gh-122133-Rework-pure-Python-socketpair-tests-to.patch b/backport-CVE-2024-3219-2-gh-122133-Rework-pure-Python-socketpair-tests-to.patch new file mode 100644 index 0000000..ac6c86e --- /dev/null +++ b/backport-CVE-2024-3219-2-gh-122133-Rework-pure-Python-socketpair-tests-to.patch @@ -0,0 +1,207 @@ +From 3f5d9d12c74787fbf3f5891835c85cc15526c86d Mon Sep 17 00:00:00 2001 +From: "Miss Islington (bot)" + <31488909+miss-islington@users.noreply.github.com> +Date: Fri, 2 Aug 2024 15:10:52 +0200 +Subject: [PATCH] [3.9] gh-122133: Rework pure Python socketpair tests to avoid + use of importlib.reload. (GH-122493) (GH-122508) + +(cherry picked from commit f071f01b7b7e19d7d6b3a4b0ec62f820ecb14660) + +Co-authored-by: Russell Keith-Magee +Co-authored-by: Gregory P. Smith +--- + Lib/socket.py | 121 +++++++++++++++++++--------------------- + Lib/test/test_socket.py | 20 ++----- + 2 files changed, 64 insertions(+), 77 deletions(-) + +diff --git a/Lib/socket.py b/Lib/socket.py +index 643f218d2f7..28360985450 100755 +--- a/Lib/socket.py ++++ b/Lib/socket.py +@@ -588,16 +588,65 @@ if hasattr(_socket.socket, "share"): + return socket(0, 0, 0, info) + __all__.append("fromshare") + +-if hasattr(_socket, "socketpair"): ++# Origin: https://gist.github.com/4325783, by Geert Jansen. Public domain. ++# This is used if _socket doesn't natively provide socketpair. It's ++# always defined so that it can be patched in for testing purposes. ++def _fallback_socketpair(family=AF_INET, type=SOCK_STREAM, proto=0): ++ if family == AF_INET: ++ host = _LOCALHOST ++ elif family == AF_INET6: ++ host = _LOCALHOST_V6 ++ else: ++ raise ValueError("Only AF_INET and AF_INET6 socket address families " ++ "are supported") ++ if type != SOCK_STREAM: ++ raise ValueError("Only SOCK_STREAM socket type is supported") ++ if proto != 0: ++ raise ValueError("Only protocol zero is supported") ++ ++ # We create a connected TCP socket. Note the trick with ++ # setblocking(False) that prevents us from having to create a thread. ++ lsock = socket(family, type, proto) ++ try: ++ lsock.bind((host, 0)) ++ lsock.listen() ++ # On IPv6, ignore flow_info and scope_id ++ addr, port = lsock.getsockname()[:2] ++ csock = socket(family, type, proto) ++ try: ++ csock.setblocking(False) ++ try: ++ csock.connect((addr, port)) ++ except (BlockingIOError, InterruptedError): ++ pass ++ csock.setblocking(True) ++ ssock, _ = lsock.accept() ++ except: ++ csock.close() ++ raise ++ finally: ++ lsock.close() + +- def socketpair(family=None, type=SOCK_STREAM, proto=0): +- """socketpair([family[, type[, proto]]]) -> (socket object, socket object) ++ # Authenticating avoids using a connection from something else ++ # able to connect to {host}:{port} instead of us. ++ # We expect only AF_INET and AF_INET6 families. ++ try: ++ if ( ++ ssock.getsockname() != csock.getpeername() ++ or csock.getsockname() != ssock.getpeername() ++ ): ++ raise ConnectionError("Unexpected peer connection") ++ except: ++ # getsockname() and getpeername() can fail ++ # if either socket isn't connected. ++ ssock.close() ++ csock.close() ++ raise + +- Create a pair of socket objects from the sockets returned by the platform +- socketpair() function. +- The arguments are the same as for socket() except the default family is +- AF_UNIX if defined on the platform; otherwise, the default is AF_INET. +- """ ++ return (ssock, csock) ++ ++if hasattr(_socket, "socketpair"): ++ def socketpair(family=None, type=SOCK_STREAM, proto=0): + if family is None: + try: + family = AF_UNIX +@@ -609,61 +658,7 @@ if hasattr(_socket, "socketpair"): + return a, b + + else: +- +- # Origin: https://gist.github.com/4325783, by Geert Jansen. Public domain. +- def socketpair(family=AF_INET, type=SOCK_STREAM, proto=0): +- if family == AF_INET: +- host = _LOCALHOST +- elif family == AF_INET6: +- host = _LOCALHOST_V6 +- else: +- raise ValueError("Only AF_INET and AF_INET6 socket address families " +- "are supported") +- if type != SOCK_STREAM: +- raise ValueError("Only SOCK_STREAM socket type is supported") +- if proto != 0: +- raise ValueError("Only protocol zero is supported") +- +- # We create a connected TCP socket. Note the trick with +- # setblocking(False) that prevents us from having to create a thread. +- lsock = socket(family, type, proto) +- try: +- lsock.bind((host, 0)) +- lsock.listen() +- # On IPv6, ignore flow_info and scope_id +- addr, port = lsock.getsockname()[:2] +- csock = socket(family, type, proto) +- try: +- csock.setblocking(False) +- try: +- csock.connect((addr, port)) +- except (BlockingIOError, InterruptedError): +- pass +- csock.setblocking(True) +- ssock, _ = lsock.accept() +- except: +- csock.close() +- raise +- finally: +- lsock.close() +- +- # Authenticating avoids using a connection from something else +- # able to connect to {host}:{port} instead of us. +- # We expect only AF_INET and AF_INET6 families. +- try: +- if ( +- ssock.getsockname() != csock.getpeername() +- or csock.getsockname() != ssock.getpeername() +- ): +- raise ConnectionError("Unexpected peer connection") +- except: +- # getsockname() and getpeername() can fail +- # if either socket isn't connected. +- ssock.close() +- csock.close() +- raise +- +- return (ssock, csock) ++ socketpair = _fallback_socketpair + __all__.append("socketpair") + + socketpair.__doc__ = """socketpair([family[, type[, proto]]]) -> (socket object, socket object) +diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py +index ea812408042..b36cb5beaec 100755 +--- a/Lib/test/test_socket.py ++++ b/Lib/test/test_socket.py +@@ -4622,7 +4622,6 @@ class BasicSocketPairTest(SocketPairTest): + + + class PurePythonSocketPairTest(SocketPairTest): +- + # Explicitly use socketpair AF_INET or AF_INET6 to ensure that is the + # code path we're using regardless platform is the pure python one where + # `_socket.socketpair` does not exist. (AF_INET does not work with +@@ -4637,28 +4636,21 @@ class PurePythonSocketPairTest(SocketPairTest): + # Local imports in this class make for easy security fix backporting. + + def setUp(self): +- import _socket +- self._orig_sp = getattr(_socket, 'socketpair', None) +- if self._orig_sp is not None: ++ if hasattr(_socket, "socketpair"): ++ self._orig_sp = socket.socketpair + # This forces the version using the non-OS provided socketpair + # emulation via an AF_INET socket in Lib/socket.py. +- del _socket.socketpair +- import importlib +- global socket +- socket = importlib.reload(socket) ++ socket.socketpair = socket._fallback_socketpair + else: +- pass # This platform already uses the non-OS provided version. ++ # This platform already uses the non-OS provided version. ++ self._orig_sp = None + super().setUp() + + def tearDown(self): + super().tearDown() +- import _socket + if self._orig_sp is not None: + # Restore the default socket.socketpair definition. +- _socket.socketpair = self._orig_sp +- import importlib +- global socket +- socket = importlib.reload(socket) ++ socket.socketpair = self._orig_sp + + def test_recv(self): + msg = self.serv.recv(1024) +-- +2.33.0 + diff --git a/backport-CVE-2024-4032-gh-113171-gh-65056-Fix-private-non-global-IP-add.patch b/backport-CVE-2024-4032-gh-113171-gh-65056-Fix-private-non-global-IP-add.patch new file mode 100644 index 0000000..302aede --- /dev/null +++ b/backport-CVE-2024-4032-gh-113171-gh-65056-Fix-private-non-global-IP-add.patch @@ -0,0 +1,402 @@ +From 22adf29da8d99933ffed8647d3e0726edd16f7f8 Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Tue, 7 May 2024 11:57:58 +0200 +Subject: [PATCH] [3.9] gh-113171: gh-65056: Fix "private" (non-global) IP + address ranges (GH-113179) (GH-113186) (GH-118177) (GH-118472) + +The _private_networks variables, used by various is_private +implementations, were missing some ranges and at the same time had +overly strict ranges (where there are more specific ranges considered +globally reachable by the IANA registries). + +This patch updates the ranges with what was missing or otherwise +incorrect. + +100.64.0.0/10 is left alone, for now, as it's been made special in [1]. + +The _address_exclude_many() call returns 8 networks for IPv4, 121 +networks for IPv6. + +[1] https://github.com/python/cpython/issues/61602 + +In 3.10 and below, is_private checks whether the network and broadcast +address are both private. +In later versions (where the test wss backported from), it checks +whether they both are in the same private network. + +For 0.0.0.0/0, both 0.0.0.0 and 255.225.255.255 are private, +but one is in 0.0.0.0/8 ("This network") and the other in +255.255.255.255/32 ("Limited broadcast"). + +--------- + +Co-authored-by: Jakub Stasiak +--- + Doc/library/ipaddress.rst | 43 ++++++++- + Doc/tools/susp-ignored.csv | 8 ++ + Doc/whatsnew/3.9.rst | 9 ++ + Lib/ipaddress.py | 95 +++++++++++++++---- + Lib/test/test_ipaddress.py | 52 ++++++++++ + ...-03-14-01-38-44.gh-issue-113171.VFnObz.rst | 9 ++ + 6 files changed, 195 insertions(+), 21 deletions(-) + create mode 100644 Misc/NEWS.d/next/Library/2024-03-14-01-38-44.gh-issue-113171.VFnObz.rst + +diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst +index 9c2dff55703..f9c1ebf3f3d 100644 +--- a/Doc/library/ipaddress.rst ++++ b/Doc/library/ipaddress.rst +@@ -188,18 +188,53 @@ write code that handles both IP versions correctly. Address objects are + + .. attribute:: is_private + +- ``True`` if the address is allocated for private networks. See ++ ``True`` if the address is defined as not globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ +- (for IPv6). ++ (for IPv6) with the following exceptions: ++ ++ * ``is_private`` is ``False`` for the shared address space (``100.64.0.0/10``) ++ * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the ++ semantics of the underlying IPv4 addresses and the following condition holds ++ (see :attr:`IPv6Address.ipv4_mapped`):: ++ ++ address.is_private == address.ipv4_mapped.is_private ++ ++ ``is_private`` has value opposite to :attr:`is_global`, except for the shared address space ++ (``100.64.0.0/10`` range) where they are both ``False``. ++ ++ .. versionchanged:: 3.9.20 ++ ++ Fixed some false positives and false negatives. ++ ++ * ``192.0.0.0/24`` is considered private with the exception of ``192.0.0.9/32`` and ++ ``192.0.0.10/32`` (previously: only the ``192.0.0.0/29`` sub-range was considered private). ++ * ``64:ff9b:1::/48`` is considered private. ++ * ``2002::/16`` is considered private. ++ * There are exceptions within ``2001::/23`` (otherwise considered private): ``2001:1::1/128``, ++ ``2001:1::2/128``, ``2001:3::/32``, ``2001:4:112::/48``, ``2001:20::/28``, ``2001:30::/28``. ++ The exceptions are not considered private. + + .. attribute:: is_global + +- ``True`` if the address is allocated for public networks. See ++ ``True`` if the address is defined as globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ +- (for IPv6). ++ (for IPv6) with the following exception: ++ ++ For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the ++ semantics of the underlying IPv4 addresses and the following condition holds ++ (see :attr:`IPv6Address.ipv4_mapped`):: ++ ++ address.is_global == address.ipv4_mapped.is_global ++ ++ ``is_global`` has value opposite to :attr:`is_private`, except for the shared address space ++ (``100.64.0.0/10`` range) where they are both ``False``. + + .. versionadded:: 3.4 + ++ .. versionchanged:: 3.9.20 ++ ++ Fixed some false positives and false negatives, see :attr:`is_private` for details. ++ + .. attribute:: is_unspecified + + ``True`` if the address is unspecified. See :RFC:`5735` (for IPv4) +diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv +index 3eb3d7954f8..de91a50bad0 100644 +--- a/Doc/tools/susp-ignored.csv ++++ b/Doc/tools/susp-ignored.csv +@@ -169,6 +169,14 @@ library/ipaddress,,:db00,2001:db00::0/24 + library/ipaddress,,::,2001:db00::0/24 + library/ipaddress,,:db00,2001:db00::0/ffff:ff00:: + library/ipaddress,,::,2001:db00::0/ffff:ff00:: ++library/ipaddress,,:ff9b,64:ff9b:1::/48 ++library/ipaddress,,::,64:ff9b:1::/48 ++library/ipaddress,,::,2001:: ++library/ipaddress,,::,2001:1:: ++library/ipaddress,,::,2001:3:: ++library/ipaddress,,::,2001:4:112:: ++library/ipaddress,,::,2001:20:: ++library/ipaddress,,::,2001:30:: + library/itertools,,:step,elements from seq[start:stop:step] + library/itertools,,:stop,elements from seq[start:stop:step] + library/itertools,,::,kernel = tuple(kernel)[::-1] +diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst +index 0064e074a3a..1756a373386 100644 +--- a/Doc/whatsnew/3.9.rst ++++ b/Doc/whatsnew/3.9.rst +@@ -1616,3 +1616,12 @@ tarfile + :exc:`DeprecationWarning`. + In Python 3.14, the default will switch to ``'data'``. + (Contributed by Petr Viktorin in :pep:`706`.) ++ ++Notable changes in 3.9.20 ++========================= ++ ++ipaddress ++--------- ++ ++* Fixed ``is_global`` and ``is_private`` behavior in ``IPv4Address``, ++ ``IPv6Address``, ``IPv4Network`` and ``IPv6Network``. +diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py +index 25f373a06a2..9b35340d9ac 100644 +--- a/Lib/ipaddress.py ++++ b/Lib/ipaddress.py +@@ -1322,18 +1322,41 @@ class IPv4Address(_BaseV4, _BaseAddress): + @property + @functools.lru_cache() + def is_private(self): +- """Test if this address is allocated for private networks. ++ """``True`` if the address is defined as not globally reachable by ++ iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ ++ (for IPv6) with the following exceptions: + +- Returns: +- A boolean, True if the address is reserved per +- iana-ipv4-special-registry. ++ * ``is_private`` is ``False`` for ``100.64.0.0/10`` ++ * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the ++ semantics of the underlying IPv4 addresses and the following condition holds ++ (see :attr:`IPv6Address.ipv4_mapped`):: ++ ++ address.is_private == address.ipv4_mapped.is_private + ++ ``is_private`` has value opposite to :attr:`is_global`, except for the ``100.64.0.0/10`` ++ IPv4 range where they are both ``False``. + """ +- return any(self in net for net in self._constants._private_networks) ++ return ( ++ any(self in net for net in self._constants._private_networks) ++ and all(self not in net for net in self._constants._private_networks_exceptions) ++ ) + + @property + @functools.lru_cache() + def is_global(self): ++ """``True`` if the address is defined as globally reachable by ++ iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ ++ (for IPv6) with the following exception: ++ ++ For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the ++ semantics of the underlying IPv4 addresses and the following condition holds ++ (see :attr:`IPv6Address.ipv4_mapped`):: ++ ++ address.is_global == address.ipv4_mapped.is_global ++ ++ ``is_global`` has value opposite to :attr:`is_private`, except for the ``100.64.0.0/10`` ++ IPv4 range where they are both ``False``. ++ """ + return self not in self._constants._public_network and not self.is_private + + @property +@@ -1537,13 +1560,15 @@ class _IPv4Constants: + + _public_network = IPv4Network('100.64.0.0/10') + ++ # Not globally reachable address blocks listed on ++ # https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml + _private_networks = [ + IPv4Network('0.0.0.0/8'), + IPv4Network('10.0.0.0/8'), + IPv4Network('127.0.0.0/8'), + IPv4Network('169.254.0.0/16'), + IPv4Network('172.16.0.0/12'), +- IPv4Network('192.0.0.0/29'), ++ IPv4Network('192.0.0.0/24'), + IPv4Network('192.0.0.170/31'), + IPv4Network('192.0.2.0/24'), + IPv4Network('192.168.0.0/16'), +@@ -1554,6 +1579,11 @@ class _IPv4Constants: + IPv4Network('255.255.255.255/32'), + ] + ++ _private_networks_exceptions = [ ++ IPv4Network('192.0.0.9/32'), ++ IPv4Network('192.0.0.10/32'), ++ ] ++ + _reserved_network = IPv4Network('240.0.0.0/4') + + _unspecified_address = IPv4Address('0.0.0.0') +@@ -1995,23 +2025,42 @@ class IPv6Address(_BaseV6, _BaseAddress): + @property + @functools.lru_cache() + def is_private(self): +- """Test if this address is allocated for private networks. ++ """``True`` if the address is defined as not globally reachable by ++ iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ ++ (for IPv6) with the following exceptions: + +- Returns: +- A boolean, True if the address is reserved per +- iana-ipv6-special-registry. ++ * ``is_private`` is ``False`` for ``100.64.0.0/10`` ++ * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the ++ semantics of the underlying IPv4 addresses and the following condition holds ++ (see :attr:`IPv6Address.ipv4_mapped`):: ++ ++ address.is_private == address.ipv4_mapped.is_private + ++ ``is_private`` has value opposite to :attr:`is_global`, except for the ``100.64.0.0/10`` ++ IPv4 range where they are both ``False``. + """ +- return any(self in net for net in self._constants._private_networks) ++ ipv4_mapped = self.ipv4_mapped ++ if ipv4_mapped is not None: ++ return ipv4_mapped.is_private ++ return ( ++ any(self in net for net in self._constants._private_networks) ++ and all(self not in net for net in self._constants._private_networks_exceptions) ++ ) + + @property + def is_global(self): +- """Test if this address is allocated for public networks. ++ """``True`` if the address is defined as globally reachable by ++ iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ ++ (for IPv6) with the following exception: + +- Returns: +- A boolean, true if the address is not reserved per +- iana-ipv6-special-registry. ++ For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the ++ semantics of the underlying IPv4 addresses and the following condition holds ++ (see :attr:`IPv6Address.ipv4_mapped`):: ++ ++ address.is_global == address.ipv4_mapped.is_global + ++ ``is_global`` has value opposite to :attr:`is_private`, except for the ``100.64.0.0/10`` ++ IPv4 range where they are both ``False``. + """ + return not self.is_private + +@@ -2252,19 +2301,31 @@ class _IPv6Constants: + + _multicast_network = IPv6Network('ff00::/8') + ++ # Not globally reachable address blocks listed on ++ # https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml + _private_networks = [ + IPv6Network('::1/128'), + IPv6Network('::/128'), + IPv6Network('::ffff:0:0/96'), ++ IPv6Network('64:ff9b:1::/48'), + IPv6Network('100::/64'), + IPv6Network('2001::/23'), +- IPv6Network('2001:2::/48'), + IPv6Network('2001:db8::/32'), +- IPv6Network('2001:10::/28'), ++ # IANA says N/A, let's consider it not globally reachable to be safe ++ IPv6Network('2002::/16'), + IPv6Network('fc00::/7'), + IPv6Network('fe80::/10'), + ] + ++ _private_networks_exceptions = [ ++ IPv6Network('2001:1::1/128'), ++ IPv6Network('2001:1::2/128'), ++ IPv6Network('2001:3::/32'), ++ IPv6Network('2001:4:112::/48'), ++ IPv6Network('2001:20::/28'), ++ IPv6Network('2001:30::/28'), ++ ] ++ + _reserved_networks = [ + IPv6Network('::/8'), IPv6Network('100::/8'), + IPv6Network('200::/7'), IPv6Network('400::/6'), +diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py +index 90897f6bedb..bd14f04f6c6 100644 +--- a/Lib/test/test_ipaddress.py ++++ b/Lib/test/test_ipaddress.py +@@ -2263,6 +2263,10 @@ class IpaddrUnitTest(unittest.TestCase): + self.assertEqual(True, ipaddress.ip_address( + '172.31.255.255').is_private) + self.assertEqual(False, ipaddress.ip_address('172.32.0.0').is_private) ++ self.assertFalse(ipaddress.ip_address('192.0.0.0').is_global) ++ self.assertTrue(ipaddress.ip_address('192.0.0.9').is_global) ++ self.assertTrue(ipaddress.ip_address('192.0.0.10').is_global) ++ self.assertFalse(ipaddress.ip_address('192.0.0.255').is_global) + + self.assertEqual(True, + ipaddress.ip_address('169.254.100.200').is_link_local) +@@ -2278,6 +2282,40 @@ class IpaddrUnitTest(unittest.TestCase): + self.assertEqual(False, ipaddress.ip_address('128.0.0.0').is_loopback) + self.assertEqual(True, ipaddress.ip_network('0.0.0.0').is_unspecified) + ++ def testPrivateNetworks(self): ++ self.assertEqual(True, ipaddress.ip_network("0.0.0.0/0").is_private) ++ self.assertEqual(False, ipaddress.ip_network("1.0.0.0/8").is_private) ++ ++ self.assertEqual(True, ipaddress.ip_network("0.0.0.0/8").is_private) ++ self.assertEqual(True, ipaddress.ip_network("10.0.0.0/8").is_private) ++ self.assertEqual(True, ipaddress.ip_network("127.0.0.0/8").is_private) ++ self.assertEqual(True, ipaddress.ip_network("169.254.0.0/16").is_private) ++ self.assertEqual(True, ipaddress.ip_network("172.16.0.0/12").is_private) ++ self.assertEqual(True, ipaddress.ip_network("192.0.0.0/29").is_private) ++ self.assertEqual(False, ipaddress.ip_network("192.0.0.9/32").is_private) ++ self.assertEqual(True, ipaddress.ip_network("192.0.0.170/31").is_private) ++ self.assertEqual(True, ipaddress.ip_network("192.0.2.0/24").is_private) ++ self.assertEqual(True, ipaddress.ip_network("192.168.0.0/16").is_private) ++ self.assertEqual(True, ipaddress.ip_network("198.18.0.0/15").is_private) ++ self.assertEqual(True, ipaddress.ip_network("198.51.100.0/24").is_private) ++ self.assertEqual(True, ipaddress.ip_network("203.0.113.0/24").is_private) ++ self.assertEqual(True, ipaddress.ip_network("240.0.0.0/4").is_private) ++ self.assertEqual(True, ipaddress.ip_network("255.255.255.255/32").is_private) ++ ++ self.assertEqual(False, ipaddress.ip_network("::/0").is_private) ++ self.assertEqual(False, ipaddress.ip_network("::ff/128").is_private) ++ ++ self.assertEqual(True, ipaddress.ip_network("::1/128").is_private) ++ self.assertEqual(True, ipaddress.ip_network("::/128").is_private) ++ self.assertEqual(True, ipaddress.ip_network("::ffff:0:0/96").is_private) ++ self.assertEqual(True, ipaddress.ip_network("100::/64").is_private) ++ self.assertEqual(True, ipaddress.ip_network("2001:2::/48").is_private) ++ self.assertEqual(False, ipaddress.ip_network("2001:3::/48").is_private) ++ self.assertEqual(True, ipaddress.ip_network("2001:db8::/32").is_private) ++ self.assertEqual(True, ipaddress.ip_network("2001:10::/28").is_private) ++ self.assertEqual(True, ipaddress.ip_network("fc00::/7").is_private) ++ self.assertEqual(True, ipaddress.ip_network("fe80::/10").is_private) ++ + def testReservedIpv6(self): + + self.assertEqual(True, ipaddress.ip_network('ffff::').is_multicast) +@@ -2351,6 +2389,20 @@ class IpaddrUnitTest(unittest.TestCase): + self.assertEqual(True, ipaddress.ip_address('0::0').is_unspecified) + self.assertEqual(False, ipaddress.ip_address('::1').is_unspecified) + ++ self.assertFalse(ipaddress.ip_address('64:ff9b:1::').is_global) ++ self.assertFalse(ipaddress.ip_address('2001::').is_global) ++ self.assertTrue(ipaddress.ip_address('2001:1::1').is_global) ++ self.assertTrue(ipaddress.ip_address('2001:1::2').is_global) ++ self.assertFalse(ipaddress.ip_address('2001:2::').is_global) ++ self.assertTrue(ipaddress.ip_address('2001:3::').is_global) ++ self.assertFalse(ipaddress.ip_address('2001:4::').is_global) ++ self.assertTrue(ipaddress.ip_address('2001:4:112::').is_global) ++ self.assertFalse(ipaddress.ip_address('2001:10::').is_global) ++ self.assertTrue(ipaddress.ip_address('2001:20::').is_global) ++ self.assertTrue(ipaddress.ip_address('2001:30::').is_global) ++ self.assertFalse(ipaddress.ip_address('2001:40::').is_global) ++ self.assertFalse(ipaddress.ip_address('2002::').is_global) ++ + # some generic IETF reserved addresses + self.assertEqual(True, ipaddress.ip_address('100::').is_reserved) + self.assertEqual(True, ipaddress.ip_network('4000::1/128').is_reserved) +diff --git a/Misc/NEWS.d/next/Library/2024-03-14-01-38-44.gh-issue-113171.VFnObz.rst b/Misc/NEWS.d/next/Library/2024-03-14-01-38-44.gh-issue-113171.VFnObz.rst +new file mode 100644 +index 00000000000..f9a72473be4 +--- /dev/null ++++ b/Misc/NEWS.d/next/Library/2024-03-14-01-38-44.gh-issue-113171.VFnObz.rst +@@ -0,0 +1,9 @@ ++Fixed various false positives and false negatives in ++ ++* :attr:`ipaddress.IPv4Address.is_private` (see these docs for details) ++* :attr:`ipaddress.IPv4Address.is_global` ++* :attr:`ipaddress.IPv6Address.is_private` ++* :attr:`ipaddress.IPv6Address.is_global` ++ ++Also in the corresponding :class:`ipaddress.IPv4Network` and :class:`ipaddress.IPv6Network` ++attributes. +-- +2.33.0 + diff --git a/backport-CVE-2024-6232-gh-121285-Remove-backtracking-when-parsing-tarf.patch b/backport-CVE-2024-6232-gh-121285-Remove-backtracking-when-parsing-tarf.patch new file mode 100644 index 0000000..8408ffc --- /dev/null +++ b/backport-CVE-2024-6232-gh-121285-Remove-backtracking-when-parsing-tarf.patch @@ -0,0 +1,248 @@ +From b4225ca91547aa97ed3aca391614afbb255bc877 Mon Sep 17 00:00:00 2001 +From: Seth Michael Larson +Date: Wed, 4 Sep 2024 10:46:01 -0500 +Subject: [PATCH] [3.9] gh-121285: Remove backtracking when parsing tarfile + headers (GH-121286) (#123641) + +* Remove backtracking when parsing tarfile headers +* Rewrite PAX header parsing to be stricter +* Optimize parsing of GNU extended sparse headers v0.0 + +(cherry picked from commit 34ddb64d088dd7ccc321f6103d23153256caa5d4) + +Co-authored-by: Seth Michael Larson +Co-authored-by: Kirill Podoprigora +Co-authored-by: Gregory P. Smith +--- + Lib/tarfile.py | 105 +++++++++++------- + Lib/test/test_tarfile.py | 42 +++++++ + ...-07-02-13-39-20.gh-issue-121285.hrl-yI.rst | 2 + + 3 files changed, 111 insertions(+), 38 deletions(-) + create mode 100644 Misc/NEWS.d/next/Security/2024-07-02-13-39-20.gh-issue-121285.hrl-yI.rst + +diff --git a/Lib/tarfile.py b/Lib/tarfile.py +index 7a6158c2eb9..d75ba50b667 100755 +--- a/Lib/tarfile.py ++++ b/Lib/tarfile.py +@@ -840,6 +840,9 @@ _NAMED_FILTERS = { + # Sentinel for replace() defaults, meaning "don't change the attribute" + _KEEP = object() + ++# Header length is digits followed by a space. ++_header_length_prefix_re = re.compile(br"([0-9]{1,20}) ") ++ + class TarInfo(object): + """Informational class which holds the details about an + archive member given by a tar header block. +@@ -1399,41 +1402,59 @@ class TarInfo(object): + else: + pax_headers = tarfile.pax_headers.copy() + +- # Check if the pax header contains a hdrcharset field. This tells us +- # the encoding of the path, linkpath, uname and gname fields. Normally, +- # these fields are UTF-8 encoded but since POSIX.1-2008 tar +- # implementations are allowed to store them as raw binary strings if +- # the translation to UTF-8 fails. +- match = re.search(br"\d+ hdrcharset=([^\n]+)\n", buf) +- if match is not None: +- pax_headers["hdrcharset"] = match.group(1).decode("utf-8") +- +- # For the time being, we don't care about anything other than "BINARY". +- # The only other value that is currently allowed by the standard is +- # "ISO-IR 10646 2000 UTF-8" in other words UTF-8. +- hdrcharset = pax_headers.get("hdrcharset") +- if hdrcharset == "BINARY": +- encoding = tarfile.encoding +- else: +- encoding = "utf-8" +- + # Parse pax header information. A record looks like that: + # "%d %s=%s\n" % (length, keyword, value). length is the size + # of the complete record including the length field itself and +- # the newline. keyword and value are both UTF-8 encoded strings. +- regex = re.compile(br"(\d+) ([^=]+)=") ++ # the newline. + pos = 0 +- while True: +- match = regex.match(buf, pos) +- if not match: +- break ++ encoding = None ++ raw_headers = [] ++ while len(buf) > pos and buf[pos] != 0x00: ++ if not (match := _header_length_prefix_re.match(buf, pos)): ++ raise InvalidHeaderError("invalid header") ++ try: ++ length = int(match.group(1)) ++ except ValueError: ++ raise InvalidHeaderError("invalid header") ++ # Headers must be at least 5 bytes, shortest being '5 x=\n'. ++ # Value is allowed to be empty. ++ if length < 5: ++ raise InvalidHeaderError("invalid header") ++ if pos + length > len(buf): ++ raise InvalidHeaderError("invalid header") + +- length, keyword = match.groups() +- length = int(length) +- if length == 0: ++ header_value_end_offset = match.start(1) + length - 1 # Last byte of the header ++ keyword_and_value = buf[match.end(1) + 1:header_value_end_offset] ++ raw_keyword, equals, raw_value = keyword_and_value.partition(b"=") ++ ++ # Check the framing of the header. The last character must be '\n' (0x0A) ++ if not raw_keyword or equals != b"=" or buf[header_value_end_offset] != 0x0A: + raise InvalidHeaderError("invalid header") +- value = buf[match.end(2) + 1:match.start(1) + length - 1] ++ raw_headers.append((length, raw_keyword, raw_value)) ++ ++ # Check if the pax header contains a hdrcharset field. This tells us ++ # the encoding of the path, linkpath, uname and gname fields. Normally, ++ # these fields are UTF-8 encoded but since POSIX.1-2008 tar ++ # implementations are allowed to store them as raw binary strings if ++ # the translation to UTF-8 fails. For the time being, we don't care about ++ # anything other than "BINARY". The only other value that is currently ++ # allowed by the standard is "ISO-IR 10646 2000 UTF-8" in other words UTF-8. ++ # Note that we only follow the initial 'hdrcharset' setting to preserve ++ # the initial behavior of the 'tarfile' module. ++ if raw_keyword == b"hdrcharset" and encoding is None: ++ if raw_value == b"BINARY": ++ encoding = tarfile.encoding ++ else: # This branch ensures only the first 'hdrcharset' header is used. ++ encoding = "utf-8" ++ ++ pos += length + ++ # If no explicit hdrcharset is set, we use UTF-8 as a default. ++ if encoding is None: ++ encoding = "utf-8" ++ ++ # After parsing the raw headers we can decode them to text. ++ for length, raw_keyword, raw_value in raw_headers: + # Normally, we could just use "utf-8" as the encoding and "strict" + # as the error handler, but we better not take the risk. For + # example, GNU tar <= 1.23 is known to store filenames it cannot +@@ -1441,17 +1462,16 @@ class TarInfo(object): + # hdrcharset=BINARY header). + # We first try the strict standard encoding, and if that fails we + # fall back on the user's encoding and error handler. +- keyword = self._decode_pax_field(keyword, "utf-8", "utf-8", ++ keyword = self._decode_pax_field(raw_keyword, "utf-8", "utf-8", + tarfile.errors) + if keyword in PAX_NAME_FIELDS: +- value = self._decode_pax_field(value, encoding, tarfile.encoding, ++ value = self._decode_pax_field(raw_value, encoding, tarfile.encoding, + tarfile.errors) + else: +- value = self._decode_pax_field(value, "utf-8", "utf-8", ++ value = self._decode_pax_field(raw_value, "utf-8", "utf-8", + tarfile.errors) + + pax_headers[keyword] = value +- pos += length + + # Fetch the next header. + try: +@@ -1466,7 +1486,7 @@ class TarInfo(object): + + elif "GNU.sparse.size" in pax_headers: + # GNU extended sparse format version 0.0. +- self._proc_gnusparse_00(next, pax_headers, buf) ++ self._proc_gnusparse_00(next, raw_headers) + + elif pax_headers.get("GNU.sparse.major") == "1" and pax_headers.get("GNU.sparse.minor") == "0": + # GNU extended sparse format version 1.0. +@@ -1488,15 +1508,24 @@ class TarInfo(object): + + return next + +- def _proc_gnusparse_00(self, next, pax_headers, buf): ++ def _proc_gnusparse_00(self, next, raw_headers): + """Process a GNU tar extended sparse header, version 0.0. + """ + offsets = [] +- for match in re.finditer(br"\d+ GNU.sparse.offset=(\d+)\n", buf): +- offsets.append(int(match.group(1))) + numbytes = [] +- for match in re.finditer(br"\d+ GNU.sparse.numbytes=(\d+)\n", buf): +- numbytes.append(int(match.group(1))) ++ for _, keyword, value in raw_headers: ++ if keyword == b"GNU.sparse.offset": ++ try: ++ offsets.append(int(value.decode())) ++ except ValueError: ++ raise InvalidHeaderError("invalid header") ++ ++ elif keyword == b"GNU.sparse.numbytes": ++ try: ++ numbytes.append(int(value.decode())) ++ except ValueError: ++ raise InvalidHeaderError("invalid header") ++ + next.sparse = list(zip(offsets, numbytes)) + + def _proc_gnusparse_01(self, next, pax_headers): +diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py +index 3df64c78032..2218401e386 100644 +--- a/Lib/test/test_tarfile.py ++++ b/Lib/test/test_tarfile.py +@@ -1113,6 +1113,48 @@ class PaxReadTest(LongnameTest, ReadTest, unittest.TestCase): + finally: + tar.close() + ++ def test_pax_header_bad_formats(self): ++ # The fields from the pax header have priority over the ++ # TarInfo. ++ pax_header_replacements = ( ++ b" foo=bar\n", ++ b"0 \n", ++ b"1 \n", ++ b"2 \n", ++ b"3 =\n", ++ b"4 =a\n", ++ b"1000000 foo=bar\n", ++ b"0 foo=bar\n", ++ b"-12 foo=bar\n", ++ b"000000000000000000000000036 foo=bar\n", ++ ) ++ pax_headers = {"foo": "bar"} ++ ++ for replacement in pax_header_replacements: ++ with self.subTest(header=replacement): ++ tar = tarfile.open(tmpname, "w", format=tarfile.PAX_FORMAT, ++ encoding="iso8859-1") ++ try: ++ t = tarfile.TarInfo() ++ t.name = "pax" # non-ASCII ++ t.uid = 1 ++ t.pax_headers = pax_headers ++ tar.addfile(t) ++ finally: ++ tar.close() ++ ++ with open(tmpname, "rb") as f: ++ data = f.read() ++ self.assertIn(b"11 foo=bar\n", data) ++ data = data.replace(b"11 foo=bar\n", replacement) ++ ++ with open(tmpname, "wb") as f: ++ f.truncate() ++ f.write(data) ++ ++ with self.assertRaisesRegex(tarfile.ReadError, r"file could not be opened successfully"): ++ tarfile.open(tmpname, encoding="iso8859-1") ++ + + class WriteTestBase(TarTest): + # Put all write tests in here that are supposed to be tested +diff --git a/Misc/NEWS.d/next/Security/2024-07-02-13-39-20.gh-issue-121285.hrl-yI.rst b/Misc/NEWS.d/next/Security/2024-07-02-13-39-20.gh-issue-121285.hrl-yI.rst +new file mode 100644 +index 00000000000..81f918bfe2b +--- /dev/null ++++ b/Misc/NEWS.d/next/Security/2024-07-02-13-39-20.gh-issue-121285.hrl-yI.rst +@@ -0,0 +1,2 @@ ++Remove backtracking from tarfile header parsing for ``hdrcharset``, PAX, and ++GNU sparse headers. +-- +2.33.0 + diff --git a/backport-gh-121650-Encode-newlines-in-headers-and-verify-head.patch b/backport-CVE-2024-6923-gh-121650-Encode-newlines-in-headers-and-verify-.patch similarity index 85% rename from backport-gh-121650-Encode-newlines-in-headers-and-verify-head.patch rename to backport-CVE-2024-6923-gh-121650-Encode-newlines-in-headers-and-verify-.patch index db3a451..f45597c 100644 --- a/backport-gh-121650-Encode-newlines-in-headers-and-verify-head.patch +++ b/backport-CVE-2024-6923-gh-121650-Encode-newlines-in-headers-and-verify-.patch @@ -1,9 +1,29 @@ -From 63f521316128957bcbf5496d9623d17d7822600c Mon Sep 17 00:00:00 2001 -From: xinsheng -Date: Tue, 3 Sep 2024 17:19:55 +0800 -Subject: [PATCH] gh-121650: Encode newlines in headers, and verify headers are - sound (GH-122233) +From f7be505d137a22528cb0fc004422c0081d5d90e6 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?=C5=81ukasz=20Langa?= +Date: Wed, 4 Sep 2024 17:39:02 +0200 +Subject: [PATCH] [3.9] gh-121650: Encode newlines in headers, and verify + headers are sound (GH-122233) (#122610) +Per RFC 2047: + +> [...] these encoding schemes allow the +> encoding of arbitrary octet values, mail readers that implement this +> decoding should also ensure that display of the decoded data on the +> recipient's terminal will not cause unwanted side-effects + +It seems that the "quoted-word" scheme is a valid way to include +a newline character in a header value, just like we already allow +undecodable bytes or control characters. +They do need to be properly quoted when serialized to text, though. + +This should fail for custom fold() implementations that aren't careful +about newlines. + +(cherry picked from commit 097633981879b3c9de9a1dd120d3aa585ecc2384) + +Co-authored-by: Petr Viktorin +Co-authored-by: Bas Bloemsaat +Co-authored-by: Serhiy Storchaka --- Doc/library/email.errors.rst | 6 ++ Doc/library/email.policy.rst | 18 ++++++ @@ -11,21 +31,21 @@ Subject: [PATCH] gh-121650: Encode newlines in headers, and verify headers are Lib/email/_header_value_parser.py | 12 +++- Lib/email/_policybase.py | 8 +++ Lib/email/errors.py | 4 ++ - Lib/email/generator.py | 14 ++++- + Lib/email/generator.py | 13 +++- Lib/test/test_email/test_generator.py | 62 +++++++++++++++++++ - Lib/test/test_email/test_policy.py | 27 ++++++++ + Lib/test/test_email/test_policy.py | 26 ++++++++ ...-07-27-16-10-41.gh-issue-121650.nf6oc9.rst | 5 ++ - 10 files changed, 163 insertions(+), 5 deletions(-) + 10 files changed, 162 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-07-27-16-10-41.gh-issue-121650.nf6oc9.rst diff --git a/Doc/library/email.errors.rst b/Doc/library/email.errors.rst -index f4b9f52..878c09b 100644 +index f4b9f525096..878c09bb040 100644 --- a/Doc/library/email.errors.rst +++ b/Doc/library/email.errors.rst @@ -59,6 +59,12 @@ The following exception classes are defined in the :mod:`email.errors` module: :class:`~email.mime.image.MIMEImage`). - - + + +.. exception:: HeaderWriteError() + + Raised when an error occurs when the :mod:`~email.generator` outputs @@ -36,13 +56,13 @@ index f4b9f52..878c09b 100644 can find while parsing messages. Note that the defects are added to the message where the problem was found, so for example, if a message nested inside a diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst -index bf53b95..57a75ce 100644 +index bf53b9520fc..57a75ce4529 100644 --- a/Doc/library/email.policy.rst +++ b/Doc/library/email.policy.rst @@ -229,6 +229,24 @@ added matters. To illustrate:: - + .. versionadded:: 3.6 - + + + .. attribute:: verify_generated_headers + @@ -63,15 +83,15 @@ index bf53b95..57a75ce 100644 + The following :class:`Policy` method is intended to be called by code using the email library to create policy instances with custom settings: - + diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst -index 3f03a40..9aca2b4 100644 +index 9383047098d..3c3cd51b7dd 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst -@@ -1599,3 +1599,15 @@ tarfile - :exc:`DeprecationWarning`. - In Python 3.14, the default will switch to ``'data'``. - (Contributed by Petr Viktorin in :pep:`706`.) +@@ -1640,3 +1640,15 @@ ipaddress + + * Fixed ``is_global`` and ``is_private`` behavior in ``IPv4Address``, + ``IPv6Address``, ``IPv4Network`` and ``IPv6Network``. + +email +----- @@ -84,9 +104,8 @@ index 3f03a40..9aca2b4 100644 + If you need to turn this safety feature off, + set :attr:`~email.policy.Policy.verify_generated_headers`. + (Contributed by Bas Bloemsaat and Petr Viktorin in :gh:`121650`.) -\ No newline at end of file diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py -index 51d355f..e579b31 100644 +index 8a8fb8bc42a..e394cfd2e19 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -92,6 +92,8 @@ TOKEN_ENDS = TSPECIALS | WSP @@ -95,7 +114,7 @@ index 51d355f..e579b31 100644 EXTENDED_ATTRIBUTE_ENDS = ATTRIBUTE_ENDS - set('%') +NLSET = {'\n', '\r'} +SPECIALSNL = SPECIALS | NLSET - + def quote_string(value): return '"'+str(value).replace('\\', '\\\\').replace('"', r'\"')+'"' @@ -2778,9 +2780,13 @@ def _refold_parse_tree(parse_tree, *, policy): @@ -116,14 +135,14 @@ index 51d355f..e579b31 100644 tstr.encode(encoding) charset = encoding diff --git a/Lib/email/_policybase.py b/Lib/email/_policybase.py -index c9cbadd..5f78928 100644 +index c9cbadd2a80..d1f48211f90 100644 --- a/Lib/email/_policybase.py +++ b/Lib/email/_policybase.py @@ -157,6 +157,13 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta): message_factory -- the class to use to create new message objects. If the value is None, the default is Message. - -+ verify_generated_headers + ++ verify_generated_headers + -- if true, the generator verifies that each header + they are properly folded, so that a parser won't + treat it as multiple headers, start-of-body, or @@ -131,24 +150,24 @@ index c9cbadd..5f78928 100644 + This is a check against custom Header & fold() + implementations. """ - + raise_on_defect = False @@ -165,6 +172,7 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta): max_line_length = 78 mangle_from_ = False message_factory = None + verify_generated_headers = True - + def handle_defect(self, obj, defect): """Based on policy, either raise defect or call register_defect. diff --git a/Lib/email/errors.py b/Lib/email/errors.py -index d28a680..1a0d5c6 100644 +index d28a6800104..1a0d5c63e60 100644 --- a/Lib/email/errors.py +++ b/Lib/email/errors.py @@ -29,6 +29,10 @@ class CharsetError(MessageError): """An illegal charset was given.""" - - + + +class HeaderWriteError(MessageError): + """Error while writing headers.""" + @@ -157,29 +176,26 @@ index d28a680..1a0d5c6 100644 class MessageDefect(ValueError): """Base class for a message defect.""" diff --git a/Lib/email/generator.py b/Lib/email/generator.py -index c9b1216..455746c 100644 +index c9b121624e0..89224ae41cb 100644 --- a/Lib/email/generator.py +++ b/Lib/email/generator.py -@@ -14,15 +14,16 @@ import random +@@ -14,12 +14,14 @@ import random from copy import deepcopy from io import StringIO, BytesIO from email.utils import _has_surrogates +from email.errors import HeaderWriteError - + UNDERSCORE = '_' NL = '\n' # XXX: no longer used by the code below. - + NLCRE = re.compile(r'\r\n|\r|\n') fcre = re.compile(r'^From ', re.MULTILINE) +NEWLINE_WITHOUT_FWSP = re.compile(r'\r\n[^ \t]|\r[^ \n\t]|\n[^ \t]') - - -- - class Generator: - """Generates output from a Message object tree. - -@@ -223,7 +224,16 @@ class Generator: - + + + +@@ -223,7 +225,16 @@ class Generator: + def _write_headers(self, msg): for h, v in msg.raw_items(): - self.write(self.policy.fold(h, v)) @@ -195,9 +211,9 @@ index c9b1216..455746c 100644 + self.write(folded) # A blank line always separates headers from body self.write(self._NL) - + diff --git a/Lib/test/test_email/test_generator.py b/Lib/test/test_email/test_generator.py -index 89e7ede..d29400f 100644 +index 89e7edeb63a..d29400f0ed1 100644 --- a/Lib/test/test_email/test_generator.py +++ b/Lib/test/test_email/test_generator.py @@ -6,6 +6,7 @@ from email.message import EmailMessage @@ -206,12 +222,12 @@ index 89e7ede..d29400f 100644 from email import policy +import email.errors from test.test_email import TestEmailBase, parameterize - - + + @@ -216,6 +217,44 @@ class TestGeneratorBase: g.flatten(msg) self.assertEqual(s.getvalue(), self.typ(expected)) - + + def test_keep_encoded_newlines(self): + msg = self.msgmaker(self.typ(textwrap.dedent("""\ + To: nobody @@ -250,13 +266,13 @@ index 89e7ede..d29400f 100644 + g.flatten(msg) + self.assertEqual(s.getvalue(), self.typ(expected)) + - + class TestGenerator(TestGeneratorBase, TestEmailBase): - + @@ -224,6 +263,29 @@ class TestGenerator(TestGeneratorBase, TestEmailBase): ioclass = io.StringIO typ = str - + + def test_verify_generated_headers(self): + """gh-121650: by default the generator prevents header injection""" + class LiteralHeader(str): @@ -280,11 +296,11 @@ index 89e7ede..d29400f 100644 + with self.assertRaises(email.errors.HeaderWriteError): + message.as_string() + - + class TestBytesGenerator(TestGeneratorBase, TestEmailBase): - + diff --git a/Lib/test/test_email/test_policy.py b/Lib/test/test_email/test_policy.py -index e87c275..cbaaa09 100644 +index e87c2755494..ff1ddf7d7a8 100644 --- a/Lib/test/test_email/test_policy.py +++ b/Lib/test/test_email/test_policy.py @@ -26,6 +26,7 @@ class PolicyAPITests(unittest.TestCase): @@ -295,11 +311,10 @@ index e87c275..cbaaa09 100644 } # These default values are the ones set on email.policy.default. # If any of these defaults change, the docs must be updated. -@@ -277,6 +278,32 @@ class PolicyAPITests(unittest.TestCase): +@@ -277,6 +278,31 @@ class PolicyAPITests(unittest.TestCase): with self.assertRaises(email.errors.HeaderParseError): policy.fold("Subject", subject) - -+ + + def test_verify_generated_headers(self): + """Turning protection off allows header injection""" + policy = email.policy.default.clone(verify_generated_headers=False) @@ -330,7 +345,7 @@ index e87c275..cbaaa09 100644 # wins), but that the order still works (right overrides left). diff --git a/Misc/NEWS.d/next/Library/2024-07-27-16-10-41.gh-issue-121650.nf6oc9.rst b/Misc/NEWS.d/next/Library/2024-07-27-16-10-41.gh-issue-121650.nf6oc9.rst new file mode 100644 -index 0000000..83dd28d +index 00000000000..83dd28d4ac5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-27-16-10-41.gh-issue-121650.nf6oc9.rst @@ -0,0 +1,5 @@ @@ -340,5 +355,5 @@ index 0000000..83dd28d +:attr:`~email.policy.Policy.verify_generated_headers`. (Contributed by Bas +Bloemsaat and Petr Viktorin in :gh:`121650`.) -- -2.43.0 +2.33.0 diff --git a/backport-gh-123067-Fix-quadratic-complexity-in-parsing-quoted.patch b/backport-CVE-2024-7592-gh-123067-Fix-quadratic-complexity-in-parsing-quoted.patch similarity index 100% rename from backport-gh-123067-Fix-quadratic-complexity-in-parsing-quoted.patch rename to backport-CVE-2024-7592-gh-123067-Fix-quadratic-complexity-in-parsing-quoted.patch diff --git a/backport-gh-123270-Replaced-SanitizedNames-with-a-more-surgic.patch b/backport-CVE-2024-8088-gh-123270-Replaced-SanitizedNames-with-a-more-surgic.patch similarity index 100% rename from backport-gh-123270-Replaced-SanitizedNames-with-a-more-surgic.patch rename to backport-CVE-2024-8088-gh-123270-Replaced-SanitizedNames-with-a-more-surgic.patch diff --git a/python3.spec b/python3.spec index 0db21ea..f8734d1 100644 --- a/python3.spec +++ b/python3.spec @@ -3,7 +3,7 @@ Summary: Interpreter of the Python3 programming language URL: https://www.python.org/ Version: 3.9.9 -Release: 35 +Release: 36 License: Python-2.0 %global branchversion 3.9 @@ -118,14 +118,18 @@ Patch6020: backport-Revert-fixes-for-CVE-2023-27043.patch Patch6021: backport-CVE-2023-27043.patch Patch6022: backport-gh-93065-Fix-HAMT-to-iterate-correctly-over-7-level-.patch Patch6023: backport-3.9-bpo-37013-Fix-the-error-handling-in-socket.if_in.patch -Patch6024: backport-3.9-gh-91133-tempfile.TemporaryDirectory-fix-symlink.patch -Patch6025: backport-3.9-gh-109858-Protect-zipfile-from-quoted-overlap-zi.patch -Patch6026: backport-3.9-gh-113659-Skip-hidden-.pth-files-GH-113660-GH-11.patch -Patch6027: backport-fix_xml_tree_assert_error.patch -Patch6028: backport-3.9-gh-114572-Fix-locking-in-cert_store_stats-and-ge.patch -Patch6029: backport-gh-121650-Encode-newlines-in-headers-and-verify-head.patch -Patch6030: backport-gh-123067-Fix-quadratic-complexity-in-parsing-quoted.patch -Patch6031: backport-gh-123270-Replaced-SanitizedNames-with-a-more-surgic.patch +Patch6024: backport-3.9-gh-113659-Skip-hidden-.pth-files-GH-113660-GH-11.patch +Patch6025: backport-fix_xml_tree_assert_error.patch +Patch6026: backport-CVE-2024-0397-gh-114572-Fix-locking-in-cert_store_stats-and-ge.patch +Patch6027: backport-CVE-2024-4032-gh-113171-gh-65056-Fix-private-non-global-IP-add.patch +Patch6028: backport-CVE-2024-6923-gh-121650-Encode-newlines-in-headers-and-verify-.patch +Patch6029: backport-CVE-2024-7592-gh-123067-Fix-quadratic-complexity-in-parsing-quoted.patch +Patch6030: backport-CVE-2024-8088-gh-123270-Replaced-SanitizedNames-with-a-more-surgic.patch +Patch6031: backport-CVE-2024-6232-gh-121285-Remove-backtracking-when-parsing-tarf.patch +Patch6032: backport-CVE-2024-3219-1-gh-122133-Authenticate-socket-connection-for-soc.patch +Patch6033: backport-CVE-2024-3219-2-gh-122133-Rework-pure-Python-socketpair-tests-to.patch +Patch6034: backport-CVE-2023-6597-gh-91133-tempfile.TemporaryDirectory-fix-symlink.patch +Patch6035: backport-CVE-2024-0450-gh-109858-Protect-zipfile-from-quoted-overlap-zi.patch Patch9000: add-the-sm3-method-for-obtaining-the-salt-value.patch Patch9001: python3-Add-sw64-architecture.patch @@ -247,6 +251,10 @@ rm -r Modules/expat %patch6029 -p1 %patch6030 -p1 %patch6031 -p1 +%patch6032 -p1 +%patch6033 -p1 +%patch6034 -p1 +%patch6035 -p1 %patch9000 -p1 %patch9001 -p1 @@ -875,6 +883,19 @@ export BEP_GTDLIST="$BEP_GTDLIST_TMP" %{_mandir}/*/* %changelog +* Tue Sep 24 2024 xinsheng - 3.9.9-36 +- Type:CVE +- CVE:CVE-2024-6232,CVE-2024-3219,CVE-2024-0450,CVE-2023-6597,CVE-2024-4032 +- SUG:NA +- DESC:fix CVE-2024-6232,CVE-2024-3219,CVE-2024-0450,CVE-2023-6597,CVE-2024-4032 + - rename all CVE patch name + - CVE-2024-6232: Remove backtracking when parsing tarfile headers + - CVE-2024-3219: patch1 Authenticate socket connection for `socket.socketpair()` fallback + - CVE-2024-3219: patch2 Rework pure Python socketpair tests to avoid use of importlib.reload. + - CVE-2024-0450: Protect zipfile from "quoted-overlap" zipbomb + - CVE-2023-6597: tempfile.TemporaryDirectory: fix symlink bug in cleanup + - CVE-2024-4032: Fix "private" (non-global) IP address ranges + * Tue Sep 03 2024 xinsheng - 3.9.9-35 - Type:CVE - CVE:NA -- Gitee