diff --git a/fix-CVE-2023-46137.patch b/fix-CVE-2023-46137.patch new file mode 100644 index 0000000000000000000000000000000000000000..afd83e2c830d4ac22d516f4d233be4c7f67773e5 --- /dev/null +++ b/fix-CVE-2023-46137.patch @@ -0,0 +1,430 @@ +From d87aababab668190d0b4c8e6c3c679d297d1efc2 Mon Sep 17 00:00:00 2001 +From: Glyph +Date: Sun, 10 Sep 2023 22:36:38 -0700 +Subject: [PATCH 01/12] test & fix + +--- + src/twisted/web/http.py | 18 ++++++++------ + src/twisted/web/test/test_web.py | 41 +++++++++++++++++++++++++++++++- + 2 files changed, 51 insertions(+), 8 deletions(-) + +diff --git a/src/twisted/web/http.py b/src/twisted/web/http.py +index 053eb601dad..e5bf6effd7d 100644 +--- a/src/twisted/web/http.py ++++ b/src/twisted/web/http.py +@@ -2250,6 +2250,9 @@ def lineReceived(self, line): + Called for each line from request until the end of headers when + it enters binary mode. + """ ++ assert ( ++ not self._handlingRequest ++ ), "when handling a request, we MUST be in raw mode to buffer the incoming data without parsing it" + self.resetTimeout() + + self._receivedHeaderSize += len(line) +@@ -2431,14 +2434,17 @@ def allContentReceived(self): + + self._handlingRequest = True + ++ # We go into raw mode here even though we will be receiving lines next ++ # in the protocol; however, this data will be buffered and then passed ++ # back to line mode in the setLineMode call in requestDone. ++ self.setRawMode() ++ + req = self.requests[-1] + req.requestReceived(command, path, version) + +- def dataReceived(self, data): +- """ +- Data was received from the network. Process it. +- """ +- # If we're currently handling a request, buffer this data. ++ def rawDataReceived(self, data): ++ # If we're currently handling a request, we'll be in raw mode. Buffer ++ # any data. + if self._handlingRequest: + self._dataBuffer.append(data) + if ( +@@ -2450,9 +2456,7 @@ def dataReceived(self, data): + # ready. See docstring for _optimisticEagerReadSize above. + self._networkProducer.pauseProducing() + return +- return basic.LineReceiver.dataReceived(self, data) + +- def rawDataReceived(self, data): + self.resetTimeout() + + try: +diff --git a/src/twisted/web/test/test_web.py b/src/twisted/web/test/test_web.py +index c196249722c..1d24f454f2e 100644 +--- a/src/twisted/web/test/test_web.py ++++ b/src/twisted/web/test/test_web.py +@@ -8,6 +8,7 @@ + import os + import zlib + from io import BytesIO ++from typing import List + + from zope.interface import implementer + from zope.interface.verify import verifyObject +@@ -15,12 +16,14 @@ + from twisted.internet import interfaces + from twisted.internet.address import IPv4Address, IPv6Address + from twisted.internet.task import Clock +-from twisted.internet.testing import EventLoggingObserver ++from twisted.internet.testing import EventLoggingObserver, StringTransport + from twisted.logger import LogLevel, globalLogPublisher + from twisted.python import failure, reflect + from twisted.python.filepath import FilePath + from twisted.trial import unittest + from twisted.web import error, http, iweb, resource, server ++from twisted.web.resource import Resource ++from twisted.web.server import NOT_DONE_YET, Request, Site + from twisted.web.static import Data + from twisted.web.test.requesthelper import DummyChannel, DummyRequest + from ._util import assertIsFilesystemTemporary +@@ -1849,3 +1852,39 @@ def test_defaultReactor(self): + + factory = http.HTTPFactory() + self.assertIs(factory.reactor, reactor) ++ ++ ++class QueueResource(Resource): ++ isLeaf = True ++ ++ def __init__(self) -> None: ++ super().__init__() ++ self.queue: List[Request] = [] ++ ++ def render_GET(self, request: Request) -> int: ++ self.queue.append(request) ++ return NOT_DONE_YET ++ ++ ++class TestRFC9112Section932(unittest.TestCase): ++ """ ++ Verify that HTTP/1.1 request ordering is preserved. ++ """ ++ ++ def test_multipleRequestsInOneSegment(self) -> None: ++ """ ++ Twisted MUST NOT respond to a second HTTP/1.1 request while the first ++ is still pending. ++ """ ++ qr = QueueResource() ++ site = Site(qr) ++ proto = site.buildProtocol(None) ++ serverTransport = StringTransport() ++ proto.makeConnection(serverTransport) ++ proto.dataReceived( ++ b"GET /first HTTP/1.1\r\nHost: a\r\n\r\n" ++ b"GET /second HTTP/1.1\r\nHost: a\r\n\r\n" ++ ) ++ self.assertEqual(len(qr.queue), 1) ++ qr.queue[0].finish() ++ self.assertEqual(len(qr.queue), 2) + +From 7f5446a379dea065dff28be5957aa59d00ab7f7e Mon Sep 17 00:00:00 2001 +From: Glyph +Date: Sun, 10 Sep 2023 23:08:46 -0700 +Subject: [PATCH 02/12] explain the change + +--- + src/twisted/web/newsfragments/11976.bugfix | 7 +++++++ + 1 file changed, 7 insertions(+) + create mode 100644 src/twisted/web/newsfragments/11976.bugfix + +diff --git a/src/twisted/web/newsfragments/11976.bugfix b/src/twisted/web/newsfragments/11976.bugfix +new file mode 100644 +index 00000000000..8ac292bef56 +--- /dev/null ++++ b/src/twisted/web/newsfragments/11976.bugfix +@@ -0,0 +1,7 @@ ++In Twisted 16.3.0, we changed twisted.web to stop dispatching HTTP/1.1 ++pipelined requests to application code. There was a bug in this change which ++still allowed clients which could send multiple full HTTP requests in a single ++TCP segment to trigger asynchronous processing of later requests, which could ++lead to out-of-order responses. This has now been corrected and twisted.web ++should never process a pipelined request over HTTP/1.1 until the previous ++request has fully completed. + +From 7de50d6b704b774d7205645512517e428b7039ce Mon Sep 17 00:00:00 2001 +From: Glyph +Date: Mon, 11 Sep 2023 08:34:19 -0700 +Subject: [PATCH 03/12] Update src/twisted/web/test/test_web.py + +Co-authored-by: Adi Roiban +--- + src/twisted/web/test/test_web.py | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/src/twisted/web/test/test_web.py b/src/twisted/web/test/test_web.py +index 1d24f454f2e..837b3f64c39 100644 +--- a/src/twisted/web/test/test_web.py ++++ b/src/twisted/web/test/test_web.py +@@ -1855,6 +1855,11 @@ def test_defaultReactor(self): + + + class QueueResource(Resource): ++ """ ++ Add all requests to an internal queue, ++ without responding to the requests. ++ You can access the requests from the queue and handle their response. ++ """ + isLeaf = True + + def __init__(self) -> None: + +From 36f8ff33e2385c35845b2745b8a89df1f06222f3 Mon Sep 17 00:00:00 2001 +From: Glyph +Date: Mon, 11 Sep 2023 08:34:38 -0700 +Subject: [PATCH 04/12] Update src/twisted/web/test/test_web.py + +Co-authored-by: Adi Roiban +--- + src/twisted/web/test/test_web.py | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/src/twisted/web/test/test_web.py b/src/twisted/web/test/test_web.py +index 837b3f64c39..19040cdb835 100644 +--- a/src/twisted/web/test/test_web.py ++++ b/src/twisted/web/test/test_web.py +@@ -1890,6 +1890,11 @@ def test_multipleRequestsInOneSegment(self) -> None: + b"GET /first HTTP/1.1\r\nHost: a\r\n\r\n" + b"GET /second HTTP/1.1\r\nHost: a\r\n\r\n" + ) ++ # The TCP data contains 2 requests, ++ # but only 1 request was dispatched, ++ # as the first request was not yet finalized. + self.assertEqual(len(qr.queue), 1) ++ # The first request is finalized and the ++ # second request is dispatched right away. + qr.queue[0].finish() + self.assertEqual(len(qr.queue), 2) + +From 731658108bbde2349a5ffc4550e602511b81167a Mon Sep 17 00:00:00 2001 +From: Glyph +Date: Mon, 11 Sep 2023 08:34:52 -0700 +Subject: [PATCH 05/12] Update src/twisted/web/test/test_web.py + +Co-authored-by: Adi Roiban +--- + src/twisted/web/test/test_web.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/twisted/web/test/test_web.py b/src/twisted/web/test/test_web.py +index 19040cdb835..54c75ef24e3 100644 +--- a/src/twisted/web/test/test_web.py ++++ b/src/twisted/web/test/test_web.py +@@ -1864,7 +1864,7 @@ class QueueResource(Resource): + + def __init__(self) -> None: + super().__init__() +- self.queue: List[Request] = [] ++ self.dispatchedRequests: List[Request] = [] + + def render_GET(self, request: Request) -> int: + self.queue.append(request) + +From d6b875b58701495725967b2c58a2dd528c429762 Mon Sep 17 00:00:00 2001 +From: "pre-commit-ci[bot]" + <66853113+pre-commit-ci[bot]@users.noreply.github.com> +Date: Mon, 11 Sep 2023 15:35:57 +0000 +Subject: [PATCH 06/12] [pre-commit.ci] auto fixes from pre-commit.com hooks + +for more information, see https://pre-commit.ci +--- + src/twisted/web/test/test_web.py | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/src/twisted/web/test/test_web.py b/src/twisted/web/test/test_web.py +index 54c75ef24e3..414e2d1e250 100644 +--- a/src/twisted/web/test/test_web.py ++++ b/src/twisted/web/test/test_web.py +@@ -1860,6 +1860,7 @@ class QueueResource(Resource): + without responding to the requests. + You can access the requests from the queue and handle their response. + """ ++ + isLeaf = True + + def __init__(self) -> None: + +From 4f6c8625a6354aa711e166b64dda15f8129b62d0 Mon Sep 17 00:00:00 2001 +From: Glyph +Date: Mon, 11 Sep 2023 10:28:50 -0700 +Subject: [PATCH 07/12] change name to correspond with suggestion + +--- + src/twisted/web/test/test_web.py | 8 ++++---- + 1 file changed, 4 insertions(+), 4 deletions(-) + +diff --git a/src/twisted/web/test/test_web.py b/src/twisted/web/test/test_web.py +index 414e2d1e250..959be653f5f 100644 +--- a/src/twisted/web/test/test_web.py ++++ b/src/twisted/web/test/test_web.py +@@ -1868,7 +1868,7 @@ def __init__(self) -> None: + self.dispatchedRequests: List[Request] = [] + + def render_GET(self, request: Request) -> int: +- self.queue.append(request) ++ self.dispatchedRequests.append(request) + return NOT_DONE_YET + + +@@ -1894,8 +1894,8 @@ def test_multipleRequestsInOneSegment(self) -> None: + # The TCP data contains 2 requests, + # but only 1 request was dispatched, + # as the first request was not yet finalized. +- self.assertEqual(len(qr.queue), 1) ++ self.assertEqual(len(qr.dispatchedRequests), 1) + # The first request is finalized and the + # second request is dispatched right away. +- qr.queue[0].finish() +- self.assertEqual(len(qr.queue), 2) ++ qr.dispatchedRequests[0].finish() ++ self.assertEqual(len(qr.dispatchedRequests), 2) + +From 70c46ba53c4e80570f0e61a4e7dda71f34c313cc Mon Sep 17 00:00:00 2001 +From: Glyph +Date: Tue, 12 Sep 2023 10:09:47 -0700 +Subject: [PATCH 08/12] Update src/twisted/web/test/test_web.py + +Co-authored-by: Adi Roiban +--- + src/twisted/web/test/test_web.py | 29 +++++++++++++++++++++++++++++ + 1 file changed, 29 insertions(+) + +diff --git a/src/twisted/web/test/test_web.py b/src/twisted/web/test/test_web.py +index 959be653f5f..0a55cfa16e3 100644 +--- a/src/twisted/web/test/test_web.py ++++ b/src/twisted/web/test/test_web.py +@@ -1899,3 +1899,32 @@ def test_multipleRequestsInOneSegment(self) -> None: + # second request is dispatched right away. + qr.dispatchedRequests[0].finish() + self.assertEqual(len(qr.dispatchedRequests), 2) ++ ++ def test_multipleRequestsInDifferentSegments(self) -> None: ++ """ ++ Twisted MUST NOT respond to a second HTTP/1.1 request while the first ++ is still pending, even if the second request is received in a separate ++ TCP package. ++ """ ++ qr = QueueResource() ++ site = Site(qr) ++ proto = site.buildProtocol(None) ++ serverTransport = StringTransport() ++ proto.makeConnection(serverTransport) ++ raw_data = ( ++ b"GET /first HTTP/1.1\r\nHost: a\r\n\r\n" ++ b"GET /second HTTP/1.1\r\nHost: a\r\n\r\n" ++ ) ++ # Just go byte by byte for the extreme case in which each byte is ++ # received in a separate TCP package. ++ for chunk in iterbytes(raw_data): ++ proto.dataReceived(chunk) ++ # The TCP data contains 2 requests, ++ # but only 1 request was dispatched, ++ # as the first request was not yet finalized. ++ self.assertEqual(len(qr.dispatchedRequests), 1) ++ # The first request is finalized and the ++ # second request is dispatched right away. ++ qr.dispatchedRequests[0].finish() ++ self.assertEqual(len(qr.dispatchedRequests), 2) ++ + + +From 159a6aa3a7f71dc4d96e4bf6c984793490b6734c Mon Sep 17 00:00:00 2001 +From: Glyph +Date: Tue, 12 Sep 2023 10:14:46 -0700 +Subject: [PATCH 10/12] docstring explaining the logic, type annotation while + we're here + +--- + src/twisted/web/http.py | 27 ++++++++++++++++++++++++--- + 1 file changed, 24 insertions(+), 3 deletions(-) + +diff --git a/src/twisted/web/http.py b/src/twisted/web/http.py +index cccca148f5e..168e825dddd 100644 +--- a/src/twisted/web/http.py ++++ b/src/twisted/web/http.py +@@ -2442,9 +2442,30 @@ def allContentReceived(self): + req = self.requests[-1] + req.requestReceived(command, path, version) + +- def rawDataReceived(self, data): +- # If we're currently handling a request, we'll be in raw mode. Buffer +- # any data. ++ def rawDataReceived(self, data: bytes) -> None: ++ """ ++ This is called when this HTTP/1.1 parser is in raw mode rather than ++ line mode. ++ ++ It may be in raw mode for one of two reasons: ++ ++ 1. All the headers of a request have been received and this ++ L{HTTPChannel} is currently receiving its body. ++ ++ 2. The full content of a request has been received and is currently ++ being processed asynchronously, and this L{HTTPChannel} is ++ buffering the data of all subsequent requests to be parsed ++ later. ++ ++ In the second state, the data will be played back later. ++ ++ @note: This isn't really a public API, and should be invoked only by ++ L{LineReceiver}'s line parsing logic. If you wish to drive an ++ L{HTTPChannel} from a custom data source, call C{dataReceived} on ++ it directly. ++ ++ @see: L{LineReceive.rawDataReceived} ++ """ + if self._handlingRequest: + self._dataBuffer.append(data) + if ( + +From 14bd26f4c68bb2b82533f68b921f596595153170 Mon Sep 17 00:00:00 2001 +From: Glyph +Date: Tue, 12 Sep 2023 10:29:41 -0700 +Subject: [PATCH 11/12] undefined name + +--- + src/twisted/web/test/test_web.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/twisted/web/test/test_web.py b/src/twisted/web/test/test_web.py +index 0a55cfa16e3..13c32916512 100644 +--- a/src/twisted/web/test/test_web.py ++++ b/src/twisted/web/test/test_web.py +@@ -19,6 +19,7 @@ + from twisted.internet.testing import EventLoggingObserver, StringTransport + from twisted.logger import LogLevel, globalLogPublisher + from twisted.python import failure, reflect ++from twisted.python.compat import iterbytes + from twisted.python.filepath import FilePath + from twisted.trial import unittest + from twisted.web import error, http, iweb, resource, server + +From 88be54dd0706457fe4db886ccc820ce0cdec00b1 Mon Sep 17 00:00:00 2001 +From: Glyph +Date: Tue, 12 Sep 2023 10:29:59 -0700 +Subject: [PATCH 12/12] remove assert + +--- + src/twisted/web/http.py | 3 --- + 1 file changed, 3 deletions(-) + +diff --git a/src/twisted/web/http.py b/src/twisted/web/http.py +index 168e825dddd..2f3c0d7bf9c 100644 +--- a/src/twisted/web/http.py ++++ b/src/twisted/web/http.py +@@ -2250,9 +2250,6 @@ def lineReceived(self, line): + Called for each line from request until the end of headers when + it enters binary mode. + """ +- assert ( +- not self._handlingRequest +- ), "when handling a request, we MUST be in raw mode to buffer the incoming data without parsing it" + self.resetTimeout() + + self._receivedHeaderSize += len(line) diff --git a/python-twisted.spec b/python-twisted.spec index 3cd36c28e275c8b32e195ec6ff845f3ca8228ed9..fffd8751c62430b2777e2499f4c8ebceeacc2e3b 100644 --- a/python-twisted.spec +++ b/python-twisted.spec @@ -6,12 +6,15 @@ Summary: Twisted is a networking engine written in Python Name: python-%{srcname} Version: 23.8.0 -Release: 1%{?dist} +Release: 2%{?dist} License: MIT URL: http://twistedmatrix.com/ Source0: https://github.com/twisted/twisted/archive/%{srcname}-%{version}/%{srcname}-%{version}.tar.gz Patch0001: 0003-11786-fix-misuse-of-mktime-in-tests.patch +#https://github.com/twisted/twisted/pull/11979 +Patch0002: fix-CVE-2023-46137.patch + Patch3001: 0001-23.8.0rc1-remove-hatch-fancy-pypi-readme.patch Patch3002: 0002-23.8.0rc1-fix-and-skip-tests-fedora.patch @@ -62,8 +65,6 @@ ln -s ./twistd %{buildroot}%{_bindir}/twistd-3 %pyproject_save_files %{srcname} echo "%ghost %{python3_sitelib}/twisted/plugins/dropin.cache" >> %{pyproject_files} -find %{buildroot} -name LICENSE | xargs -I {} rm -f {} - %check %if %{with test} PATH=%{buildroot}%{_bindir}:$PATH PYTHONPATH="$PWD/src:%{?with_python_bootstrap:$PYTHONPATH}" \ @@ -94,6 +95,10 @@ PATH=%{buildroot}%{_bindir}:$PATH PYTHONPATH="$PWD/src:%{?with_python_bootstrap: %{_mandir}/man1/twistd.1* %changelog +* Thu Jun 27 2024 Weiyao Feng - 23.8.0-2 +- [type] security +- [desc] fix CVE-2023-46137 + * Mon Sep 25 2023 Shuo Wang - 23.8.0-1 - update to 23.8.0