diff --git a/backport-20104-Add-flag-capabilities-to-posix_spawn-GH-66.patch b/backport-20104-Add-flag-capabilities-to-posix_spawn-GH-66.patch new file mode 100644 index 0000000000000000000000000000000000000000..5260953abc23f02fc2b6b460466ec4dfd5ecde88 --- /dev/null +++ b/backport-20104-Add-flag-capabilities-to-posix_spawn-GH-66.patch @@ -0,0 +1,460 @@ +From 52628b241a1559840479e72c32214cf4a6b4a92f Mon Sep 17 00:00:00 2001 +From: Pablo Galindo +Date: Fri, 7 Sep 2018 16:44:24 +0100 +Subject: [PATCH] bpo-20104: Add flag capabilities to posix_spawn (GH-6693) + +Implement the "attributes objects" parameter of `os.posix_spawn` to complete the implementation and fully cover the underlying API. + +Conflict:NA +Reference:https://github.com/python/cpython/commit/254a4663d8c5970ae2928185c50ebaa6c7e62c80 + +Signed-off-by: hanxinke +--- + Lib/test/test_posix.py | 143 ++++++++++++++++++ + .../2018-05-05-23-26-58.bpo-20104.tDBciE.rst | 2 + + Modules/clinic/posixmodule.c.h | 37 ++++- + Modules/posixmodule.c | 140 ++++++++++++++++- + 4 files changed, 310 insertions(+), 12 deletions(-) + create mode 100644 Misc/NEWS.d/next/Core and Builtins/2018-05-05-23-26-58.bpo-20104.tDBciE.rst + +diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py +index a22baac..1103ff3 100644 +--- a/Lib/test/test_posix.py ++++ b/Lib/test/test_posix.py +@@ -8,6 +8,7 @@ posix = support.import_module('posix') + + import errno + import sys ++import signal + import time + import os + import platform +@@ -16,6 +17,7 @@ import stat + import tempfile + import unittest + import warnings ++import textwrap + + _DUMMY_SYMLINK = os.path.join(tempfile.gettempdir(), + support.TESTFN + '-dummy-symlink') +@@ -1558,6 +1560,147 @@ class TestPosixSpawn(unittest.TestCase): + ) + self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + ++ def test_resetids_explicit_default(self): ++ pid = posix.posix_spawn( ++ sys.executable, ++ [sys.executable, '-c', 'pass'], ++ os.environ, ++ resetids=False ++ ) ++ self.assertEqual(os.waitpid(pid, 0), (pid, 0)) ++ ++ def test_resetids(self): ++ pid = posix.posix_spawn( ++ sys.executable, ++ [sys.executable, '-c', 'pass'], ++ os.environ, ++ resetids=True ++ ) ++ self.assertEqual(os.waitpid(pid, 0), (pid, 0)) ++ ++ def test_resetids_wrong_type(self): ++ with self.assertRaises(TypeError): ++ posix.posix_spawn(sys.executable, ++ [sys.executable, "-c", "pass"], ++ os.environ, resetids=None) ++ ++ def test_setpgroup(self): ++ pid = posix.posix_spawn( ++ sys.executable, ++ [sys.executable, '-c', 'pass'], ++ os.environ, ++ setpgroup=os.getpgrp() ++ ) ++ self.assertEqual(os.waitpid(pid, 0), (pid, 0)) ++ ++ def test_setpgroup_wrong_type(self): ++ with self.assertRaises(TypeError): ++ posix.posix_spawn(sys.executable, ++ [sys.executable, "-c", "pass"], ++ os.environ, setpgroup="023") ++ ++ @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), ++ 'need signal.pthread_sigmask()') ++ def test_setsigmask(self): ++ code = textwrap.dedent("""\ ++ import _testcapi, signal ++ _testcapi.raise_signal(signal.SIGUSR1)""") ++ ++ pid = posix.posix_spawn( ++ sys.executable, ++ [sys.executable, '-c', code], ++ os.environ, ++ setsigmask=[signal.SIGUSR1] ++ ) ++ self.assertEqual(os.waitpid(pid, 0), (pid, 0)) ++ ++ def test_setsigmask_wrong_type(self): ++ with self.assertRaises(TypeError): ++ posix.posix_spawn(sys.executable, ++ [sys.executable, "-c", "pass"], ++ os.environ, setsigmask=34) ++ with self.assertRaises(TypeError): ++ posix.posix_spawn(sys.executable, ++ [sys.executable, "-c", "pass"], ++ os.environ, setsigmask=["j"]) ++ with self.assertRaises(ValueError): ++ posix.posix_spawn(sys.executable, ++ [sys.executable, "-c", "pass"], ++ os.environ, setsigmask=[signal.NSIG, ++ signal.NSIG+1]) ++ ++ @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), ++ 'need signal.pthread_sigmask()') ++ def test_setsigdef(self): ++ original_handler = signal.signal(signal.SIGUSR1, signal.SIG_IGN) ++ code = textwrap.dedent("""\ ++ import _testcapi, signal ++ _testcapi.raise_signal(signal.SIGUSR1)""") ++ try: ++ pid = posix.posix_spawn( ++ sys.executable, ++ [sys.executable, '-c', code], ++ os.environ, ++ setsigdef=[signal.SIGUSR1] ++ ) ++ finally: ++ signal.signal(signal.SIGUSR1, original_handler) ++ ++ pid2, status = os.waitpid(pid, 0) ++ self.assertEqual(pid2, pid) ++ self.assertTrue(os.WIFSIGNALED(status), status) ++ self.assertEqual(os.WTERMSIG(status), signal.SIGUSR1) ++ ++ def test_setsigdef_wrong_type(self): ++ with self.assertRaises(TypeError): ++ posix.posix_spawn(sys.executable, ++ [sys.executable, "-c", "pass"], ++ os.environ, setsigdef=34) ++ with self.assertRaises(TypeError): ++ posix.posix_spawn(sys.executable, ++ [sys.executable, "-c", "pass"], ++ os.environ, setsigdef=["j"]) ++ with self.assertRaises(ValueError): ++ posix.posix_spawn(sys.executable, ++ [sys.executable, "-c", "pass"], ++ os.environ, setsigdef=[signal.NSIG, signal.NSIG+1]) ++ ++ @unittest.skipUnless(hasattr(posix, 'sched_setscheduler'), "can't change scheduler") ++ def test_setscheduler_only_param(self): ++ policy = os.sched_getscheduler(0) ++ priority = os.sched_get_priority_min(policy) ++ code = textwrap.dedent(f"""\ ++ import os ++ if os.sched_getscheduler(0) != {policy}: ++ os.exit(101) ++ if os.sched_getparam(0).sched_priority != {priority}: ++ os.exit(102)""") ++ pid = posix.posix_spawn( ++ sys.executable, ++ [sys.executable, '-c', code], ++ os.environ, ++ scheduler=(None, os.sched_param(priority)) ++ ) ++ self.assertEqual(os.waitpid(pid, 0), (pid, 0)) ++ ++ @unittest.skipUnless(hasattr(posix, 'sched_setscheduler'), "can't change scheduler") ++ def test_setscheduler_with_policy(self): ++ policy = os.sched_getscheduler(0) ++ priority = os.sched_get_priority_min(policy) ++ code = textwrap.dedent(f"""\ ++ import os ++ if os.sched_getscheduler(0) != {policy}: ++ os.exit(101) ++ if os.sched_getparam(0).sched_priority != {priority}: ++ os.exit(102)""") ++ pid = posix.posix_spawn( ++ sys.executable, ++ [sys.executable, '-c', code], ++ os.environ, ++ scheduler=(policy, os.sched_param(priority)) ++ ) ++ self.assertEqual(os.waitpid(pid, 0), (pid, 0)) ++ + def test_multiple_file_actions(self): + file_actions = [ + (os.POSIX_SPAWN_OPEN, 3, os.path.realpath(__file__), os.O_RDONLY, 0), +diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-05-05-23-26-58.bpo-20104.tDBciE.rst b/Misc/NEWS.d/next/Core and Builtins/2018-05-05-23-26-58.bpo-20104.tDBciE.rst +new file mode 100644 +index 0000000..1d725ba +--- /dev/null ++++ b/Misc/NEWS.d/next/Core and Builtins/2018-05-05-23-26-58.bpo-20104.tDBciE.rst +@@ -0,0 +1,2 @@ ++Added support for the `setpgroup`, `resetids`, `setsigmask`, `setsigdef` and ++`scheduler` parameters of `posix_spawn`. Patch by Pablo Galindo. +diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h +index 1371ef6..a3b5a8b 100644 +--- a/Modules/clinic/posixmodule.c.h ++++ b/Modules/clinic/posixmodule.c.h +@@ -1730,7 +1730,9 @@ exit: + #if defined(HAVE_POSIX_SPAWN) + + PyDoc_STRVAR(os_posix_spawn__doc__, +-"posix_spawn($module, path, argv, env, file_actions=None, /)\n" ++"posix_spawn($module, path, argv, env, file_actions=None, /, *,\n" ++" setpgroup=None, resetids=False, setsigmask=(),\n" ++" setsigdef=(), scheduler=None)\n" + "--\n" + "\n" + "Execute the program specified by path in a new process.\n" +@@ -1742,29 +1744,48 @@ PyDoc_STRVAR(os_posix_spawn__doc__, + " env\n" + " Dictionary of strings mapping to strings.\n" + " file_actions\n" +-" A sequence of file action tuples."); ++" A sequence of file action tuples.\n" ++" setpgroup\n" ++" The pgroup to use with the POSIX_SPAWN_SETPGROUP flag.\n" ++" resetids\n" ++" If the value is `True` the POSIX_SPAWN_RESETIDS will be activated.\n" ++" setsigmask\n" ++" The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag.\n" ++" setsigdef\n" ++" The sigmask to use with the POSIX_SPAWN_SETSIGDEF flag.\n" ++" scheduler\n" ++" A tuple with the scheduler policy (optional) and parameters."); + + #define OS_POSIX_SPAWN_METHODDEF \ +- {"posix_spawn", (PyCFunction)os_posix_spawn, METH_FASTCALL, os_posix_spawn__doc__}, ++ {"posix_spawn", (PyCFunction)os_posix_spawn, METH_FASTCALL|METH_KEYWORDS, os_posix_spawn__doc__}, + + static PyObject * + os_posix_spawn_impl(PyObject *module, path_t *path, PyObject *argv, +- PyObject *env, PyObject *file_actions); ++ PyObject *env, PyObject *file_actions, ++ PyObject *setpgroup, int resetids, PyObject *setsigmask, ++ PyObject *setsigdef, PyObject *scheduler); + + static PyObject * +-os_posix_spawn(PyObject *module, PyObject *const *args, Py_ssize_t nargs) ++os_posix_spawn(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) + { + PyObject *return_value = NULL; ++ static const char * const _keywords[] = {"", "", "", "", "setpgroup", "resetids", "setsigmask", "setsigdef", "scheduler", NULL}; ++ static _PyArg_Parser _parser = {"O&OO|O$OiOOO:posix_spawn", _keywords, 0}; + path_t path = PATH_T_INITIALIZE("posix_spawn", "path", 0, 0); + PyObject *argv; + PyObject *env; + PyObject *file_actions = Py_None; ++ PyObject *setpgroup = NULL; ++ int resetids = 0; ++ PyObject *setsigmask = NULL; ++ PyObject *setsigdef = NULL; ++ PyObject *scheduler = NULL; + +- if (!_PyArg_ParseStack(args, nargs, "O&OO|O:posix_spawn", +- path_converter, &path, &argv, &env, &file_actions)) { ++ if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, ++ path_converter, &path, &argv, &env, &file_actions, &setpgroup, &resetids, &setsigmask, &setsigdef, &scheduler)) { + goto exit; + } +- return_value = os_posix_spawn_impl(module, &path, argv, env, file_actions); ++ return_value = os_posix_spawn_impl(module, &path, argv, env, file_actions, setpgroup, resetids, setsigmask, setsigdef, scheduler); + + exit: + /* Cleanup for path */ +diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c +index 441c82e..a1808f6 100644 +--- a/Modules/posixmodule.c ++++ b/Modules/posixmodule.c +@@ -5138,6 +5138,114 @@ enum posix_spawn_file_actions_identifier { + POSIX_SPAWN_DUP2 + }; + ++static int ++convert_sched_param(PyObject *param, struct sched_param *res); ++ ++static int ++parse_posix_spawn_flags(PyObject *setpgroup, int resetids, PyObject *setsigmask, ++ PyObject *setsigdef, PyObject *scheduler, ++ posix_spawnattr_t *attrp) ++{ ++ long all_flags = 0; ++ ++ errno = posix_spawnattr_init(attrp); ++ if (errno) { ++ posix_error(); ++ return -1; ++ } ++ ++ if (setpgroup) { ++ pid_t pgid = PyLong_AsPid(setpgroup); ++ if (pgid == (pid_t)-1 && PyErr_Occurred()) { ++ goto fail; ++ } ++ errno = posix_spawnattr_setpgroup(attrp, pgid); ++ if (errno) { ++ posix_error(); ++ goto fail; ++ } ++ all_flags |= POSIX_SPAWN_SETPGROUP; ++ } ++ ++ if (resetids) { ++ all_flags |= POSIX_SPAWN_RESETIDS; ++ } ++ ++ if (setsigmask) { ++ sigset_t set; ++ if (!_Py_Sigset_Converter(setsigmask, &set)) { ++ goto fail; ++ } ++ errno = posix_spawnattr_setsigmask(attrp, &set); ++ if (errno) { ++ posix_error(); ++ goto fail; ++ } ++ all_flags |= POSIX_SPAWN_SETSIGMASK; ++ } ++ ++ if (setsigdef) { ++ sigset_t set; ++ if (!_Py_Sigset_Converter(setsigdef, &set)) { ++ goto fail; ++ } ++ errno = posix_spawnattr_setsigdefault(attrp, &set); ++ if (errno) { ++ posix_error(); ++ goto fail; ++ } ++ all_flags |= POSIX_SPAWN_SETSIGDEF; ++ } ++ ++ if (scheduler) { ++#ifdef POSIX_SPAWN_SETSCHEDULER ++ PyObject *py_schedpolicy; ++ struct sched_param schedparam; ++ ++ if (!PyArg_ParseTuple(scheduler, "OO&" ++ ";A scheduler tuple must have two elements", ++ &py_schedpolicy, convert_sched_param, &schedparam)) { ++ goto fail; ++ } ++ if (py_schedpolicy != Py_None) { ++ int schedpolicy = _PyLong_AsInt(py_schedpolicy); ++ ++ if (schedpolicy == -1 && PyErr_Occurred()) { ++ goto fail; ++ } ++ errno = posix_spawnattr_setschedpolicy(attrp, schedpolicy); ++ if (errno) { ++ posix_error(); ++ goto fail; ++ } ++ all_flags |= POSIX_SPAWN_SETSCHEDULER; ++ } ++ errno = posix_spawnattr_setschedparam(attrp, &schedparam); ++ if (errno) { ++ posix_error(); ++ goto fail; ++ } ++ all_flags |= POSIX_SPAWN_SETSCHEDPARAM; ++#else ++ PyErr_SetString(PyExc_NotImplementedError, ++ "The scheduler option is not supported in this system."); ++ goto fail; ++#endif ++ } ++ ++ errno = posix_spawnattr_setflags(attrp, all_flags); ++ if (errno) { ++ posix_error(); ++ goto fail; ++ } ++ ++ return 0; ++ ++fail: ++ (void)posix_spawnattr_destroy(attrp); ++ return -1; ++} ++ + static int + parse_file_actions(PyObject *file_actions, + posix_spawn_file_actions_t *file_actionsp, +@@ -5238,6 +5346,7 @@ parse_file_actions(PyObject *file_actions, + } + Py_DECREF(file_action); + } ++ + Py_DECREF(seq); + return 0; + +@@ -5260,19 +5369,33 @@ os.posix_spawn + file_actions: object = None + A sequence of file action tuples. + / +- ++ * ++ setpgroup: object = NULL ++ The pgroup to use with the POSIX_SPAWN_SETPGROUP flag. ++ resetids: bool(accept={int}) = False ++ If the value is `True` the POSIX_SPAWN_RESETIDS will be activated. ++ setsigmask: object(c_default='NULL') = () ++ The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag. ++ setsigdef: object(c_default='NULL') = () ++ The sigmask to use with the POSIX_SPAWN_SETSIGDEF flag. ++ scheduler: object = NULL ++ A tuple with the scheduler policy (optional) and parameters. + Execute the program specified by path in a new process. + [clinic start generated code]*/ + + static PyObject * + os_posix_spawn_impl(PyObject *module, path_t *path, PyObject *argv, +- PyObject *env, PyObject *file_actions) +-/*[clinic end generated code: output=d023521f541c709c input=a3db1021d33230dc]*/ ++ PyObject *env, PyObject *file_actions, ++ PyObject *setpgroup, int resetids, PyObject *setsigmask, ++ PyObject *setsigdef, PyObject *scheduler) ++/*[clinic end generated code: output=45dfa4c515d09f2c input=2d7a7578430a90f0]*/ + { + EXECV_CHAR **argvlist = NULL; + EXECV_CHAR **envlist = NULL; + posix_spawn_file_actions_t file_actions_buf; + posix_spawn_file_actions_t *file_actionsp = NULL; ++ posix_spawnattr_t attr; ++ posix_spawnattr_t *attrp = NULL; + Py_ssize_t argc, envc; + PyObject *result = NULL; + PyObject *temp_buffer = NULL; +@@ -5334,9 +5457,15 @@ os_posix_spawn_impl(PyObject *module, path_t *path, PyObject *argv, + file_actionsp = &file_actions_buf; + } + ++ if (parse_posix_spawn_flags(setpgroup, resetids, setsigmask, ++ setsigdef, scheduler, &attr)) { ++ goto exit; ++ } ++ attrp = &attr; ++ + _Py_BEGIN_SUPPRESS_IPH + err_code = posix_spawn(&pid, path->narrow, +- file_actionsp, NULL, argvlist, envlist); ++ file_actionsp, attrp, argvlist, envlist); + _Py_END_SUPPRESS_IPH + if (err_code) { + errno = err_code; +@@ -5349,6 +5478,9 @@ exit: + if (file_actionsp) { + (void)posix_spawn_file_actions_destroy(file_actionsp); + } ++ if (attrp) { ++ (void)posix_spawnattr_destroy(attrp); ++ } + if (envlist) { + free_string_array(envlist, envc); + } +-- +2.23.0 + diff --git a/backport-20104-Change-the-file_actions-parameter-of-os.po.patch b/backport-20104-Change-the-file_actions-parameter-of-os.po.patch new file mode 100644 index 0000000000000000000000000000000000000000..10046144134b36bc78495f2711db60078c2cc534 --- /dev/null +++ b/backport-20104-Change-the-file_actions-parameter-of-os.po.patch @@ -0,0 +1,209 @@ +From 80c7ad17e8004096e508db5fc78d3b69b51e354e Mon Sep 17 00:00:00 2001 +From: Serhiy Storchaka +Date: Sat, 8 Sep 2018 14:48:18 +0300 +Subject: [PATCH] bpo-20104: Change the file_actions parameter of + os.posix_spawn(). (GH-6725) + +* Make its default value an empty tuple instead of None. +* Make it a keyword-only parameter. + +Conflict:NA +Reference:https://github.com/python/cpython/commit/d700f97b627989d41cd4629dc02969f9a6b56d2f + +Signed-off-by: hanxinke +--- + Lib/test/test_posix.py | 57 +++++++++++++++++----------------- + Modules/clinic/posixmodule.c.h | 8 ++--- + Modules/posixmodule.c | 9 +++--- + 3 files changed, 37 insertions(+), 37 deletions(-) + +diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py +index 77fedb1..ee3c5f0 100644 +--- a/Lib/test/test_posix.py ++++ b/Lib/test/test_posix.py +@@ -1527,8 +1527,7 @@ class TestPosixSpawn(unittest.TestCase): + pidfile.write(str(os.getpid())) + """ + args = self.python_args('-c', script) +- pid = posix.posix_spawn(args[0], args, +- os.environ) ++ pid = posix.posix_spawn(args[0], args, os.environ) + self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + with open(pidfile) as f: + self.assertEqual(f.read(), str(pid)) +@@ -1566,7 +1565,7 @@ class TestPosixSpawn(unittest.TestCase): + self.NOOP_PROGRAM[0], + self.NOOP_PROGRAM, + os.environ, +- [] ++ file_actions=[] + ) + self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + +@@ -1719,37 +1718,38 @@ class TestPosixSpawn(unittest.TestCase): + ] + pid = posix.posix_spawn(self.NOOP_PROGRAM[0], + self.NOOP_PROGRAM, +- os.environ, file_actions) ++ os.environ, ++ file_actions=file_actions) + self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + + def test_bad_file_actions(self): + args = self.NOOP_PROGRAM + with self.assertRaises(TypeError): +- posix.posix_spawn(args[0], args, +- os.environ, [None]) ++ posix.posix_spawn(args[0], args, os.environ, ++ file_actions=[None]) + with self.assertRaises(TypeError): +- posix.posix_spawn(args[0], args, +- os.environ, [()]) ++ posix.posix_spawn(args[0], args, os.environ, ++ file_actions=[()]) + with self.assertRaises(TypeError): +- posix.posix_spawn(args[0], args, +- os.environ, [(None,)]) ++ posix.posix_spawn(args[0], args, os.environ, ++ file_actions=[(None,)]) + with self.assertRaises(TypeError): +- posix.posix_spawn(args[0], args, +- os.environ, [(12345,)]) ++ posix.posix_spawn(args[0], args, os.environ, ++ file_actions=[(12345,)]) + with self.assertRaises(TypeError): +- posix.posix_spawn(args[0], args, +- os.environ, [(os.POSIX_SPAWN_CLOSE,)]) ++ posix.posix_spawn(args[0], args, os.environ, ++ file_actions=[(os.POSIX_SPAWN_CLOSE,)]) + with self.assertRaises(TypeError): +- posix.posix_spawn(args[0], args, +- os.environ, [(os.POSIX_SPAWN_CLOSE, 1, 2)]) ++ posix.posix_spawn(args[0], args, os.environ, ++ file_actions=[(os.POSIX_SPAWN_CLOSE, 1, 2)]) + with self.assertRaises(TypeError): +- posix.posix_spawn(args[0], args, +- os.environ, [(os.POSIX_SPAWN_CLOSE, None)]) ++ posix.posix_spawn(args[0], args, os.environ, ++ file_actions=[(os.POSIX_SPAWN_CLOSE, None)]) + with self.assertRaises(ValueError): +- posix.posix_spawn(args[0], args, +- os.environ, +- [(os.POSIX_SPAWN_OPEN, 3, __file__ + '\0', +- os.O_RDONLY, 0)]) ++ posix.posix_spawn(args[0], args, os.environ, ++ file_actions=[(os.POSIX_SPAWN_OPEN, ++ 3, __file__ + '\0', ++ os.O_RDONLY, 0)]) + + def test_open_file(self): + outfile = support.TESTFN +@@ -1764,8 +1764,8 @@ class TestPosixSpawn(unittest.TestCase): + stat.S_IRUSR | stat.S_IWUSR), + ] + args = self.python_args('-c', script) +- pid = posix.posix_spawn(args[0], args, +- os.environ, file_actions) ++ pid = posix.posix_spawn(args[0], args, os.environ, ++ file_actions=file_actions) + self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + with open(outfile) as f: + self.assertEqual(f.read(), 'hello') +@@ -1782,9 +1782,8 @@ class TestPosixSpawn(unittest.TestCase): + closefile.write('is closed %d' % e.errno) + """ + args = self.python_args('-c', script) +- pid = posix.posix_spawn(args[0], args, +- os.environ, +- [(os.POSIX_SPAWN_CLOSE, 0),]) ++ pid = posix.posix_spawn(args[0], args, os.environ, ++ file_actions=[(os.POSIX_SPAWN_CLOSE, 0),]) + self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + with open(closefile) as f: + self.assertEqual(f.read(), 'is closed %d' % errno.EBADF) +@@ -1801,8 +1800,8 @@ class TestPosixSpawn(unittest.TestCase): + (os.POSIX_SPAWN_DUP2, childfile.fileno(), 1), + ] + args = self.python_args('-c', script) +- pid = posix.posix_spawn(args[0], args, +- os.environ, file_actions) ++ pid = posix.posix_spawn(args[0], args, os.environ, ++ file_actions=file_actions) + self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + with open(dupfile) as f: + self.assertEqual(f.read(), 'hello') +diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h +index a3b5a8b..133abd7 100644 +--- a/Modules/clinic/posixmodule.c.h ++++ b/Modules/clinic/posixmodule.c.h +@@ -1730,7 +1730,7 @@ exit: + #if defined(HAVE_POSIX_SPAWN) + + PyDoc_STRVAR(os_posix_spawn__doc__, +-"posix_spawn($module, path, argv, env, file_actions=None, /, *,\n" ++"posix_spawn($module, path, argv, env, /, *, file_actions=(),\n" + " setpgroup=None, resetids=False, setsigmask=(),\n" + " setsigdef=(), scheduler=None)\n" + "--\n" +@@ -1769,12 +1769,12 @@ static PyObject * + os_posix_spawn(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) + { + PyObject *return_value = NULL; +- static const char * const _keywords[] = {"", "", "", "", "setpgroup", "resetids", "setsigmask", "setsigdef", "scheduler", NULL}; +- static _PyArg_Parser _parser = {"O&OO|O$OiOOO:posix_spawn", _keywords, 0}; ++ static const char * const _keywords[] = {"", "", "", "file_actions", "setpgroup", "resetids", "setsigmask", "setsigdef", "scheduler", NULL}; ++ static _PyArg_Parser _parser = {"O&OO|$OOiOOO:posix_spawn", _keywords, 0}; + path_t path = PATH_T_INITIALIZE("posix_spawn", "path", 0, 0); + PyObject *argv; + PyObject *env; +- PyObject *file_actions = Py_None; ++ PyObject *file_actions = NULL; + PyObject *setpgroup = NULL; + int resetids = 0; + PyObject *setsigmask = NULL; +diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c +index a1808f6..dc6a22f 100644 +--- a/Modules/posixmodule.c ++++ b/Modules/posixmodule.c +@@ -5366,10 +5366,10 @@ os.posix_spawn + Tuple or list of strings. + env: object + Dictionary of strings mapping to strings. +- file_actions: object = None +- A sequence of file action tuples. + / + * ++ file_actions: object(c_default='NULL') = () ++ A sequence of file action tuples. + setpgroup: object = NULL + The pgroup to use with the POSIX_SPAWN_SETPGROUP flag. + resetids: bool(accept={int}) = False +@@ -5380,6 +5380,7 @@ os.posix_spawn + The sigmask to use with the POSIX_SPAWN_SETSIGDEF flag. + scheduler: object = NULL + A tuple with the scheduler policy (optional) and parameters. ++ + Execute the program specified by path in a new process. + [clinic start generated code]*/ + +@@ -5388,7 +5389,7 @@ os_posix_spawn_impl(PyObject *module, path_t *path, PyObject *argv, + PyObject *env, PyObject *file_actions, + PyObject *setpgroup, int resetids, PyObject *setsigmask, + PyObject *setsigdef, PyObject *scheduler) +-/*[clinic end generated code: output=45dfa4c515d09f2c input=2d7a7578430a90f0]*/ ++/*[clinic end generated code: output=45dfa4c515d09f2c input=2891c2f1d457e39b]*/ + { + EXECV_CHAR **argvlist = NULL; + EXECV_CHAR **envlist = NULL; +@@ -5438,7 +5439,7 @@ os_posix_spawn_impl(PyObject *module, path_t *path, PyObject *argv, + goto exit; + } + +- if (file_actions != Py_None) { ++ if (file_actions != NULL) { + /* There is a bug in old versions of glibc that makes some of the + * helper functions for manipulating file actions not copy the provided + * buffers. The problem is that posix_spawn_file_actions_addopen does not +-- +2.23.0 + diff --git a/backport-20104-Expose-posix_spawn-in-the-os-module-GH-510.patch b/backport-20104-Expose-posix_spawn-in-the-os-module-GH-510.patch new file mode 100644 index 0000000000000000000000000000000000000000..ffb733351f3ae0dab8c47a9b7ee417168e75050c --- /dev/null +++ b/backport-20104-Expose-posix_spawn-in-the-os-module-GH-510.patch @@ -0,0 +1,369 @@ +From a4faef5368c5a15a2a4bb83aee451708ed622aee Mon Sep 17 00:00:00 2001 +From: Pablo Galindo +Date: Mon, 29 Jan 2018 01:56:10 +0000 +Subject: [PATCH] bpo-20104: Expose `posix_spawn` in the os module (GH-5109) + +Add os.posix_spawn to wrap the low level POSIX API of the same name. + +Contributed by Pablo Galindo. + +Conflict:NA +Reference:https://github.com/python/cpython/commit/6c6ddf97c402709713d668d0ed53836a7749ba99 + +Signed-off-by: hanxinke +--- + Lib/test/test_posix.py | 16 ++ + .../2018-01-06-01-14-53.bpo-20104.9DkKb8.rst | 1 + + Modules/clinic/posixmodule.c.h | 52 +++++ + Modules/posixmodule.c | 202 ++++++++++++++++++ + 4 files changed, 271 insertions(+) + create mode 100644 Misc/NEWS.d/next/Core and Builtins/2018-01-06-01-14-53.bpo-20104.9DkKb8.rst + +diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py +index 08306b2..c67087c 100644 +--- a/Lib/test/test_posix.py ++++ b/Lib/test/test_posix.py +@@ -192,6 +192,22 @@ class PosixTester(unittest.TestCase): + os.close(fp) + + ++ @unittest.skipUnless(hasattr(os, 'posix_spawn'), "test needs os.posix_spawn") ++ def test_posix_spawn(self): ++ pid = posix.posix_spawn(sys.executable, [sys.executable, "-c", "pass"], os.environ,[]) ++ self.assertEqual(os.waitpid(pid,0),(pid,0)) ++ ++ ++ @unittest.skipUnless(hasattr(os, 'posix_spawn'), "test needs os.posix_spawn") ++ def test_posix_spawn_file_actions(self): ++ file_actions = [] ++ file_actions.append((0,3,os.path.realpath(__file__),0,0)) ++ file_actions.append((os.POSIX_SPAWN_CLOSE,2)) ++ file_actions.append((os.POSIX_SPAWN_DUP2,1,4)) ++ pid = posix.posix_spawn(sys.executable, [sys.executable, "-c", "pass"], os.environ, file_actions) ++ self.assertEqual(os.waitpid(pid,0),(pid,0)) ++ ++ + @unittest.skipUnless(hasattr(posix, 'waitid'), "test needs posix.waitid()") + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + def test_waitid(self): +diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-01-06-01-14-53.bpo-20104.9DkKb8.rst b/Misc/NEWS.d/next/Core and Builtins/2018-01-06-01-14-53.bpo-20104.9DkKb8.rst +new file mode 100644 +index 0000000..cb69f32 +--- /dev/null ++++ b/Misc/NEWS.d/next/Core and Builtins/2018-01-06-01-14-53.bpo-20104.9DkKb8.rst +@@ -0,0 +1 @@ ++Expose posix_spawn as a low level API in the os module. +diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h +index 22eef68..07be916 100644 +--- a/Modules/clinic/posixmodule.c.h ++++ b/Modules/clinic/posixmodule.c.h +@@ -1727,6 +1727,54 @@ exit: + + #endif /* defined(HAVE_EXECV) */ + ++#if defined(HAVE_POSIX_SPAWN) ++ ++PyDoc_STRVAR(os_posix_spawn__doc__, ++"posix_spawn($module, path, argv, env, file_actions=None, /)\n" ++"--\n" ++"\n" ++"Execute the program specified by path in a new process.\n" ++"\n" ++" path\n" ++" Path of executable file.\n" ++" argv\n" ++" Tuple or list of strings.\n" ++" env\n" ++" Dictionary of strings mapping to strings.\n" ++" file_actions\n" ++" FileActions object."); ++ ++#define OS_POSIX_SPAWN_METHODDEF \ ++ {"posix_spawn", (PyCFunction)os_posix_spawn, METH_FASTCALL, os_posix_spawn__doc__}, ++ ++static PyObject * ++os_posix_spawn_impl(PyObject *module, path_t *path, PyObject *argv, ++ PyObject *env, PyObject *file_actions); ++ ++static PyObject * ++os_posix_spawn(PyObject *module, PyObject *const *args, Py_ssize_t nargs) ++{ ++ PyObject *return_value = NULL; ++ path_t path = PATH_T_INITIALIZE("posix_spawn", "path", 0, 0); ++ PyObject *argv; ++ PyObject *env; ++ PyObject *file_actions = Py_None; ++ ++ if (!_PyArg_ParseStack(args, nargs, "O&OO|O:posix_spawn", ++ path_converter, &path, &argv, &env, &file_actions)) { ++ goto exit; ++ } ++ return_value = os_posix_spawn_impl(module, &path, argv, env, file_actions); ++ ++exit: ++ /* Cleanup for path */ ++ path_cleanup(&path); ++ ++ return return_value; ++} ++ ++#endif /* defined(HAVE_POSIX_SPAWN) */ ++ + #if (defined(HAVE_SPAWNV) || defined(HAVE_WSPAWNV)) + + PyDoc_STRVAR(os_spawnv__doc__, +@@ -6147,6 +6195,10 @@ exit: + #define OS_EXECVE_METHODDEF + #endif /* !defined(OS_EXECVE_METHODDEF) */ + ++#ifndef OS_POSIX_SPAWN_METHODDEF ++ #define OS_POSIX_SPAWN_METHODDEF ++#endif /* !defined(OS_POSIX_SPAWN_METHODDEF) */ ++ + #ifndef OS_SPAWNV_METHODDEF + #define OS_SPAWNV_METHODDEF + #endif /* !defined(OS_SPAWNV_METHODDEF) */ +diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c +index 43d4302..fcb22c2 100644 +--- a/Modules/posixmodule.c ++++ b/Modules/posixmodule.c +@@ -176,6 +176,7 @@ corresponding Unix manual entries for more information on calls."); + #else + /* Unix functions that the configure script doesn't check for */ + #define HAVE_EXECV 1 ++#define HAVE_POSIX_SPAWN 1 + #define HAVE_FORK 1 + #if defined(__USLC__) && defined(__SCO_VERSION__) /* SCO UDK Compiler */ + #define HAVE_FORK1 1 +@@ -246,6 +247,10 @@ extern int lstat(const char *, struct stat *); + + #endif /* !_MSC_VER */ + ++#ifdef HAVE_POSIX_SPAWN ++#include ++#endif ++ + #ifdef HAVE_UTIME_H + #include + #endif /* HAVE_UTIME_H */ +@@ -5126,6 +5131,195 @@ os_execve_impl(PyObject *module, path_t *path, PyObject *argv, PyObject *env) + + #endif /* HAVE_EXECV */ + ++#ifdef HAVE_POSIX_SPAWN ++ ++enum posix_spawn_file_actions_identifier { ++ POSIX_SPAWN_OPEN, ++ POSIX_SPAWN_CLOSE, ++ POSIX_SPAWN_DUP2 ++}; ++ ++/*[clinic input] ++ ++os.posix_spawn ++ path: path_t ++ Path of executable file. ++ argv: object ++ Tuple or list of strings. ++ env: object ++ Dictionary of strings mapping to strings. ++ file_actions: object = None ++ FileActions object. ++ / ++ ++Execute the program specified by path in a new process. ++[clinic start generated code]*/ ++ ++static PyObject * ++os_posix_spawn_impl(PyObject *module, path_t *path, PyObject *argv, ++ PyObject *env, PyObject *file_actions) ++/*[clinic end generated code: output=d023521f541c709c input=0ec9f1cfdc890be5]*/ ++{ ++ EXECV_CHAR **argvlist = NULL; ++ EXECV_CHAR **envlist; ++ Py_ssize_t argc, envc; ++ ++ /* posix_spawn has three arguments: (path, argv, env), where ++ argv is a list or tuple of strings and env is a dictionary ++ like posix.environ. */ ++ ++ if (!PySequence_Check(argv)){ ++ PyErr_SetString(PyExc_TypeError, ++ "posix_spawn: argv must be a tuple or list"); ++ goto fail; ++ } ++ argc = PySequence_Size(argv); ++ if (argc < 1) { ++ PyErr_SetString(PyExc_ValueError, "posix_spawn: argv must not be empty"); ++ return NULL; ++ } ++ ++ if (!PyMapping_Check(env)) { ++ PyErr_SetString(PyExc_TypeError, ++ "posix_spawn: environment must be a mapping object"); ++ goto fail; ++ } ++ ++ argvlist = parse_arglist(argv, &argc); ++ if (argvlist == NULL) { ++ goto fail; ++ } ++ if (!argvlist[0][0]) { ++ PyErr_SetString(PyExc_ValueError, ++ "posix_spawn: argv first element cannot be empty"); ++ goto fail; ++ } ++ ++ envlist = parse_envlist(env, &envc); ++ if (envlist == NULL) ++ goto fail; ++ ++ pid_t pid; ++ posix_spawn_file_actions_t *file_actionsp = NULL; ++ if (file_actions != NULL && file_actions != Py_None){ ++ posix_spawn_file_actions_t _file_actions; ++ if(posix_spawn_file_actions_init(&_file_actions) != 0){ ++ PyErr_SetString(PyExc_TypeError, ++ "Error initializing file actions"); ++ goto fail; ++ } ++ ++ ++ file_actionsp = &_file_actions; ++ ++ ++ PyObject* seq = PySequence_Fast(file_actions, "file_actions must be a sequence"); ++ if(seq == NULL){ ++ goto fail; ++ } ++ PyObject* file_actions_obj; ++ PyObject* mode_obj; ++ ++ for (int i = 0; i < PySequence_Fast_GET_SIZE(seq); ++i) { ++ file_actions_obj = PySequence_Fast_GET_ITEM(seq, i); ++ ++ if(!PySequence_Check(file_actions_obj) | !PySequence_Size(file_actions_obj)){ ++ PyErr_SetString(PyExc_TypeError,"Each file_action element must be a non empty sequence"); ++ goto fail; ++ } ++ ++ ++ mode_obj = PySequence_Fast_GET_ITEM(file_actions_obj, 0); ++ int mode = PyLong_AsLong(mode_obj); ++ ++ /* Populate the file_actions object */ ++ ++ switch(mode) { ++ ++ case POSIX_SPAWN_OPEN: ++ if(PySequence_Size(file_actions_obj) != 5){ ++ PyErr_SetString(PyExc_TypeError,"A open file_action object must have 5 elements"); ++ goto fail; ++ } ++ ++ long open_fd = PyLong_AsLong(PySequence_GetItem(file_actions_obj, 1)); ++ if(PyErr_Occurred()) { ++ goto fail; ++ } ++ const char* open_path = PyUnicode_AsUTF8(PySequence_GetItem(file_actions_obj, 2)); ++ if(open_path == NULL){ ++ goto fail; ++ } ++ long open_oflag = PyLong_AsLong(PySequence_GetItem(file_actions_obj, 3)); ++ if(PyErr_Occurred()) { ++ goto fail; ++ } ++ long open_mode = PyLong_AsLong(PySequence_GetItem(file_actions_obj, 4)); ++ if(PyErr_Occurred()) { ++ goto fail; ++ } ++ posix_spawn_file_actions_addopen(file_actionsp, open_fd, open_path, open_oflag, open_mode); ++ break; ++ ++ case POSIX_SPAWN_CLOSE: ++ if(PySequence_Size(file_actions_obj) != 2){ ++ PyErr_SetString(PyExc_TypeError,"A close file_action object must have 2 elements"); ++ goto fail; ++ } ++ ++ long close_fd = PyLong_AsLong(PySequence_GetItem(file_actions_obj, 1)); ++ if(PyErr_Occurred()) { ++ goto fail; ++ } ++ posix_spawn_file_actions_addclose(file_actionsp, close_fd); ++ break; ++ ++ case POSIX_SPAWN_DUP2: ++ if(PySequence_Size(file_actions_obj) != 3){ ++ PyErr_SetString(PyExc_TypeError,"A dup2 file_action object must have 3 elements"); ++ goto fail; ++ } ++ ++ long fd1 = PyLong_AsLong(PySequence_GetItem(file_actions_obj, 1)); ++ if(PyErr_Occurred()) { ++ goto fail; ++ } ++ long fd2 = PyLong_AsLong(PySequence_GetItem(file_actions_obj, 2)); ++ if(PyErr_Occurred()) { ++ goto fail; ++ } ++ posix_spawn_file_actions_adddup2(file_actionsp, fd1, fd2); ++ break; ++ ++ default: ++ PyErr_SetString(PyExc_TypeError,"Unknown file_actions identifier"); ++ goto fail; ++ } ++ } ++ Py_DECREF(seq); ++} ++ ++ _Py_BEGIN_SUPPRESS_IPH ++ posix_spawn(&pid, path->narrow, file_actionsp, NULL, argvlist, envlist); ++ return PyLong_FromPid(pid); ++ _Py_END_SUPPRESS_IPH ++ ++ path_error(path); ++ ++ free_string_array(envlist, envc); ++ ++fail: ++ ++ if (argvlist) { ++ free_string_array(argvlist, argc); ++ } ++ return NULL; ++ ++ ++} ++#endif /* HAVE_POSIX_SPAWN */ ++ ++ + #if defined(HAVE_SPAWNV) || defined(HAVE_WSPAWNV) + /*[clinic input] + os.spawnv +@@ -12687,6 +12881,7 @@ static PyMethodDef posix_methods[] = { + OS_NICE_METHODDEF + OS_GETPRIORITY_METHODDEF + OS_SETPRIORITY_METHODDEF ++ OS_POSIX_SPAWN_METHODDEF + #ifdef HAVE_READLINK + {"readlink", (PyCFunction)posix_readlink, + METH_VARARGS | METH_KEYWORDS, +@@ -13241,6 +13436,13 @@ all_ins(PyObject *m) + if (PyModule_AddIntConstant(m, "RWF_NOWAIT", RWF_NOWAIT)) return -1; + #endif + ++/* constants for posix_spawn */ ++#ifdef HAVE_POSIX_SPAWN ++ if (PyModule_AddIntConstant(m, "POSIX_SPAWN_OPEN", POSIX_SPAWN_OPEN)) return -1; ++ if (PyModule_AddIntConstant(m, "POSIX_SPAWN_CLOSE", POSIX_SPAWN_CLOSE)) return -1; ++ if (PyModule_AddIntConstant(m, "POSIX_SPAWN_DUP2", POSIX_SPAWN_DUP2)) return -1; ++#endif ++ + #ifdef HAVE_SPAWNV + if (PyModule_AddIntConstant(m, "P_WAIT", _P_WAIT)) return -1; + if (PyModule_AddIntConstant(m, "P_NOWAIT", _P_NOWAIT)) return -1; +-- +2.23.0 + diff --git a/backport-20104-Fix-leaks-and-errors-in-new-os.posix_spawn.patch b/backport-20104-Fix-leaks-and-errors-in-new-os.posix_spawn.patch new file mode 100644 index 0000000000000000000000000000000000000000..76a37df251e973e5b6a19e91e982ff6c33123213 --- /dev/null +++ b/backport-20104-Fix-leaks-and-errors-in-new-os.posix_spawn.patch @@ -0,0 +1,257 @@ +From 94083ce4d60abd390fbba615de333bdec07edc2c Mon Sep 17 00:00:00 2001 +From: Pablo Galindo +Date: Mon, 29 Jan 2018 20:34:42 +0000 +Subject: [PATCH] bpo-20104: Fix leaks and errors in new os.posix_spawn + (GH-5418) + +* Fix memory leaks and error handling in posix spawn +* Improve error handling when destroying the file_actions object +* Py_DECREF the result of PySequence_Fast on error +* Handle uninitialized pid +* Use OSError if file actions fails to initialize +* Move _file_actions to outer scope to avoid undefined behaviour +* Remove HAVE_POSIX_SPAWN define in Modules/posixmodule.c +* Unshadow exception and clean error message + +Conflict:NA +Reference:https://github.com/python/cpython/commit/0cd6bca65519109a8a7862d38ba1b8924e432a16 + +Signed-off-by: hanxinke +--- + Modules/posixmodule.c | 111 +++++++++++++++++++++++++----------------- + 1 file changed, 66 insertions(+), 45 deletions(-) + +diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c +index fcb22c2..b5636f5 100644 +--- a/Modules/posixmodule.c ++++ b/Modules/posixmodule.c +@@ -176,7 +176,6 @@ corresponding Unix manual entries for more information on calls."); + #else + /* Unix functions that the configure script doesn't check for */ + #define HAVE_EXECV 1 +-#define HAVE_POSIX_SPAWN 1 + #define HAVE_FORK 1 + #if defined(__USLC__) && defined(__SCO_VERSION__) /* SCO UDK Compiler */ + #define HAVE_FORK1 1 +@@ -5161,17 +5160,22 @@ os_posix_spawn_impl(PyObject *module, path_t *path, PyObject *argv, + /*[clinic end generated code: output=d023521f541c709c input=0ec9f1cfdc890be5]*/ + { + EXECV_CHAR **argvlist = NULL; +- EXECV_CHAR **envlist; ++ EXECV_CHAR **envlist = NULL; ++ posix_spawn_file_actions_t _file_actions; ++ posix_spawn_file_actions_t *file_actionsp = NULL; + Py_ssize_t argc, envc; ++ PyObject* result = NULL; ++ PyObject* seq = NULL; ++ + + /* posix_spawn has three arguments: (path, argv, env), where + argv is a list or tuple of strings and env is a dictionary + like posix.environ. */ + +- if (!PySequence_Check(argv)){ ++ if (!PySequence_Check(argv)) { + PyErr_SetString(PyExc_TypeError, + "posix_spawn: argv must be a tuple or list"); +- goto fail; ++ goto exit; + } + argc = PySequence_Size(argv); + if (argc < 1) { +@@ -5182,40 +5186,37 @@ os_posix_spawn_impl(PyObject *module, path_t *path, PyObject *argv, + if (!PyMapping_Check(env)) { + PyErr_SetString(PyExc_TypeError, + "posix_spawn: environment must be a mapping object"); +- goto fail; ++ goto exit; + } + + argvlist = parse_arglist(argv, &argc); + if (argvlist == NULL) { +- goto fail; ++ goto exit; + } + if (!argvlist[0][0]) { + PyErr_SetString(PyExc_ValueError, + "posix_spawn: argv first element cannot be empty"); +- goto fail; ++ goto exit; + } + + envlist = parse_envlist(env, &envc); +- if (envlist == NULL) +- goto fail; ++ if (envlist == NULL) { ++ goto exit; ++ } + + pid_t pid; +- posix_spawn_file_actions_t *file_actionsp = NULL; +- if (file_actions != NULL && file_actions != Py_None){ +- posix_spawn_file_actions_t _file_actions; ++ if (file_actions != NULL && file_actions != Py_None) { + if(posix_spawn_file_actions_init(&_file_actions) != 0){ +- PyErr_SetString(PyExc_TypeError, ++ PyErr_SetString(PyExc_OSError, + "Error initializing file actions"); +- goto fail; ++ goto exit; + } + +- + file_actionsp = &_file_actions; + +- +- PyObject* seq = PySequence_Fast(file_actions, "file_actions must be a sequence"); +- if(seq == NULL){ +- goto fail; ++ seq = PySequence_Fast(file_actions, "file_actions must be a sequence"); ++ if(seq == NULL) { ++ goto exit; + } + PyObject* file_actions_obj; + PyObject* mode_obj; +@@ -5223,9 +5224,9 @@ os_posix_spawn_impl(PyObject *module, path_t *path, PyObject *argv, + for (int i = 0; i < PySequence_Fast_GET_SIZE(seq); ++i) { + file_actions_obj = PySequence_Fast_GET_ITEM(seq, i); + +- if(!PySequence_Check(file_actions_obj) | !PySequence_Size(file_actions_obj)){ ++ if(!PySequence_Check(file_actions_obj) | !PySequence_Size(file_actions_obj)) { + PyErr_SetString(PyExc_TypeError,"Each file_action element must be a non empty sequence"); +- goto fail; ++ goto exit; + } + + +@@ -5237,83 +5238,103 @@ os_posix_spawn_impl(PyObject *module, path_t *path, PyObject *argv, + switch(mode) { + + case POSIX_SPAWN_OPEN: +- if(PySequence_Size(file_actions_obj) != 5){ ++ if(PySequence_Size(file_actions_obj) != 5) { + PyErr_SetString(PyExc_TypeError,"A open file_action object must have 5 elements"); +- goto fail; ++ goto exit; + } + + long open_fd = PyLong_AsLong(PySequence_GetItem(file_actions_obj, 1)); + if(PyErr_Occurred()) { +- goto fail; ++ goto exit; + } + const char* open_path = PyUnicode_AsUTF8(PySequence_GetItem(file_actions_obj, 2)); +- if(open_path == NULL){ +- goto fail; ++ if(open_path == NULL) { ++ goto exit; + } + long open_oflag = PyLong_AsLong(PySequence_GetItem(file_actions_obj, 3)); + if(PyErr_Occurred()) { +- goto fail; ++ goto exit; + } + long open_mode = PyLong_AsLong(PySequence_GetItem(file_actions_obj, 4)); + if(PyErr_Occurred()) { +- goto fail; ++ goto exit; ++ } ++ if(posix_spawn_file_actions_addopen(file_actionsp, open_fd, open_path, open_oflag, open_mode)) { ++ PyErr_SetString(PyExc_OSError,"Failed to add open file action"); ++ goto exit; + } +- posix_spawn_file_actions_addopen(file_actionsp, open_fd, open_path, open_oflag, open_mode); ++ + break; + + case POSIX_SPAWN_CLOSE: + if(PySequence_Size(file_actions_obj) != 2){ + PyErr_SetString(PyExc_TypeError,"A close file_action object must have 2 elements"); +- goto fail; ++ goto exit; + } + + long close_fd = PyLong_AsLong(PySequence_GetItem(file_actions_obj, 1)); + if(PyErr_Occurred()) { +- goto fail; ++ goto exit; ++ } ++ if(posix_spawn_file_actions_addclose(file_actionsp, close_fd)) { ++ PyErr_SetString(PyExc_OSError,"Failed to add close file action"); ++ goto exit; + } +- posix_spawn_file_actions_addclose(file_actionsp, close_fd); + break; + + case POSIX_SPAWN_DUP2: + if(PySequence_Size(file_actions_obj) != 3){ + PyErr_SetString(PyExc_TypeError,"A dup2 file_action object must have 3 elements"); +- goto fail; ++ goto exit; + } + + long fd1 = PyLong_AsLong(PySequence_GetItem(file_actions_obj, 1)); + if(PyErr_Occurred()) { +- goto fail; ++ goto exit; + } + long fd2 = PyLong_AsLong(PySequence_GetItem(file_actions_obj, 2)); + if(PyErr_Occurred()) { +- goto fail; ++ goto exit; ++ } ++ if(posix_spawn_file_actions_adddup2(file_actionsp, fd1, fd2)) { ++ PyErr_SetString(PyExc_OSError,"Failed to add dup2 file action"); ++ goto exit; + } +- posix_spawn_file_actions_adddup2(file_actionsp, fd1, fd2); + break; + + default: + PyErr_SetString(PyExc_TypeError,"Unknown file_actions identifier"); +- goto fail; ++ goto exit; + } + } +- Py_DECREF(seq); +-} ++ } + + _Py_BEGIN_SUPPRESS_IPH +- posix_spawn(&pid, path->narrow, file_actionsp, NULL, argvlist, envlist); +- return PyLong_FromPid(pid); ++ int err_code = posix_spawn(&pid, path->narrow, file_actionsp, NULL, argvlist, envlist); + _Py_END_SUPPRESS_IPH ++ if(err_code) { ++ PyErr_SetString(PyExc_OSError,"posix_spawn call failed"); ++ goto exit; ++ } ++ result = PyLong_FromPid(pid); + +- path_error(path); ++exit: + +- free_string_array(envlist, envc); ++ Py_XDECREF(seq); + +-fail: ++ if(file_actionsp) { ++ posix_spawn_file_actions_destroy(file_actionsp); ++ } ++ ++ if (envlist) { ++ free_string_array(envlist, envc); ++ } + + if (argvlist) { + free_string_array(argvlist, argc); + } +- return NULL; ++ ++ return result; + + + } +-- +2.23.0 + diff --git a/backport-20104-Improve-error-handling-and-fix-a-reference.patch b/backport-20104-Improve-error-handling-and-fix-a-reference.patch new file mode 100644 index 0000000000000000000000000000000000000000..26ec85290a06e1a3792c0b05b0152b31fbea7611 --- /dev/null +++ b/backport-20104-Improve-error-handling-and-fix-a-reference.patch @@ -0,0 +1,543 @@ +From bebc98800f4a760d03c3ee9fe49e2943132b47c6 Mon Sep 17 00:00:00 2001 +From: Serhiy Storchaka +Date: Tue, 1 May 2018 16:45:04 +0300 +Subject: [PATCH] bpo-20104: Improve error handling and fix a reference leak in + os.posix_spawn(). (#6332) + +Conflict:NA +Reference:https://github.com/python/cpython/commit/ef347535f289baad22c0601e12a36b2dcd155c3a + +Signed-off-by: hanxinke +--- + Lib/test/test_posix.py | 177 +++++++++++-- + .../2018-04-01-19-21-04.bpo-20104.-AKcGa.rst | 1 + + Modules/clinic/posixmodule.c.h | 2 +- + Modules/posixmodule.c | 245 +++++++++--------- + 4 files changed, 285 insertions(+), 140 deletions(-) + create mode 100644 Misc/NEWS.d/next/Library/2018-04-01-19-21-04.bpo-20104.-AKcGa.rst + +diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py +index c67087c..a22baac 100644 +--- a/Lib/test/test_posix.py ++++ b/Lib/test/test_posix.py +@@ -192,22 +192,6 @@ class PosixTester(unittest.TestCase): + os.close(fp) + + +- @unittest.skipUnless(hasattr(os, 'posix_spawn'), "test needs os.posix_spawn") +- def test_posix_spawn(self): +- pid = posix.posix_spawn(sys.executable, [sys.executable, "-c", "pass"], os.environ,[]) +- self.assertEqual(os.waitpid(pid,0),(pid,0)) +- +- +- @unittest.skipUnless(hasattr(os, 'posix_spawn'), "test needs os.posix_spawn") +- def test_posix_spawn_file_actions(self): +- file_actions = [] +- file_actions.append((0,3,os.path.realpath(__file__),0,0)) +- file_actions.append((os.POSIX_SPAWN_CLOSE,2)) +- file_actions.append((os.POSIX_SPAWN_DUP2,1,4)) +- pid = posix.posix_spawn(sys.executable, [sys.executable, "-c", "pass"], os.environ, file_actions) +- self.assertEqual(os.waitpid(pid,0),(pid,0)) +- +- + @unittest.skipUnless(hasattr(posix, 'waitid'), "test needs posix.waitid()") + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + def test_waitid(self): +@@ -1519,9 +1503,168 @@ class PosixGroupsTester(unittest.TestCase): + posix.setgroups(groups) + self.assertListEqual(groups, posix.getgroups()) + ++ ++@unittest.skipUnless(hasattr(os, 'posix_spawn'), "test needs os.posix_spawn") ++class TestPosixSpawn(unittest.TestCase): ++ def test_returns_pid(self): ++ pidfile = support.TESTFN ++ self.addCleanup(support.unlink, pidfile) ++ script = f"""if 1: ++ import os ++ with open({pidfile!r}, "w") as pidfile: ++ pidfile.write(str(os.getpid())) ++ """ ++ pid = posix.posix_spawn(sys.executable, ++ [sys.executable, '-c', script], ++ os.environ) ++ self.assertEqual(os.waitpid(pid, 0), (pid, 0)) ++ with open(pidfile) as f: ++ self.assertEqual(f.read(), str(pid)) ++ ++ def test_no_such_executable(self): ++ no_such_executable = 'no_such_executable' ++ try: ++ pid = posix.posix_spawn(no_such_executable, ++ [no_such_executable], ++ os.environ) ++ except FileNotFoundError as exc: ++ self.assertEqual(exc.filename, no_such_executable) ++ else: ++ pid2, status = os.waitpid(pid, 0) ++ self.assertEqual(pid2, pid) ++ self.assertNotEqual(status, 0) ++ ++ def test_specify_environment(self): ++ envfile = support.TESTFN ++ self.addCleanup(support.unlink, envfile) ++ script = f"""if 1: ++ import os ++ with open({envfile!r}, "w") as envfile: ++ envfile.write(os.environ['foo']) ++ """ ++ pid = posix.posix_spawn(sys.executable, ++ [sys.executable, '-c', script], ++ {'foo': 'bar'}) ++ self.assertEqual(os.waitpid(pid, 0), (pid, 0)) ++ with open(envfile) as f: ++ self.assertEqual(f.read(), 'bar') ++ ++ def test_empty_file_actions(self): ++ pid = posix.posix_spawn( ++ sys.executable, ++ [sys.executable, '-c', 'pass'], ++ os.environ, ++ [] ++ ) ++ self.assertEqual(os.waitpid(pid, 0), (pid, 0)) ++ ++ def test_multiple_file_actions(self): ++ file_actions = [ ++ (os.POSIX_SPAWN_OPEN, 3, os.path.realpath(__file__), os.O_RDONLY, 0), ++ (os.POSIX_SPAWN_CLOSE, 0), ++ (os.POSIX_SPAWN_DUP2, 1, 4), ++ ] ++ pid = posix.posix_spawn(sys.executable, ++ [sys.executable, "-c", "pass"], ++ os.environ, file_actions) ++ self.assertEqual(os.waitpid(pid, 0), (pid, 0)) ++ ++ def test_bad_file_actions(self): ++ with self.assertRaises(TypeError): ++ posix.posix_spawn(sys.executable, ++ [sys.executable, "-c", "pass"], ++ os.environ, [None]) ++ with self.assertRaises(TypeError): ++ posix.posix_spawn(sys.executable, ++ [sys.executable, "-c", "pass"], ++ os.environ, [()]) ++ with self.assertRaises(TypeError): ++ posix.posix_spawn(sys.executable, ++ [sys.executable, "-c", "pass"], ++ os.environ, [(None,)]) ++ with self.assertRaises(TypeError): ++ posix.posix_spawn(sys.executable, ++ [sys.executable, "-c", "pass"], ++ os.environ, [(12345,)]) ++ with self.assertRaises(TypeError): ++ posix.posix_spawn(sys.executable, ++ [sys.executable, "-c", "pass"], ++ os.environ, [(os.POSIX_SPAWN_CLOSE,)]) ++ with self.assertRaises(TypeError): ++ posix.posix_spawn(sys.executable, ++ [sys.executable, "-c", "pass"], ++ os.environ, [(os.POSIX_SPAWN_CLOSE, 1, 2)]) ++ with self.assertRaises(TypeError): ++ posix.posix_spawn(sys.executable, ++ [sys.executable, "-c", "pass"], ++ os.environ, [(os.POSIX_SPAWN_CLOSE, None)]) ++ with self.assertRaises(ValueError): ++ posix.posix_spawn(sys.executable, ++ [sys.executable, "-c", "pass"], ++ os.environ, ++ [(os.POSIX_SPAWN_OPEN, 3, __file__ + '\0', ++ os.O_RDONLY, 0)]) ++ ++ def test_open_file(self): ++ outfile = support.TESTFN ++ self.addCleanup(support.unlink, outfile) ++ script = """if 1: ++ import sys ++ sys.stdout.write("hello") ++ """ ++ file_actions = [ ++ (os.POSIX_SPAWN_OPEN, 1, outfile, ++ os.O_WRONLY | os.O_CREAT | os.O_TRUNC, ++ stat.S_IRUSR | stat.S_IWUSR), ++ ] ++ pid = posix.posix_spawn(sys.executable, ++ [sys.executable, '-c', script], ++ os.environ, file_actions) ++ self.assertEqual(os.waitpid(pid, 0), (pid, 0)) ++ with open(outfile) as f: ++ self.assertEqual(f.read(), 'hello') ++ ++ def test_close_file(self): ++ closefile = support.TESTFN ++ self.addCleanup(support.unlink, closefile) ++ script = f"""if 1: ++ import os ++ try: ++ os.fstat(0) ++ except OSError as e: ++ with open({closefile!r}, 'w') as closefile: ++ closefile.write('is closed %d' % e.errno) ++ """ ++ pid = posix.posix_spawn(sys.executable, ++ [sys.executable, '-c', script], ++ os.environ, ++ [(os.POSIX_SPAWN_CLOSE, 0),]) ++ self.assertEqual(os.waitpid(pid, 0), (pid, 0)) ++ with open(closefile) as f: ++ self.assertEqual(f.read(), 'is closed %d' % errno.EBADF) ++ ++ def test_dup2(self): ++ dupfile = support.TESTFN ++ self.addCleanup(support.unlink, dupfile) ++ script = """if 1: ++ import sys ++ sys.stdout.write("hello") ++ """ ++ with open(dupfile, "wb") as childfile: ++ file_actions = [ ++ (os.POSIX_SPAWN_DUP2, childfile.fileno(), 1), ++ ] ++ pid = posix.posix_spawn(sys.executable, ++ [sys.executable, '-c', script], ++ os.environ, file_actions) ++ self.assertEqual(os.waitpid(pid, 0), (pid, 0)) ++ with open(dupfile) as f: ++ self.assertEqual(f.read(), 'hello') ++ ++ + def test_main(): + try: +- support.run_unittest(PosixTester, PosixGroupsTester) ++ support.run_unittest(PosixTester, PosixGroupsTester, TestPosixSpawn) + finally: + support.reap_children() + +diff --git a/Misc/NEWS.d/next/Library/2018-04-01-19-21-04.bpo-20104.-AKcGa.rst b/Misc/NEWS.d/next/Library/2018-04-01-19-21-04.bpo-20104.-AKcGa.rst +new file mode 100644 +index 0000000..150401d +--- /dev/null ++++ b/Misc/NEWS.d/next/Library/2018-04-01-19-21-04.bpo-20104.-AKcGa.rst +@@ -0,0 +1 @@ ++Improved error handling and fixed a reference leak in :func:`os.posix_spawn()`. +diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h +index 07be916..1371ef6 100644 +--- a/Modules/clinic/posixmodule.c.h ++++ b/Modules/clinic/posixmodule.c.h +@@ -1742,7 +1742,7 @@ PyDoc_STRVAR(os_posix_spawn__doc__, + " env\n" + " Dictionary of strings mapping to strings.\n" + " file_actions\n" +-" FileActions object."); ++" A sequence of file action tuples."); + + #define OS_POSIX_SPAWN_METHODDEF \ + {"posix_spawn", (PyCFunction)os_posix_spawn, METH_FASTCALL, os_posix_spawn__doc__}, +diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c +index b5636f5..f10fe2b 100644 +--- a/Modules/posixmodule.c ++++ b/Modules/posixmodule.c +@@ -5138,6 +5138,111 @@ enum posix_spawn_file_actions_identifier { + POSIX_SPAWN_DUP2 + }; + ++static int ++parse_file_actions(PyObject *file_actions, ++ posix_spawn_file_actions_t *file_actionsp) ++{ ++ PyObject *seq; ++ PyObject *file_action = NULL; ++ PyObject *tag_obj; ++ ++ seq = PySequence_Fast(file_actions, ++ "file_actions must be a sequence or None"); ++ if (seq == NULL) { ++ return -1; ++ } ++ ++ errno = posix_spawn_file_actions_init(file_actionsp); ++ if (errno) { ++ posix_error(); ++ Py_DECREF(seq); ++ return -1; ++ } ++ ++ for (int i = 0; i < PySequence_Fast_GET_SIZE(seq); ++i) { ++ file_action = PySequence_Fast_GET_ITEM(seq, i); ++ Py_INCREF(file_action); ++ if (!PyTuple_Check(file_action) || !PyTuple_GET_SIZE(file_action)) { ++ PyErr_SetString(PyExc_TypeError, ++ "Each file_actions element must be a non-empty tuple"); ++ goto fail; ++ } ++ long tag = PyLong_AsLong(PyTuple_GET_ITEM(file_action, 0)); ++ if (tag == -1 && PyErr_Occurred()) { ++ goto fail; ++ } ++ ++ /* Populate the file_actions object */ ++ switch (tag) { ++ case POSIX_SPAWN_OPEN: { ++ int fd, oflag; ++ PyObject *path; ++ unsigned long mode; ++ if (!PyArg_ParseTuple(file_action, "OiO&ik" ++ ";A open file_action tuple must have 5 elements", ++ &tag_obj, &fd, PyUnicode_FSConverter, &path, ++ &oflag, &mode)) ++ { ++ goto fail; ++ } ++ errno = posix_spawn_file_actions_addopen(file_actionsp, ++ fd, PyBytes_AS_STRING(path), oflag, (mode_t)mode); ++ Py_DECREF(path); /* addopen copied it. */ ++ if (errno) { ++ posix_error(); ++ goto fail; ++ } ++ break; ++ } ++ case POSIX_SPAWN_CLOSE: { ++ int fd; ++ if (!PyArg_ParseTuple(file_action, "Oi" ++ ";A close file_action tuple must have 2 elements", ++ &tag_obj, &fd)) ++ { ++ goto fail; ++ } ++ errno = posix_spawn_file_actions_addclose(file_actionsp, fd); ++ if (errno) { ++ posix_error(); ++ goto fail; ++ } ++ break; ++ } ++ case POSIX_SPAWN_DUP2: { ++ int fd1, fd2; ++ if (!PyArg_ParseTuple(file_action, "Oii" ++ ";A dup2 file_action tuple must have 3 elements", ++ &tag_obj, &fd1, &fd2)) ++ { ++ goto fail; ++ } ++ errno = posix_spawn_file_actions_adddup2(file_actionsp, ++ fd1, fd2); ++ if (errno) { ++ posix_error(); ++ goto fail; ++ } ++ break; ++ } ++ default: { ++ PyErr_SetString(PyExc_TypeError, ++ "Unknown file_actions identifier"); ++ goto fail; ++ } ++ } ++ Py_DECREF(file_action); ++ } ++ Py_DECREF(seq); ++ return 0; ++ ++fail: ++ Py_DECREF(seq); ++ Py_DECREF(file_action); ++ (void)posix_spawn_file_actions_destroy(file_actionsp); ++ return -1; ++} ++ + /*[clinic input] + + os.posix_spawn +@@ -5148,7 +5253,7 @@ os.posix_spawn + env: object + Dictionary of strings mapping to strings. + file_actions: object = None +- FileActions object. ++ A sequence of file action tuples. + / + + Execute the program specified by path in a new process. +@@ -5157,22 +5262,22 @@ Execute the program specified by path in a new process. + static PyObject * + os_posix_spawn_impl(PyObject *module, path_t *path, PyObject *argv, + PyObject *env, PyObject *file_actions) +-/*[clinic end generated code: output=d023521f541c709c input=0ec9f1cfdc890be5]*/ ++/*[clinic end generated code: output=d023521f541c709c input=a3db1021d33230dc]*/ + { + EXECV_CHAR **argvlist = NULL; + EXECV_CHAR **envlist = NULL; +- posix_spawn_file_actions_t _file_actions; ++ posix_spawn_file_actions_t file_actions_buf; + posix_spawn_file_actions_t *file_actionsp = NULL; + Py_ssize_t argc, envc; +- PyObject* result = NULL; +- PyObject* seq = NULL; +- ++ PyObject *result = NULL; ++ pid_t pid; ++ int err_code; + + /* posix_spawn has three arguments: (path, argv, env), where +- argv is a list or tuple of strings and env is a dictionary ++ argv is a list or tuple of strings and env is a dictionary + like posix.environ. */ + +- if (!PySequence_Check(argv)) { ++ if (!PyList_Check(argv) && !PyTuple_Check(argv)) { + PyErr_SetString(PyExc_TypeError, + "posix_spawn: argv must be a tuple or list"); + goto exit; +@@ -5204,139 +5309,35 @@ os_posix_spawn_impl(PyObject *module, path_t *path, PyObject *argv, + goto exit; + } + +- pid_t pid; +- if (file_actions != NULL && file_actions != Py_None) { +- if(posix_spawn_file_actions_init(&_file_actions) != 0){ +- PyErr_SetString(PyExc_OSError, +- "Error initializing file actions"); +- goto exit; +- } +- +- file_actionsp = &_file_actions; +- +- seq = PySequence_Fast(file_actions, "file_actions must be a sequence"); +- if(seq == NULL) { ++ if (file_actions != Py_None) { ++ if (parse_file_actions(file_actions, &file_actions_buf)) { + goto exit; + } +- PyObject* file_actions_obj; +- PyObject* mode_obj; +- +- for (int i = 0; i < PySequence_Fast_GET_SIZE(seq); ++i) { +- file_actions_obj = PySequence_Fast_GET_ITEM(seq, i); +- +- if(!PySequence_Check(file_actions_obj) | !PySequence_Size(file_actions_obj)) { +- PyErr_SetString(PyExc_TypeError,"Each file_action element must be a non empty sequence"); +- goto exit; +- } +- +- +- mode_obj = PySequence_Fast_GET_ITEM(file_actions_obj, 0); +- int mode = PyLong_AsLong(mode_obj); +- +- /* Populate the file_actions object */ +- +- switch(mode) { +- +- case POSIX_SPAWN_OPEN: +- if(PySequence_Size(file_actions_obj) != 5) { +- PyErr_SetString(PyExc_TypeError,"A open file_action object must have 5 elements"); +- goto exit; +- } +- +- long open_fd = PyLong_AsLong(PySequence_GetItem(file_actions_obj, 1)); +- if(PyErr_Occurred()) { +- goto exit; +- } +- const char* open_path = PyUnicode_AsUTF8(PySequence_GetItem(file_actions_obj, 2)); +- if(open_path == NULL) { +- goto exit; +- } +- long open_oflag = PyLong_AsLong(PySequence_GetItem(file_actions_obj, 3)); +- if(PyErr_Occurred()) { +- goto exit; +- } +- long open_mode = PyLong_AsLong(PySequence_GetItem(file_actions_obj, 4)); +- if(PyErr_Occurred()) { +- goto exit; +- } +- if(posix_spawn_file_actions_addopen(file_actionsp, open_fd, open_path, open_oflag, open_mode)) { +- PyErr_SetString(PyExc_OSError,"Failed to add open file action"); +- goto exit; +- } +- +- break; +- +- case POSIX_SPAWN_CLOSE: +- if(PySequence_Size(file_actions_obj) != 2){ +- PyErr_SetString(PyExc_TypeError,"A close file_action object must have 2 elements"); +- goto exit; +- } +- +- long close_fd = PyLong_AsLong(PySequence_GetItem(file_actions_obj, 1)); +- if(PyErr_Occurred()) { +- goto exit; +- } +- if(posix_spawn_file_actions_addclose(file_actionsp, close_fd)) { +- PyErr_SetString(PyExc_OSError,"Failed to add close file action"); +- goto exit; +- } +- break; +- +- case POSIX_SPAWN_DUP2: +- if(PySequence_Size(file_actions_obj) != 3){ +- PyErr_SetString(PyExc_TypeError,"A dup2 file_action object must have 3 elements"); +- goto exit; +- } +- +- long fd1 = PyLong_AsLong(PySequence_GetItem(file_actions_obj, 1)); +- if(PyErr_Occurred()) { +- goto exit; +- } +- long fd2 = PyLong_AsLong(PySequence_GetItem(file_actions_obj, 2)); +- if(PyErr_Occurred()) { +- goto exit; +- } +- if(posix_spawn_file_actions_adddup2(file_actionsp, fd1, fd2)) { +- PyErr_SetString(PyExc_OSError,"Failed to add dup2 file action"); +- goto exit; +- } +- break; +- +- default: +- PyErr_SetString(PyExc_TypeError,"Unknown file_actions identifier"); +- goto exit; +- } +- } ++ file_actionsp = &file_actions_buf; + } + + _Py_BEGIN_SUPPRESS_IPH +- int err_code = posix_spawn(&pid, path->narrow, file_actionsp, NULL, argvlist, envlist); ++ err_code = posix_spawn(&pid, path->narrow, ++ file_actionsp, NULL, argvlist, envlist); + _Py_END_SUPPRESS_IPH +- if(err_code) { +- PyErr_SetString(PyExc_OSError,"posix_spawn call failed"); ++ if (err_code) { ++ errno = err_code; ++ PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, path->object); + goto exit; + } + result = PyLong_FromPid(pid); + + exit: +- +- Py_XDECREF(seq); +- +- if(file_actionsp) { +- posix_spawn_file_actions_destroy(file_actionsp); ++ if (file_actionsp) { ++ (void)posix_spawn_file_actions_destroy(file_actionsp); + } +- + if (envlist) { + free_string_array(envlist, envc); + } +- + if (argvlist) { + free_string_array(argvlist, argc); + } +- + return result; +- +- + } + #endif /* HAVE_POSIX_SPAWN */ + +-- +2.23.0 + diff --git a/backport-33332-Add-signal.valid_signals-GH-6581.patch b/backport-33332-Add-signal.valid_signals-GH-6581.patch new file mode 100644 index 0000000000000000000000000000000000000000..71231f4006d569c043f6af8e7fb204fa11b75a99 --- /dev/null +++ b/backport-33332-Add-signal.valid_signals-GH-6581.patch @@ -0,0 +1,458 @@ +From 2d0f8adba1764f0b242b15c4f0c67a791fb11e46 Mon Sep 17 00:00:00 2001 +From: Antoine Pitrou +Date: Fri, 4 May 2018 13:00:50 +0200 +Subject: [PATCH] bpo-33332: Add signal.valid_signals() (GH-6581) + +Conflict:NA +Reference:https://github.com/python/cpython/commit/9d3627e311211a1b4abcda29c36fe4afe2c46532 + +Signed-off-by: hanxinke +--- + Doc/library/signal.rst | 9 ++- + Lib/asyncio/unix_events.py | 4 +- + Lib/multiprocessing/resource_sharer.py | 2 +- + Lib/signal.py | 10 ++- + Lib/test/support/__init__.py | 2 +- + Lib/test/test_asyncio/test_unix_events.py | 12 ++++ + Lib/test/test_signal.py | 30 +++++++++ + .../2018-04-23-21-41-30.bpo-33332.Y6OZ8Z.rst | 2 + + Modules/clinic/signalmodule.c.h | 29 +++++++++ + Modules/signalmodule.c | 64 +++++++++++++++++-- + configure | 2 +- + configure.ac | 2 +- + pyconfig.h.in | 3 + + 13 files changed, 156 insertions(+), 15 deletions(-) + create mode 100644 Misc/NEWS.d/next/Library/2018-04-23-21-41-30.bpo-33332.Y6OZ8Z.rst + +diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst +index 7f48f8c..f39e0b2 100644 +--- a/Doc/library/signal.rst ++++ b/Doc/library/signal.rst +@@ -313,6 +313,11 @@ The :mod:`signal` module defines the following functions: + previously in use, and ``None`` means that the previous signal handler was not + installed from Python. + ++.. function:: valid_signals() ++ ++ Return the set of valid signal numbers on this platform. This can be ++ less than ``range(1, NSIG)`` if some signals are reserved by the system ++ for internal use. + + .. function:: pause() + +@@ -368,8 +373,8 @@ The :mod:`signal` module defines the following functions: + argument. + + *mask* is a set of signal numbers (e.g. {:const:`signal.SIGINT`, +- :const:`signal.SIGTERM`}). Use ``range(1, signal.NSIG)`` for a full mask +- including all signals. ++ :const:`signal.SIGTERM`}). Use :func:`~signal.valid_signals` for a full ++ mask including all signals. + + For example, ``signal.pthread_sigmask(signal.SIG_BLOCK, [])`` reads the + signal mask of the calling thread. +diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py +index e037e12..4c1bf40 100644 +--- a/Lib/asyncio/unix_events.py ++++ b/Lib/asyncio/unix_events.py +@@ -168,8 +168,8 @@ class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): + if not isinstance(sig, int): + raise TypeError(f'sig must be an int, not {sig!r}') + +- if not (1 <= sig < signal.NSIG): +- raise ValueError(f'sig {sig} out of range(1, {signal.NSIG})') ++ if sig not in signal.valid_signals(): ++ raise ValueError(f'invalid signal number {sig}') + + def _make_read_pipe_transport(self, pipe, protocol, waiter=None, + extra=None): +diff --git a/Lib/multiprocessing/resource_sharer.py b/Lib/multiprocessing/resource_sharer.py +index c8f18ea..8d5c990 100644 +--- a/Lib/multiprocessing/resource_sharer.py ++++ b/Lib/multiprocessing/resource_sharer.py +@@ -136,7 +136,7 @@ class _ResourceSharer(object): + + def _serve(self): + if hasattr(signal, 'pthread_sigmask'): +- signal.pthread_sigmask(signal.SIG_BLOCK, range(1, signal.NSIG)) ++ signal.pthread_sigmask(signal.SIG_BLOCK, signal.valid_signals()) + while 1: + try: + with self._listener.accept() as conn: +diff --git a/Lib/signal.py b/Lib/signal.py +index 9f05c91..826b62c 100644 +--- a/Lib/signal.py ++++ b/Lib/signal.py +@@ -65,8 +65,7 @@ if 'pthread_sigmask' in _globals: + if 'sigpending' in _globals: + @_wraps(_signal.sigpending) + def sigpending(): +- sigs = _signal.sigpending() +- return set(_int_to_enum(x, Signals) for x in sigs) ++ return {_int_to_enum(x, Signals) for x in _signal.sigpending()} + + + if 'sigwait' in _globals: +@@ -76,4 +75,11 @@ if 'sigwait' in _globals: + return _int_to_enum(retsig, Signals) + sigwait.__doc__ = _signal.sigwait + ++ ++if 'valid_signals' in _globals: ++ @_wraps(_signal.valid_signals) ++ def valid_signals(): ++ return {_int_to_enum(x, Signals) for x in _signal.valid_signals()} ++ ++ + del _globals, _wraps +diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py +index b78451b..f0c6327 100644 +--- a/Lib/test/support/__init__.py ++++ b/Lib/test/support/__init__.py +@@ -2901,7 +2901,7 @@ class SaveSignals: + def __init__(self): + import signal + self.signal = signal +- self.signals = list(range(1, signal.NSIG)) ++ self.signals = signal.valid_signals() + # SIGKILL and SIGSTOP signals cannot be ignored nor caught + for signame in ('SIGKILL', 'SIGSTOP'): + try: +diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py +index 66a6bc1..bbdfd16 100644 +--- a/Lib/test/test_asyncio/test_unix_events.py ++++ b/Lib/test/test_asyncio/test_unix_events.py +@@ -69,6 +69,7 @@ class SelectorEventLoopSignalTests(test_utils.TestCase): + @mock.patch('asyncio.unix_events.signal') + def test_add_signal_handler_setup_error(self, m_signal): + m_signal.NSIG = signal.NSIG ++ m_signal.valid_signals = signal.valid_signals + m_signal.set_wakeup_fd.side_effect = ValueError + + self.assertRaises( +@@ -96,6 +97,7 @@ class SelectorEventLoopSignalTests(test_utils.TestCase): + @mock.patch('asyncio.unix_events.signal') + def test_add_signal_handler(self, m_signal): + m_signal.NSIG = signal.NSIG ++ m_signal.valid_signals = signal.valid_signals + + cb = lambda: True + self.loop.add_signal_handler(signal.SIGHUP, cb) +@@ -106,6 +108,7 @@ class SelectorEventLoopSignalTests(test_utils.TestCase): + @mock.patch('asyncio.unix_events.signal') + def test_add_signal_handler_install_error(self, m_signal): + m_signal.NSIG = signal.NSIG ++ m_signal.valid_signals = signal.valid_signals + + def set_wakeup_fd(fd): + if fd == -1: +@@ -125,6 +128,7 @@ class SelectorEventLoopSignalTests(test_utils.TestCase): + @mock.patch('asyncio.base_events.logger') + def test_add_signal_handler_install_error2(self, m_logging, m_signal): + m_signal.NSIG = signal.NSIG ++ m_signal.valid_signals = signal.valid_signals + + class Err(OSError): + errno = errno.EINVAL +@@ -145,6 +149,7 @@ class SelectorEventLoopSignalTests(test_utils.TestCase): + errno = errno.EINVAL + m_signal.signal.side_effect = Err + m_signal.NSIG = signal.NSIG ++ m_signal.valid_signals = signal.valid_signals + + self.assertRaises( + RuntimeError, +@@ -156,6 +161,7 @@ class SelectorEventLoopSignalTests(test_utils.TestCase): + @mock.patch('asyncio.unix_events.signal') + def test_remove_signal_handler(self, m_signal): + m_signal.NSIG = signal.NSIG ++ m_signal.valid_signals = signal.valid_signals + + self.loop.add_signal_handler(signal.SIGHUP, lambda: True) + +@@ -170,6 +176,7 @@ class SelectorEventLoopSignalTests(test_utils.TestCase): + def test_remove_signal_handler_2(self, m_signal): + m_signal.NSIG = signal.NSIG + m_signal.SIGINT = signal.SIGINT ++ m_signal.valid_signals = signal.valid_signals + + self.loop.add_signal_handler(signal.SIGINT, lambda: True) + self.loop._signal_handlers[signal.SIGHUP] = object() +@@ -187,6 +194,7 @@ class SelectorEventLoopSignalTests(test_utils.TestCase): + @mock.patch('asyncio.base_events.logger') + def test_remove_signal_handler_cleanup_error(self, m_logging, m_signal): + m_signal.NSIG = signal.NSIG ++ m_signal.valid_signals = signal.valid_signals + self.loop.add_signal_handler(signal.SIGHUP, lambda: True) + + m_signal.set_wakeup_fd.side_effect = ValueError +@@ -197,6 +205,7 @@ class SelectorEventLoopSignalTests(test_utils.TestCase): + @mock.patch('asyncio.unix_events.signal') + def test_remove_signal_handler_error(self, m_signal): + m_signal.NSIG = signal.NSIG ++ m_signal.valid_signals = signal.valid_signals + self.loop.add_signal_handler(signal.SIGHUP, lambda: True) + + m_signal.signal.side_effect = OSError +@@ -207,6 +216,7 @@ class SelectorEventLoopSignalTests(test_utils.TestCase): + @mock.patch('asyncio.unix_events.signal') + def test_remove_signal_handler_error2(self, m_signal): + m_signal.NSIG = signal.NSIG ++ m_signal.valid_signals = signal.valid_signals + self.loop.add_signal_handler(signal.SIGHUP, lambda: True) + + class Err(OSError): +@@ -219,6 +229,7 @@ class SelectorEventLoopSignalTests(test_utils.TestCase): + @mock.patch('asyncio.unix_events.signal') + def test_close(self, m_signal): + m_signal.NSIG = signal.NSIG ++ m_signal.valid_signals = signal.valid_signals + + self.loop.add_signal_handler(signal.SIGHUP, lambda: True) + self.loop.add_signal_handler(signal.SIGCHLD, lambda: True) +@@ -236,6 +247,7 @@ class SelectorEventLoopSignalTests(test_utils.TestCase): + @mock.patch('asyncio.unix_events.signal') + def test_close_on_finalizing(self, m_signal, m_sys): + m_signal.NSIG = signal.NSIG ++ m_signal.valid_signals = signal.valid_signals + self.loop.add_signal_handler(signal.SIGHUP, lambda: True) + + self.assertEqual(len(self.loop._signal_handlers), 1) +diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py +index 406684b..0244650 100644 +--- a/Lib/test/test_signal.py ++++ b/Lib/test/test_signal.py +@@ -61,9 +61,28 @@ class PosixTests(unittest.TestCase): + script = os.path.join(dirname, 'signalinterproctester.py') + assert_python_ok(script) + ++ def test_valid_signals(self): ++ s = signal.valid_signals() ++ self.assertIsInstance(s, set) ++ self.assertIn(signal.Signals.SIGINT, s) ++ self.assertIn(signal.Signals.SIGALRM, s) ++ self.assertNotIn(0, s) ++ self.assertNotIn(signal.NSIG, s) ++ self.assertLess(len(s), signal.NSIG) ++ + + @unittest.skipUnless(sys.platform == "win32", "Windows specific") + class WindowsSignalTests(unittest.TestCase): ++ ++ def test_valid_signals(self): ++ s = signal.valid_signals() ++ self.assertIsInstance(s, set) ++ self.assertGreaterEqual(len(s), 6) ++ self.assertIn(signal.Signals.SIGINT, s) ++ self.assertNotIn(0, s) ++ self.assertNotIn(signal.NSIG, s) ++ self.assertLess(len(s), signal.NSIG) ++ + def test_issue9324(self): + # Updated for issue #10003, adding SIGBREAK + handler = lambda x, y: None +@@ -941,6 +960,17 @@ class PendingSignalsTests(unittest.TestCase): + self.assertRaises(TypeError, signal.pthread_sigmask, 1) + self.assertRaises(TypeError, signal.pthread_sigmask, 1, 2, 3) + self.assertRaises(OSError, signal.pthread_sigmask, 1700, []) ++ with self.assertRaises(ValueError): ++ signal.pthread_sigmask(signal.SIG_BLOCK, [signal.NSIG]) ++ ++ @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), ++ 'need signal.pthread_sigmask()') ++ def test_pthread_sigmask_valid_signals(self): ++ s = signal.pthread_sigmask(signal.SIG_BLOCK, signal.valid_signals()) ++ self.addCleanup(signal.pthread_sigmask, signal.SIG_SETMASK, s) ++ # Get current blocked set ++ s = signal.pthread_sigmask(signal.SIG_UNBLOCK, signal.valid_signals()) ++ self.assertLessEqual(s, signal.valid_signals()) + + @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), + 'need signal.pthread_sigmask()') +diff --git a/Misc/NEWS.d/next/Library/2018-04-23-21-41-30.bpo-33332.Y6OZ8Z.rst b/Misc/NEWS.d/next/Library/2018-04-23-21-41-30.bpo-33332.Y6OZ8Z.rst +new file mode 100644 +index 0000000..05dca54 +--- /dev/null ++++ b/Misc/NEWS.d/next/Library/2018-04-23-21-41-30.bpo-33332.Y6OZ8Z.rst +@@ -0,0 +1,2 @@ ++Add ``signal.valid_signals()`` to expose the POSIX sigfillset() ++functionality. +diff --git a/Modules/clinic/signalmodule.c.h b/Modules/clinic/signalmodule.c.h +index dc3aadf..ac97e80 100644 +--- a/Modules/clinic/signalmodule.c.h ++++ b/Modules/clinic/signalmodule.c.h +@@ -311,6 +311,31 @@ PyDoc_STRVAR(signal_sigwait__doc__, + + #endif /* defined(HAVE_SIGWAIT) */ + ++#if (defined(HAVE_SIGFILLSET) || defined(MS_WINDOWS)) ++ ++PyDoc_STRVAR(signal_valid_signals__doc__, ++"valid_signals($module, /)\n" ++"--\n" ++"\n" ++"Return a set of valid signal numbers on this platform.\n" ++"\n" ++"The signal numbers returned by this function can be safely passed to\n" ++"functions like `pthread_sigmask`."); ++ ++#define SIGNAL_VALID_SIGNALS_METHODDEF \ ++ {"valid_signals", (PyCFunction)signal_valid_signals, METH_NOARGS, signal_valid_signals__doc__}, ++ ++static PyObject * ++signal_valid_signals_impl(PyObject *module); ++ ++static PyObject * ++signal_valid_signals(PyObject *module, PyObject *Py_UNUSED(ignored)) ++{ ++ return signal_valid_signals_impl(module); ++} ++ ++#endif /* (defined(HAVE_SIGFILLSET) || defined(MS_WINDOWS)) */ ++ + #if defined(HAVE_SIGWAITINFO) + + PyDoc_STRVAR(signal_sigwaitinfo__doc__, +@@ -429,6 +454,10 @@ exit: + #define SIGNAL_SIGWAIT_METHODDEF + #endif /* !defined(SIGNAL_SIGWAIT_METHODDEF) */ + ++#ifndef SIGNAL_VALID_SIGNALS_METHODDEF ++ #define SIGNAL_VALID_SIGNALS_METHODDEF ++#endif /* !defined(SIGNAL_VALID_SIGNALS_METHODDEF) */ ++ + #ifndef SIGNAL_SIGWAITINFO_METHODDEF + #define SIGNAL_SIGWAITINFO_METHODDEF + #endif /* !defined(SIGNAL_SIGWAITINFO_METHODDEF) */ +diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c +index a0722b7..b6b5bf1 100644 +--- a/Modules/signalmodule.c ++++ b/Modules/signalmodule.c +@@ -774,11 +774,21 @@ iterable_to_sigset(PyObject *iterable, sigset_t *mask) + if (signum == -1 && PyErr_Occurred()) + goto error; + if (0 < signum && signum < NSIG) { +- /* bpo-33329: ignore sigaddset() return value as it can fail +- * for some reserved signals, but we want the `range(1, NSIG)` +- * idiom to allow selecting all valid signals. +- */ +- (void) sigaddset(mask, (int)signum); ++ if (sigaddset(mask, (int)signum)) { ++ if (errno != EINVAL) { ++ /* Probably impossible */ ++ PyErr_SetFromErrno(PyExc_OSError); ++ goto error; ++ } ++ /* For backwards compatibility, allow idioms such as ++ * `range(1, NSIG)` but warn about invalid signal numbers ++ */ ++ const char *msg = ++ "invalid signal number %ld, please use valid_signals()"; ++ if (PyErr_WarnFormat(PyExc_RuntimeWarning, 1, msg, signum)) { ++ goto error; ++ } ++ } + } + else { + PyErr_Format(PyExc_ValueError, +@@ -934,6 +944,47 @@ signal_sigwait(PyObject *module, PyObject *sigset) + #endif /* #ifdef HAVE_SIGWAIT */ + + ++#if defined(HAVE_SIGFILLSET) || defined(MS_WINDOWS) ++ ++/*[clinic input] ++signal.valid_signals ++ ++Return a set of valid signal numbers on this platform. ++ ++The signal numbers returned by this function can be safely passed to ++functions like `pthread_sigmask`. ++[clinic start generated code]*/ ++ ++static PyObject * ++signal_valid_signals_impl(PyObject *module) ++/*[clinic end generated code: output=1609cffbcfcf1314 input=86a3717ff25288f2]*/ ++{ ++#ifdef MS_WINDOWS ++#ifdef SIGBREAK ++ PyObject *tup = Py_BuildValue("(iiiiiii)", SIGABRT, SIGBREAK, SIGFPE, ++ SIGILL, SIGINT, SIGSEGV, SIGTERM); ++#else ++ PyObject *tup = Py_BuildValue("(iiiiii)", SIGABRT, SIGFPE, SIGILL, ++ SIGINT, SIGSEGV, SIGTERM); ++#endif ++ if (tup == NULL) { ++ return NULL; ++ } ++ PyObject *set = PySet_New(tup); ++ Py_DECREF(tup); ++ return set; ++#else ++ sigset_t mask; ++ if (sigemptyset(&mask) || sigfillset(&mask)) { ++ return PyErr_SetFromErrno(PyExc_OSError); ++ } ++ return sigset_to_set(mask); ++#endif ++} ++ ++#endif /* #if defined(HAVE_SIGFILLSET) || defined(MS_WINDOWS) */ ++ ++ + #if defined(HAVE_SIGWAITINFO) || defined(HAVE_SIGTIMEDWAIT) + static int initialized; + static PyStructSequence_Field struct_siginfo_fields[] = { +@@ -1157,6 +1208,9 @@ static PyMethodDef signal_methods[] = { + SIGNAL_SIGWAIT_METHODDEF + SIGNAL_SIGWAITINFO_METHODDEF + SIGNAL_SIGTIMEDWAIT_METHODDEF ++#if defined(HAVE_SIGFILLSET) || defined(MS_WINDOWS) ++ SIGNAL_VALID_SIGNALS_METHODDEF ++#endif + {NULL, NULL} /* sentinel */ + }; + +diff --git a/configure b/configure +index 829dd69..d30ccf5 100755 +--- a/configure ++++ b/configure +@@ -11506,7 +11506,7 @@ for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ + setlocale setregid setreuid setresuid setresgid setsid setpgid setpgrp setpriority setuid setvbuf \ + sched_get_priority_max sched_setaffinity sched_setscheduler sched_setparam \ + sched_rr_get_interval \ +- sigaction sigaltstack siginterrupt sigpending sigrelse \ ++ sigaction sigaltstack sigfillset siginterrupt sigpending sigrelse \ + sigtimedwait sigwait sigwaitinfo snprintf strftime strlcpy symlinkat sync \ + sysconf tcgetpgrp tcsetpgrp tempnam timegm times tmpfile tmpnam tmpnam_r \ + truncate uname unlinkat unsetenv utimensat utimes waitid waitpid wait3 wait4 \ +diff --git a/configure.ac b/configure.ac +index f1cc8e9..9d4c889 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -3590,7 +3590,7 @@ AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ + setlocale setregid setreuid setresuid setresgid setsid setpgid setpgrp setpriority setuid setvbuf \ + sched_get_priority_max sched_setaffinity sched_setscheduler sched_setparam \ + sched_rr_get_interval \ +- sigaction sigaltstack siginterrupt sigpending sigrelse \ ++ sigaction sigaltstack sigfillset siginterrupt sigpending sigrelse \ + sigtimedwait sigwait sigwaitinfo snprintf strftime strlcpy symlinkat sync \ + sysconf tcgetpgrp tcsetpgrp tempnam timegm times tmpfile tmpnam tmpnam_r \ + truncate uname unlinkat unsetenv utimensat utimes waitid waitpid wait3 wait4 \ +diff --git a/pyconfig.h.in b/pyconfig.h.in +index ebab5ff..70cead7 100644 +--- a/pyconfig.h.in ++++ b/pyconfig.h.in +@@ -899,6 +899,9 @@ + /* Define to 1 if you have the `sigaltstack' function. */ + #undef HAVE_SIGALTSTACK + ++/* Define to 1 if you have the `sigfillset' function. */ ++#undef HAVE_SIGFILLSET ++ + /* Define to 1 if `si_band' is a member of `siginfo_t'. */ + #undef HAVE_SIGINFO_T_SI_BAND + +-- +2.23.0 + diff --git a/backport-33441-Make-the-sigset_t-converter-available-in-o.patch b/backport-33441-Make-the-sigset_t-converter-available-in-o.patch new file mode 100644 index 0000000000000000000000000000000000000000..b0464b49940bd0bed2081c2d2a170cb9fa32e4cc --- /dev/null +++ b/backport-33441-Make-the-sigset_t-converter-available-in-o.patch @@ -0,0 +1,459 @@ +From cc17fe8bed98cae0bcb7739c953933d1f379dd12 Mon Sep 17 00:00:00 2001 +From: Serhiy Storchaka +Date: Tue, 8 May 2018 07:48:50 +0300 +Subject: [PATCH] bpo-33441: Make the sigset_t converter available in other + modules. (GH-6720) + +* Expose the sigset_t converter via private API _Py_Sigset_Converter(). +* Use Argument Clinic for parsing sigset_t in signalmodule.c. +* Raise ValueError instead OverflowError for integers out of + the C long range. + +Based on patch by Pablo Galindo Salgado. + +Conflict:NA +Reference:https://github.com/python/cpython/commit/d54cfb160c626626394e2f171d3ccfe03309f34e + +Signed-off-by: hanxinke +--- + Lib/test/test_signal.py | 4 ++ + Modules/clinic/signalmodule.c.h | 53 +++++++++++--- + Modules/posixmodule.c | 59 ++++++++++++++++ + Modules/posixmodule.h | 9 +++ + Modules/signalmodule.c | 120 +++++++------------------------- + 5 files changed, 141 insertions(+), 104 deletions(-) + +diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py +index 0244650..152f803 100644 +--- a/Lib/test/test_signal.py ++++ b/Lib/test/test_signal.py +@@ -962,6 +962,10 @@ class PendingSignalsTests(unittest.TestCase): + self.assertRaises(OSError, signal.pthread_sigmask, 1700, []) + with self.assertRaises(ValueError): + signal.pthread_sigmask(signal.SIG_BLOCK, [signal.NSIG]) ++ with self.assertRaises(ValueError): ++ signal.pthread_sigmask(signal.SIG_BLOCK, [0]) ++ with self.assertRaises(ValueError): ++ signal.pthread_sigmask(signal.SIG_BLOCK, [1<<1000]) + + @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), + 'need signal.pthread_sigmask()') +diff --git a/Modules/clinic/signalmodule.c.h b/Modules/clinic/signalmodule.c.h +index ac97e80..811b252 100644 +--- a/Modules/clinic/signalmodule.c.h ++++ b/Modules/clinic/signalmodule.c.h +@@ -248,17 +248,17 @@ PyDoc_STRVAR(signal_pthread_sigmask__doc__, + {"pthread_sigmask", (PyCFunction)signal_pthread_sigmask, METH_FASTCALL, signal_pthread_sigmask__doc__}, + + static PyObject * +-signal_pthread_sigmask_impl(PyObject *module, int how, PyObject *mask); ++signal_pthread_sigmask_impl(PyObject *module, int how, sigset_t mask); + + static PyObject * + signal_pthread_sigmask(PyObject *module, PyObject *const *args, Py_ssize_t nargs) + { + PyObject *return_value = NULL; + int how; +- PyObject *mask; ++ sigset_t mask; + +- if (!_PyArg_ParseStack(args, nargs, "iO:pthread_sigmask", +- &how, &mask)) { ++ if (!_PyArg_ParseStack(args, nargs, "iO&:pthread_sigmask", ++ &how, _Py_Sigset_Converter, &mask)) { + goto exit; + } + return_value = signal_pthread_sigmask_impl(module, how, mask); +@@ -309,6 +309,24 @@ PyDoc_STRVAR(signal_sigwait__doc__, + #define SIGNAL_SIGWAIT_METHODDEF \ + {"sigwait", (PyCFunction)signal_sigwait, METH_O, signal_sigwait__doc__}, + ++static PyObject * ++signal_sigwait_impl(PyObject *module, sigset_t sigset); ++ ++static PyObject * ++signal_sigwait(PyObject *module, PyObject *arg) ++{ ++ PyObject *return_value = NULL; ++ sigset_t sigset; ++ ++ if (!PyArg_Parse(arg, "O&:sigwait", _Py_Sigset_Converter, &sigset)) { ++ goto exit; ++ } ++ return_value = signal_sigwait_impl(module, sigset); ++ ++exit: ++ return return_value; ++} ++ + #endif /* defined(HAVE_SIGWAIT) */ + + #if (defined(HAVE_SIGFILLSET) || defined(MS_WINDOWS)) +@@ -349,6 +367,24 @@ PyDoc_STRVAR(signal_sigwaitinfo__doc__, + #define SIGNAL_SIGWAITINFO_METHODDEF \ + {"sigwaitinfo", (PyCFunction)signal_sigwaitinfo, METH_O, signal_sigwaitinfo__doc__}, + ++static PyObject * ++signal_sigwaitinfo_impl(PyObject *module, sigset_t sigset); ++ ++static PyObject * ++signal_sigwaitinfo(PyObject *module, PyObject *arg) ++{ ++ PyObject *return_value = NULL; ++ sigset_t sigset; ++ ++ if (!PyArg_Parse(arg, "O&:sigwaitinfo", _Py_Sigset_Converter, &sigset)) { ++ goto exit; ++ } ++ return_value = signal_sigwaitinfo_impl(module, sigset); ++ ++exit: ++ return return_value; ++} ++ + #endif /* defined(HAVE_SIGWAITINFO) */ + + #if defined(HAVE_SIGTIMEDWAIT) +@@ -365,19 +401,18 @@ PyDoc_STRVAR(signal_sigtimedwait__doc__, + {"sigtimedwait", (PyCFunction)signal_sigtimedwait, METH_FASTCALL, signal_sigtimedwait__doc__}, + + static PyObject * +-signal_sigtimedwait_impl(PyObject *module, PyObject *sigset, ++signal_sigtimedwait_impl(PyObject *module, sigset_t sigset, + PyObject *timeout_obj); + + static PyObject * + signal_sigtimedwait(PyObject *module, PyObject *const *args, Py_ssize_t nargs) + { + PyObject *return_value = NULL; +- PyObject *sigset; ++ sigset_t sigset; + PyObject *timeout_obj; + +- if (!_PyArg_UnpackStack(args, nargs, "sigtimedwait", +- 2, 2, +- &sigset, &timeout_obj)) { ++ if (!_PyArg_ParseStack(args, nargs, "O&O:sigtimedwait", ++ _Py_Sigset_Converter, &sigset, &timeout_obj)) { + goto exit; + } + return_value = signal_sigtimedwait_impl(module, sigset, timeout_obj); +diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c +index 441c82e..5689f93 100644 +--- a/Modules/posixmodule.c ++++ b/Modules/posixmodule.c +@@ -1278,6 +1278,65 @@ PyLong_FromPy_off_t(Py_off_t offset) + #endif + } + ++#ifdef HAVE_SIGSET_T ++/* Convert an iterable of integers to a sigset. ++ Return 1 on success, return 0 and raise an exception on error. */ ++int ++_Py_Sigset_Converter(PyObject *obj, void *addr) ++{ ++ sigset_t *mask = (sigset_t *)addr; ++ PyObject *iterator, *item; ++ long signum; ++ int overflow; ++ ++ if (sigemptyset(mask)) { ++ /* Probably only if mask == NULL. */ ++ PyErr_SetFromErrno(PyExc_OSError); ++ return 0; ++ } ++ ++ iterator = PyObject_GetIter(obj); ++ if (iterator == NULL) { ++ return 0; ++ } ++ ++ while ((item = PyIter_Next(iterator)) != NULL) { ++ signum = PyLong_AsLongAndOverflow(item, &overflow); ++ Py_DECREF(item); ++ if (signum <= 0 || signum >= NSIG) { ++ if (overflow || signum != -1 || !PyErr_Occurred()) { ++ PyErr_Format(PyExc_ValueError, ++ "signal number %ld out of range", signum); ++ } ++ goto error; ++ } ++ if (sigaddset(mask, (int)signum)) { ++ if (errno != EINVAL) { ++ /* Probably impossible */ ++ PyErr_SetFromErrno(PyExc_OSError); ++ goto error; ++ } ++ /* For backwards compatibility, allow idioms such as ++ * `range(1, NSIG)` but warn about invalid signal numbers ++ */ ++ const char msg[] = ++ "invalid signal number %ld, please use valid_signals()"; ++ if (PyErr_WarnFormat(PyExc_RuntimeWarning, 1, msg, signum)) { ++ goto error; ++ } ++ } ++ } ++ if (!PyErr_Occurred()) { ++ Py_DECREF(iterator); ++ return 1; ++ } ++ ++error: ++ Py_DECREF(iterator); ++ return 0; ++} ++#endif /* HAVE_SIGSET_T */ ++ + #ifdef MS_WINDOWS + + static int +diff --git a/Modules/posixmodule.h b/Modules/posixmodule.h +index 1ec1833..1e00562 100644 +--- a/Modules/posixmodule.h ++++ b/Modules/posixmodule.h +@@ -17,8 +17,17 @@ PyAPI_FUNC(PyObject *) _PyLong_FromGid(gid_t); + PyAPI_FUNC(int) _Py_Uid_Converter(PyObject *, void *); + PyAPI_FUNC(int) _Py_Gid_Converter(PyObject *, void *); + #endif /* MS_WINDOWS */ ++ ++#if defined(PYPTHREAD_SIGMASK) || defined(HAVE_SIGWAIT) || \ ++ defined(HAVE_SIGWAITINFO) || defined(HAVE_SIGTIMEDWAIT) ++# define HAVE_SIGSET_T + #endif + ++#ifdef HAVE_SIGSET_T ++PyAPI_FUNC(int) _Py_Sigset_Converter(PyObject *, void *); ++#endif /* HAVE_SIGSET_T */ ++#endif /* Py_LIMITED_API */ ++ + #ifdef __cplusplus + } + #endif +diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c +index b6b5bf1..0a5f190 100644 +--- a/Modules/signalmodule.c ++++ b/Modules/signalmodule.c +@@ -59,6 +59,14 @@ module signal + [clinic start generated code]*/ + /*[clinic end generated code: output=da39a3ee5e6b4b0d input=b0301a3bde5fe9d3]*/ + ++/*[python input] ++ ++class sigset_t_converter(CConverter): ++ type = 'sigset_t' ++ converter = '_Py_Sigset_Converter' ++ ++[python start generated code]*/ ++/*[python end generated code: output=da39a3ee5e6b4b0d input=b5689d14466b6823]*/ + + /* + NOTES ON THE INTERACTION BETWEEN SIGNALS AND THREADS +@@ -741,69 +749,6 @@ signal_getitimer_impl(PyObject *module, int which) + + #endif + +-#if defined(PYPTHREAD_SIGMASK) || defined(HAVE_SIGWAIT) || \ +- defined(HAVE_SIGWAITINFO) || defined(HAVE_SIGTIMEDWAIT) +-/* Convert an iterable to a sigset. +- Return 0 on success, return -1 and raise an exception on error. */ +- +-static int +-iterable_to_sigset(PyObject *iterable, sigset_t *mask) +-{ +- int result = -1; +- PyObject *iterator, *item; +- long signum; +- +- sigemptyset(mask); +- +- iterator = PyObject_GetIter(iterable); +- if (iterator == NULL) +- goto error; +- +- while (1) +- { +- item = PyIter_Next(iterator); +- if (item == NULL) { +- if (PyErr_Occurred()) +- goto error; +- else +- break; +- } +- +- signum = PyLong_AsLong(item); +- Py_DECREF(item); +- if (signum == -1 && PyErr_Occurred()) +- goto error; +- if (0 < signum && signum < NSIG) { +- if (sigaddset(mask, (int)signum)) { +- if (errno != EINVAL) { +- /* Probably impossible */ +- PyErr_SetFromErrno(PyExc_OSError); +- goto error; +- } +- /* For backwards compatibility, allow idioms such as +- * `range(1, NSIG)` but warn about invalid signal numbers +- */ +- const char *msg = +- "invalid signal number %ld, please use valid_signals()"; +- if (PyErr_WarnFormat(PyExc_RuntimeWarning, 1, msg, signum)) { +- goto error; +- } +- } +- } +- else { +- PyErr_Format(PyExc_ValueError, +- "signal number %ld out of range", signum); +- goto error; +- } +- } +- result = 0; +- +-error: +- Py_XDECREF(iterator); +- return result; +-} +-#endif +- + #if defined(PYPTHREAD_SIGMASK) || defined(HAVE_SIGPENDING) + static PyObject* + sigset_to_set(sigset_t mask) +@@ -846,23 +791,20 @@ sigset_to_set(sigset_t mask) + signal.pthread_sigmask + + how: int +- mask: object ++ mask: sigset_t + / + + Fetch and/or change the signal mask of the calling thread. + [clinic start generated code]*/ + + static PyObject * +-signal_pthread_sigmask_impl(PyObject *module, int how, PyObject *mask) +-/*[clinic end generated code: output=ff640fe092bc9181 input=f3b7d7a61b7b8283]*/ ++signal_pthread_sigmask_impl(PyObject *module, int how, sigset_t mask) ++/*[clinic end generated code: output=0562c0fb192981a8 input=85bcebda442fa77f]*/ + { +- sigset_t newmask, previous; ++ sigset_t previous; + int err; + +- if (iterable_to_sigset(mask, &newmask)) +- return NULL; +- +- err = pthread_sigmask(how, &newmask, &previous); ++ err = pthread_sigmask(how, &mask, &previous); + if (err != 0) { + errno = err; + PyErr_SetFromErrno(PyExc_OSError); +@@ -910,7 +852,7 @@ signal_sigpending_impl(PyObject *module) + /*[clinic input] + signal.sigwait + +- sigset: object ++ sigset: sigset_t + / + + Wait for a signal. +@@ -921,17 +863,13 @@ and returns the signal number. + [clinic start generated code]*/ + + static PyObject * +-signal_sigwait(PyObject *module, PyObject *sigset) +-/*[clinic end generated code: output=557173647424f6e4 input=11af2d82d83c2e94]*/ ++signal_sigwait_impl(PyObject *module, sigset_t sigset) ++/*[clinic end generated code: output=f43770699d682f96 input=a6fbd47b1086d119]*/ + { +- sigset_t set; + int err, signum; + +- if (iterable_to_sigset(sigset, &set)) +- return NULL; +- + Py_BEGIN_ALLOW_THREADS +- err = sigwait(&set, &signum); ++ err = sigwait(&sigset, &signum); + Py_END_ALLOW_THREADS + if (err) { + errno = err; +@@ -1046,7 +984,7 @@ fill_siginfo(siginfo_t *si) + /*[clinic input] + signal.sigwaitinfo + +- sigset: object ++ sigset: sigset_t + / + + Wait synchronously until one of the signals in *sigset* is delivered. +@@ -1055,20 +993,16 @@ Returns a struct_siginfo containing information about the signal. + [clinic start generated code]*/ + + static PyObject * +-signal_sigwaitinfo(PyObject *module, PyObject *sigset) +-/*[clinic end generated code: output=c40f27b269cd2309 input=f3779a74a991e171]*/ ++signal_sigwaitinfo_impl(PyObject *module, sigset_t sigset) ++/*[clinic end generated code: output=1eb2f1fa236fdbca input=3d1a7e1f27fc664c]*/ + { +- sigset_t set; + siginfo_t si; + int err; + int async_err = 0; + +- if (iterable_to_sigset(sigset, &set)) +- return NULL; +- + do { + Py_BEGIN_ALLOW_THREADS +- err = sigwaitinfo(&set, &si); ++ err = sigwaitinfo(&sigset, &si); + Py_END_ALLOW_THREADS + } while (err == -1 + && errno == EINTR && !(async_err = PyErr_CheckSignals())); +@@ -1085,7 +1019,7 @@ signal_sigwaitinfo(PyObject *module, PyObject *sigset) + /*[clinic input] + signal.sigtimedwait + +- sigset: object ++ sigset: sigset_t + timeout as timeout_obj: object + / + +@@ -1095,12 +1029,11 @@ The timeout is specified in seconds, with floating point numbers allowed. + [clinic start generated code]*/ + + static PyObject * +-signal_sigtimedwait_impl(PyObject *module, PyObject *sigset, ++signal_sigtimedwait_impl(PyObject *module, sigset_t sigset, + PyObject *timeout_obj) +-/*[clinic end generated code: output=f7eff31e679f4312 input=53fd4ea3e3724eb8]*/ ++/*[clinic end generated code: output=59c8971e8ae18a64 input=87fd39237cf0b7ba]*/ + { + struct timespec ts; +- sigset_t set; + siginfo_t si; + int res; + _PyTime_t timeout, deadline, monotonic; +@@ -1114,9 +1047,6 @@ signal_sigtimedwait_impl(PyObject *module, PyObject *sigset, + return NULL; + } + +- if (iterable_to_sigset(sigset, &set)) +- return NULL; +- + deadline = _PyTime_GetMonotonicClock() + timeout; + + do { +@@ -1124,7 +1054,7 @@ signal_sigtimedwait_impl(PyObject *module, PyObject *sigset, + return NULL; + + Py_BEGIN_ALLOW_THREADS +- res = sigtimedwait(&set, &si, &ts); ++ res = sigtimedwait(&sigset, &si, &ts); + Py_END_ALLOW_THREADS + + if (res != -1) +-- +2.23.0 + diff --git a/backport-33455-Pass-os.environ-in-test_posix-test_specify.patch b/backport-33455-Pass-os.environ-in-test_posix-test_specify.patch new file mode 100644 index 0000000000000000000000000000000000000000..78b5beccc4b0a4c766f5181182d618656d5f4904 --- /dev/null +++ b/backport-33455-Pass-os.environ-in-test_posix-test_specify.patch @@ -0,0 +1,35 @@ +From af40fb24e5ae3a462da9ea43529030eb05f487c8 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= +Date: Fri, 11 May 2018 07:40:43 +0200 +Subject: [PATCH] bpo-33455: Pass os.environ in + test_posix::test_specify_environment. (GH-6753) + +Pass os.environ's copy to new process created at test_posix: +test_specify_environment. Otherwise important variables such as +LD_LIBRARY_PATH are not set and the child process might not work at all +in an environment where such variables are required for Python to function. + +Conflict:NA +Reference:https://github.com/python/cpython/commit/7ec8f28656ea9d84048e9b5655375c6a74a59f53 + +Signed-off-by: hanxinke +--- + Lib/test/test_posix.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py +index 5063a56..e2cda33 100644 +--- a/Lib/test/test_posix.py ++++ b/Lib/test/test_posix.py +@@ -1545,7 +1545,7 @@ class TestPosixSpawn(unittest.TestCase): + """ + pid = posix.posix_spawn(sys.executable, + [sys.executable, '-c', script], +- {'foo': 'bar'}) ++ {**os.environ, 'foo': 'bar'}) + self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + with open(envfile) as f: + self.assertEqual(f.read(), 'bar') +-- +2.23.0 + diff --git a/backport-33630-Fix-using-of-freed-memory-in-old-versions-.patch b/backport-33630-Fix-using-of-freed-memory-in-old-versions-.patch new file mode 100644 index 0000000000000000000000000000000000000000..50cac949f770b1ad8e692075b2cb23f07eb596de --- /dev/null +++ b/backport-33630-Fix-using-of-freed-memory-in-old-versions-.patch @@ -0,0 +1,83 @@ +From d2b779f242d19fa0f2c1d8d55014f8dd0e7ca1cf Mon Sep 17 00:00:00 2001 +From: Pablo Galindo +Date: Tue, 19 Jun 2018 09:19:50 +0100 +Subject: [PATCH] bpo-33630: Fix using of freed memory in old versions of glicb + for posix_spawn(). (GH-7685) + +Conflict:NA +Reference:https://github.com/python/cpython/commit/cb970730e3ca2522e9b1700dcaf0a06b7e898db6 + +Signed-off-by: hanxinke +--- + Modules/posixmodule.c | 25 ++++++++++++++++++++++--- + 1 file changed, 22 insertions(+), 3 deletions(-) + +diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c +index f10fe2b..441c82e 100644 +--- a/Modules/posixmodule.c ++++ b/Modules/posixmodule.c +@@ -5140,7 +5140,8 @@ enum posix_spawn_file_actions_identifier { + + static int + parse_file_actions(PyObject *file_actions, +- posix_spawn_file_actions_t *file_actionsp) ++ posix_spawn_file_actions_t *file_actionsp, ++ PyObject *temp_buffer) + { + PyObject *seq; + PyObject *file_action = NULL; +@@ -5185,9 +5186,13 @@ parse_file_actions(PyObject *file_actions, + { + goto fail; + } ++ if (PyList_Append(temp_buffer, path)) { ++ Py_DECREF(path); ++ goto fail; ++ } + errno = posix_spawn_file_actions_addopen(file_actionsp, + fd, PyBytes_AS_STRING(path), oflag, (mode_t)mode); +- Py_DECREF(path); /* addopen copied it. */ ++ Py_DECREF(path); + if (errno) { + posix_error(); + goto fail; +@@ -5270,6 +5275,7 @@ os_posix_spawn_impl(PyObject *module, path_t *path, PyObject *argv, + posix_spawn_file_actions_t *file_actionsp = NULL; + Py_ssize_t argc, envc; + PyObject *result = NULL; ++ PyObject *temp_buffer = NULL; + pid_t pid; + int err_code; + +@@ -5310,7 +5316,19 @@ os_posix_spawn_impl(PyObject *module, path_t *path, PyObject *argv, + } + + if (file_actions != Py_None) { +- if (parse_file_actions(file_actions, &file_actions_buf)) { ++ /* There is a bug in old versions of glibc that makes some of the ++ * helper functions for manipulating file actions not copy the provided ++ * buffers. The problem is that posix_spawn_file_actions_addopen does not ++ * copy the value of path for some old versions of glibc (<2.20). ++ * The use of temp_buffer here is a workaround that keeps the ++ * python objects that own the buffers alive until posix_spawn gets called. ++ * Check https://bugs.python.org/issue33630 and ++ * https://sourceware.org/bugzilla/show_bug.cgi?id=17048 for more info.*/ ++ temp_buffer = PyList_New(0); ++ if (!temp_buffer) { ++ goto exit; ++ } ++ if (parse_file_actions(file_actions, &file_actions_buf, temp_buffer)) { + goto exit; + } + file_actionsp = &file_actions_buf; +@@ -5337,6 +5355,7 @@ exit: + if (argvlist) { + free_string_array(argvlist, argc); + } ++ Py_XDECREF(temp_buffer); + return result; + } + #endif /* HAVE_POSIX_SPAWN */ +-- +2.23.0 + diff --git a/backport-34862-Guard-definition-of-convert_sched_p.patch b/backport-34862-Guard-definition-of-convert_sched_p.patch new file mode 100644 index 0000000000000000000000000000000000000000..c124bed7f82bebaa4dd9cf424edb9008612b6690 --- /dev/null +++ b/backport-34862-Guard-definition-of-convert_sched_p.patch @@ -0,0 +1,93 @@ +From 26c316254e347dffe64944dcf9ffeb6ee981dc0d Mon Sep 17 00:00:00 2001 +From: William Orr +Date: Mon, 1 Oct 2018 22:19:56 -0700 +Subject: [PATCH] closes bpo-34862: Guard definition of convert_sched_param + with POSIX_SPAWN_SETSCHEDULER. (GH-9658) + +Fixes broken build on OpenBSD-current. + +Conflict:NA +Reference:https://github.com/python/cpython/commit/81574b80e92554adf75c13fa42415beb8be383cb + +Signed-off-by: hanxinke +--- + Modules/clinic/posixmodule.c.h | 4 ++-- + Modules/posixmodule.c | 10 ++++++---- + 2 files changed, 8 insertions(+), 6 deletions(-) + +diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h +index 0448aa5..51b30d9 100644 +--- a/Modules/clinic/posixmodule.c.h ++++ b/Modules/clinic/posixmodule.c.h +@@ -2154,7 +2154,7 @@ exit: + + #endif /* defined(HAVE_SCHED_H) && defined(HAVE_SCHED_SETSCHEDULER) */ + +-#if defined(HAVE_SCHED_H) && (defined(HAVE_SCHED_SETSCHEDULER) || defined(HAVE_SCHED_SETPARAM)) ++#if defined(HAVE_SCHED_H) && (defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDPARAM)) + + PyDoc_STRVAR(os_sched_param__doc__, + "sched_param(sched_priority)\n" +@@ -2186,7 +2186,7 @@ exit: + return return_value; + } + +-#endif /* defined(HAVE_SCHED_H) && (defined(HAVE_SCHED_SETSCHEDULER) || defined(HAVE_SCHED_SETPARAM)) */ ++#endif /* defined(HAVE_SCHED_H) && (defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDPARAM)) */ + + #if defined(HAVE_SCHED_H) && defined(HAVE_SCHED_SETSCHEDULER) + +diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c +index 8d0e312..ba7fbef 100644 +--- a/Modules/posixmodule.c ++++ b/Modules/posixmodule.c +@@ -1937,7 +1937,7 @@ static PyTypeObject WaitidResultType; + static int initialized; + static PyTypeObject StatResultType; + static PyTypeObject StatVFSResultType; +-#if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) ++#if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDPARAM) + static PyTypeObject SchedParamType; + #endif + static newfunc structseq_new; +@@ -5138,8 +5138,10 @@ enum posix_spawn_file_actions_identifier { + POSIX_SPAWN_DUP2 + }; + ++#if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDPARAM) + static int + convert_sched_param(PyObject *param, struct sched_param *res); ++#endif + + static int + parse_posix_spawn_flags(PyObject *setpgroup, int resetids, PyObject *setsigmask, +@@ -5961,7 +5963,7 @@ os_sched_getscheduler_impl(PyObject *module, pid_t pid) + #endif /* HAVE_SCHED_SETSCHEDULER */ + + +-#if defined(HAVE_SCHED_SETSCHEDULER) || defined(HAVE_SCHED_SETPARAM) ++#if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDPARAM) + /*[clinic input] + class os.sched_param "PyObject *" "&SchedParamType" + +@@ -6022,7 +6024,7 @@ convert_sched_param(PyObject *param, struct sched_param *res) + res->sched_priority = Py_SAFE_DOWNCAST(priority, long, int); + return 1; + } +-#endif /* defined(HAVE_SCHED_SETSCHEDULER) || defined(HAVE_SCHED_SETPARAM) */ ++#endif /* defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDPARAM) */ + + + #ifdef HAVE_SCHED_SETSCHEDULER +@@ -13977,7 +13979,7 @@ INITFUNC(void) + # endif + #endif + +-#if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) ++#if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDPARAM) + sched_param_desc.name = MODNAME ".sched_param"; + if (PyStructSequence_InitType2(&SchedParamType, &sched_param_desc) < 0) + return NULL; +-- +2.23.0 + diff --git a/backport-35537-Add-setsid-parameter-to-os.posix_spawn-and.patch b/backport-35537-Add-setsid-parameter-to-os.posix_spawn-and.patch new file mode 100644 index 0000000000000000000000000000000000000000..fa49b56f5fd38be62545e8d4f122ef04c3e6c3db --- /dev/null +++ b/backport-35537-Add-setsid-parameter-to-os.posix_spawn-and.patch @@ -0,0 +1,298 @@ +From 28e39f4fff94e3f17ba49dce6b6f812a60f3d9dc Mon Sep 17 00:00:00 2001 +From: Joannah Nanjekye <33177550+nanjekyejoannah@users.noreply.github.com> +Date: Fri, 1 Feb 2019 13:05:22 +0300 +Subject: [PATCH] bpo-35537: Add setsid parameter to os.posix_spawn() and + os.posix_spawnp() (GH-11608) + +Conflict:NA +Reference:https://github.com/python/cpython/commit/80c5dfe74b4402d0a220c9227f262ec6fde1d7fc + +Signed-off-by: hanxinke +--- + Lib/test/test_posix.py | 16 +++++++ + .../2019-01-18-13-44-13.bpo-35537.R1lbTl.rst | 1 + + Modules/clinic/posixmodule.c.h | 42 +++++++++++------- + Modules/posixmodule.c | 44 +++++++++++++------ + 4 files changed, 73 insertions(+), 30 deletions(-) + create mode 100644 Misc/NEWS.d/next/Library/2019-01-18-13-44-13.bpo-35537.R1lbTl.rst + +diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py +index ec72f9e..ccc16b5 100644 +--- a/Lib/test/test_posix.py ++++ b/Lib/test/test_posix.py +@@ -1638,6 +1638,22 @@ class _PosixSpawnMixin: + os.environ, setsigmask=[signal.NSIG, + signal.NSIG+1]) + ++ def test_start_new_session(self): ++ # For code coverage of calling setsid(). We don't care if we get an ++ # EPERM error from it depending on the test execution environment, that ++ # still indicates that it was called. ++ code = "import os; print(os.getpgid(os.getpid()))" ++ try: ++ self.spawn_func(sys.executable, ++ [sys.executable, "-c", code], ++ os.environ, setsid=True) ++ except NotImplementedError as exc: ++ self.skipTest("setsid is not supported: %s" % exc) ++ else: ++ parent_pgid = os.getpgid(os.getpid()) ++ child_pgid = int(output) ++ self.assertNotEqual(parent_pgid, child_pgid) ++ + @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), + 'need signal.pthread_sigmask()') + def test_setsigdef(self): +diff --git a/Misc/NEWS.d/next/Library/2019-01-18-13-44-13.bpo-35537.R1lbTl.rst b/Misc/NEWS.d/next/Library/2019-01-18-13-44-13.bpo-35537.R1lbTl.rst +new file mode 100644 +index 0000000..56f23a1 +--- /dev/null ++++ b/Misc/NEWS.d/next/Library/2019-01-18-13-44-13.bpo-35537.R1lbTl.rst +@@ -0,0 +1 @@ ++:func:`os.posix_spawn` and :func:`os.posix_spawnp` now have a *setsid* parameter. +\ No newline at end of file +diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h +index 51b30d9..e06ddec 100644 +--- a/Modules/clinic/posixmodule.c.h ++++ b/Modules/clinic/posixmodule.c.h +@@ -1731,8 +1731,8 @@ exit: + + PyDoc_STRVAR(os_posix_spawn__doc__, + "posix_spawn($module, path, argv, env, /, *, file_actions=(),\n" +-" setpgroup=None, resetids=False, setsigmask=(),\n" +-" setsigdef=(), scheduler=None)\n" ++" setpgroup=None, resetids=False, setsid=False,\n" ++" setsigmask=(), setsigdef=(), scheduler=None)\n" + "--\n" + "\n" + "Execute the program specified by path in a new process.\n" +@@ -1748,7 +1748,9 @@ PyDoc_STRVAR(os_posix_spawn__doc__, + " setpgroup\n" + " The pgroup to use with the POSIX_SPAWN_SETPGROUP flag.\n" + " resetids\n" +-" If the value is `True` the POSIX_SPAWN_RESETIDS will be activated.\n" ++" If the value is `true` the POSIX_SPAWN_RESETIDS will be activated.\n" ++" setsid\n" ++" If the value is `true` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP will be activated.\n" + " setsigmask\n" + " The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag.\n" + " setsigdef\n" +@@ -1762,30 +1764,32 @@ PyDoc_STRVAR(os_posix_spawn__doc__, + static PyObject * + os_posix_spawn_impl(PyObject *module, path_t *path, PyObject *argv, + PyObject *env, PyObject *file_actions, +- PyObject *setpgroup, int resetids, PyObject *setsigmask, +- PyObject *setsigdef, PyObject *scheduler); ++ PyObject *setpgroup, int resetids, int setsid, ++ PyObject *setsigmask, PyObject *setsigdef, ++ PyObject *scheduler); + + static PyObject * + os_posix_spawn(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) + { + PyObject *return_value = NULL; +- static const char * const _keywords[] = {"", "", "", "file_actions", "setpgroup", "resetids", "setsigmask", "setsigdef", "scheduler", NULL}; +- static _PyArg_Parser _parser = {"O&OO|$OOiOOO:posix_spawn", _keywords, 0}; ++ static const char * const _keywords[] = {"", "", "", "file_actions", "setpgroup", "resetids", "setsid", "setsigmask", "setsigdef", "scheduler", NULL}; ++ static _PyArg_Parser _parser = {"O&OO|$OOiiOOO:posix_spawn", _keywords, 0}; + path_t path = PATH_T_INITIALIZE("posix_spawn", "path", 0, 0); + PyObject *argv; + PyObject *env; + PyObject *file_actions = NULL; + PyObject *setpgroup = NULL; + int resetids = 0; ++ int setsid = 0; + PyObject *setsigmask = NULL; + PyObject *setsigdef = NULL; + PyObject *scheduler = NULL; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, +- path_converter, &path, &argv, &env, &file_actions, &setpgroup, &resetids, &setsigmask, &setsigdef, &scheduler)) { ++ path_converter, &path, &argv, &env, &file_actions, &setpgroup, &resetids, &setsid, &setsigmask, &setsigdef, &scheduler)) { + goto exit; + } +- return_value = os_posix_spawn_impl(module, &path, argv, env, file_actions, setpgroup, resetids, setsigmask, setsigdef, scheduler); ++ return_value = os_posix_spawn_impl(module, &path, argv, env, file_actions, setpgroup, resetids, setsid, setsigmask, setsigdef, scheduler); + + exit: + /* Cleanup for path */ +@@ -1800,8 +1804,8 @@ exit: + + PyDoc_STRVAR(os_posix_spawnp__doc__, + "posix_spawnp($module, path, argv, env, /, *, file_actions=(),\n" +-" setpgroup=None, resetids=False, setsigmask=(),\n" +-" setsigdef=(), scheduler=None)\n" ++" setpgroup=None, resetids=False, setsid=False,\n" ++" setsigmask=(), setsigdef=(), scheduler=None)\n" + "--\n" + "\n" + "Execute the program specified by path in a new process.\n" +@@ -1818,6 +1822,8 @@ PyDoc_STRVAR(os_posix_spawnp__doc__, + " The pgroup to use with the POSIX_SPAWN_SETPGROUP flag.\n" + " resetids\n" + " If the value is `True` the POSIX_SPAWN_RESETIDS will be activated.\n" ++" setsid\n" ++" If the value is `True` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP will be activated.\n" + " setsigmask\n" + " The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag.\n" + " setsigdef\n" +@@ -1831,30 +1837,32 @@ PyDoc_STRVAR(os_posix_spawnp__doc__, + static PyObject * + os_posix_spawnp_impl(PyObject *module, path_t *path, PyObject *argv, + PyObject *env, PyObject *file_actions, +- PyObject *setpgroup, int resetids, PyObject *setsigmask, +- PyObject *setsigdef, PyObject *scheduler); ++ PyObject *setpgroup, int resetids, int setsid, ++ PyObject *setsigmask, PyObject *setsigdef, ++ PyObject *scheduler); + + static PyObject * + os_posix_spawnp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) + { + PyObject *return_value = NULL; +- static const char * const _keywords[] = {"", "", "", "file_actions", "setpgroup", "resetids", "setsigmask", "setsigdef", "scheduler", NULL}; +- static _PyArg_Parser _parser = {"O&OO|$OOiOOO:posix_spawnp", _keywords, 0}; ++ static const char * const _keywords[] = {"", "", "", "file_actions", "setpgroup", "resetids", "setsid", "setsigmask", "setsigdef", "scheduler", NULL}; ++ static _PyArg_Parser _parser = {"O&OO|$OOiiOOO:posix_spawnp", _keywords, 0}; + path_t path = PATH_T_INITIALIZE("posix_spawnp", "path", 0, 0); + PyObject *argv; + PyObject *env; + PyObject *file_actions = NULL; + PyObject *setpgroup = NULL; + int resetids = 0; ++ int setsid = 0; + PyObject *setsigmask = NULL; + PyObject *setsigdef = NULL; + PyObject *scheduler = NULL; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, +- path_converter, &path, &argv, &env, &file_actions, &setpgroup, &resetids, &setsigmask, &setsigdef, &scheduler)) { ++ path_converter, &path, &argv, &env, &file_actions, &setpgroup, &resetids, &setsid, &setsigmask, &setsigdef, &scheduler)) { + goto exit; + } +- return_value = os_posix_spawnp_impl(module, &path, argv, env, file_actions, setpgroup, resetids, setsigmask, setsigdef, scheduler); ++ return_value = os_posix_spawnp_impl(module, &path, argv, env, file_actions, setpgroup, resetids, setsid, setsigmask, setsigdef, scheduler); + + exit: + /* Cleanup for path */ +diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c +index ba7fbef..834ead4 100644 +--- a/Modules/posixmodule.c ++++ b/Modules/posixmodule.c +@@ -5144,10 +5144,11 @@ convert_sched_param(PyObject *param, struct sched_param *res); + #endif + + static int +-parse_posix_spawn_flags(PyObject *setpgroup, int resetids, PyObject *setsigmask, ++parse_posix_spawn_flags(PyObject *setpgroup, int resetids, int setsid, PyObject *setsigmask, + PyObject *setsigdef, PyObject *scheduler, + posix_spawnattr_t *attrp) + { ++ const char *func_name = "posix_spawnp"; + long all_flags = 0; + + errno = posix_spawnattr_init(attrp); +@@ -5173,6 +5174,17 @@ parse_posix_spawn_flags(PyObject *setpgroup, int resetids, PyObject *setsigmask, + all_flags |= POSIX_SPAWN_RESETIDS; + } + ++ if (setsid) { ++#ifdef POSIX_SPAWN_SETSID ++ all_flags |= POSIX_SPAWN_SETSID; ++#elif defined(POSIX_SPAWN_SETSID_NP) ++ all_flags |= POSIX_SPAWN_SETSID_NP; ++#else ++ argument_unavailable_error(func_name, "setsid"); ++ return -1; ++#endif ++ } ++ + if (setsigmask) { + sigset_t set; + if (!_Py_Sigset_Converter(setsigmask, &set)) { +@@ -5363,7 +5375,7 @@ fail: + static PyObject * + py_posix_spawn(int use_posix_spawnp, PyObject *module, path_t *path, PyObject *argv, + PyObject *env, PyObject *file_actions, +- PyObject *setpgroup, int resetids, PyObject *setsigmask, ++ PyObject *setpgroup, int resetids, int setsid, PyObject *setsigmask, + PyObject *setsigdef, PyObject *scheduler) + { + EXECV_CHAR **argvlist = NULL; +@@ -5378,7 +5390,7 @@ py_posix_spawn(int use_posix_spawnp, PyObject *module, path_t *path, PyObject *a + pid_t pid; + int err_code; + +- /* posix_spawn has three arguments: (path, argv, env), where ++ /* posix_spawn and posix_spawnp have three arguments: (path, argv, env), where + argv is a list or tuple of strings and env is a dictionary + like posix.environ. */ + +@@ -5433,7 +5445,7 @@ py_posix_spawn(int use_posix_spawnp, PyObject *module, path_t *path, PyObject *a + file_actionsp = &file_actions_buf; + } + +- if (parse_posix_spawn_flags(setpgroup, resetids, setsigmask, ++ if (parse_posix_spawn_flags(setpgroup, resetids, setsid, setsigmask, + setsigdef, scheduler, &attr)) { + goto exit; + } +@@ -5495,6 +5507,8 @@ os.posix_spawn + The pgroup to use with the POSIX_SPAWN_SETPGROUP flag. + resetids: bool(accept={int}) = False + If the value is `True` the POSIX_SPAWN_RESETIDS will be activated. ++ setsid: bool(accept={int}) = False ++ If the value is `True` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP will be activated. + setsigmask: object(c_default='NULL') = () + The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag. + setsigdef: object(c_default='NULL') = () +@@ -5508,12 +5522,13 @@ Execute the program specified by path in a new process. + static PyObject * + os_posix_spawn_impl(PyObject *module, path_t *path, PyObject *argv, + PyObject *env, PyObject *file_actions, +- PyObject *setpgroup, int resetids, PyObject *setsigmask, +- PyObject *setsigdef, PyObject *scheduler) +-/*[clinic end generated code: output=45dfa4c515d09f2c input=2891c2f1d457e39b]*/ ++ PyObject *setpgroup, int resetids, int setsid, ++ PyObject *setsigmask, PyObject *setsigdef, ++ PyObject *scheduler) ++/*[clinic end generated code: output=14a1098c566bc675 input=8c6305619a00ad04]*/ + { + return py_posix_spawn(0, module, path, argv, env, file_actions, +- setpgroup, resetids, setsigmask, setsigdef, ++ setpgroup, resetids, setsid, setsigmask, setsigdef, + scheduler); + } + #endif /* HAVE_POSIX_SPAWN */ +@@ -5537,7 +5552,9 @@ os.posix_spawnp + setpgroup: object = NULL + The pgroup to use with the POSIX_SPAWN_SETPGROUP flag. + resetids: bool(accept={int}) = False +- If the value is `True` the POSIX_SPAWN_RESETIDS will be activated. ++ If the value is `true` the POSIX_SPAWN_RESETIDS will be activated. ++ setsid: bool(accept={int}) = False ++ If the value is `true` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP will be activated. + setsigmask: object(c_default='NULL') = () + The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag. + setsigdef: object(c_default='NULL') = () +@@ -5551,12 +5568,13 @@ Execute the program specified by path in a new process. + static PyObject * + os_posix_spawnp_impl(PyObject *module, path_t *path, PyObject *argv, + PyObject *env, PyObject *file_actions, +- PyObject *setpgroup, int resetids, PyObject *setsigmask, +- PyObject *setsigdef, PyObject *scheduler) +-/*[clinic end generated code: output=7955dc0edc82b8c3 input=b7576eb25b1ed9eb]*/ ++ PyObject *setpgroup, int resetids, int setsid, ++ PyObject *setsigmask, PyObject *setsigdef, ++ PyObject *scheduler) ++/*[clinic end generated code: output=7b9aaefe3031238d input=c1911043a22028da]*/ + { + return py_posix_spawn(1, module, path, argv, env, file_actions, +- setpgroup, resetids, setsigmask, setsigdef, ++ setpgroup, resetids, setsid, setsigmask, setsigdef, + scheduler); + } + #endif /* HAVE_POSIX_SPAWNP */ +-- +2.23.0 + diff --git a/backport-35537-Fix-function-name-in-os.posix_spawnp-error.patch b/backport-35537-Fix-function-name-in-os.posix_spawnp-error.patch new file mode 100644 index 0000000000000000000000000000000000000000..86e9dc7ba9a38b16c9b6f6e8d8bef9a692e0b1e8 --- /dev/null +++ b/backport-35537-Fix-function-name-in-os.posix_spawnp-error.patch @@ -0,0 +1,91 @@ +From e601dbfea5dc592618863bd77cf7ae2dafda466b Mon Sep 17 00:00:00 2001 +From: Victor Stinner +Date: Fri, 1 Feb 2019 15:47:24 +0100 +Subject: [PATCH] bpo-35537: Fix function name in os.posix_spawnp() errors + (GH-11719) + +Conflict:NA +Reference:https://github.com/python/cpython/commit/325e4bac5ab49f47ec60242d3242647605193a2e + +Signed-off-by: hanxinke +--- + Modules/posixmodule.c | 24 +++++++++++++----------- + 1 file changed, 13 insertions(+), 11 deletions(-) + +diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c +index 834ead4..cc1f0be 100644 +--- a/Modules/posixmodule.c ++++ b/Modules/posixmodule.c +@@ -5144,11 +5144,11 @@ convert_sched_param(PyObject *param, struct sched_param *res); + #endif + + static int +-parse_posix_spawn_flags(PyObject *setpgroup, int resetids, int setsid, PyObject *setsigmask, ++parse_posix_spawn_flags(const char *func_name, PyObject *setpgroup, ++ int resetids, int setsid, PyObject *setsigmask, + PyObject *setsigdef, PyObject *scheduler, + posix_spawnattr_t *attrp) + { +- const char *func_name = "posix_spawnp"; + long all_flags = 0; + + errno = posix_spawnattr_init(attrp); +@@ -5378,6 +5378,7 @@ py_posix_spawn(int use_posix_spawnp, PyObject *module, path_t *path, PyObject *a + PyObject *setpgroup, int resetids, int setsid, PyObject *setsigmask, + PyObject *setsigdef, PyObject *scheduler) + { ++ const char *func_name = use_posix_spawnp ? "posix_spawnp" : "posix_spawn"; + EXECV_CHAR **argvlist = NULL; + EXECV_CHAR **envlist = NULL; + posix_spawn_file_actions_t file_actions_buf; +@@ -5395,19 +5396,20 @@ py_posix_spawn(int use_posix_spawnp, PyObject *module, path_t *path, PyObject *a + like posix.environ. */ + + if (!PyList_Check(argv) && !PyTuple_Check(argv)) { +- PyErr_SetString(PyExc_TypeError, +- "posix_spawn: argv must be a tuple or list"); ++ PyErr_Format(PyExc_TypeError, ++ "%s: argv must be a tuple or list", func_name); + goto exit; + } + argc = PySequence_Size(argv); + if (argc < 1) { +- PyErr_SetString(PyExc_ValueError, "posix_spawn: argv must not be empty"); ++ PyErr_Format(PyExc_ValueError, ++ "%s: argv must not be empty", func_name); + return NULL; + } + + if (!PyMapping_Check(env)) { +- PyErr_SetString(PyExc_TypeError, +- "posix_spawn: environment must be a mapping object"); ++ PyErr_Format(PyExc_TypeError, ++ "%s: environment must be a mapping object", func_name); + goto exit; + } + +@@ -5416,8 +5418,8 @@ py_posix_spawn(int use_posix_spawnp, PyObject *module, path_t *path, PyObject *a + goto exit; + } + if (!argvlist[0][0]) { +- PyErr_SetString(PyExc_ValueError, +- "posix_spawn: argv first element cannot be empty"); ++ PyErr_Format(PyExc_ValueError, ++ "%s: argv first element cannot be empty", func_name); + goto exit; + } + +@@ -5445,8 +5447,8 @@ py_posix_spawn(int use_posix_spawnp, PyObject *module, path_t *path, PyObject *a + file_actionsp = &file_actions_buf; + } + +- if (parse_posix_spawn_flags(setpgroup, resetids, setsid, setsigmask, +- setsigdef, scheduler, &attr)) { ++ if (parse_posix_spawn_flags(func_name, setpgroup, resetids, setsid, ++ setsigmask, setsigdef, scheduler, &attr)) { + goto exit; + } + attrp = &attr; +-- +2.23.0 + diff --git a/backport-35537-Rewrite-setsid-test-for-os.posix_spawn-GH-.patch b/backport-35537-Rewrite-setsid-test-for-os.posix_spawn-GH-.patch new file mode 100644 index 0000000000000000000000000000000000000000..113616e0bc204331f6bcda566aa77de6f713c116 --- /dev/null +++ b/backport-35537-Rewrite-setsid-test-for-os.posix_spawn-GH-.patch @@ -0,0 +1,101 @@ +From 7940f5c6e508ccb5a4d647094bf40f7a2f57b097 Mon Sep 17 00:00:00 2001 +From: Victor Stinner +Date: Fri, 14 Jun 2019 19:31:43 +0200 +Subject: [PATCH] bpo-35537: Rewrite setsid test for os.posix_spawn (GH-11721) + +bpo-35537, bpo-35876: Fix also test_start_new_session() of +test_subprocess: use os.getsid() rather than os.getpgid(). + +Conflict:NA +Reference:https://github.com/python/cpython/commit/5884043252473ac733aba1d3251d4debe72511e5 + +Signed-off-by: hanxinke +--- + Lib/test/test_posix.py | 44 +++++++++++++++++++++++-------------- + Lib/test/test_subprocess.py | 9 ++++---- + 2 files changed, 32 insertions(+), 21 deletions(-) + +diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py +index 83accd4..e816946 100644 +--- a/Lib/test/test_posix.py ++++ b/Lib/test/test_posix.py +@@ -1638,23 +1638,35 @@ class _PosixSpawnMixin: + os.environ, setsigmask=[signal.NSIG, + signal.NSIG+1]) + +- @unittest.skipIf(True, +- "FIXME: bpo-35537: test fails is setsid is supported") +- def test_start_new_session(self): +- # For code coverage of calling setsid(). We don't care if we get an +- # EPERM error from it depending on the test execution environment, that +- # still indicates that it was called. +- code = "import os; print(os.getpgid(os.getpid()))" ++ def test_setsid(self): ++ rfd, wfd = os.pipe() ++ self.addCleanup(os.close, rfd) + try: +- self.spawn_func(sys.executable, +- [sys.executable, "-c", code], +- os.environ, setsid=True) +- except NotImplementedError as exc: +- self.skipTest("setsid is not supported: %s" % exc) +- else: +- parent_pgid = os.getpgid(os.getpid()) +- child_pgid = int(output) +- self.assertNotEqual(parent_pgid, child_pgid) ++ os.set_inheritable(wfd, True) ++ ++ code = textwrap.dedent(f""" ++ import os ++ fd = {wfd} ++ sid = os.getsid(0) ++ os.write(fd, str(sid).encode()) ++ """) ++ ++ try: ++ pid = self.spawn_func(sys.executable, ++ [sys.executable, "-c", code], ++ os.environ, setsid=True) ++ except NotImplementedError as exc: ++ self.skipTest(f"setsid is not supported: {exc!r}") ++ except PermissionError as exc: ++ self.skipTest(f"setsid failed with: {exc!r}") ++ finally: ++ os.close(wfd) ++ ++ self.assertEqual(os.waitpid(pid, 0), (pid, 0)) ++ output = os.read(rfd, 100) ++ child_sid = int(output) ++ parent_sid = os.getsid(os.getpid()) ++ self.assertNotEqual(parent_sid, child_sid) + + @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), + 'need signal.pthread_sigmask()') +diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py +index 89dcb8f..eebd348 100644 +--- a/Lib/test/test_subprocess.py ++++ b/Lib/test/test_subprocess.py +@@ -1671,16 +1671,15 @@ class POSIXProcessTestCase(BaseTestCase): + # still indicates that it was called. + try: + output = subprocess.check_output( +- [sys.executable, "-c", +- "import os; print(os.getpgid(os.getpid()))"], ++ [sys.executable, "-c", "import os; print(os.getsid(0))"], + start_new_session=True) + except OSError as e: + if e.errno != errno.EPERM: + raise + else: +- parent_pgid = os.getpgid(os.getpid()) +- child_pgid = int(output) +- self.assertNotEqual(parent_pgid, child_pgid) ++ parent_sid = os.getsid(0) ++ child_sid = int(output) ++ self.assertNotEqual(parent_sid, child_sid) + + def test_run_abort(self): + # returncode handles signal termination +-- +2.23.0 + diff --git a/backport-35537-Skip-test_start_new_session-of-posix_spawn.patch b/backport-35537-Skip-test_start_new_session-of-posix_spawn.patch new file mode 100644 index 0000000000000000000000000000000000000000..7c19ef67309336a30ebdc0782a6639abe2ce42cd --- /dev/null +++ b/backport-35537-Skip-test_start_new_session-of-posix_spawn.patch @@ -0,0 +1,32 @@ +From 41c5fb1900f673029d1b6ecf24c743ac1ce4114f Mon Sep 17 00:00:00 2001 +From: Victor Stinner +Date: Fri, 1 Feb 2019 11:40:26 +0100 +Subject: [PATCH] bpo-35537: Skip test_start_new_session() of posix_spawn + (GH-11718) + +The test fails. Skip the test until a fix can be found. + +Conflict:NA +Reference:https://github.com/python/cpython/commit/1e39b83f24ffade3e0ca1a8004b461354f64ae08 + +Signed-off-by: hanxinke +--- + Lib/test/test_posix.py | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py +index ccc16b5..83accd4 100644 +--- a/Lib/test/test_posix.py ++++ b/Lib/test/test_posix.py +@@ -1638,6 +1638,8 @@ class _PosixSpawnMixin: + os.environ, setsigmask=[signal.NSIG, + signal.NSIG+1]) + ++ @unittest.skipIf(True, ++ "FIXME: bpo-35537: test fails is setsid is supported") + def test_start_new_session(self): + # For code coverage of calling setsid(). We don't care if we get an + # EPERM error from it depending on the test execution environment, that +-- +2.23.0 + diff --git a/backport-35537-subprocess-can-use-posix_spawn-with-pipes-.patch b/backport-35537-subprocess-can-use-posix_spawn-with-pipes-.patch new file mode 100644 index 0000000000000000000000000000000000000000..8fdf12215ea509ab4aef2fa00cb21dfee9f0c2df --- /dev/null +++ b/backport-35537-subprocess-can-use-posix_spawn-with-pipes-.patch @@ -0,0 +1,162 @@ +From e2f485341b9b99e33a10738191cc933f68509096 Mon Sep 17 00:00:00 2001 +From: Victor Stinner +Date: Wed, 23 Jan 2019 19:00:39 +0100 +Subject: [PATCH] bpo-35537: subprocess can use posix_spawn with pipes + (GH-11575) + +* subprocess.Popen can now also use os.posix_spawn() with pipes, + but only if pipe file descriptors are greater than 2. +* Fix Popen._posix_spawn(): set '_child_created' attribute to True. +* Add Popen._close_pipe_fds() helper function to factorize the code. + +Conflict:NA +Reference:https://github.com/python/cpython/commit/f6243ac1e4828299fe5a8e943d7bd41cab1f34cd + +Signed-off-by: hanxinke +--- + Lib/subprocess.py | 92 ++++++++++++++++++++++++++++++++--------------- + 1 file changed, 64 insertions(+), 28 deletions(-) + +diff --git a/Lib/subprocess.py b/Lib/subprocess.py +index d711ddb..b02b701 100644 +--- a/Lib/subprocess.py ++++ b/Lib/subprocess.py +@@ -1085,6 +1085,34 @@ class Popen(object): + pass + raise # resume the KeyboardInterrupt + ++ def _close_pipe_fds(self, ++ p2cread, p2cwrite, ++ c2pread, c2pwrite, ++ errread, errwrite): ++ # self._devnull is not always defined. ++ devnull_fd = getattr(self, '_devnull', None) ++ ++ if _mswindows: ++ if p2cread != -1: ++ p2cread.Close() ++ if c2pwrite != -1: ++ c2pwrite.Close() ++ if errwrite != -1: ++ errwrite.Close() ++ else: ++ if p2cread != -1 and p2cwrite != -1 and p2cread != devnull_fd: ++ os.close(p2cread) ++ if c2pwrite != -1 and c2pread != -1 and c2pwrite != devnull_fd: ++ os.close(c2pwrite) ++ if errwrite != -1 and errread != -1 and errwrite != devnull_fd: ++ os.close(errwrite) ++ ++ if devnull_fd is not None: ++ os.close(devnull_fd) ++ ++ # Prevent a double close of these handles/fds from __init__ on error. ++ self._closed_child_pipe_fds = True ++ + + if _mswindows: + # +@@ -1263,17 +1291,9 @@ class Popen(object): + # output pipe are maintained in this process or else the + # pipe will not close when the child process exits and the + # ReadFile will hang. +- if p2cread != -1: +- p2cread.Close() +- if c2pwrite != -1: +- c2pwrite.Close() +- if errwrite != -1: +- errwrite.Close() +- if hasattr(self, '_devnull'): +- os.close(self._devnull) +- # Prevent a double close of these handles/fds from __init__ +- # on error. +- self._closed_child_pipe_fds = True ++ self._close_pipe_fds(p2cread, p2cwrite, ++ c2pread, c2pwrite, ++ errread, errwrite) + + # Retain the process handle, but close the thread handle + self._child_created = True +@@ -1460,7 +1480,10 @@ class Popen(object): + errread, errwrite) + + +- def _posix_spawn(self, args, executable, env, restore_signals): ++ def _posix_spawn(self, args, executable, env, restore_signals, ++ p2cread, p2cwrite, ++ c2pread, c2pwrite, ++ errread, errwrite): + """Execute program using os.posix_spawn().""" + if env is None: + env = os.environ +@@ -1475,7 +1498,26 @@ class Popen(object): + sigset.append(signum) + kwargs['setsigdef'] = sigset + ++ file_actions = [] ++ for fd in (p2cwrite, c2pread, errread): ++ if fd != -1: ++ file_actions.append((os.POSIX_SPAWN_CLOSE, fd)) ++ for fd, fd2 in ( ++ (p2cread, 0), ++ (c2pwrite, 1), ++ (errwrite, 2), ++ ): ++ if fd != -1: ++ file_actions.append((os.POSIX_SPAWN_DUP2, fd, fd2)) ++ if file_actions: ++ kwargs['file_actions'] = file_actions ++ + self.pid = os.posix_spawn(executable, args, env, **kwargs) ++ self._child_created = True ++ ++ self._close_pipe_fds(p2cread, p2cwrite, ++ c2pread, c2pwrite, ++ errread, errwrite) + + def _execute_child(self, args, executable, preexec_fn, close_fds, + pass_fds, cwd, env, +@@ -1508,11 +1550,14 @@ class Popen(object): + and not close_fds + and not pass_fds + and cwd is None +- and p2cread == p2cwrite == -1 +- and c2pread == c2pwrite == -1 +- and errread == errwrite == -1 ++ and (p2cread == -1 or p2cread > 2) ++ and (c2pwrite == -1 or c2pwrite > 2) ++ and (errwrite == -1 or errwrite > 2) + and not start_new_session): +- self._posix_spawn(args, executable, env, restore_signals) ++ self._posix_spawn(args, executable, env, restore_signals, ++ p2cread, p2cwrite, ++ c2pread, c2pwrite, ++ errread, errwrite) + return + + orig_executable = executable +@@ -1567,18 +1612,9 @@ class Popen(object): + # be sure the FD is closed no matter what + os.close(errpipe_write) + +- # self._devnull is not always defined. +- devnull_fd = getattr(self, '_devnull', None) +- if p2cread != -1 and p2cwrite != -1 and p2cread != devnull_fd: +- os.close(p2cread) +- if c2pwrite != -1 and c2pread != -1 and c2pwrite != devnull_fd: +- os.close(c2pwrite) +- if errwrite != -1 and errread != -1 and errwrite != devnull_fd: +- os.close(errwrite) +- if devnull_fd is not None: +- os.close(devnull_fd) +- # Prevent a double close of these fds from __init__ on error. +- self._closed_child_pipe_fds = True ++ self._close_pipe_fds(p2cread, p2cwrite, ++ c2pread, c2pwrite, ++ errread, errwrite) + + # Wait for exec to fail or succeed; possibly raising an + # exception (limited in size) +-- +2.23.0 + diff --git a/backport-35537-subprocess-uses-os.posix_spawn-in-some-cas.patch b/backport-35537-subprocess-uses-os.posix_spawn-in-some-cas.patch new file mode 100644 index 0000000000000000000000000000000000000000..43af4355de4908ac7f573f91dce09eb22a5760f1 --- /dev/null +++ b/backport-35537-subprocess-uses-os.posix_spawn-in-some-cas.patch @@ -0,0 +1,144 @@ +From 41bd027d617de09a2c58972312b36f7be2779db6 Mon Sep 17 00:00:00 2001 +From: Victor Stinner +Date: Wed, 16 Jan 2019 00:02:35 +0100 +Subject: [PATCH] bpo-35537: subprocess uses os.posix_spawn in some cases + (GH-11452) + +The subprocess module can now use the os.posix_spawn() function +in some cases for better performance. Currently, it is only used on macOS +and Linux (using glibc 2.24 or newer) if all these conditions are met: + +* executable path contains a directory +* close_fds=False +* preexec_fn, pass_fds, cwd, stdin, stdout, stderr + and start_new_session parameters are not set + +Conflict:NA +Reference:https://github.com/python/cpython/commit/9daecf37a571e98aaf43a387bcc9e41a7132f477 + +Co-authored-by: Joannah Nanjekye +Signed-off-by: hanxinke +--- + Lib/subprocess.py | 82 +++++++++++++++++++ + .../2018-12-20-16-24-51.bpo-35537.z4E7aA.rst | 2 + + 2 files changed, 84 insertions(+) + create mode 100644 Misc/NEWS.d/next/Library/2018-12-20-16-24-51.bpo-35537.z4E7aA.rst + +diff --git a/Lib/subprocess.py b/Lib/subprocess.py +index 3f99be5..d711ddb 100644 +--- a/Lib/subprocess.py ++++ b/Lib/subprocess.py +@@ -630,6 +630,57 @@ def getoutput(cmd): + return getstatusoutput(cmd)[1] + + ++def _use_posix_spawn(): ++ """Check is posix_spawn() can be used for subprocess. ++ ++ subprocess requires a posix_spawn() implementation that reports properly ++ errors to the parent process, set errno on the following failures: ++ ++ * process attribute actions failed ++ * file actions failed ++ * exec() failed ++ ++ Prefer an implementation which can use vfork in some cases for best ++ performances. ++ """ ++ if _mswindows or not hasattr(os, 'posix_spawn'): ++ # os.posix_spawn() is not available ++ return False ++ ++ if sys.platform == 'darwin': ++ # posix_spawn() is a syscall on macOS and properly reports errors ++ return True ++ ++ # Check libc name and runtime libc version ++ try: ++ ver = os.confstr('CS_GNU_LIBC_VERSION') ++ # parse 'glibc 2.28' as ('glibc', (2, 28)) ++ parts = ver.split(maxsplit=1) ++ if len(parts) != 2: ++ # reject unknown format ++ raise ValueError ++ libc = parts[0] ++ version = tuple(map(int, parts[1].split('.'))) ++ ++ if sys.platform == 'linux' and libc == 'glibc' and version >= (2, 24): ++ # glibc 2.24 has a new Linux posix_spawn implementation using vfork ++ # which properly reports errors to the parent process. ++ return True ++ # Note: Don't use the POSIX implementation of glibc because it doesn't ++ # use vfork (even if glibc 2.26 added a pipe to properly report errors ++ # to the parent process). ++ except (AttributeError, ValueError, OSError): ++ # os.confstr() or CS_GNU_LIBC_VERSION value not available ++ pass ++ ++ # By default, consider that the implementation does not properly report ++ # errors. ++ return False ++ ++ ++_USE_POSIX_SPAWN = _use_posix_spawn() ++ ++ + class Popen(object): + """ Execute a child program in a new process. + +@@ -1409,6 +1460,23 @@ class Popen(object): + errread, errwrite) + + ++ def _posix_spawn(self, args, executable, env, restore_signals): ++ """Execute program using os.posix_spawn().""" ++ if env is None: ++ env = os.environ ++ ++ kwargs = {} ++ if restore_signals: ++ # See _Py_RestoreSignals() in Python/pylifecycle.c ++ sigset = [] ++ for signame in ('SIGPIPE', 'SIGXFZ', 'SIGXFSZ'): ++ signum = getattr(signal, signame, None) ++ if signum is not None: ++ sigset.append(signum) ++ kwargs['setsigdef'] = sigset ++ ++ self.pid = os.posix_spawn(executable, args, env, **kwargs) ++ + def _execute_child(self, args, executable, preexec_fn, close_fds, + pass_fds, cwd, env, + startupinfo, creationflags, shell, +@@ -1433,6 +1501,20 @@ class Popen(object): + + if executable is None: + executable = args[0] ++ ++ if (_USE_POSIX_SPAWN ++ and os.path.dirname(executable) ++ and preexec_fn is None ++ and not close_fds ++ and not pass_fds ++ and cwd is None ++ and p2cread == p2cwrite == -1 ++ and c2pread == c2pwrite == -1 ++ and errread == errwrite == -1 ++ and not start_new_session): ++ self._posix_spawn(args, executable, env, restore_signals) ++ return ++ + orig_executable = executable + + # For transferring possible exec failure from child to parent. +diff --git a/Misc/NEWS.d/next/Library/2018-12-20-16-24-51.bpo-35537.z4E7aA.rst b/Misc/NEWS.d/next/Library/2018-12-20-16-24-51.bpo-35537.z4E7aA.rst +new file mode 100644 +index 0000000..b14d749 +--- /dev/null ++++ b/Misc/NEWS.d/next/Library/2018-12-20-16-24-51.bpo-35537.z4E7aA.rst +@@ -0,0 +1,2 @@ ++The :mod:`subprocess` module can now use the :func:`os.posix_spawn` function in ++some cases for better performance. +-- +2.23.0 + diff --git a/backport-35674-Add-os.posix_spawnp-GH-11554.patch b/backport-35674-Add-os.posix_spawnp-GH-11554.patch new file mode 100644 index 0000000000000000000000000000000000000000..48f84dab29f4f0f01461056f47af41fe3a6c13db --- /dev/null +++ b/backport-35674-Add-os.posix_spawnp-GH-11554.patch @@ -0,0 +1,686 @@ +From cfe32a7f819ed8d802141fcad2842bbbf356670b Mon Sep 17 00:00:00 2001 +From: Joannah Nanjekye <33177550+nanjekyejoannah@users.noreply.github.com> +Date: Wed, 16 Jan 2019 16:29:26 +0300 +Subject: [PATCH] bpo-35674: Add os.posix_spawnp() (GH-11554) + +Add a new os.posix_spawnp() function. + +Conflict:NA +Reference:https://github.com/python/cpython/commit/92b8322e7ea04b239cb1cb87b78d952f13ddfebb + +Signed-off-by: hanxinke +--- + Lib/test/test_posix.py | 190 +++++++++++------- + .../2019-01-14-14-13-08.bpo-35674.kamWqz.rst | 2 + + Modules/clinic/posixmodule.c.h | 73 +++++++ + Modules/posixmodule.c | 135 +++++++++---- + configure | 2 +- + configure.ac | 2 +- + pyconfig.h.in | 3 + + 7 files changed, 300 insertions(+), 107 deletions(-) + create mode 100644 Misc/NEWS.d/next/Library/2019-01-14-14-13-08.bpo-35674.kamWqz.rst + +diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py +index ee3c5f0..ec72f9e 100644 +--- a/Lib/test/test_posix.py ++++ b/Lib/test/test_posix.py +@@ -1505,10 +1505,10 @@ class PosixGroupsTester(unittest.TestCase): + self.assertListEqual(groups, posix.getgroups()) + + +-@unittest.skipUnless(hasattr(os, 'posix_spawn'), "test needs os.posix_spawn") +-class TestPosixSpawn(unittest.TestCase): +- # Program which does nothing and exit with status 0 (success) ++class _PosixSpawnMixin: ++ # Program which does nothing and exits with status 0 (success) + NOOP_PROGRAM = (sys.executable, '-I', '-S', '-c', 'pass') ++ spawn_func = None + + def python_args(self, *args): + # Disable site module to avoid side effects. For example, +@@ -1527,7 +1527,7 @@ class TestPosixSpawn(unittest.TestCase): + pidfile.write(str(os.getpid())) + """ + args = self.python_args('-c', script) +- pid = posix.posix_spawn(args[0], args, os.environ) ++ pid = self.spawn_func(args[0], args, os.environ) + self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + with open(pidfile) as f: + self.assertEqual(f.read(), str(pid)) +@@ -1535,9 +1535,9 @@ class TestPosixSpawn(unittest.TestCase): + def test_no_such_executable(self): + no_such_executable = 'no_such_executable' + try: +- pid = posix.posix_spawn(no_such_executable, +- [no_such_executable], +- os.environ) ++ pid = self.spawn_func(no_such_executable, ++ [no_such_executable], ++ os.environ) + except FileNotFoundError as exc: + self.assertEqual(exc.filename, no_such_executable) + else: +@@ -1554,14 +1554,14 @@ class TestPosixSpawn(unittest.TestCase): + envfile.write(os.environ['foo']) + """ + args = self.python_args('-c', script) +- pid = posix.posix_spawn(args[0], args, +- {**os.environ, 'foo': 'bar'}) ++ pid = self.spawn_func(args[0], args, ++ {**os.environ, 'foo': 'bar'}) + self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + with open(envfile) as f: + self.assertEqual(f.read(), 'bar') + + def test_empty_file_actions(self): +- pid = posix.posix_spawn( ++ pid = self.spawn_func( + self.NOOP_PROGRAM[0], + self.NOOP_PROGRAM, + os.environ, +@@ -1570,7 +1570,7 @@ class TestPosixSpawn(unittest.TestCase): + self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + + def test_resetids_explicit_default(self): +- pid = posix.posix_spawn( ++ pid = self.spawn_func( + sys.executable, + [sys.executable, '-c', 'pass'], + os.environ, +@@ -1579,7 +1579,7 @@ class TestPosixSpawn(unittest.TestCase): + self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + + def test_resetids(self): +- pid = posix.posix_spawn( ++ pid = self.spawn_func( + sys.executable, + [sys.executable, '-c', 'pass'], + os.environ, +@@ -1589,12 +1589,12 @@ class TestPosixSpawn(unittest.TestCase): + + def test_resetids_wrong_type(self): + with self.assertRaises(TypeError): +- posix.posix_spawn(sys.executable, +- [sys.executable, "-c", "pass"], +- os.environ, resetids=None) ++ self.spawn_func(sys.executable, ++ [sys.executable, "-c", "pass"], ++ os.environ, resetids=None) + + def test_setpgroup(self): +- pid = posix.posix_spawn( ++ pid = self.spawn_func( + sys.executable, + [sys.executable, '-c', 'pass'], + os.environ, +@@ -1604,9 +1604,9 @@ class TestPosixSpawn(unittest.TestCase): + + def test_setpgroup_wrong_type(self): + with self.assertRaises(TypeError): +- posix.posix_spawn(sys.executable, +- [sys.executable, "-c", "pass"], +- os.environ, setpgroup="023") ++ self.spawn_func(sys.executable, ++ [sys.executable, "-c", "pass"], ++ os.environ, setpgroup="023") + + @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), + 'need signal.pthread_sigmask()') +@@ -1615,7 +1615,7 @@ class TestPosixSpawn(unittest.TestCase): + import _testcapi, signal + _testcapi.raise_signal(signal.SIGUSR1)""") + +- pid = posix.posix_spawn( ++ pid = self.spawn_func( + sys.executable, + [sys.executable, '-c', code], + os.environ, +@@ -1625,18 +1625,18 @@ class TestPosixSpawn(unittest.TestCase): + + def test_setsigmask_wrong_type(self): + with self.assertRaises(TypeError): +- posix.posix_spawn(sys.executable, +- [sys.executable, "-c", "pass"], +- os.environ, setsigmask=34) ++ self.spawn_func(sys.executable, ++ [sys.executable, "-c", "pass"], ++ os.environ, setsigmask=34) + with self.assertRaises(TypeError): +- posix.posix_spawn(sys.executable, +- [sys.executable, "-c", "pass"], +- os.environ, setsigmask=["j"]) ++ self.spawn_func(sys.executable, ++ [sys.executable, "-c", "pass"], ++ os.environ, setsigmask=["j"]) + with self.assertRaises(ValueError): +- posix.posix_spawn(sys.executable, +- [sys.executable, "-c", "pass"], +- os.environ, setsigmask=[signal.NSIG, +- signal.NSIG+1]) ++ self.spawn_func(sys.executable, ++ [sys.executable, "-c", "pass"], ++ os.environ, setsigmask=[signal.NSIG, ++ signal.NSIG+1]) + + @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), + 'need signal.pthread_sigmask()') +@@ -1646,7 +1646,7 @@ class TestPosixSpawn(unittest.TestCase): + import _testcapi, signal + _testcapi.raise_signal(signal.SIGUSR1)""") + try: +- pid = posix.posix_spawn( ++ pid = self.spawn_func( + sys.executable, + [sys.executable, '-c', code], + os.environ, +@@ -1662,17 +1662,17 @@ class TestPosixSpawn(unittest.TestCase): + + def test_setsigdef_wrong_type(self): + with self.assertRaises(TypeError): +- posix.posix_spawn(sys.executable, +- [sys.executable, "-c", "pass"], +- os.environ, setsigdef=34) ++ self.spawn_func(sys.executable, ++ [sys.executable, "-c", "pass"], ++ os.environ, setsigdef=34) + with self.assertRaises(TypeError): +- posix.posix_spawn(sys.executable, +- [sys.executable, "-c", "pass"], +- os.environ, setsigdef=["j"]) ++ self.spawn_func(sys.executable, ++ [sys.executable, "-c", "pass"], ++ os.environ, setsigdef=["j"]) + with self.assertRaises(ValueError): +- posix.posix_spawn(sys.executable, +- [sys.executable, "-c", "pass"], +- os.environ, setsigdef=[signal.NSIG, signal.NSIG+1]) ++ self.spawn_func(sys.executable, ++ [sys.executable, "-c", "pass"], ++ os.environ, setsigdef=[signal.NSIG, signal.NSIG+1]) + + @unittest.skipUnless(hasattr(posix, 'sched_setscheduler'), "can't change scheduler") + def test_setscheduler_only_param(self): +@@ -1684,7 +1684,7 @@ class TestPosixSpawn(unittest.TestCase): + os.exit(101) + if os.sched_getparam(0).sched_priority != {priority}: + os.exit(102)""") +- pid = posix.posix_spawn( ++ pid = self.spawn_func( + sys.executable, + [sys.executable, '-c', code], + os.environ, +@@ -1702,7 +1702,7 @@ class TestPosixSpawn(unittest.TestCase): + os.exit(101) + if os.sched_getparam(0).sched_priority != {priority}: + os.exit(102)""") +- pid = posix.posix_spawn( ++ pid = self.spawn_func( + sys.executable, + [sys.executable, '-c', code], + os.environ, +@@ -1716,40 +1716,40 @@ class TestPosixSpawn(unittest.TestCase): + (os.POSIX_SPAWN_CLOSE, 0), + (os.POSIX_SPAWN_DUP2, 1, 4), + ] +- pid = posix.posix_spawn(self.NOOP_PROGRAM[0], +- self.NOOP_PROGRAM, +- os.environ, +- file_actions=file_actions) ++ pid = self.spawn_func(self.NOOP_PROGRAM[0], ++ self.NOOP_PROGRAM, ++ os.environ, ++ file_actions=file_actions) + self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + + def test_bad_file_actions(self): + args = self.NOOP_PROGRAM + with self.assertRaises(TypeError): +- posix.posix_spawn(args[0], args, os.environ, +- file_actions=[None]) ++ self.spawn_func(args[0], args, os.environ, ++ file_actions=[None]) + with self.assertRaises(TypeError): +- posix.posix_spawn(args[0], args, os.environ, +- file_actions=[()]) ++ self.spawn_func(args[0], args, os.environ, ++ file_actions=[()]) + with self.assertRaises(TypeError): +- posix.posix_spawn(args[0], args, os.environ, +- file_actions=[(None,)]) ++ self.spawn_func(args[0], args, os.environ, ++ file_actions=[(None,)]) + with self.assertRaises(TypeError): +- posix.posix_spawn(args[0], args, os.environ, +- file_actions=[(12345,)]) ++ self.spawn_func(args[0], args, os.environ, ++ file_actions=[(12345,)]) + with self.assertRaises(TypeError): +- posix.posix_spawn(args[0], args, os.environ, +- file_actions=[(os.POSIX_SPAWN_CLOSE,)]) ++ self.spawn_func(args[0], args, os.environ, ++ file_actions=[(os.POSIX_SPAWN_CLOSE,)]) + with self.assertRaises(TypeError): +- posix.posix_spawn(args[0], args, os.environ, +- file_actions=[(os.POSIX_SPAWN_CLOSE, 1, 2)]) ++ self.spawn_func(args[0], args, os.environ, ++ file_actions=[(os.POSIX_SPAWN_CLOSE, 1, 2)]) + with self.assertRaises(TypeError): +- posix.posix_spawn(args[0], args, os.environ, +- file_actions=[(os.POSIX_SPAWN_CLOSE, None)]) ++ self.spawn_func(args[0], args, os.environ, ++ file_actions=[(os.POSIX_SPAWN_CLOSE, None)]) + with self.assertRaises(ValueError): +- posix.posix_spawn(args[0], args, os.environ, +- file_actions=[(os.POSIX_SPAWN_OPEN, +- 3, __file__ + '\0', +- os.O_RDONLY, 0)]) ++ self.spawn_func(args[0], args, os.environ, ++ file_actions=[(os.POSIX_SPAWN_OPEN, ++ 3, __file__ + '\0', ++ os.O_RDONLY, 0)]) + + def test_open_file(self): + outfile = support.TESTFN +@@ -1764,8 +1764,8 @@ class TestPosixSpawn(unittest.TestCase): + stat.S_IRUSR | stat.S_IWUSR), + ] + args = self.python_args('-c', script) +- pid = posix.posix_spawn(args[0], args, os.environ, +- file_actions=file_actions) ++ pid = self.spawn_func(args[0], args, os.environ, ++ file_actions=file_actions) + self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + with open(outfile) as f: + self.assertEqual(f.read(), 'hello') +@@ -1782,8 +1782,8 @@ class TestPosixSpawn(unittest.TestCase): + closefile.write('is closed %d' % e.errno) + """ + args = self.python_args('-c', script) +- pid = posix.posix_spawn(args[0], args, os.environ, +- file_actions=[(os.POSIX_SPAWN_CLOSE, 0),]) ++ pid = self.spawn_func(args[0], args, os.environ, ++ file_actions=[(os.POSIX_SPAWN_CLOSE, 0)]) + self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + with open(closefile) as f: + self.assertEqual(f.read(), 'is closed %d' % errno.EBADF) +@@ -1800,16 +1800,64 @@ class TestPosixSpawn(unittest.TestCase): + (os.POSIX_SPAWN_DUP2, childfile.fileno(), 1), + ] + args = self.python_args('-c', script) +- pid = posix.posix_spawn(args[0], args, os.environ, +- file_actions=file_actions) ++ pid = self.spawn_func(args[0], args, os.environ, ++ file_actions=file_actions) + self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + with open(dupfile) as f: + self.assertEqual(f.read(), 'hello') + + ++@unittest.skipUnless(hasattr(os, 'posix_spawn'), "test needs os.posix_spawn") ++class TestPosixSpawn(unittest.TestCase, _PosixSpawnMixin): ++ spawn_func = getattr(posix, 'posix_spawn', None) ++ ++ ++@unittest.skipUnless(hasattr(os, 'posix_spawnp'), "test needs os.posix_spawnp") ++class TestPosixSpawnP(unittest.TestCase, _PosixSpawnMixin): ++ spawn_func = getattr(posix, 'posix_spawnp', None) ++ ++ @support.skip_unless_symlink ++ def test_posix_spawnp(self): ++ # Use a symlink to create a program in its own temporary directory ++ temp_dir = tempfile.mkdtemp() ++ self.addCleanup(support.rmtree, temp_dir) ++ ++ program = 'posix_spawnp_test_program.exe' ++ program_fullpath = os.path.join(temp_dir, program) ++ os.symlink(sys.executable, program_fullpath) ++ ++ try: ++ path = os.pathsep.join((temp_dir, os.environ['PATH'])) ++ except KeyError: ++ path = temp_dir # PATH is not set ++ ++ spawn_args = (program, '-I', '-S', '-c', 'pass') ++ code = textwrap.dedent(""" ++ import os ++ args = %a ++ pid = os.posix_spawnp(args[0], args, os.environ) ++ pid2, status = os.waitpid(pid, 0) ++ if pid2 != pid: ++ raise Exception(f"pid {pid2} != {pid}") ++ if status != 0: ++ raise Exception(f"status {status} != 0") ++ """ % (spawn_args,)) ++ ++ # Use a subprocess to test os.posix_spawnp() with a modified PATH ++ # environment variable: posix_spawnp() uses the current environment ++ # to locate the program, not its environment argument. ++ args = ('-c', code) ++ assert_python_ok(*args, PATH=path) ++ ++ + def test_main(): + try: +- support.run_unittest(PosixTester, PosixGroupsTester, TestPosixSpawn) ++ support.run_unittest( ++ PosixTester, ++ PosixGroupsTester, ++ TestPosixSpawn, ++ TestPosixSpawnP, ++ ) + finally: + support.reap_children() + +diff --git a/Misc/NEWS.d/next/Library/2019-01-14-14-13-08.bpo-35674.kamWqz.rst b/Misc/NEWS.d/next/Library/2019-01-14-14-13-08.bpo-35674.kamWqz.rst +new file mode 100644 +index 0000000..02d170e +--- /dev/null ++++ b/Misc/NEWS.d/next/Library/2019-01-14-14-13-08.bpo-35674.kamWqz.rst +@@ -0,0 +1,2 @@ ++Add a new :func:`os.posix_spawnp` function. ++Patch by Joannah Nanjekye. +\ No newline at end of file +diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h +index 133abd7..0448aa5 100644 +--- a/Modules/clinic/posixmodule.c.h ++++ b/Modules/clinic/posixmodule.c.h +@@ -1796,6 +1796,75 @@ exit: + + #endif /* defined(HAVE_POSIX_SPAWN) */ + ++#if defined(HAVE_POSIX_SPAWNP) ++ ++PyDoc_STRVAR(os_posix_spawnp__doc__, ++"posix_spawnp($module, path, argv, env, /, *, file_actions=(),\n" ++" setpgroup=None, resetids=False, setsigmask=(),\n" ++" setsigdef=(), scheduler=None)\n" ++"--\n" ++"\n" ++"Execute the program specified by path in a new process.\n" ++"\n" ++" path\n" ++" Path of executable file.\n" ++" argv\n" ++" Tuple or list of strings.\n" ++" env\n" ++" Dictionary of strings mapping to strings.\n" ++" file_actions\n" ++" A sequence of file action tuples.\n" ++" setpgroup\n" ++" The pgroup to use with the POSIX_SPAWN_SETPGROUP flag.\n" ++" resetids\n" ++" If the value is `True` the POSIX_SPAWN_RESETIDS will be activated.\n" ++" setsigmask\n" ++" The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag.\n" ++" setsigdef\n" ++" The sigmask to use with the POSIX_SPAWN_SETSIGDEF flag.\n" ++" scheduler\n" ++" A tuple with the scheduler policy (optional) and parameters."); ++ ++#define OS_POSIX_SPAWNP_METHODDEF \ ++ {"posix_spawnp", (PyCFunction)(void(*)(void))os_posix_spawnp, METH_FASTCALL|METH_KEYWORDS, os_posix_spawnp__doc__}, ++ ++static PyObject * ++os_posix_spawnp_impl(PyObject *module, path_t *path, PyObject *argv, ++ PyObject *env, PyObject *file_actions, ++ PyObject *setpgroup, int resetids, PyObject *setsigmask, ++ PyObject *setsigdef, PyObject *scheduler); ++ ++static PyObject * ++os_posix_spawnp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) ++{ ++ PyObject *return_value = NULL; ++ static const char * const _keywords[] = {"", "", "", "file_actions", "setpgroup", "resetids", "setsigmask", "setsigdef", "scheduler", NULL}; ++ static _PyArg_Parser _parser = {"O&OO|$OOiOOO:posix_spawnp", _keywords, 0}; ++ path_t path = PATH_T_INITIALIZE("posix_spawnp", "path", 0, 0); ++ PyObject *argv; ++ PyObject *env; ++ PyObject *file_actions = NULL; ++ PyObject *setpgroup = NULL; ++ int resetids = 0; ++ PyObject *setsigmask = NULL; ++ PyObject *setsigdef = NULL; ++ PyObject *scheduler = NULL; ++ ++ if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, ++ path_converter, &path, &argv, &env, &file_actions, &setpgroup, &resetids, &setsigmask, &setsigdef, &scheduler)) { ++ goto exit; ++ } ++ return_value = os_posix_spawnp_impl(module, &path, argv, env, file_actions, setpgroup, resetids, setsigmask, setsigdef, scheduler); ++ ++exit: ++ /* Cleanup for path */ ++ path_cleanup(&path); ++ ++ return return_value; ++} ++ ++#endif /* defined(HAVE_POSIX_SPAWNP) */ ++ + #if (defined(HAVE_SPAWNV) || defined(HAVE_WSPAWNV)) + + PyDoc_STRVAR(os_spawnv__doc__, +@@ -6220,6 +6289,10 @@ exit: + #define OS_POSIX_SPAWN_METHODDEF + #endif /* !defined(OS_POSIX_SPAWN_METHODDEF) */ + ++#ifndef OS_POSIX_SPAWNP_METHODDEF ++ #define OS_POSIX_SPAWNP_METHODDEF ++#endif /* !defined(OS_POSIX_SPAWNP_METHODDEF) */ ++ + #ifndef OS_SPAWNV_METHODDEF + #define OS_SPAWNV_METHODDEF + #endif /* !defined(OS_SPAWNV_METHODDEF) */ +diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c +index dc6a22f..8d0e312 100644 +--- a/Modules/posixmodule.c ++++ b/Modules/posixmodule.c +@@ -5357,39 +5357,12 @@ fail: + return -1; + } + +-/*[clinic input] +- +-os.posix_spawn +- path: path_t +- Path of executable file. +- argv: object +- Tuple or list of strings. +- env: object +- Dictionary of strings mapping to strings. +- / +- * +- file_actions: object(c_default='NULL') = () +- A sequence of file action tuples. +- setpgroup: object = NULL +- The pgroup to use with the POSIX_SPAWN_SETPGROUP flag. +- resetids: bool(accept={int}) = False +- If the value is `True` the POSIX_SPAWN_RESETIDS will be activated. +- setsigmask: object(c_default='NULL') = () +- The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag. +- setsigdef: object(c_default='NULL') = () +- The sigmask to use with the POSIX_SPAWN_SETSIGDEF flag. +- scheduler: object = NULL +- A tuple with the scheduler policy (optional) and parameters. +- +-Execute the program specified by path in a new process. +-[clinic start generated code]*/ + + static PyObject * +-os_posix_spawn_impl(PyObject *module, path_t *path, PyObject *argv, +- PyObject *env, PyObject *file_actions, +- PyObject *setpgroup, int resetids, PyObject *setsigmask, +- PyObject *setsigdef, PyObject *scheduler) +-/*[clinic end generated code: output=45dfa4c515d09f2c input=2891c2f1d457e39b]*/ ++py_posix_spawn(int use_posix_spawnp, PyObject *module, path_t *path, PyObject *argv, ++ PyObject *env, PyObject *file_actions, ++ PyObject *setpgroup, int resetids, PyObject *setsigmask, ++ PyObject *setsigdef, PyObject *scheduler) + { + EXECV_CHAR **argvlist = NULL; + EXECV_CHAR **envlist = NULL; +@@ -5465,9 +5438,19 @@ os_posix_spawn_impl(PyObject *module, path_t *path, PyObject *argv, + attrp = &attr; + + _Py_BEGIN_SUPPRESS_IPH +- err_code = posix_spawn(&pid, path->narrow, +- file_actionsp, attrp, argvlist, envlist); ++#ifdef HAVE_POSIX_SPAWNP ++ if (use_posix_spawnp) { ++ err_code = posix_spawnp(&pid, path->narrow, ++ file_actionsp, attrp, argvlist, envlist); ++ } ++ else ++#endif /* HAVE_POSIX_SPAWNP */ ++ { ++ err_code = posix_spawn(&pid, path->narrow, ++ file_actionsp, attrp, argvlist, envlist); ++ } + _Py_END_SUPPRESS_IPH ++ + if (err_code) { + errno = err_code; + PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, path->object); +@@ -5491,7 +5474,90 @@ exit: + Py_XDECREF(temp_buffer); + return result; + } +-#endif /* HAVE_POSIX_SPAWN */ ++ ++ ++/*[clinic input] ++ ++os.posix_spawn ++ path: path_t ++ Path of executable file. ++ argv: object ++ Tuple or list of strings. ++ env: object ++ Dictionary of strings mapping to strings. ++ / ++ * ++ file_actions: object(c_default='NULL') = () ++ A sequence of file action tuples. ++ setpgroup: object = NULL ++ The pgroup to use with the POSIX_SPAWN_SETPGROUP flag. ++ resetids: bool(accept={int}) = False ++ If the value is `True` the POSIX_SPAWN_RESETIDS will be activated. ++ setsigmask: object(c_default='NULL') = () ++ The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag. ++ setsigdef: object(c_default='NULL') = () ++ The sigmask to use with the POSIX_SPAWN_SETSIGDEF flag. ++ scheduler: object = NULL ++ A tuple with the scheduler policy (optional) and parameters. ++ ++Execute the program specified by path in a new process. ++[clinic start generated code]*/ ++ ++static PyObject * ++os_posix_spawn_impl(PyObject *module, path_t *path, PyObject *argv, ++ PyObject *env, PyObject *file_actions, ++ PyObject *setpgroup, int resetids, PyObject *setsigmask, ++ PyObject *setsigdef, PyObject *scheduler) ++/*[clinic end generated code: output=45dfa4c515d09f2c input=2891c2f1d457e39b]*/ ++{ ++ return py_posix_spawn(0, module, path, argv, env, file_actions, ++ setpgroup, resetids, setsigmask, setsigdef, ++ scheduler); ++} ++ #endif /* HAVE_POSIX_SPAWN */ ++ ++ ++ ++#ifdef HAVE_POSIX_SPAWNP ++/*[clinic input] ++ ++os.posix_spawnp ++ path: path_t ++ Path of executable file. ++ argv: object ++ Tuple or list of strings. ++ env: object ++ Dictionary of strings mapping to strings. ++ / ++ * ++ file_actions: object(c_default='NULL') = () ++ A sequence of file action tuples. ++ setpgroup: object = NULL ++ The pgroup to use with the POSIX_SPAWN_SETPGROUP flag. ++ resetids: bool(accept={int}) = False ++ If the value is `True` the POSIX_SPAWN_RESETIDS will be activated. ++ setsigmask: object(c_default='NULL') = () ++ The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag. ++ setsigdef: object(c_default='NULL') = () ++ The sigmask to use with the POSIX_SPAWN_SETSIGDEF flag. ++ scheduler: object = NULL ++ A tuple with the scheduler policy (optional) and parameters. ++ ++Execute the program specified by path in a new process. ++[clinic start generated code]*/ ++ ++static PyObject * ++os_posix_spawnp_impl(PyObject *module, path_t *path, PyObject *argv, ++ PyObject *env, PyObject *file_actions, ++ PyObject *setpgroup, int resetids, PyObject *setsigmask, ++ PyObject *setsigdef, PyObject *scheduler) ++/*[clinic end generated code: output=7955dc0edc82b8c3 input=b7576eb25b1ed9eb]*/ ++{ ++ return py_posix_spawn(1, module, path, argv, env, file_actions, ++ setpgroup, resetids, setsigmask, setsigdef, ++ scheduler); ++} ++#endif /* HAVE_POSIX_SPAWNP */ + + + #if defined(HAVE_SPAWNV) || defined(HAVE_WSPAWNV) +@@ -13056,6 +13122,7 @@ static PyMethodDef posix_methods[] = { + OS_GETPRIORITY_METHODDEF + OS_SETPRIORITY_METHODDEF + OS_POSIX_SPAWN_METHODDEF ++ OS_POSIX_SPAWNP_METHODDEF + #ifdef HAVE_READLINK + {"readlink", (PyCFunction)posix_readlink, + METH_VARARGS | METH_KEYWORDS, +diff --git a/configure b/configure +index 829dd69..820fa5e 100755 +--- a/configure ++++ b/configure +@@ -11499,7 +11499,7 @@ for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ + initgroups kill killpg lchown lockf linkat lstat lutimes mmap \ + memrchr mbrtowc mkdirat mkfifo \ + mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \ +- posix_fallocate posix_fadvise posix_spawn pread preadv preadv2 \ ++ posix_fallocate posix_fadvise posix_spawn posix_spawnp pread preadv preadv2 \ + pthread_init pthread_kill putenv pwrite pwritev pwritev2 readlink readlinkat readv realpath renameat \ + sem_open sem_timedwait sem_getvalue sem_unlink sendfile setegid seteuid \ + setgid sethostname \ +diff --git a/configure.ac b/configure.ac +index f1cc8e9..493d28f 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -3583,7 +3583,7 @@ AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ + initgroups kill killpg lchown lockf linkat lstat lutimes mmap \ + memrchr mbrtowc mkdirat mkfifo \ + mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \ +- posix_fallocate posix_fadvise posix_spawn pread preadv preadv2 \ ++ posix_fallocate posix_fadvise posix_spawn posix_spawnp pread preadv preadv2 \ + pthread_init pthread_kill putenv pwrite pwritev pwritev2 readlink readlinkat readv realpath renameat \ + sem_open sem_timedwait sem_getvalue sem_unlink sendfile setegid seteuid \ + setgid sethostname \ +diff --git a/pyconfig.h.in b/pyconfig.h.in +index ebab5ff..d6ab0b5 100644 +--- a/pyconfig.h.in ++++ b/pyconfig.h.in +@@ -713,6 +713,9 @@ + /* Define to 1 if you have the `posix_spawn' function. */ + #undef HAVE_POSIX_SPAWN + ++/* Define to 1 if you have the `posix_spawnp' function. */ ++#undef HAVE_POSIX_SPAWNP ++ + /* Define to 1 if you have the `pread' function. */ + #undef HAVE_PREAD + +-- +2.23.0 + diff --git a/backport-35823-Allow-setsid-after-vfork-on-Linux.-GH-2294.patch b/backport-35823-Allow-setsid-after-vfork-on-Linux.-GH-2294.patch new file mode 100644 index 0000000000000000000000000000000000000000..5cfd2c72c8448a249c3ab3925519452e1600e927 --- /dev/null +++ b/backport-35823-Allow-setsid-after-vfork-on-Linux.-GH-2294.patch @@ -0,0 +1,56 @@ +From 27f95e26df82f2a9cfd3bdf517eeeb0449606538 Mon Sep 17 00:00:00 2001 +From: "Gregory P. Smith" +Date: Sat, 24 Oct 2020 12:07:35 -0700 +Subject: [PATCH] bpo-35823: Allow setsid() after vfork() on Linux. (GH-22945) + +It should just be a syscall updating a couple of fields in the kernel side +process info. Confirming, in glibc is appears to be a shim for the setsid +syscall (based on not finding any code implementing anything special for it) +and in uclibc (*much* easier to read) it is clearly just a setsid syscall shim. + +A breadcrumb _suggesting_ that it is not allowed on Darwin/macOS comes from +a commit in emacs: https://lists.gnu.org/archive/html/bug-gnu-emacs/2017-04/msg00297.html +but I don't have a way to verify if that is true or not. +As we are not supporting vfork on macOS today I just left a note in a comment. + +Conflict:NA +Reference:https://github.com/python/cpython/commit/be3c3a0e468237430ad7d19a33c60d306199a7f2 + +Signed-off-by: hanxinke +--- + Modules/_posixsubprocess.c | 5 +++-- + 1 file changed, 3 insertions(+), 2 deletions(-) + +diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c +index 3caa8f0..5845445 100644 +--- a/Modules/_posixsubprocess.c ++++ b/Modules/_posixsubprocess.c +@@ -37,6 +37,8 @@ + + #if defined(__linux__) && defined(HAVE_VFORK) && defined(HAVE_SIGNAL_H) && \ + defined(HAVE_PTHREAD_SIGMASK) && !defined(HAVE_BROKEN_PTHREAD_SIGMASK) ++/* If this is ever expanded to non-Linux platforms, verify what calls are ++ * allowed after vfork(). Ex: setsid() may be disallowed on macOS? */ + # include + # define VFORK_USABLE 1 + #endif +@@ -699,7 +701,6 @@ do_fork_exec(char *const exec_array[], + #ifdef VFORK_USABLE + if (child_sigmask) { + /* These are checked by our caller; verify them in debug builds. */ +- assert(!call_setsid); + assert(!call_setuid); + assert(!call_setgid); + assert(!call_setgroups); +@@ -969,7 +970,7 @@ subprocess_fork_exec(PyObject* self, PyObject *args) + /* Use vfork() only if it's safe. See the comment above child_exec(). */ + sigset_t old_sigs; + if (preexec_fn == Py_None && +- !call_setuid && !call_setgid && !call_setgroups && !call_setsid) { ++ !call_setuid && !call_setgid && !call_setgroups) { + /* Block all signals to ensure that no signal handlers are run in the + * child process while it shares memory with us. Note that signals + * used internally by C libraries won't be blocked by +-- +2.23.0 + diff --git a/backport-35823-subprocess-Fix-handling-of-pthread_sigmask.patch b/backport-35823-subprocess-Fix-handling-of-pthread_sigmask.patch new file mode 100644 index 0000000000000000000000000000000000000000..0f8781a004d99f002e502514cc296192a0a246cb --- /dev/null +++ b/backport-35823-subprocess-Fix-handling-of-pthread_sigmask.patch @@ -0,0 +1,68 @@ +From a51a04b531a7f5b7ce4079dc17cc19b7e5cec2ed Mon Sep 17 00:00:00 2001 +From: Alexey Izbyshev +Date: Sat, 24 Oct 2020 20:47:38 +0300 +Subject: [PATCH] bpo-35823: subprocess: Fix handling of pthread_sigmask() + errors (GH-22944) + +Using POSIX_CALL() is incorrect since pthread_sigmask() returns +the error number instead of setting errno. + +Also handle failure of the first call to pthread_sigmask() +in the parent process, and explain why we don't handle failure +of the second call in a comment. + +Conflict:NA +Reference:https://github.com/python/cpython/commit/473db47747bb8bc986d88ad81799bcbd88153ac5 + +Signed-off-by: hanxinke +--- + Modules/_posixsubprocess.c | 19 +++++++++++++++---- + 1 file changed, 15 insertions(+), 4 deletions(-) + +diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c +index a8fb851..3caa8f0 100644 +--- a/Modules/_posixsubprocess.c ++++ b/Modules/_posixsubprocess.c +@@ -568,7 +568,9 @@ child_exec(char *const exec_array[], + #ifdef VFORK_USABLE + if (child_sigmask) { + reset_signal_handlers(child_sigmask); +- POSIX_CALL(pthread_sigmask(SIG_SETMASK, child_sigmask, NULL)); ++ if ((errno = pthread_sigmask(SIG_SETMASK, child_sigmask, NULL))) { ++ goto error; ++ } + } + #endif + +@@ -979,7 +981,11 @@ subprocess_fork_exec(PyObject* self, PyObject *args) + */ + sigset_t all_sigs; + sigfillset(&all_sigs); +- pthread_sigmask(SIG_BLOCK, &all_sigs, &old_sigs); ++ if ((saved_errno = pthread_sigmask(SIG_BLOCK, &all_sigs, &old_sigs))) { ++ errno = saved_errno; ++ PyErr_SetFromErrno(PyExc_OSError); ++ goto cleanup; ++ } + old_sigmask = &old_sigs; + } + #endif +@@ -1006,8 +1012,13 @@ subprocess_fork_exec(PyObject* self, PyObject *args) + * Note that in environments where vfork() is implemented as fork(), + * such as QEMU user-mode emulation, the parent won't be blocked, + * but it won't share the address space with the child, +- * so it's still safe to unblock the signals. */ +- pthread_sigmask(SIG_SETMASK, old_sigmask, NULL); ++ * so it's still safe to unblock the signals. ++ * ++ * We don't handle errors here because this call can't fail ++ * if valid arguments are given, and because there is no good ++ * way for the caller to deal with a failure to restore ++ * the thread signal mask. */ ++ (void) pthread_sigmask(SIG_SETMASK, old_sigmask, NULL); + } + #endif + +-- +2.23.0 + diff --git a/backport-35823-subprocess-Use-vfork-instead-of-fork-on-Li.patch b/backport-35823-subprocess-Use-vfork-instead-of-fork-on-Li.patch new file mode 100644 index 0000000000000000000000000000000000000000..e2955c332d0df7ae904a28050b6496d286455fb4 --- /dev/null +++ b/backport-35823-subprocess-Use-vfork-instead-of-fork-on-Li.patch @@ -0,0 +1,421 @@ +From d227db75c726e3881c538de4723400ba740af69a Mon Sep 17 00:00:00 2001 +From: Alexey Izbyshev +Date: Sat, 24 Oct 2020 03:47:01 +0300 +Subject: [PATCH] bpo-35823: subprocess: Use vfork() instead of fork() on Linux + when safe (GH-11671) + +* bpo-35823: subprocess: Use vfork() instead of fork() on Linux when safe + +When used to run a new executable image, fork() is not a good choice +for process creation, especially if the parent has a large working set: +fork() needs to copy page tables, which is slow, and may fail on systems +where overcommit is disabled, despite that the child is not going to +touch most of its address space. + +Currently, subprocess is capable of using posix_spawn() instead, which +normally provides much better performance. However, posix_spawn() does not +support many of child setup operations exposed by subprocess.Popen(). +Most notably, it's not possible to express `close_fds=True`, which +happens to be the default, via posix_spawn(). As a result, most users +can't benefit from faster process creation, at least not without +changing their code. + +However, Linux provides vfork() system call, which creates a new process +without copying the address space of the parent, and which is actually +used by C libraries to efficiently implement posix_spawn(). Due to sharing +of the address space and even the stack with the parent, extreme care +is required to use vfork(). At least the following restrictions must hold: + +* No signal handlers must execute in the child process. Otherwise, they + might clobber memory shared with the parent, potentially confusing it. + +* Any library function called after vfork() in the child must be + async-signal-safe (as for fork()), but it must also not interact with any + library state in a way that might break due to address space sharing + and/or lack of any preparations performed by libraries on normal fork(). + POSIX.1 permits to call only execve() and _exit(), and later revisions + remove vfork() specification entirely. In practice, however, almost all + operations needed by subprocess.Popen() can be safely implemented on + Linux. + +* Due to sharing of the stack with the parent, the child must be careful + not to clobber local variables that are alive across vfork() call. + Compilers are normally aware of this and take extra care with vfork() + (and setjmp(), which has a similar problem). + +* In case the parent is privileged, special attention must be paid to vfork() + use, because sharing an address space across different privilege domains + is insecure[1]. + +This patch adds support for using vfork() instead of fork() on Linux +when it's possible to do safely given the above. In particular: + +* vfork() is not used if credential switch is requested. The reverse case + (simple subprocess.Popen() but another application thread switches + credentials concurrently) is not possible for pure-Python apps because + subprocess.Popen() and functions like os.setuid() are mutually excluded + via GIL. We might also consider to add a way to opt-out of vfork() (and + posix_spawn() on platforms where it might be implemented via vfork()) in + a future PR. + +* vfork() is not used if `preexec_fn != None`. + +With this change, subprocess will still use posix_spawn() if possible, but +will fallback to vfork() on Linux in most cases, and, failing that, +to fork(). + +[1] https://ewontfix.com/7 + +Conflict:NA +Reference:https://github.com/python/cpython/commit/976da903a746a5455998e9ca45fbc4d3ad3479d8 + +Co-authored-by: Gregory P. Smith [Google LLC] +Signed-off-by: hanxinke +--- + .../2020-10-16-07-45-35.bpo-35823.SNQo56.rst | 2 + + Modules/_posixsubprocess.c | 224 +++++++++++++++--- + configure | 2 +- + configure.ac | 2 +- + pyconfig.h.in | 3 + + 5 files changed, 204 insertions(+), 29 deletions(-) + create mode 100644 Misc/NEWS.d/next/Library/2020-10-16-07-45-35.bpo-35823.SNQo56.rst + +diff --git a/Misc/NEWS.d/next/Library/2020-10-16-07-45-35.bpo-35823.SNQo56.rst b/Misc/NEWS.d/next/Library/2020-10-16-07-45-35.bpo-35823.SNQo56.rst +new file mode 100644 +index 0000000..cd428d3 +--- /dev/null ++++ b/Misc/NEWS.d/next/Library/2020-10-16-07-45-35.bpo-35823.SNQo56.rst +@@ -0,0 +1,2 @@ ++Use ``vfork()`` instead of ``fork()`` for :func:`subprocess.Popen` on Linux ++to improve performance in cases where it is deemed safe. +diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c +index 9e5e7c6..a8fb851 100644 +--- a/Modules/_posixsubprocess.c ++++ b/Modules/_posixsubprocess.c +@@ -35,6 +35,12 @@ + # define SYS_getdents64 __NR_getdents64 + #endif + ++#if defined(__linux__) && defined(HAVE_VFORK) && defined(HAVE_SIGNAL_H) && \ ++ defined(HAVE_PTHREAD_SIGMASK) && !defined(HAVE_BROKEN_PTHREAD_SIGMASK) ++# include ++# define VFORK_USABLE 1 ++#endif ++ + #if defined(__sun) && defined(__SVR4) + /* readdir64 is used to work around Solaris 9 bug 6395699. */ + # define readdir readdir64 +@@ -394,9 +400,53 @@ _close_open_fds_maybe_unsafe(long start_fd, PyObject* py_fds_to_keep) + #endif /* else NOT (defined(__linux__) && defined(HAVE_SYS_SYSCALL_H)) */ + + ++#ifdef VFORK_USABLE ++/* Reset dispositions for all signals to SIG_DFL except for ignored ++ * signals. This way we ensure that no signal handlers can run ++ * after we unblock signals in a child created by vfork(). ++ */ ++static void ++reset_signal_handlers(const sigset_t *child_sigmask) ++{ ++ struct sigaction sa_dfl = {.sa_handler = SIG_DFL}; ++ for (int sig = 1; sig < _NSIG; sig++) { ++ /* Dispositions for SIGKILL and SIGSTOP can't be changed. */ ++ if (sig == SIGKILL || sig == SIGSTOP) { ++ continue; ++ } ++ ++ /* There is no need to reset the disposition of signals that will ++ * remain blocked across execve() since the kernel will do it. */ ++ if (sigismember(child_sigmask, sig) == 1) { ++ continue; ++ } ++ ++ struct sigaction sa; ++ /* C libraries usually return EINVAL for signals used ++ * internally (e.g. for thread cancellation), so simply ++ * skip errors here. */ ++ if (sigaction(sig, NULL, &sa) == -1) { ++ continue; ++ } ++ ++ /* void *h works as these fields are both pointer types already. */ ++ void *h = (sa.sa_flags & SA_SIGINFO ? (void *)sa.sa_sigaction : ++ (void *)sa.sa_handler); ++ if (h == SIG_IGN || h == SIG_DFL) { ++ continue; ++ } ++ ++ /* This call can't reasonably fail, but if it does, terminating ++ * the child seems to be too harsh, so ignore errors. */ ++ (void) sigaction(sig, &sa_dfl, NULL); ++ } ++} ++#endif /* VFORK_USABLE */ ++ ++ + /* +- * This function is code executed in the child process immediately after fork +- * to set things up and call exec(). ++ * This function is code executed in the child process immediately after ++ * (v)fork to set things up and call exec(). + * + * All of the code in this function must only use async-signal-safe functions, + * listed at `man 7 signal` or +@@ -404,8 +454,28 @@ _close_open_fds_maybe_unsafe(long start_fd, PyObject* py_fds_to_keep) + * + * This restriction is documented at + * http://www.opengroup.org/onlinepubs/009695399/functions/fork.html. ++ * ++ * If this function is called after vfork(), even more care must be taken. ++ * The lack of preparations that C libraries normally take on fork(), ++ * as well as sharing the address space with the parent, might make even ++ * async-signal-safe functions vfork-unsafe. In particular, on Linux, ++ * set*id() and setgroups() library functions must not be called, since ++ * they have to interact with the library-level thread list and send ++ * library-internal signals to implement per-process credentials semantics ++ * required by POSIX but not supported natively on Linux. Another reason to ++ * avoid this family of functions is that sharing an address space between ++ * processes running with different privileges is inherently insecure. ++ * See bpo-35823 for further discussion and references. ++ * ++ * In some C libraries, setrlimit() has the same thread list/signalling ++ * behavior since resource limits were per-thread attributes before ++ * Linux 2.6.10. Musl, as of 1.2.1, is known to have this issue ++ * (https://www.openwall.com/lists/musl/2020/10/15/6). ++ * ++ * If vfork-unsafe functionality is desired after vfork(), consider using ++ * syscall() to obtain it. + */ +-static void ++_Py_NO_INLINE static void + child_exec(char *const exec_array[], + char *const argv[], + char *const envp[], +@@ -419,6 +489,7 @@ child_exec(char *const exec_array[], + int call_setgid, gid_t gid, + int call_setgroups, size_t groups_size, const gid_t *groups, + int call_setuid, uid_t uid, int child_umask, ++ const void *child_sigmask, + PyObject *py_fds_to_keep, + PyObject *preexec_fn, + PyObject *preexec_fn_args_tuple) +@@ -494,6 +565,13 @@ child_exec(char *const exec_array[], + if (restore_signals) + _Py_RestoreSignals(); + ++#ifdef VFORK_USABLE ++ if (child_sigmask) { ++ reset_signal_handlers(child_sigmask); ++ POSIX_CALL(pthread_sigmask(SIG_SETMASK, child_sigmask, NULL)); ++ } ++#endif ++ + #ifdef HAVE_SETSID + if (call_setsid) + POSIX_CALL(setsid()); +@@ -586,6 +664,81 @@ error: + } + + ++/* The main purpose of this wrapper function is to isolate vfork() from both ++ * subprocess_fork_exec() and child_exec(). A child process created via ++ * vfork() executes on the same stack as the parent process while the latter is ++ * suspended, so this function should not be inlined to avoid compiler bugs ++ * that might clobber data needed by the parent later. Additionally, ++ * child_exec() should not be inlined to avoid spurious -Wclobber warnings from ++ * GCC (see bpo-35823). ++ */ ++_Py_NO_INLINE static pid_t ++do_fork_exec(char *const exec_array[], ++ char *const argv[], ++ char *const envp[], ++ const char *cwd, ++ int p2cread, int p2cwrite, ++ int c2pread, int c2pwrite, ++ int errread, int errwrite, ++ int errpipe_read, int errpipe_write, ++ int close_fds, int restore_signals, ++ int call_setsid, ++ int call_setgid, gid_t gid, ++ int call_setgroups, size_t groups_size, const gid_t *groups, ++ int call_setuid, uid_t uid, int child_umask, ++ const void *child_sigmask, ++ PyObject *py_fds_to_keep, ++ PyObject *preexec_fn, ++ PyObject *preexec_fn_args_tuple) ++{ ++ ++ pid_t pid; ++ ++#ifdef VFORK_USABLE ++ if (child_sigmask) { ++ /* These are checked by our caller; verify them in debug builds. */ ++ assert(!call_setsid); ++ assert(!call_setuid); ++ assert(!call_setgid); ++ assert(!call_setgroups); ++ assert(preexec_fn == Py_None); ++ ++ pid = vfork(); ++ } else ++#endif ++ { ++ pid = fork(); ++ } ++ ++ if (pid != 0) { ++ return pid; ++ } ++ ++ /* Child process. ++ * See the comment above child_exec() for restrictions imposed on ++ * the code below. ++ */ ++ ++ if (preexec_fn != Py_None) { ++ /* We'll be calling back into Python later so we need to do this. ++ * This call may not be async-signal-safe but neither is calling ++ * back into Python. The user asked us to use hope as a strategy ++ * to avoid deadlock... */ ++ PyOS_AfterFork_Child(); ++ } ++ ++ child_exec(exec_array, argv, envp, cwd, ++ p2cread, p2cwrite, c2pread, c2pwrite, ++ errread, errwrite, errpipe_read, errpipe_write, ++ close_fds, restore_signals, call_setsid, ++ call_setgid, gid, call_setgroups, groups_size, groups, ++ call_setuid, uid, child_umask, child_sigmask, ++ py_fds_to_keep, preexec_fn, preexec_fn_args_tuple); ++ _exit(255); ++ return 0; /* Dead code to avoid a potential compiler warning. */ ++} ++ ++ + static PyObject * + subprocess_fork_exec(PyObject* self, PyObject *args) + { +@@ -808,39 +961,56 @@ subprocess_fork_exec(PyObject* self, PyObject *args) + need_after_fork = 1; + } + +- pid = fork(); +- if (pid == 0) { +- /* Child process */ +- /* +- * Code from here to _exit() must only use async-signal-safe functions, +- * listed at `man 7 signal` or +- * http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_04.html. ++ /* NOTE: When old_sigmask is non-NULL, do_fork_exec() may use vfork(). */ ++ const void *old_sigmask = NULL; ++#ifdef VFORK_USABLE ++ /* Use vfork() only if it's safe. See the comment above child_exec(). */ ++ sigset_t old_sigs; ++ if (preexec_fn == Py_None && ++ !call_setuid && !call_setgid && !call_setgroups && !call_setsid) { ++ /* Block all signals to ensure that no signal handlers are run in the ++ * child process while it shares memory with us. Note that signals ++ * used internally by C libraries won't be blocked by ++ * pthread_sigmask(), but signal handlers installed by C libraries ++ * normally service only signals originating from *within the process*, ++ * so it should be sufficient to consider any library function that ++ * might send such a signal to be vfork-unsafe and do not call it in ++ * the child. + */ ++ sigset_t all_sigs; ++ sigfillset(&all_sigs); ++ pthread_sigmask(SIG_BLOCK, &all_sigs, &old_sigs); ++ old_sigmask = &old_sigs; ++ } ++#endif + +- if (preexec_fn != Py_None) { +- /* We'll be calling back into Python later so we need to do this. +- * This call may not be async-signal-safe but neither is calling +- * back into Python. The user asked us to use hope as a strategy +- * to avoid deadlock... */ +- PyOS_AfterFork_Child(); +- } ++ pid = do_fork_exec(exec_array, argv, envp, cwd, ++ p2cread, p2cwrite, c2pread, c2pwrite, ++ errread, errwrite, errpipe_read, errpipe_write, ++ close_fds, restore_signals, call_setsid, ++ call_setgid, gid, call_setgroups, num_groups, groups, ++ call_setuid, uid, child_umask, old_sigmask, ++ py_fds_to_keep, preexec_fn, preexec_fn_args_tuple); + +- child_exec(exec_array, argv, envp, cwd, +- p2cread, p2cwrite, c2pread, c2pwrite, +- errread, errwrite, errpipe_read, errpipe_write, +- close_fds, restore_signals, call_setsid, +- call_setgid, gid, call_setgroups, num_groups, groups, +- call_setuid, uid, child_umask, +- py_fds_to_keep, preexec_fn, preexec_fn_args_tuple); +- _exit(255); +- return NULL; /* Dead code to avoid a potential compiler warning. */ +- } + /* Parent (original) process */ + if (pid == -1) { + /* Capture errno for the exception. */ + saved_errno = errno; + } + ++#ifdef VFORK_USABLE ++ if (old_sigmask) { ++ /* vfork() semantics guarantees that the parent is blocked ++ * until the child performs _exit() or execve(), so it is safe ++ * to unblock signals once we're here. ++ * Note that in environments where vfork() is implemented as fork(), ++ * such as QEMU user-mode emulation, the parent won't be blocked, ++ * but it won't share the address space with the child, ++ * so it's still safe to unblock the signals. */ ++ pthread_sigmask(SIG_SETMASK, old_sigmask, NULL); ++ } ++#endif ++ + Py_XDECREF(cwd_obj2); + + if (need_after_fork) +diff --git a/configure b/configure +index 93615b6..35b4d8a 100755 +--- a/configure ++++ b/configure +@@ -11509,7 +11509,7 @@ for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ + sigaction sigaltstack sigfillset siginterrupt sigpending sigrelse \ + sigtimedwait sigwait sigwaitinfo snprintf strftime strlcpy symlinkat sync \ + sysconf tcgetpgrp tcsetpgrp tempnam timegm times tmpfile tmpnam tmpnam_r \ +- truncate uname unlinkat unsetenv utimensat utimes waitid waitpid wait3 wait4 \ ++ truncate uname unlinkat unsetenv utimensat utimes vfork waitid waitpid wait3 wait4 \ + wcscoll wcsftime wcsxfrm wmemcmp writev _getpty + do : + as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh` +diff --git a/configure.ac b/configure.ac +index c5ec7a9..c2e9fbb 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -3593,7 +3593,7 @@ AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ + sigaction sigaltstack sigfillset siginterrupt sigpending sigrelse \ + sigtimedwait sigwait sigwaitinfo snprintf strftime strlcpy symlinkat sync \ + sysconf tcgetpgrp tcsetpgrp tempnam timegm times tmpfile tmpnam tmpnam_r \ +- truncate uname unlinkat unsetenv utimensat utimes waitid waitpid wait3 wait4 \ ++ truncate uname unlinkat unsetenv utimensat utimes vfork waitid waitpid wait3 wait4 \ + wcscoll wcsftime wcsxfrm wmemcmp writev _getpty) + + # Force lchmod off for Linux. Linux disallows changing the mode of symbolic +diff --git a/pyconfig.h.in b/pyconfig.h.in +index 55f3fca..5ef1a34 100644 +--- a/pyconfig.h.in ++++ b/pyconfig.h.in +@@ -1233,6 +1233,9 @@ + /* Define to 1 if you have the header file. */ + #undef HAVE_UUID_UUID_H + ++/* Define to 1 if you have the `vfork' function. */ ++#undef HAVE_VFORK ++ + /* Define to 1 if you have the `wait3' function. */ + #undef HAVE_WAIT3 + +-- +2.23.0 + diff --git a/backport-36046-Add-user-and-group-parameters-to-subproces.patch b/backport-36046-Add-user-and-group-parameters-to-subproces.patch new file mode 100644 index 0000000000000000000000000000000000000000..84a89f1640259961679f4a19d9aaceb6bbc718b8 --- /dev/null +++ b/backport-36046-Add-user-and-group-parameters-to-subproces.patch @@ -0,0 +1,739 @@ +From df76b86c86a24a0b7faefac03e22d702359eb6b5 Mon Sep 17 00:00:00 2001 +From: Patrick McLean <47801044+patrick-mclean@users.noreply.github.com> +Date: Thu, 12 Sep 2019 10:15:44 -0700 +Subject: [PATCH] bpo-36046: Add user and group parameters to subprocess + (GH-11950) + +* subprocess: Add user, group and extra_groups paremeters to subprocess.Popen + +This adds a `user` parameter to the Popen constructor that will call +setreuid() in the child before calling exec(). This allows processes +running as root to safely drop privileges before running the subprocess +without having to use a preexec_fn. + +This also adds a `group` parameter that will call setregid() in +the child process before calling exec(). + +Finally an `extra_groups` parameter was added that will call +setgroups() to set the supplimental groups. + +Conflict:NA +Reference:https://github.com/python/cpython/commit/2b2ead74382513d0bb9ef34504e283a71e6a706f + +Signed-off-by: hanxinke +--- + Doc/library/subprocess.rst | 32 +++- + Lib/multiprocessing/util.py | 2 +- + Lib/subprocess.py | 106 +++++++++++- + Lib/test/test_capi.py | 6 +- + Lib/test/test_subprocess.py | 163 +++++++++++++++++- + .../2019-02-19-17-32-45.bpo-36046.fX9OPj.rst | 2 + + Modules/_posixsubprocess.c | 136 ++++++++++++++- + 7 files changed, 428 insertions(+), 19 deletions(-) + create mode 100644 Misc/NEWS.d/next/Library/2019-02-19-17-32-45.bpo-36046.fX9OPj.rst + +diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst +index 9f2a056..3461297 100644 +--- a/Doc/library/subprocess.rst ++++ b/Doc/library/subprocess.rst +@@ -339,8 +339,9 @@ functions. + stderr=None, preexec_fn=None, close_fds=True, shell=False, \ + cwd=None, env=None, universal_newlines=None, \ + startupinfo=None, creationflags=0, restore_signals=True, \ +- start_new_session=False, pass_fds=(), *, \ +- encoding=None, errors=None, text=None) ++ start_new_session=False, pass_fds=(), *, group=None, \ ++ extra_groups=None, user=None, encoding=None, errors=None, \ ++ text=None) + + Execute a child program in a new process. On POSIX, the class uses + :meth:`os.execvp`-like behavior to execute the child program. On Windows, +@@ -520,6 +521,33 @@ functions. + .. versionchanged:: 3.2 + *start_new_session* was added. + ++ If *group* is not ``None``, the setregid() system call will be made in the ++ child process prior to the execution of the subprocess. If the provided ++ value is a string, it will be looked up via :func:`grp.getgrnam()` and ++ the value in ``gr_gid`` will be used. If the value is an integer, it ++ will be passed verbatim. (POSIX only) ++ ++ .. availability:: POSIX ++ .. versionadded:: 3.9 ++ ++ If *extra_groups* is not ``None``, the setgroups() system call will be ++ made in the child process prior to the execution of the subprocess. ++ Strings provided in *extra_groups* will be looked up via ++ :func:`grp.getgrnam()` and the values in ``gr_gid`` will be used. ++ Integer values will be passed verbatim. (POSIX only) ++ ++ .. availability:: POSIX ++ .. versionadded:: 3.9 ++ ++ If *user* is not ``None``, the setreuid() system call will be made in the ++ child process prior to the execution of the subprocess. If the provided ++ value is a string, it will be looked up via :func:`pwd.getpwnam()` and ++ the value in ``pw_uid`` will be used. If the value is an integer, it will ++ be passed verbatim. (POSIX only) ++ ++ .. availability:: POSIX ++ .. versionadded:: 3.9 ++ + If *env* is not ``None``, it must be a mapping that defines the environment + variables for the new process; these are used instead of the default + behavior of inheriting the current process' environment. +diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py +index 327fe42..c7e9b85 100644 +--- a/Lib/multiprocessing/util.py ++++ b/Lib/multiprocessing/util.py +@@ -452,7 +452,7 @@ def spawnv_passfds(path, args, passfds): + return _posixsubprocess.fork_exec( + args, [os.fsencode(path)], True, passfds, None, None, + -1, -1, -1, -1, -1, -1, errpipe_read, errpipe_write, +- False, False, None) ++ False, False, None, None, None, None) + finally: + os.close(errpipe_read) + os.close(errpipe_write) +diff --git a/Lib/subprocess.py b/Lib/subprocess.py +index 332c19f..12ca9ef 100644 +--- a/Lib/subprocess.py ++++ b/Lib/subprocess.py +@@ -54,6 +54,15 @@ import errno + import contextlib + from time import monotonic as _time + ++try: ++ import pwd ++except ImportError: ++ pwd = None ++try: ++ import grp ++except ImportError: ++ grp = None ++ + # Exception classes used by this module. + class SubprocessError(Exception): pass + +@@ -720,6 +729,12 @@ class Popen(object): + + start_new_session (POSIX only) + ++ group (POSIX only) ++ ++ extra_groups (POSIX only) ++ ++ user (POSIX only) ++ + pass_fds (POSIX only) + + encoding and errors: Text mode encoding and error handling to use for +@@ -736,7 +751,8 @@ class Popen(object): + shell=False, cwd=None, env=None, universal_newlines=None, + startupinfo=None, creationflags=0, + restore_signals=True, start_new_session=False, +- pass_fds=(), *, encoding=None, errors=None, text=None): ++ pass_fds=(), *, user=None, group=None, extra_groups=None, ++ encoding=None, errors=None, text=None): + """Create new Popen instance.""" + _cleanup() + # Held while anything is calling waitpid before returncode has been +@@ -825,6 +841,78 @@ class Popen(object): + + self._closed_child_pipe_fds = False + ++ gid = None ++ if group is not None: ++ if not hasattr(os, 'setregid'): ++ raise ValueError("The 'group' parameter is not supported on the " ++ "current platform") ++ ++ elif isinstance(group, str): ++ if grp is None: ++ raise ValueError("The group parameter cannot be a string " ++ "on systems without the grp module") ++ ++ gid = grp.getgrnam(group).gr_gid ++ elif isinstance(group, int): ++ gid = group ++ else: ++ raise TypeError("Group must be a string or an integer, not {}" ++ .format(type(group))) ++ ++ if gid < 0: ++ raise ValueError(f"Group ID cannot be negative, got {gid}") ++ ++ gids = None ++ if extra_groups is not None: ++ if not hasattr(os, 'setgroups'): ++ raise ValueError("The 'extra_groups' parameter is not " ++ "supported on the current platform") ++ ++ elif isinstance(extra_groups, str): ++ raise ValueError("Groups must be a list, not a string") ++ ++ gids = [] ++ for extra_group in extra_groups: ++ if isinstance(extra_group, str): ++ if grp is None: ++ raise ValueError("Items in extra_groups cannot be " ++ "strings on systems without the " ++ "grp module") ++ ++ gids.append(grp.getgrnam(extra_group).gr_gid) ++ elif isinstance(extra_group, int): ++ gids.append(extra_group) ++ else: ++ raise TypeError("Items in extra_groups must be a string " ++ "or integer, not {}" ++ .format(type(extra_group))) ++ ++ # make sure that the gids are all positive here so we can do less ++ # checking in the C code ++ for gid_check in gids: ++ if gid_check < 0: ++ raise ValueError(f"Group ID cannot be negative, got {gid_check}") ++ ++ uid = None ++ if user is not None: ++ if not hasattr(os, 'setreuid'): ++ raise ValueError("The 'user' parameter is not supported on " ++ "the current platform") ++ ++ elif isinstance(user, str): ++ if pwd is None: ++ raise ValueError("The user parameter cannot be a string " ++ "on systems without the pwd module") ++ ++ uid = pwd.getpwnam(user).pw_uid ++ elif isinstance(user, int): ++ uid = user ++ else: ++ raise TypeError("User must be a string or an integer") ++ ++ if uid < 0: ++ raise ValueError(f"User ID cannot be negative, got {uid}") ++ + try: + if p2cwrite != -1: + self.stdin = io.open(p2cwrite, 'wb', bufsize) +@@ -849,7 +937,9 @@ class Popen(object): + p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite, +- restore_signals, start_new_session) ++ restore_signals, ++ gid, gids, uid, ++ start_new_session) + except: + # Cleanup if the child failed starting. + for f in filter(None, (self.stdin, self.stdout, self.stderr)): +@@ -1219,7 +1309,9 @@ class Popen(object): + p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite, +- unused_restore_signals, unused_start_new_session): ++ unused_restore_signals, ++ unused_gid, unused_gids, unused_uid, ++ unused_start_new_session): + """Execute program (MS Windows version)""" + + assert not pass_fds, "pass_fds not supported on Windows." +@@ -1526,7 +1618,9 @@ class Popen(object): + p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite, +- restore_signals, start_new_session): ++ restore_signals, ++ gid, gids, uid, ++ start_new_session): + """Execute program (POSIX version)""" + + if isinstance(args, (str, bytes)): +@@ -1607,7 +1701,9 @@ class Popen(object): + p2cread, p2cwrite, c2pread, c2pwrite, + errread, errwrite, + errpipe_read, errpipe_write, +- restore_signals, start_new_session, preexec_fn) ++ restore_signals, start_new_session, ++ gid, gids, uid, ++ preexec_fn) + self._child_created = True + finally: + # be sure the FD is closed no matter what +diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py +index 3ed2263..dcff095 100644 +--- a/Lib/test/test_capi.py ++++ b/Lib/test/test_capi.py +@@ -96,7 +96,7 @@ class CAPITest(unittest.TestCase): + def __len__(self): + return 1 + self.assertRaises(TypeError, _posixsubprocess.fork_exec, +- 1,Z(),3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17) ++ 1,Z(),3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20) + # Issue #15736: overflow in _PySequence_BytesToCharpArray() + class Z(object): + def __len__(self): +@@ -104,7 +104,7 @@ class CAPITest(unittest.TestCase): + def __getitem__(self, i): + return b'x' + self.assertRaises(MemoryError, _posixsubprocess.fork_exec, +- 1,Z(),3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17) ++ 1,Z(),3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20) + + @unittest.skipUnless(_posixsubprocess, '_posixsubprocess required for this test.') + def test_subprocess_fork_exec(self): +@@ -114,7 +114,7 @@ class CAPITest(unittest.TestCase): + + # Issue #15738: crash in subprocess_fork_exec() + self.assertRaises(TypeError, _posixsubprocess.fork_exec, +- Z(),[b'1'],3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17) ++ Z(),[b'1'],3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20) + + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") +diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py +index eebd348..6c2fd61 100644 +--- a/Lib/test/test_subprocess.py ++++ b/Lib/test/test_subprocess.py +@@ -19,6 +19,7 @@ import shutil + import threading + import gc + import textwrap ++import json + from test.support import FakePath + + try: +@@ -33,6 +34,15 @@ try: + except ImportError: + _testcapi = None + ++try: ++ import pwd ++except ImportError: ++ pwd = None ++try: ++ import grp ++except ImportError: ++ grp = None ++ + if support.PGO: + raise unittest.SkipTest("test is not helpful for PGO") + +@@ -1681,6 +1691,139 @@ class POSIXProcessTestCase(BaseTestCase): + child_sid = int(output) + self.assertNotEqual(parent_sid, child_sid) + ++ @unittest.skipUnless(hasattr(os, 'setreuid'), 'no setreuid on platform') ++ def test_user(self): ++ # For code coverage of the user parameter. We don't care if we get an ++ # EPERM error from it depending on the test execution environment, that ++ # still indicates that it was called. ++ ++ uid = os.geteuid() ++ test_users = [65534 if uid != 65534 else 65533, uid] ++ name_uid = "nobody" if sys.platform != 'darwin' else "unknown" ++ ++ if pwd is not None: ++ test_users.append(name_uid) ++ ++ for user in test_users: ++ with self.subTest(user=user): ++ try: ++ output = subprocess.check_output( ++ [sys.executable, "-c", ++ "import os; print(os.getuid())"], ++ user=user) ++ except OSError as e: ++ if e.errno != errno.EPERM: ++ raise ++ else: ++ if isinstance(user, str): ++ user_uid = pwd.getpwnam(user).pw_uid ++ else: ++ user_uid = user ++ child_user = int(output) ++ self.assertEqual(child_user, user_uid) ++ ++ with self.assertRaises(ValueError): ++ subprocess.check_call([sys.executable, "-c", "pass"], user=-1) ++ ++ if pwd is None: ++ with self.assertRaises(ValueError): ++ subprocess.check_call([sys.executable, "-c", "pass"], user=name_uid) ++ ++ @unittest.skipIf(hasattr(os, 'setreuid'), 'setreuid() available on platform') ++ def test_user_error(self): ++ with self.assertRaises(ValueError): ++ subprocess.check_call([sys.executable, "-c", "pass"], user=65535) ++ ++ @unittest.skipUnless(hasattr(os, 'setregid'), 'no setregid() on platform') ++ def test_group(self): ++ gid = os.getegid() ++ group_list = [65534 if gid != 65534 else 65533] ++ name_group = "nogroup" if sys.platform != 'darwin' else "staff" ++ ++ if grp is not None: ++ group_list.append(name_group) ++ ++ for group in group_list + [gid]: ++ with self.subTest(group=group): ++ try: ++ output = subprocess.check_output( ++ [sys.executable, "-c", ++ "import os; print(os.getgid())"], ++ group=group) ++ except OSError as e: ++ if e.errno != errno.EPERM: ++ raise ++ else: ++ if isinstance(group, str): ++ group_gid = grp.getgrnam(group).gr_gid ++ else: ++ group_gid = group ++ ++ child_group = int(output) ++ self.assertEqual(child_group, group_gid) ++ ++ # make sure we bomb on negative values ++ with self.assertRaises(ValueError): ++ subprocess.check_call([sys.executable, "-c", "pass"], group=-1) ++ ++ if grp is None: ++ with self.assertRaises(ValueError): ++ subprocess.check_call([sys.executable, "-c", "pass"], group=name_group) ++ ++ @unittest.skipIf(hasattr(os, 'setregid'), 'setregid() available on platform') ++ def test_group_error(self): ++ with self.assertRaises(ValueError): ++ subprocess.check_call([sys.executable, "-c", "pass"], group=65535) ++ ++ @unittest.skipUnless(hasattr(os, 'setgroups'), 'no setgroups() on platform') ++ def test_extra_groups(self): ++ gid = os.getegid() ++ group_list = [65534 if gid != 65534 else 65533] ++ name_group = "nogroup" if sys.platform != 'darwin' else "staff" ++ perm_error = False ++ ++ if grp is not None: ++ group_list.append(name_group) ++ ++ try: ++ output = subprocess.check_output( ++ [sys.executable, "-c", ++ "import os, sys, json; json.dump(os.getgroups(), sys.stdout)"], ++ extra_groups=group_list) ++ except OSError as ex: ++ if ex.errno != errno.EPERM: ++ raise ++ perm_error = True ++ ++ else: ++ parent_groups = os.getgroups() ++ child_groups = json.loads(output) ++ ++ if grp is not None: ++ desired_gids = [grp.getgrnam(g).gr_gid if isinstance(g, str) else g ++ for g in group_list] ++ else: ++ desired_gids = group_list ++ ++ if perm_error: ++ self.assertEqual(set(child_groups), set(parent_groups)) ++ else: ++ self.assertEqual(set(desired_gids), set(child_groups)) ++ ++ # make sure we bomb on negative values ++ with self.assertRaises(ValueError): ++ subprocess.check_call([sys.executable, "-c", "pass"], extra_groups=[-1]) ++ ++ if grp is None: ++ with self.assertRaises(ValueError): ++ subprocess.check_call([sys.executable, "-c", "pass"], ++ extra_groups=[name_group]) ++ ++ @unittest.skipIf(hasattr(os, 'setgroups'), 'setgroups() available on platform') ++ def test_extra_groups_error(self): ++ with self.assertRaises(ValueError): ++ subprocess.check_call([sys.executable, "-c", "pass"], extra_groups=[]) ++ + def test_run_abort(self): + # returncode handles signal termination + with support.SuppressCrashReport(): +@@ -2730,13 +2873,23 @@ class POSIXProcessTestCase(BaseTestCase): + ([b"arg"], [b"exe"], 123, [b"env"]), + ([b"arg"], [b"exe"], None, 123), + ): +- with self.assertRaises(TypeError): ++ with self.assertRaises(TypeError) as err: + _posixsubprocess.fork_exec( + args, exe_list, + True, (), cwd, env_list, + -1, -1, -1, -1, + 1, 2, 3, 4, +- True, True, func) ++ True, True, ++ False, [], 0, ++ func) ++ # Attempt to prevent ++ # "TypeError: fork_exec() takes exactly N arguments (M given)" ++ # from passing the test. More refactoring to have us start ++ # with a valid *args list, confirm a good call with that works ++ # before mutating it in various ways to ensure that bad calls ++ # with individual arg type errors raise a typeerror would be ++ # ideal. Saving that for a future PR... ++ self.assertNotIn('takes exactly', str(err.exception)) + finally: + if not gc_enabled: + gc.disable() +@@ -2775,7 +2928,9 @@ class POSIXProcessTestCase(BaseTestCase): + True, fds_to_keep, None, [b"env"], + -1, -1, -1, -1, + 1, 2, 3, 4, +- True, True, None) ++ True, True, ++ None, None, None, ++ None) + self.assertIn('fds_to_keep', str(c.exception)) + finally: + if not gc_enabled: +@@ -3198,7 +3353,7 @@ class MiscTests(unittest.TestCase): + + def test__all__(self): + """Ensure that __all__ is populated properly.""" +- intentionally_excluded = {"list2cmdline", "Handle"} ++ intentionally_excluded = {"list2cmdline", "Handle", "pwd", "grp"} + exported = set(subprocess.__all__) + possible_exports = set() + import types +diff --git a/Misc/NEWS.d/next/Library/2019-02-19-17-32-45.bpo-36046.fX9OPj.rst b/Misc/NEWS.d/next/Library/2019-02-19-17-32-45.bpo-36046.fX9OPj.rst +new file mode 100644 +index 0000000..5e809d6 +--- /dev/null ++++ b/Misc/NEWS.d/next/Library/2019-02-19-17-32-45.bpo-36046.fX9OPj.rst +@@ -0,0 +1,2 @@ ++Added ``user``, ``group`` and ``extra_groups`` parameters to the ++subprocess.Popen constructor. Patch by Patrick McLean. +diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c +index 3cf0683..caa8d7a 100644 +--- a/Modules/_posixsubprocess.c ++++ b/Modules/_posixsubprocess.c +@@ -20,6 +20,11 @@ + #ifdef HAVE_DIRENT_H + #include + #endif ++#ifdef HAVE_GRP_H ++#include ++#endif /* HAVE_GRP_H */ ++ ++#include "posixmodule.h" + + #ifdef _Py_MEMORY_SANITIZER + # include +@@ -47,6 +52,12 @@ + # define FD_DIR "/proc/self/fd" + #endif + ++#ifdef NGROUPS_MAX ++#define MAX_GROUPS NGROUPS_MAX ++#else ++#define MAX_GROUPS 64 ++#endif ++ + #define POSIX_CALL(call) do { if ((call) == -1) goto error; } while (0) + + +@@ -405,6 +416,9 @@ child_exec(char *const exec_array[], + int errpipe_read, int errpipe_write, + int close_fds, int restore_signals, + int call_setsid, ++ int call_setgid, gid_t gid, ++ int call_setgroups, size_t groups_size, const gid_t *groups, ++ int call_setuid, uid_t uid, + PyObject *py_fds_to_keep, + PyObject *preexec_fn, + PyObject *preexec_fn_args_tuple) +@@ -482,6 +496,22 @@ child_exec(char *const exec_array[], + POSIX_CALL(setsid()); + #endif + ++#ifdef HAVE_SETGROUPS ++ if (call_setgroups) ++ POSIX_CALL(setgroups(groups_size, groups)); ++#endif /* HAVE_SETGROUPS */ ++ ++#ifdef HAVE_SETREGID ++ if (call_setgid) ++ POSIX_CALL(setregid(gid, gid)); ++#endif /* HAVE_SETREGID */ ++ ++#ifdef HAVE_SETREUID ++ if (call_setuid) ++ POSIX_CALL(setreuid(uid, uid)); ++#endif /* HAVE_SETREUID */ ++ ++ + reached_preexec = 1; + if (preexec_fn != Py_None && preexec_fn_args_tuple) { + /* This is where the user has asked us to deadlock their program. */ +@@ -561,26 +591,33 @@ subprocess_fork_exec(PyObject* self, PyObject *args) + PyObject *env_list, *preexec_fn; + PyObject *process_args, *converted_args = NULL, *fast_args = NULL; + PyObject *preexec_fn_args_tuple = NULL; ++ PyObject *groups_list; ++ PyObject *uid_object, *gid_object; + int p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite; + int errpipe_read, errpipe_write, close_fds, restore_signals; + int call_setsid; ++ int call_setgid = 0, call_setgroups = 0, call_setuid = 0; ++ uid_t uid; ++ gid_t gid, *groups = NULL; + PyObject *cwd_obj, *cwd_obj2; + const char *cwd; + pid_t pid; + int need_to_reenable_gc = 0; + char *const *exec_array, *const *argv = NULL, *const *envp = NULL; +- Py_ssize_t arg_num; ++ Py_ssize_t arg_num, num_groups = 0; + int need_after_fork = 0; + int saved_errno = 0; + + if (!PyArg_ParseTuple( +- args, "OOpO!OOiiiiiiiiiiO:fork_exec", ++ args, "OOpO!OOiiiiiiiiiiOOOO:fork_exec", + &process_args, &executable_list, + &close_fds, &PyTuple_Type, &py_fds_to_keep, + &cwd_obj, &env_list, + &p2cread, &p2cwrite, &c2pread, &c2pwrite, + &errread, &errwrite, &errpipe_read, &errpipe_write, +- &restore_signals, &call_setsid, &preexec_fn)) ++ &restore_signals, &call_setsid, ++ &gid_object, &groups_list, &uid_object, ++ &preexec_fn)) + return NULL; + + if (close_fds && errpipe_write < 3) { /* precondition */ +@@ -672,6 +709,90 @@ subprocess_fork_exec(PyObject* self, PyObject *args) + cwd_obj2 = NULL; + } + ++ if (groups_list != Py_None) { ++#ifdef HAVE_SETGROUPS ++ Py_ssize_t i; ++ unsigned long gid; ++ ++ if (!PyList_Check(groups_list)) { ++ PyErr_SetString(PyExc_TypeError, ++ "setgroups argument must be a list"); ++ goto cleanup; ++ } ++ num_groups = PySequence_Size(groups_list); ++ ++ if (num_groups < 0) ++ goto cleanup; ++ ++ if (num_groups > MAX_GROUPS) { ++ PyErr_SetString(PyExc_ValueError, "too many groups"); ++ goto cleanup; ++ } ++ ++ if ((groups = PyMem_RawMalloc(num_groups * sizeof(gid_t))) == NULL) { ++ PyErr_SetString(PyExc_MemoryError, ++ "failed to allocate memory for group list"); ++ goto cleanup; ++ } ++ ++ for (i = 0; i < num_groups; i++) { ++ PyObject *elem; ++ elem = PySequence_GetItem(groups_list, i); ++ if (!elem) ++ goto cleanup; ++ if (!PyLong_Check(elem)) { ++ PyErr_SetString(PyExc_TypeError, ++ "groups must be integers"); ++ Py_DECREF(elem); ++ goto cleanup; ++ } else { ++ /* In posixmodule.c UnsignedLong is used as a fallback value ++ * if the value provided does not fit in a Long. Since we are ++ * already doing the bounds checking on the Python side, we ++ * can go directly to an UnsignedLong here. */ ++ if (!_Py_Gid_Converter(elem, &gid)) { ++ Py_DECREF(elem); ++ PyErr_SetString(PyExc_ValueError, "invalid group id"); ++ goto cleanup; ++ } ++ groups[i] = gid; ++ } ++ Py_DECREF(elem); ++ } ++ call_setgroups = 1; ++ ++#else /* HAVE_SETGROUPS */ ++ PyErr_BadInternalCall(); ++ goto cleanup; ++#endif /* HAVE_SETGROUPS */ ++ } ++ ++ if (gid_object != Py_None) { ++#ifdef HAVE_SETREGID ++ if (!_Py_Gid_Converter(gid_object, &gid)) ++ goto cleanup; ++ ++ call_setgid = 1; ++ ++#else /* HAVE_SETREGID */ ++ PyErr_BadInternalCall(); ++ goto cleanup; ++#endif /* HAVE_SETREUID */ ++ } ++ ++ if (uid_object != Py_None) { ++#ifdef HAVE_SETREUID ++ if (!_Py_Uid_Converter(uid_object, &uid)) ++ goto cleanup; ++ ++ call_setuid = 1; ++ ++#else /* HAVE_SETREUID */ ++ PyErr_BadInternalCall(); ++ goto cleanup; ++#endif /* HAVE_SETREUID */ ++ } ++ + /* This must be the last thing done before fork() because we do not + * want to call PyOS_BeforeFork() if there is any chance of another + * error leading to the cleanup: code without calling fork(). */ +@@ -704,6 +825,8 @@ subprocess_fork_exec(PyObject* self, PyObject *args) + p2cread, p2cwrite, c2pread, c2pwrite, + errread, errwrite, errpipe_read, errpipe_write, + close_fds, restore_signals, call_setsid, ++ call_setgid, gid, call_setgroups, num_groups, groups, ++ call_setuid, uid, + py_fds_to_keep, preexec_fn, preexec_fn_args_tuple); + _exit(255); + return NULL; /* Dead code to avoid a potential compiler warning. */ +@@ -748,6 +871,8 @@ cleanup: + _Py_FreeCharPArray(argv); + if (exec_array) + _Py_FreeCharPArray(exec_array); ++ ++ PyMem_RawFree(groups); + Py_XDECREF(converted_args); + Py_XDECREF(fast_args); + Py_XDECREF(preexec_fn_args_tuple); +@@ -761,7 +886,10 @@ PyDoc_STRVAR(subprocess_fork_exec_doc, + "fork_exec(args, executable_list, close_fds, cwd, env,\n\ + p2cread, p2cwrite, c2pread, c2pwrite,\n\ + errread, errwrite, errpipe_read, errpipe_write,\n\ +- restore_signals, call_setsid, preexec_fn)\n\ ++ restore_signals, call_setsid,\n\ ++ call_setgid, gid, groups_size, gids,\n\ ++ call_setuid, uid,\n\ ++ preexec_fn)\n\ + \n\ + Forks a child process, closes parent file descriptors as appropriate in the\n\ + child and dups the few that are needed before calling exec() in the child\n\ +-- +2.23.0 + diff --git a/backport-36046-Fix-buildbot-failures-GH-16091.patch b/backport-36046-Fix-buildbot-failures-GH-16091.patch new file mode 100644 index 0000000000000000000000000000000000000000..4235a7c1d78f68879588b431c3dead0a817dda8c --- /dev/null +++ b/backport-36046-Fix-buildbot-failures-GH-16091.patch @@ -0,0 +1,71 @@ +From f0deaf10e67b96413be55e18c768b897de02dea2 Mon Sep 17 00:00:00 2001 +From: "Gregory P. Smith" +Date: Fri, 13 Sep 2019 14:43:35 +0100 +Subject: [PATCH] bpo-36046: Fix buildbot failures (GH-16091) + +Varying user/group/permission check needs on platforms. + +Conflict:NA +Reference:https://github.com/python/cpython/commit/693aa80a434590ea7dcd35c000209e53d01b9425 + +Signed-off-by: hanxinke +--- + Lib/test/test_subprocess.py | 20 +++++++++++++++++--- + 1 file changed, 17 insertions(+), 3 deletions(-) + +diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py +index 6c2fd61..aa2f539 100644 +--- a/Lib/test/test_subprocess.py ++++ b/Lib/test/test_subprocess.py +@@ -1539,6 +1539,18 @@ class RunFuncTestCase(BaseTestCase): + f"{stacks}```") + + ++def _get_test_grp_name(): ++ for name_group in ('staff', 'nogroup', 'grp'): ++ if grp: ++ try: ++ grp.getgrnam(name_group) ++ except KeyError: ++ continue ++ return name_group ++ else: ++ raise unittest.SkipTest('No identified group name to use for this test on this platform.') ++ ++ + @unittest.skipIf(mswindows, "POSIX specific tests") + class POSIXProcessTestCase(BaseTestCase): + +@@ -1711,8 +1723,10 @@ class POSIXProcessTestCase(BaseTestCase): + [sys.executable, "-c", + "import os; print(os.getuid())"], + user=user) ++ except PermissionError: # errno.EACCES ++ pass + except OSError as e: +- if e.errno != errno.EPERM: ++ if e.errno not in (errno.EACCES, errno.EPERM): + raise + else: + if isinstance(user, str): +@@ -1738,7 +1752,7 @@ class POSIXProcessTestCase(BaseTestCase): + def test_group(self): + gid = os.getegid() + group_list = [65534 if gid != 65534 else 65533] +- name_group = "nogroup" if sys.platform != 'darwin' else "staff" ++ name_group = _get_test_grp_name() + + if grp is not None: + group_list.append(name_group) +@@ -1779,7 +1793,7 @@ class POSIXProcessTestCase(BaseTestCase): + def test_extra_groups(self): + gid = os.getegid() + group_list = [65534 if gid != 65534 else 65533] +- name_group = "nogroup" if sys.platform != 'darwin' else "staff" ++ name_group = _get_test_grp_name() + perm_error = False + + if grp is not None: +-- +2.23.0 + diff --git a/backport-36046-posix_spawn-doesn-t-support-uid-gid-GH-163.patch b/backport-36046-posix_spawn-doesn-t-support-uid-gid-GH-163.patch new file mode 100644 index 0000000000000000000000000000000000000000..0657a07522970a420c1a7a1e31f0e9cc2db61087 --- /dev/null +++ b/backport-36046-posix_spawn-doesn-t-support-uid-gid-GH-163.patch @@ -0,0 +1,136 @@ +From 39129f265d74f4ed4aa424b8bc54075621622d07 Mon Sep 17 00:00:00 2001 +From: Victor Stinner +Date: Wed, 25 Sep 2019 15:52:49 +0200 +Subject: [PATCH] bpo-36046: posix_spawn() doesn't support uid/gid (GH-16384) + +* subprocess.Popen now longer uses posix_spawn() if uid, gid or gids are set. +* test_subprocess: add "nobody" and "nfsnobody" group names for test_group(). +* test_subprocess: test_user() and test_group() are now also tested with close_fds=False. + +Conflict:NA +Reference:https://github.com/python/cpython/commit/faca8553425c231d867dcabf6a69a9dd21118b6c + +Signed-off-by: hanxinke +--- + Lib/subprocess.py | 5 ++- + Lib/test/test_subprocess.py | 71 ++++++++++++++++++++----------------- + 2 files changed, 42 insertions(+), 34 deletions(-) + +diff --git a/Lib/subprocess.py b/Lib/subprocess.py +index 6e0eaf9..c80d07e 100644 +--- a/Lib/subprocess.py ++++ b/Lib/subprocess.py +@@ -1646,7 +1646,10 @@ class Popen(object): + and (p2cread == -1 or p2cread > 2) + and (c2pwrite == -1 or c2pwrite > 2) + and (errwrite == -1 or errwrite > 2) +- and not start_new_session): ++ and not start_new_session ++ and gid is None ++ and gids is None ++ and uid is None): + self._posix_spawn(args, executable, env, restore_signals, + p2cread, p2cwrite, + c2pread, c2pwrite, +diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py +index aa2f539..8360c6f 100644 +--- a/Lib/test/test_subprocess.py ++++ b/Lib/test/test_subprocess.py +@@ -1540,7 +1540,7 @@ class RunFuncTestCase(BaseTestCase): + + + def _get_test_grp_name(): +- for name_group in ('staff', 'nogroup', 'grp'): ++ for name_group in ('staff', 'nogroup', 'grp', 'nobody', 'nfsnobody'): + if grp: + try: + grp.getgrnam(name_group) +@@ -1717,24 +1717,27 @@ class POSIXProcessTestCase(BaseTestCase): + test_users.append(name_uid) + + for user in test_users: +- with self.subTest(user=user): +- try: +- output = subprocess.check_output( +- [sys.executable, "-c", +- "import os; print(os.getuid())"], +- user=user) +- except PermissionError: # errno.EACCES +- pass +- except OSError as e: +- if e.errno not in (errno.EACCES, errno.EPERM): +- raise +- else: +- if isinstance(user, str): +- user_uid = pwd.getpwnam(user).pw_uid ++ # posix_spawn() may be used with close_fds=False ++ for close_fds in (False, True): ++ with self.subTest(user=user, close_fds=close_fds): ++ try: ++ output = subprocess.check_output( ++ [sys.executable, "-c", ++ "import os; print(os.getuid())"], ++ user=user, ++ close_fds=close_fds) ++ except PermissionError: # (EACCES, EPERM) ++ pass ++ except OSError as e: ++ if e.errno not in (errno.EACCES, errno.EPERM): ++ raise + else: +- user_uid = user +- child_user = int(output) +- self.assertEqual(child_user, user_uid) ++ if isinstance(user, str): ++ user_uid = pwd.getpwnam(user).pw_uid ++ else: ++ user_uid = user ++ child_user = int(output) ++ self.assertEqual(child_user, user_uid) + + with self.assertRaises(ValueError): + subprocess.check_call([sys.executable, "-c", "pass"], user=-1) +@@ -1758,23 +1761,25 @@ class POSIXProcessTestCase(BaseTestCase): + group_list.append(name_group) + + for group in group_list + [gid]: +- with self.subTest(group=group): +- try: +- output = subprocess.check_output( +- [sys.executable, "-c", +- "import os; print(os.getgid())"], +- group=group) +- except OSError as e: +- if e.errno != errno.EPERM: +- raise +- else: +- if isinstance(group, str): +- group_gid = grp.getgrnam(group).gr_gid ++ # posix_spawn() may be used with close_fds=False ++ for close_fds in (False, True): ++ with self.subTest(group=group, close_fds=close_fds): ++ try: ++ output = subprocess.check_output( ++ [sys.executable, "-c", ++ "import os; print(os.getgid())"], ++ group=group, ++ close_fds=close_fds) ++ except PermissionError: # (EACCES, EPERM) ++ pass + else: +- group_gid = group ++ if isinstance(group, str): ++ group_gid = grp.getgrnam(group).gr_gid ++ else: ++ group_gid = group + +- child_group = int(output) +- self.assertEqual(child_group, group_gid) ++ child_group = int(output) ++ self.assertEqual(child_group, group_gid) + + # make sure we bomb on negative values + with self.assertRaises(ValueError): +-- +2.23.0 + diff --git a/backport-36814-ensure-os.posix_spawn-handles-None-GH-1314.patch b/backport-36814-ensure-os.posix_spawn-handles-None-GH-1314.patch new file mode 100644 index 0000000000000000000000000000000000000000..16e4f087acc1ffd632bb69e845d6e37f5965ac28 --- /dev/null +++ b/backport-36814-ensure-os.posix_spawn-handles-None-GH-1314.patch @@ -0,0 +1,63 @@ +From e63da81ac5d562dfad72fded544fd08566f58de8 Mon Sep 17 00:00:00 2001 +From: Anthony Shaw +Date: Fri, 10 May 2019 12:00:06 +1000 +Subject: [PATCH] bpo-36814: ensure os.posix_spawn() handles None (GH-13144) + +Fix an issue where os.posix_spawn() would incorrectly raise a TypeError +when file_actions is None. + +Conflict:NA +Reference:https://github.com/python/cpython/commit/948ed8c96b6912541a608591efe3e00fb520429a + +Signed-off-by: hanxinke +--- + Lib/test/test_posix.py | 9 +++++++++ + .../Library/2019-05-06-23-13-26.bpo-36814.dSeMz_.rst | 1 + + Modules/posixmodule.c | 2 +- + 3 files changed, 11 insertions(+), 1 deletion(-) + create mode 100644 Misc/NEWS.d/next/Library/2019-05-06-23-13-26.bpo-36814.dSeMz_.rst + +diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py +index 83accd4..9440083 100644 +--- a/Lib/test/test_posix.py ++++ b/Lib/test/test_posix.py +@@ -1560,6 +1560,15 @@ class _PosixSpawnMixin: + with open(envfile) as f: + self.assertEqual(f.read(), 'bar') + ++ def test_none_file_actions(self): ++ pid = self.spawn_func( ++ self.NOOP_PROGRAM[0], ++ self.NOOP_PROGRAM, ++ os.environ, ++ file_actions=None ++ ) ++ self.assertEqual(os.waitpid(pid, 0), (pid, 0)) ++ + def test_empty_file_actions(self): + pid = self.spawn_func( + self.NOOP_PROGRAM[0], +diff --git a/Misc/NEWS.d/next/Library/2019-05-06-23-13-26.bpo-36814.dSeMz_.rst b/Misc/NEWS.d/next/Library/2019-05-06-23-13-26.bpo-36814.dSeMz_.rst +new file mode 100644 +index 0000000..3f40011 +--- /dev/null ++++ b/Misc/NEWS.d/next/Library/2019-05-06-23-13-26.bpo-36814.dSeMz_.rst +@@ -0,0 +1 @@ ++Fix an issue where os.posix_spawnp() would incorrectly raise a TypeError when file_actions is None. +\ No newline at end of file +diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c +index 129de76..f1ab030 100644 +--- a/Modules/posixmodule.c ++++ b/Modules/posixmodule.c +@@ -5487,7 +5487,7 @@ py_posix_spawn(int use_posix_spawnp, PyObject *module, path_t *path, PyObject *a + goto exit; + } + +- if (file_actions != NULL) { ++ if (file_actions != NULL && file_actions != Py_None) { + /* There is a bug in old versions of glibc that makes some of the + * helper functions for manipulating file actions not copy the provided + * buffers. The problem is that posix_spawn_file_actions_addopen does not +-- +2.23.0 + diff --git a/backport-38417-Add-umask-support-to-subprocess-GH-16726.patch b/backport-38417-Add-umask-support-to-subprocess-GH-16726.patch new file mode 100644 index 0000000000000000000000000000000000000000..0a26812fc35158ba029fe1fcf9b5d2fdb009ad90 --- /dev/null +++ b/backport-38417-Add-umask-support-to-subprocess-GH-16726.patch @@ -0,0 +1,291 @@ +From 7ea255a0878c2486a02d0982fea1cb8e72ebd52f Mon Sep 17 00:00:00 2001 +From: "Gregory P. Smith" +Date: Sat, 12 Oct 2019 13:24:56 -0700 +Subject: [PATCH] bpo-38417: Add umask support to subprocess (GH-16726) + +On POSIX systems, allow the umask to be set in the child process before we exec. + +Conflict:NA +Reference:https://github.com/python/cpython/commit/f3751efb5c8b53b37efbbf75d9422c1d11c01646 + +Signed-off-by: hanxinke +--- + Doc/library/subprocess.rst | 12 ++++++--- + Lib/multiprocessing/util.py | 2 +- + Lib/subprocess.py | 14 ++++++---- + Lib/test/test_capi.py | 6 ++--- + Lib/test/test_subprocess.py | 26 +++++++++++++++++-- + .../2019-10-12-00-13-47.bpo-38417.W7x_aS.rst | 2 ++ + Modules/_posixsubprocess.c | 14 ++++++---- + 7 files changed, 57 insertions(+), 19 deletions(-) + create mode 100644 Misc/NEWS.d/next/Library/2019-10-12-00-13-47.bpo-38417.W7x_aS.rst + +diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst +index 3461297..009de63 100644 +--- a/Doc/library/subprocess.rst ++++ b/Doc/library/subprocess.rst +@@ -339,9 +339,9 @@ functions. + stderr=None, preexec_fn=None, close_fds=True, shell=False, \ + cwd=None, env=None, universal_newlines=None, \ + startupinfo=None, creationflags=0, restore_signals=True, \ +- start_new_session=False, pass_fds=(), *, group=None, \ +- extra_groups=None, user=None, encoding=None, errors=None, \ +- text=None) ++ start_new_session=False, pass_fds=(), \*, group=None, \ ++ extra_groups=None, user=None, umask=-1, \ ++ encoding=None, errors=None, text=None) + + Execute a child program in a new process. On POSIX, the class uses + :meth:`os.execvp`-like behavior to execute the child program. On Windows, +@@ -548,6 +548,12 @@ functions. + .. availability:: POSIX + .. versionadded:: 3.9 + ++ If *umask* is not negative, the umask() system call will be made in the ++ child process prior to the execution of the subprocess. ++ ++ .. availability:: POSIX ++ .. versionadded:: 3.9 ++ + If *env* is not ``None``, it must be a mapping that defines the environment + variables for the new process; these are used instead of the default + behavior of inheriting the current process' environment. +diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py +index c7e9b85..ff8b4ec 100644 +--- a/Lib/multiprocessing/util.py ++++ b/Lib/multiprocessing/util.py +@@ -452,7 +452,7 @@ def spawnv_passfds(path, args, passfds): + return _posixsubprocess.fork_exec( + args, [os.fsencode(path)], True, passfds, None, None, + -1, -1, -1, -1, -1, -1, errpipe_read, errpipe_write, +- False, False, None, None, None, None) ++ False, False, None, None, None, -1, None) + finally: + os.close(errpipe_read) + os.close(errpipe_write) +diff --git a/Lib/subprocess.py b/Lib/subprocess.py +index c80d07e..300ad58 100644 +--- a/Lib/subprocess.py ++++ b/Lib/subprocess.py +@@ -734,6 +734,8 @@ class Popen(object): + + user (POSIX only) + ++ umask (POSIX only) ++ + pass_fds (POSIX only) + + encoding and errors: Text mode encoding and error handling to use for +@@ -751,7 +753,7 @@ class Popen(object): + startupinfo=None, creationflags=0, + restore_signals=True, start_new_session=False, + pass_fds=(), *, user=None, group=None, extra_groups=None, +- encoding=None, errors=None, text=None): ++ encoding=None, errors=None, text=None, umask=-1): + """Create new Popen instance.""" + _cleanup() + # Held while anything is calling waitpid before returncode has been +@@ -936,7 +938,7 @@ class Popen(object): + c2pread, c2pwrite, + errread, errwrite, + restore_signals, +- gid, gids, uid, ++ gid, gids, uid, umask, + start_new_session) + except: + # Cleanup if the child failed starting. +@@ -1309,6 +1311,7 @@ class Popen(object): + errread, errwrite, + unused_restore_signals, + unused_gid, unused_gids, unused_uid, ++ unused_umask, + unused_start_new_session): + """Execute program (MS Windows version)""" + +@@ -1617,7 +1620,7 @@ class Popen(object): + c2pread, c2pwrite, + errread, errwrite, + restore_signals, +- gid, gids, uid, ++ gid, gids, uid, umask, + start_new_session): + """Execute program (POSIX version)""" + +@@ -1649,7 +1652,8 @@ class Popen(object): + and not start_new_session + and gid is None + and gids is None +- and uid is None): ++ and uid is None ++ and umask < 0): + self._posix_spawn(args, executable, env, restore_signals, + p2cread, p2cwrite, + c2pread, c2pwrite, +@@ -1703,7 +1707,7 @@ class Popen(object): + errread, errwrite, + errpipe_read, errpipe_write, + restore_signals, start_new_session, +- gid, gids, uid, ++ gid, gids, uid, umask, + preexec_fn) + self._child_created = True + finally: +diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py +index dcff095..ebcb692 100644 +--- a/Lib/test/test_capi.py ++++ b/Lib/test/test_capi.py +@@ -96,7 +96,7 @@ class CAPITest(unittest.TestCase): + def __len__(self): + return 1 + self.assertRaises(TypeError, _posixsubprocess.fork_exec, +- 1,Z(),3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20) ++ 1,Z(),3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21) + # Issue #15736: overflow in _PySequence_BytesToCharpArray() + class Z(object): + def __len__(self): +@@ -104,7 +104,7 @@ class CAPITest(unittest.TestCase): + def __getitem__(self, i): + return b'x' + self.assertRaises(MemoryError, _posixsubprocess.fork_exec, +- 1,Z(),3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20) ++ 1,Z(),3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21) + + @unittest.skipUnless(_posixsubprocess, '_posixsubprocess required for this test.') + def test_subprocess_fork_exec(self): +@@ -114,7 +114,7 @@ class CAPITest(unittest.TestCase): + + # Issue #15738: crash in subprocess_fork_exec() + self.assertRaises(TypeError, _posixsubprocess.fork_exec, +- Z(),[b'1'],3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20) ++ Z(),[b'1'],3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21) + + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") +diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py +index 8360c6f..059a007 100644 +--- a/Lib/test/test_subprocess.py ++++ b/Lib/test/test_subprocess.py +@@ -1843,6 +1843,28 @@ class POSIXProcessTestCase(BaseTestCase): + with self.assertRaises(ValueError): + subprocess.check_call([sys.executable, "-c", "pass"], extra_groups=[]) + ++ @unittest.skipIf(mswindows or not hasattr(os, 'umask'), ++ 'POSIX umask() is not available.') ++ def test_umask(self): ++ tmpdir = None ++ try: ++ tmpdir = tempfile.mkdtemp() ++ name = os.path.join(tmpdir, "beans") ++ # We set an unusual umask in the child so as a unique mode ++ # for us to test the child's touched file for. ++ subprocess.check_call( ++ [sys.executable, "-c", f"open({name!r}, 'w')"], # touch ++ umask=0o053) ++ # Ignore execute permissions entirely in our test, ++ # filesystems could be mounted to ignore or force that. ++ st_mode = os.stat(name).st_mode & 0o666 ++ expected_mode = 0o624 ++ self.assertEqual(expected_mode, st_mode, ++ msg=f'{oct(expected_mode)} != {oct(st_mode)}') ++ finally: ++ if tmpdir is not None: ++ shutil.rmtree(tmpdir) ++ + def test_run_abort(self): + # returncode handles signal termination + with support.SuppressCrashReport(): +@@ -2899,7 +2921,7 @@ class POSIXProcessTestCase(BaseTestCase): + -1, -1, -1, -1, + 1, 2, 3, 4, + True, True, +- False, [], 0, ++ False, [], 0, -1, + func) + # Attempt to prevent + # "TypeError: fork_exec() takes exactly N arguments (M given)" +@@ -2948,7 +2970,7 @@ class POSIXProcessTestCase(BaseTestCase): + -1, -1, -1, -1, + 1, 2, 3, 4, + True, True, +- None, None, None, ++ None, None, None, -1, + None) + self.assertIn('fds_to_keep', str(c.exception)) + finally: +diff --git a/Misc/NEWS.d/next/Library/2019-10-12-00-13-47.bpo-38417.W7x_aS.rst b/Misc/NEWS.d/next/Library/2019-10-12-00-13-47.bpo-38417.W7x_aS.rst +new file mode 100644 +index 0000000..c2356dd +--- /dev/null ++++ b/Misc/NEWS.d/next/Library/2019-10-12-00-13-47.bpo-38417.W7x_aS.rst +@@ -0,0 +1,2 @@ ++Added support for setting the umask in the child process to the subprocess ++module on POSIX systems. +diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c +index caa8d7a..9e5e7c6 100644 +--- a/Modules/_posixsubprocess.c ++++ b/Modules/_posixsubprocess.c +@@ -8,7 +8,7 @@ + #ifdef HAVE_SYS_TYPES_H + #include + #endif +-#if defined(HAVE_SYS_STAT_H) && defined(__FreeBSD__) ++#if defined(HAVE_SYS_STAT_H) + #include + #endif + #ifdef HAVE_SYS_SYSCALL_H +@@ -418,7 +418,7 @@ child_exec(char *const exec_array[], + int call_setsid, + int call_setgid, gid_t gid, + int call_setgroups, size_t groups_size, const gid_t *groups, +- int call_setuid, uid_t uid, ++ int call_setuid, uid_t uid, int child_umask, + PyObject *py_fds_to_keep, + PyObject *preexec_fn, + PyObject *preexec_fn_args_tuple) +@@ -488,6 +488,9 @@ child_exec(char *const exec_array[], + if (cwd) + POSIX_CALL(chdir(cwd)); + ++ if (child_umask >= 0) ++ umask(child_umask); /* umask() always succeeds. */ ++ + if (restore_signals) + _Py_RestoreSignals(); + +@@ -599,6 +602,7 @@ subprocess_fork_exec(PyObject* self, PyObject *args) + int call_setgid = 0, call_setgroups = 0, call_setuid = 0; + uid_t uid; + gid_t gid, *groups = NULL; ++ int child_umask; + PyObject *cwd_obj, *cwd_obj2; + const char *cwd; + pid_t pid; +@@ -609,14 +613,14 @@ subprocess_fork_exec(PyObject* self, PyObject *args) + int saved_errno = 0; + + if (!PyArg_ParseTuple( +- args, "OOpO!OOiiiiiiiiiiOOOO:fork_exec", ++ args, "OOpO!OOiiiiiiiiiiOOOiO:fork_exec", + &process_args, &executable_list, + &close_fds, &PyTuple_Type, &py_fds_to_keep, + &cwd_obj, &env_list, + &p2cread, &p2cwrite, &c2pread, &c2pwrite, + &errread, &errwrite, &errpipe_read, &errpipe_write, + &restore_signals, &call_setsid, +- &gid_object, &groups_list, &uid_object, ++ &gid_object, &groups_list, &uid_object, &child_umask, + &preexec_fn)) + return NULL; + +@@ -826,7 +830,7 @@ subprocess_fork_exec(PyObject* self, PyObject *args) + errread, errwrite, errpipe_read, errpipe_write, + close_fds, restore_signals, call_setsid, + call_setgid, gid, call_setgroups, num_groups, groups, +- call_setuid, uid, ++ call_setuid, uid, child_umask, + py_fds_to_keep, preexec_fn, preexec_fn_args_tuple); + _exit(255); + return NULL; /* Dead code to avoid a potential compiler warning. */ +-- +2.23.0 + diff --git a/backport-38456-Handle-the-case-when-there-is-no-true-comm.patch b/backport-38456-Handle-the-case-when-there-is-no-true-comm.patch new file mode 100644 index 0000000000000000000000000000000000000000..ec57cb650113b30c3fbe133ea27487f8737706c3 --- /dev/null +++ b/backport-38456-Handle-the-case-when-there-is-no-true-comm.patch @@ -0,0 +1,30 @@ +From 70975d9a05f6d957c669a521071f207dca6002bc Mon Sep 17 00:00:00 2001 +From: Pablo Galindo +Date: Sun, 13 Oct 2019 02:40:24 +0100 +Subject: [PATCH] bpo-38456: Handle the case when there is no 'true' command + (GH-16739) + +Conflict:NA +Reference:https://github.com/python/cpython/commit/46113e0cf32748f66cf64cd633984d143b433cd1 + +Signed-off-by: hanxinke +--- + Lib/test/test_subprocess.py | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py +index 9820507..ac45436 100644 +--- a/Lib/test/test_subprocess.py ++++ b/Lib/test/test_subprocess.py +@@ -67,6 +67,8 @@ ZERO_RETURN_CMD = (sys.executable, '-c', 'pass') + + def setUpModule(): + shell_true = shutil.which('true') ++ if shell_true is None: ++ return + if (os.access(shell_true, os.X_OK) and + subprocess.run([shell_true]).returncode == 0): + global ZERO_RETURN_CMD +-- +2.23.0 + diff --git a/backport-38456-Use-bin-true-in-test_subprocess-GH-16736.patch b/backport-38456-Use-bin-true-in-test_subprocess-GH-16736.patch new file mode 100644 index 0000000000000000000000000000000000000000..b89e63d2a0c515ff3a9f3f0d72231c28b989b80b --- /dev/null +++ b/backport-38456-Use-bin-true-in-test_subprocess-GH-16736.patch @@ -0,0 +1,422 @@ +From 080b02c44bb09cdcf0af439681250d8c71a4f245 Mon Sep 17 00:00:00 2001 +From: "Gregory P. Smith" +Date: Sat, 12 Oct 2019 16:35:53 -0700 +Subject: [PATCH] bpo-38456: Use /bin/true in test_subprocess (GH-16736) + +* bpo-38456: Use /bin/true in test_subprocess. + +Instead of sys.executable, "-c", "pass" or "import sys; sys.exit(0)" +use /bin/true when it is available. On a reasonable machine this +shaves up to two seconds wall time off the otherwise ~40sec execution +on a --with-pydebug build. It should be more notable on many +buildbots or overloaded slower I/O systems (CI, etc). + +Conflict:NA +Reference:https://github.com/python/cpython/commit/67b93f80c764bca01c81c989d74a99df208bea4d + +Signed-off-by: hanxinke +--- + Lib/test/test_subprocess.py | 108 ++++++++++++++++++++---------------- + 1 file changed, 59 insertions(+), 49 deletions(-) + +diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py +index 059a007..9820507 100644 +--- a/Lib/test/test_subprocess.py ++++ b/Lib/test/test_subprocess.py +@@ -62,6 +62,16 @@ NONEXISTING_CMD = ('nonexisting_i_hope',) + # Ignore errors that indicate the command was not found + NONEXISTING_ERRORS = (FileNotFoundError, NotADirectoryError, PermissionError) + ++ZERO_RETURN_CMD = (sys.executable, '-c', 'pass') ++ ++ ++def setUpModule(): ++ shell_true = shutil.which('true') ++ if (os.access(shell_true, os.X_OK) and ++ subprocess.run([shell_true]).returncode == 0): ++ global ZERO_RETURN_CMD ++ ZERO_RETURN_CMD = (shell_true,) # Faster than Python startup. ++ + + class BaseTestCase(unittest.TestCase): + def setUp(self): +@@ -106,7 +116,7 @@ class PopenExecuteChildRaises(subprocess.Popen): + class ProcessTestCase(BaseTestCase): + + def test_io_buffered_by_default(self): +- p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], ++ p = subprocess.Popen(ZERO_RETURN_CMD, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + try: +@@ -120,7 +130,7 @@ class ProcessTestCase(BaseTestCase): + p.wait() + + def test_io_unbuffered_works(self): +- p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], ++ p = subprocess.Popen(ZERO_RETURN_CMD, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, bufsize=0) + try: +@@ -150,8 +160,7 @@ class ProcessTestCase(BaseTestCase): + + def test_check_call_zero(self): + # check_call() function with zero return code +- rc = subprocess.check_call([sys.executable, "-c", +- "import sys; sys.exit(0)"]) ++ rc = subprocess.check_call(ZERO_RETURN_CMD) + self.assertEqual(rc, 0) + + def test_check_call_nonzero(self): +@@ -689,19 +698,19 @@ class ProcessTestCase(BaseTestCase): + newenv = os.environ.copy() + newenv["FRUIT\0VEGETABLE"] = "cabbage" + with self.assertRaises(ValueError): +- subprocess.Popen([sys.executable, "-c", "pass"], env=newenv) ++ subprocess.Popen(ZERO_RETURN_CMD, env=newenv) + + # null character in the environment variable value + newenv = os.environ.copy() + newenv["FRUIT"] = "orange\0VEGETABLE=cabbage" + with self.assertRaises(ValueError): +- subprocess.Popen([sys.executable, "-c", "pass"], env=newenv) ++ subprocess.Popen(ZERO_RETURN_CMD, env=newenv) + + # equal character in the environment variable name + newenv = os.environ.copy() + newenv["FRUIT=ORANGE"] = "lemon" + with self.assertRaises(ValueError): +- subprocess.Popen([sys.executable, "-c", "pass"], env=newenv) ++ subprocess.Popen(ZERO_RETURN_CMD, env=newenv) + + # equal character in the environment variable value + newenv = os.environ.copy() +@@ -802,7 +811,7 @@ class ProcessTestCase(BaseTestCase): + options['stderr'] = subprocess.PIPE + if not options: + continue +- p = subprocess.Popen((sys.executable, "-c", "pass"), **options) ++ p = subprocess.Popen(ZERO_RETURN_CMD, **options) + p.communicate() + if p.stdin is not None: + self.assertTrue(p.stdin.closed) +@@ -941,7 +950,7 @@ class ProcessTestCase(BaseTestCase): + # + # We set stdout to PIPE because, as of this writing, a different + # code path is tested when the number of pipes is zero or one. +- p = subprocess.Popen([sys.executable, "-c", "pass"], ++ p = subprocess.Popen(ZERO_RETURN_CMD, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + universal_newlines=True) +@@ -1089,7 +1098,7 @@ class ProcessTestCase(BaseTestCase): + self.assertEqual(p.poll(), 0) + + def test_wait(self): +- p = subprocess.Popen([sys.executable, "-c", "pass"]) ++ p = subprocess.Popen(ZERO_RETURN_CMD) + self.assertEqual(p.wait(), 0) + # Subsequent invocations should just return the returncode + self.assertEqual(p.wait(), 0) +@@ -1108,14 +1117,14 @@ class ProcessTestCase(BaseTestCase): + # an invalid type of the bufsize argument should raise + # TypeError. + with self.assertRaises(TypeError): +- subprocess.Popen([sys.executable, "-c", "pass"], "orange") ++ subprocess.Popen(ZERO_RETURN_CMD, "orange") + + def test_bufsize_is_none(self): + # bufsize=None should be the same as bufsize=0. +- p = subprocess.Popen([sys.executable, "-c", "pass"], None) ++ p = subprocess.Popen(ZERO_RETURN_CMD, None) + self.assertEqual(p.wait(), 0) + # Again with keyword arg +- p = subprocess.Popen([sys.executable, "-c", "pass"], bufsize=None) ++ p = subprocess.Popen(ZERO_RETURN_CMD, bufsize=None) + self.assertEqual(p.wait(), 0) + + def _test_bufsize_equal_one(self, line, expected, universal_newlines): +@@ -1319,7 +1328,7 @@ class ProcessTestCase(BaseTestCase): + + def test_communicate_epipe(self): + # Issue 10963: communicate() should hide EPIPE +- p = subprocess.Popen([sys.executable, "-c", 'pass'], ++ p = subprocess.Popen(ZERO_RETURN_CMD, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) +@@ -1330,7 +1339,7 @@ class ProcessTestCase(BaseTestCase): + + def test_communicate_epipe_only_stdin(self): + # Issue 10963: communicate() should hide EPIPE +- p = subprocess.Popen([sys.executable, "-c", 'pass'], ++ p = subprocess.Popen(ZERO_RETURN_CMD, + stdin=subprocess.PIPE) + self.addCleanup(p.stdin.close) + p.wait() +@@ -1369,7 +1378,7 @@ class ProcessTestCase(BaseTestCase): + fds_before_popen = os.listdir(fd_directory) + with self.assertRaises(PopenTestException): + PopenExecuteChildRaises( +- [sys.executable, '-c', 'pass'], stdin=subprocess.PIPE, ++ ZERO_RETURN_CMD, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + # NOTE: This test doesn't verify that the real _execute_child +@@ -1412,7 +1421,7 @@ class RunFuncTestCase(BaseTestCase): + + def test_check_zero(self): + # check_returncode shouldn't raise when returncode is zero +- cp = self.run_python("import sys; sys.exit(0)", check=True) ++ cp = subprocess.run(ZERO_RETURN_CMD, check=True) + self.assertEqual(cp.returncode, 0) + + def test_timeout(self): +@@ -1740,16 +1749,16 @@ class POSIXProcessTestCase(BaseTestCase): + self.assertEqual(child_user, user_uid) + + with self.assertRaises(ValueError): +- subprocess.check_call([sys.executable, "-c", "pass"], user=-1) ++ subprocess.check_call(ZERO_RETURN_CMD, user=-1) + + if pwd is None: + with self.assertRaises(ValueError): +- subprocess.check_call([sys.executable, "-c", "pass"], user=name_uid) ++ subprocess.check_call(ZERO_RETURN_CMD, user=name_uid) + + @unittest.skipIf(hasattr(os, 'setreuid'), 'setreuid() available on platform') + def test_user_error(self): + with self.assertRaises(ValueError): +- subprocess.check_call([sys.executable, "-c", "pass"], user=65535) ++ subprocess.check_call(ZERO_RETURN_CMD, user=65535) + + @unittest.skipUnless(hasattr(os, 'setregid'), 'no setregid() on platform') + def test_group(self): +@@ -1783,16 +1792,16 @@ class POSIXProcessTestCase(BaseTestCase): + + # make sure we bomb on negative values + with self.assertRaises(ValueError): +- subprocess.check_call([sys.executable, "-c", "pass"], group=-1) ++ subprocess.check_call(ZERO_RETURN_CMD, group=-1) + + if grp is None: + with self.assertRaises(ValueError): +- subprocess.check_call([sys.executable, "-c", "pass"], group=name_group) ++ subprocess.check_call(ZERO_RETURN_CMD, group=name_group) + + @unittest.skipIf(hasattr(os, 'setregid'), 'setregid() available on platform') + def test_group_error(self): + with self.assertRaises(ValueError): +- subprocess.check_call([sys.executable, "-c", "pass"], group=65535) ++ subprocess.check_call(ZERO_RETURN_CMD, group=65535) + + @unittest.skipUnless(hasattr(os, 'setgroups'), 'no setgroups() on platform') + def test_extra_groups(self): +@@ -1831,17 +1840,17 @@ class POSIXProcessTestCase(BaseTestCase): + + # make sure we bomb on negative values + with self.assertRaises(ValueError): +- subprocess.check_call([sys.executable, "-c", "pass"], extra_groups=[-1]) ++ subprocess.check_call(ZERO_RETURN_CMD, extra_groups=[-1]) + + if grp is None: + with self.assertRaises(ValueError): +- subprocess.check_call([sys.executable, "-c", "pass"], ++ subprocess.check_call(ZERO_RETURN_CMD, + extra_groups=[name_group]) + + @unittest.skipIf(hasattr(os, 'setgroups'), 'setgroups() available on platform') + def test_extra_groups_error(self): + with self.assertRaises(ValueError): +- subprocess.check_call([sys.executable, "-c", "pass"], extra_groups=[]) ++ subprocess.check_call(ZERO_RETURN_CMD, extra_groups=[]) + + @unittest.skipIf(mswindows or not hasattr(os, 'umask'), + 'POSIX umask() is not available.') +@@ -1853,7 +1862,7 @@ class POSIXProcessTestCase(BaseTestCase): + # We set an unusual umask in the child so as a unique mode + # for us to test the child's touched file for. + subprocess.check_call( +- [sys.executable, "-c", f"open({name!r}, 'w')"], # touch ++ [sys.executable, "-c", f"open({name!r}, 'w').close()"], + umask=0o053) + # Ignore execute permissions entirely in our test, + # filesystems could be mounted to ignore or force that. +@@ -1956,7 +1965,7 @@ class POSIXProcessTestCase(BaseTestCase): + + with self.assertRaises(subprocess.SubprocessError): + self._TestExecuteChildPopen( +- self, [sys.executable, "-c", "pass"], ++ self, ZERO_RETURN_CMD, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, preexec_fn=raise_it) + +@@ -2413,7 +2422,7 @@ class POSIXProcessTestCase(BaseTestCase): + + try: + subprocess.call( +- [sys.executable, "-c", "pass"], ++ ZERO_RETURN_CMD, + preexec_fn=prepare) + except ValueError as err: + # Pure Python implementations keeps the message +@@ -2456,29 +2465,30 @@ class POSIXProcessTestCase(BaseTestCase): + self.assertEqual(stdout.decode('ascii'), ascii(encoded_value)) + + def test_bytes_program(self): +- abs_program = os.fsencode(sys.executable) +- path, program = os.path.split(sys.executable) ++ abs_program = os.fsencode(ZERO_RETURN_CMD[0]) ++ args = list(ZERO_RETURN_CMD[1:]) ++ path, program = os.path.split(ZERO_RETURN_CMD[0]) + program = os.fsencode(program) + + # absolute bytes path +- exitcode = subprocess.call([abs_program, "-c", "pass"]) ++ exitcode = subprocess.call([abs_program]+args) + self.assertEqual(exitcode, 0) + + # absolute bytes path as a string +- cmd = b"'" + abs_program + b"' -c pass" ++ cmd = b"'%s' %s" % (abs_program, " ".join(args).encode("utf-8")) + exitcode = subprocess.call(cmd, shell=True) + self.assertEqual(exitcode, 0) + + # bytes program, unicode PATH + env = os.environ.copy() + env["PATH"] = path +- exitcode = subprocess.call([program, "-c", "pass"], env=env) ++ exitcode = subprocess.call([program]+args, env=env) + self.assertEqual(exitcode, 0) + + # bytes program, bytes PATH + envb = os.environb.copy() + envb[b"PATH"] = os.fsencode(path) +- exitcode = subprocess.call([program, "-c", "pass"], env=envb) ++ exitcode = subprocess.call([program]+args, env=envb) + self.assertEqual(exitcode, 0) + + def test_pipe_cloexec(self): +@@ -2706,7 +2716,7 @@ class POSIXProcessTestCase(BaseTestCase): + # pass_fds overrides close_fds with a warning. + with self.assertWarns(RuntimeWarning) as context: + self.assertFalse(subprocess.call( +- [sys.executable, "-c", "import sys; sys.exit(0)"], ++ ZERO_RETURN_CMD, + close_fds=False, pass_fds=(fd, ))) + self.assertIn('overriding close_fds', str(context.warning)) + +@@ -2768,19 +2778,19 @@ class POSIXProcessTestCase(BaseTestCase): + + def test_stdout_stdin_are_single_inout_fd(self): + with io.open(os.devnull, "r+") as inout: +- p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], ++ p = subprocess.Popen(ZERO_RETURN_CMD, + stdout=inout, stdin=inout) + p.wait() + + def test_stdout_stderr_are_single_inout_fd(self): + with io.open(os.devnull, "r+") as inout: +- p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], ++ p = subprocess.Popen(ZERO_RETURN_CMD, + stdout=inout, stderr=inout) + p.wait() + + def test_stderr_stdin_are_single_inout_fd(self): + with io.open(os.devnull, "r+") as inout: +- p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], ++ p = subprocess.Popen(ZERO_RETURN_CMD, + stderr=inout, stdin=inout) + p.wait() + +@@ -2980,7 +2990,7 @@ class POSIXProcessTestCase(BaseTestCase): + def test_communicate_BrokenPipeError_stdin_close(self): + # By not setting stdout or stderr or a timeout we force the fast path + # that just calls _stdin_write() internally due to our mock. +- proc = subprocess.Popen([sys.executable, '-c', 'pass']) ++ proc = subprocess.Popen(ZERO_RETURN_CMD) + with proc, mock.patch.object(proc, 'stdin') as mock_proc_stdin: + mock_proc_stdin.close.side_effect = BrokenPipeError + proc.communicate() # Should swallow BrokenPipeError from close. +@@ -2989,7 +2999,7 @@ class POSIXProcessTestCase(BaseTestCase): + def test_communicate_BrokenPipeError_stdin_write(self): + # By not setting stdout or stderr or a timeout we force the fast path + # that just calls _stdin_write() internally due to our mock. +- proc = subprocess.Popen([sys.executable, '-c', 'pass']) ++ proc = subprocess.Popen(ZERO_RETURN_CMD) + with proc, mock.patch.object(proc, 'stdin') as mock_proc_stdin: + mock_proc_stdin.write.side_effect = BrokenPipeError + proc.communicate(b'stuff') # Should swallow the BrokenPipeError. +@@ -3028,7 +3038,7 @@ class POSIXProcessTestCase(BaseTestCase): + 'need _testcapi.W_STOPCODE') + def test_stopped(self): + """Test wait() behavior when waitpid returns WIFSTOPPED; issue29335.""" +- args = [sys.executable, '-c', 'pass'] ++ args = ZERO_RETURN_CMD + proc = subprocess.Popen(args) + + # Wait until the real process completes to avoid zombie process +@@ -3069,7 +3079,7 @@ class Win32ProcessTestCase(BaseTestCase): + # Since Python is a console process, it won't be affected + # by wShowWindow, but the argument should be silently + # ignored +- subprocess.call([sys.executable, "-c", "import sys; sys.exit(0)"], ++ subprocess.call(ZERO_RETURN_CMD, + startupinfo=startupinfo) + + def test_startupinfo_keywords(self): +@@ -3085,7 +3095,7 @@ class Win32ProcessTestCase(BaseTestCase): + # Since Python is a console process, it won't be affected + # by wShowWindow, but the argument should be silently + # ignored +- subprocess.call([sys.executable, "-c", "import sys; sys.exit(0)"], ++ subprocess.call(ZERO_RETURN_CMD, + startupinfo=startupinfo) + + def test_startupinfo_copy(self): +@@ -3097,7 +3107,7 @@ class Win32ProcessTestCase(BaseTestCase): + # Call Popen() twice with the same startupinfo object to make sure + # that it's not modified + for _ in range(2): +- cmd = [sys.executable, "-c", "pass"] ++ cmd = ZERO_RETURN_CMD + with open(os.devnull, 'w') as null: + proc = subprocess.Popen(cmd, + stdout=null, +@@ -3137,7 +3147,7 @@ class Win32ProcessTestCase(BaseTestCase): + class BadEnv(dict): + keys = None + with self.assertRaises(TypeError): +- subprocess.Popen([sys.executable, "-c", "pass"], env=BadEnv()) ++ subprocess.Popen(ZERO_RETURN_CMD, env=BadEnv()) + + def test_close_fds(self): + # close file descriptors +@@ -3198,13 +3208,13 @@ class Win32ProcessTestCase(BaseTestCase): + def test_empty_attribute_list(self): + startupinfo = subprocess.STARTUPINFO() + startupinfo.lpAttributeList = {} +- subprocess.call([sys.executable, "-c", "import sys; sys.exit(0)"], ++ subprocess.call(ZERO_RETURN_CMD, + startupinfo=startupinfo) + + def test_empty_handle_list(self): + startupinfo = subprocess.STARTUPINFO() + startupinfo.lpAttributeList = {"handle_list": []} +- subprocess.call([sys.executable, "-c", "import sys; sys.exit(0)"], ++ subprocess.call(ZERO_RETURN_CMD, + startupinfo=startupinfo) + + def test_shell_sequence(self): +@@ -3503,7 +3513,7 @@ class ContextManagerTests(BaseTestCase): + + def test_broken_pipe_cleanup(self): + """Broken pipe error should not prevent wait() (Issue 21619)""" +- proc = subprocess.Popen([sys.executable, '-c', 'pass'], ++ proc = subprocess.Popen(ZERO_RETURN_CMD, + stdin=subprocess.PIPE, + bufsize=support.PIPE_MAX_SIZE*2) + proc = proc.__enter__() +-- +2.23.0 + diff --git a/backport-39855-Fix-test_subprocess-if-nobody-user-doesn-t.patch b/backport-39855-Fix-test_subprocess-if-nobody-user-doesn-t.patch new file mode 100644 index 0000000000000000000000000000000000000000..d404c5ea99042fe8666465930c3051925cf22687 --- /dev/null +++ b/backport-39855-Fix-test_subprocess-if-nobody-user-doesn-t.patch @@ -0,0 +1,59 @@ +From c70f250a25f240cb6595a8eb8ff80389ae472e45 Mon Sep 17 00:00:00 2001 +From: Victor Stinner +Date: Thu, 5 Mar 2020 14:28:40 +0100 +Subject: [PATCH] bpo-39855: Fix test_subprocess if nobody user doesn't exist + (GH-18781) + +test_subprocess.test_user() now skips the test on an user name if the +user name doesn't exist. For example, skip the test if the user +"nobody" doesn't exist on Linux. + +Conflict:NA +Reference:https://github.com/python/cpython/commit/f7b5d419bf871d9cc898982c7b6b4c043f7d5e9d + +Signed-off-by: hanxinke +--- + Lib/test/test_subprocess.py | 9 +++++++-- + .../next/Tests/2020-03-04-23-03-01.bpo-39855.Ql5xv8.rst | 3 +++ + 2 files changed, 10 insertions(+), 2 deletions(-) + create mode 100644 Misc/NEWS.d/next/Tests/2020-03-04-23-03-01.bpo-39855.Ql5xv8.rst + +diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py +index ac45436..bced1e7 100644 +--- a/Lib/test/test_subprocess.py ++++ b/Lib/test/test_subprocess.py +@@ -1725,7 +1725,12 @@ class POSIXProcessTestCase(BaseTestCase): + name_uid = "nobody" if sys.platform != 'darwin' else "unknown" + + if pwd is not None: +- test_users.append(name_uid) ++ try: ++ pwd.getpwnam(name_uid) ++ test_users.append(name_uid) ++ except KeyError: ++ # unknown user name ++ name_uid = None + + for user in test_users: + # posix_spawn() may be used with close_fds=False +@@ -1753,7 +1758,7 @@ class POSIXProcessTestCase(BaseTestCase): + with self.assertRaises(ValueError): + subprocess.check_call(ZERO_RETURN_CMD, user=-1) + +- if pwd is None: ++ if pwd is None and name_uid is not None: + with self.assertRaises(ValueError): + subprocess.check_call(ZERO_RETURN_CMD, user=name_uid) + +diff --git a/Misc/NEWS.d/next/Tests/2020-03-04-23-03-01.bpo-39855.Ql5xv8.rst b/Misc/NEWS.d/next/Tests/2020-03-04-23-03-01.bpo-39855.Ql5xv8.rst +new file mode 100644 +index 0000000..0601241 +--- /dev/null ++++ b/Misc/NEWS.d/next/Tests/2020-03-04-23-03-01.bpo-39855.Ql5xv8.rst +@@ -0,0 +1,3 @@ ++test_subprocess.test_user() now skips the test on an user name if the user ++name doesn't exist. For example, skip the test if the user "nobody" doesn't ++exist on Linux. +-- +2.23.0 + diff --git a/backport-42146-Fix-memory-leak-in-subprocess.Popen-in-cas.patch b/backport-42146-Fix-memory-leak-in-subprocess.Popen-in-cas.patch new file mode 100644 index 0000000000000000000000000000000000000000..31ca60ccdd70190934b4f4b814e40e74d521e822 --- /dev/null +++ b/backport-42146-Fix-memory-leak-in-subprocess.Popen-in-cas.patch @@ -0,0 +1,103 @@ +From ee0f0b4f066eb9e44e0e0270eba23ddb60e8b44a Mon Sep 17 00:00:00 2001 +From: Alexey Izbyshev +Date: Mon, 26 Oct 2020 03:09:32 +0300 +Subject: [PATCH] bpo-42146: Fix memory leak in subprocess.Popen() in case of + uid/gid overflow (GH-22966) + +Fix memory leak in subprocess.Popen() in case of uid/gid overflow + +Also add a test that would catch this leak with `--huntrleaks`. + +Alas, the test for `extra_groups` also exposes an inconsistency +in our error reporting: we use a custom ValueError for `extra_groups`, +but propagate OverflowError for `user` and `group`. + +Conflict:NA +Reference:https://github.com/python/cpython/commit/c0590c0033e86f98cdf5f2ca6898656f98ab4053 + +Signed-off-by: hanxinke +--- + Lib/test/test_subprocess.py | 13 +++++++++++++ + .../2020-10-25-19-25-02.bpo-42146.6A8uvS.rst | 2 ++ + Modules/_posixsubprocess.c | 4 ++-- + 3 files changed, 17 insertions(+), 2 deletions(-) + create mode 100644 Misc/NEWS.d/next/Library/2020-10-25-19-25-02.bpo-42146.6A8uvS.rst + +diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py +index bced1e7..e401656 100644 +--- a/Lib/test/test_subprocess.py ++++ b/Lib/test/test_subprocess.py +@@ -1758,6 +1758,10 @@ class POSIXProcessTestCase(BaseTestCase): + with self.assertRaises(ValueError): + subprocess.check_call(ZERO_RETURN_CMD, user=-1) + ++ with self.assertRaises(OverflowError): ++ subprocess.check_call(ZERO_RETURN_CMD, ++ cwd=os.curdir, env=os.environ, user=2**64) ++ + if pwd is None and name_uid is not None: + with self.assertRaises(ValueError): + subprocess.check_call(ZERO_RETURN_CMD, user=name_uid) +@@ -1801,6 +1805,10 @@ class POSIXProcessTestCase(BaseTestCase): + with self.assertRaises(ValueError): + subprocess.check_call(ZERO_RETURN_CMD, group=-1) + ++ with self.assertRaises(OverflowError): ++ subprocess.check_call(ZERO_RETURN_CMD, ++ cwd=os.curdir, env=os.environ, group=2**64) ++ + if grp is None: + with self.assertRaises(ValueError): + subprocess.check_call(ZERO_RETURN_CMD, group=name_group) +@@ -1849,6 +1857,11 @@ class POSIXProcessTestCase(BaseTestCase): + with self.assertRaises(ValueError): + subprocess.check_call(ZERO_RETURN_CMD, extra_groups=[-1]) + ++ with self.assertRaises(ValueError): ++ subprocess.check_call(ZERO_RETURN_CMD, ++ cwd=os.curdir, env=os.environ, ++ extra_groups=[2**64]) ++ + if grp is None: + with self.assertRaises(ValueError): + subprocess.check_call(ZERO_RETURN_CMD, +diff --git a/Misc/NEWS.d/next/Library/2020-10-25-19-25-02.bpo-42146.6A8uvS.rst b/Misc/NEWS.d/next/Library/2020-10-25-19-25-02.bpo-42146.6A8uvS.rst +new file mode 100644 +index 0000000..0418098 +--- /dev/null ++++ b/Misc/NEWS.d/next/Library/2020-10-25-19-25-02.bpo-42146.6A8uvS.rst +@@ -0,0 +1,2 @@ ++Fix memory leak in :func:`subprocess.Popen` in case an uid (gid) specified in ++`user` (`group`, `extra_groups`) overflows `uid_t` (`gid_t`). +diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c +index 5845445..2981253 100644 +--- a/Modules/_posixsubprocess.c ++++ b/Modules/_posixsubprocess.c +@@ -759,7 +759,7 @@ subprocess_fork_exec(PyObject* self, PyObject *args) + uid_t uid; + gid_t gid, *groups = NULL; + int child_umask; +- PyObject *cwd_obj, *cwd_obj2; ++ PyObject *cwd_obj, *cwd_obj2 = NULL; + const char *cwd; + pid_t pid; + int need_to_reenable_gc = 0; +@@ -866,7 +866,6 @@ subprocess_fork_exec(PyObject* self, PyObject *args) + cwd = PyBytes_AsString(cwd_obj2); + } else { + cwd = NULL; +- cwd_obj2 = NULL; + } + + if (groups_list != Py_None) { +@@ -1051,6 +1050,7 @@ subprocess_fork_exec(PyObject* self, PyObject *args) + return PyLong_FromPid(pid); + + cleanup: ++ Py_XDECREF(cwd_obj2); + if (envp) + _Py_FreeCharPArray(envp); + if (argv) +-- +2.23.0 + diff --git a/backport-42146-Unify-cleanup-in-subprocess_fork_exec-GH-2.patch b/backport-42146-Unify-cleanup-in-subprocess_fork_exec-GH-2.patch new file mode 100644 index 0000000000000000000000000000000000000000..85c0c0811770c80625e041e4b964f715342534ab --- /dev/null +++ b/backport-42146-Unify-cleanup-in-subprocess_fork_exec-GH-2.patch @@ -0,0 +1,139 @@ +From e8aadd0680a208a4963aa801a5a9d505b4af7add Mon Sep 17 00:00:00 2001 +From: Alexey Izbyshev +Date: Sun, 1 Nov 2020 08:33:08 +0300 +Subject: [PATCH] bpo-42146: Unify cleanup in subprocess_fork_exec() (GH-22970) + +* bpo-42146: Unify cleanup in subprocess_fork_exec() + +Also ignore errors from _enable_gc(): +* They are always suppressed by the current code due to a bug. +* _enable_gc() is only used if `preexec_fn != None`, which is unsafe. +* We don't have a good way to handle errors in case we successfully + created a child process. + +Conflict:NA +Reference:https://github.com/python/cpython/commit/d3b4e068077dd26927ae7485bd0303e09d962c02 + +Co-authored-by: Gregory P. Smith +Signed-off-by: hanxinke +--- + Modules/_posixsubprocess.c | 52 +++++++++++++------------------------- + 1 file changed, 18 insertions(+), 34 deletions(-) + +diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c +index 2981253..fe801e9 100644 +--- a/Modules/_posixsubprocess.c ++++ b/Modules/_posixsubprocess.c +@@ -69,8 +69,8 @@ + #define POSIX_CALL(call) do { if ((call) == -1) goto error; } while (0) + + +-/* If gc was disabled, call gc.enable(). Return 0 on success. */ +-static int ++/* If gc was disabled, call gc.enable(). Ignore errors. */ ++static void + _enable_gc(int need_to_reenable_gc, PyObject *gc_module) + { + PyObject *result; +@@ -80,15 +80,17 @@ _enable_gc(int need_to_reenable_gc, PyObject *gc_module) + if (need_to_reenable_gc) { + PyErr_Fetch(&exctype, &val, &tb); + result = _PyObject_CallMethodId(gc_module, &PyId_enable, NULL); ++ if (result == NULL) { ++ /* We might have created a child process at this point, we ++ * we have no good way to handle a failure to reenable GC ++ * and return information about the child process. */ ++ PyErr_Print(); ++ } ++ Py_XDECREF(result); + if (exctype != NULL) { + PyErr_Restore(exctype, val, tb); + } +- if (result == NULL) { +- return 1; +- } +- Py_DECREF(result); + } +- return 0; + } + + +@@ -761,7 +763,7 @@ subprocess_fork_exec(PyObject* self, PyObject *args) + int child_umask; + PyObject *cwd_obj, *cwd_obj2 = NULL; + const char *cwd; +- pid_t pid; ++ pid_t pid = -1; + int need_to_reenable_gc = 0; + char *const *exec_array, *const *argv = NULL, *const *envp = NULL; + Py_ssize_t arg_num, num_groups = 0; +@@ -982,8 +984,6 @@ subprocess_fork_exec(PyObject* self, PyObject *args) + sigset_t all_sigs; + sigfillset(&all_sigs); + if ((saved_errno = pthread_sigmask(SIG_BLOCK, &all_sigs, &old_sigs))) { +- errno = saved_errno; +- PyErr_SetFromErrno(PyExc_OSError); + goto cleanup; + } + old_sigmask = &old_sigs; +@@ -1022,49 +1022,33 @@ subprocess_fork_exec(PyObject* self, PyObject *args) + } + #endif + +- Py_XDECREF(cwd_obj2); +- + if (need_after_fork) + PyOS_AfterFork_Parent(); +- if (envp) +- _Py_FreeCharPArray(envp); +- if (argv) +- _Py_FreeCharPArray(argv); +- _Py_FreeCharPArray(exec_array); + +- /* Reenable gc in the parent process (or if fork failed). */ +- if (_enable_gc(need_to_reenable_gc, gc_module)) { +- pid = -1; +- } +- Py_XDECREF(preexec_fn_args_tuple); +- Py_XDECREF(gc_module); +- +- if (pid == -1) { ++cleanup: ++ if (saved_errno != 0) { + errno = saved_errno; + /* We can't call this above as PyOS_AfterFork_Parent() calls back + * into Python code which would see the unreturned error. */ + PyErr_SetFromErrno(PyExc_OSError); +- return NULL; /* fork() failed. */ + } + +- return PyLong_FromPid(pid); +- +-cleanup: ++ Py_XDECREF(preexec_fn_args_tuple); ++ PyMem_RawFree(groups); + Py_XDECREF(cwd_obj2); + if (envp) + _Py_FreeCharPArray(envp); ++ Py_XDECREF(converted_args); ++ Py_XDECREF(fast_args); + if (argv) + _Py_FreeCharPArray(argv); + if (exec_array) + _Py_FreeCharPArray(exec_array); + +- PyMem_RawFree(groups); +- Py_XDECREF(converted_args); +- Py_XDECREF(fast_args); +- Py_XDECREF(preexec_fn_args_tuple); + _enable_gc(need_to_reenable_gc, gc_module); + Py_XDECREF(gc_module); +- return NULL; ++ ++ return pid == -1 ? NULL : PyLong_FromPid(pid); + } + + +-- +2.23.0 + diff --git a/backport-Fix-TestPosixSpawn.test_close_file-GH-8992.patch b/backport-Fix-TestPosixSpawn.test_close_file-GH-8992.patch new file mode 100644 index 0000000000000000000000000000000000000000..3411254d276db2b6fc3406d70a9a99d4e0f6b3b6 --- /dev/null +++ b/backport-Fix-TestPosixSpawn.test_close_file-GH-8992.patch @@ -0,0 +1,166 @@ +From 34014f7bcfd422c7fd7d3077f9c65dc50e0210c9 Mon Sep 17 00:00:00 2001 +From: Victor Stinner +Date: Thu, 30 Aug 2018 01:21:11 +0200 +Subject: [PATCH] Fix TestPosixSpawn.test_close_file() (GH-8992) + +Modify TestPosixSpawn to run Python using -I and -S options. + +Disable site module to avoid side effects. For example, on Fedora 28, +if the HOME environment variable is not set, site._getuserbase() +calls pwd.getpwuid() which opens /var/lib/sss/mc/passwd, but then +leaves the file open which makes test_close_file() to fail. + +Conflict:NA +Reference:https://github.com/python/cpython/commit/0382406fccbb31aa993de118b60e7fd4ec264968 + +Signed-off-by: hanxinke +--- + Lib/test/test_posix.py | 64 ++++++++++++++++++++++-------------------- + 1 file changed, 34 insertions(+), 30 deletions(-) + +diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py +index e2cda33..77fedb1 100644 +--- a/Lib/test/test_posix.py ++++ b/Lib/test/test_posix.py +@@ -1507,6 +1507,17 @@ class PosixGroupsTester(unittest.TestCase): + + @unittest.skipUnless(hasattr(os, 'posix_spawn'), "test needs os.posix_spawn") + class TestPosixSpawn(unittest.TestCase): ++ # Program which does nothing and exit with status 0 (success) ++ NOOP_PROGRAM = (sys.executable, '-I', '-S', '-c', 'pass') ++ ++ def python_args(self, *args): ++ # Disable site module to avoid side effects. For example, ++ # on Fedora 28, if the HOME environment variable is not set, ++ # site._getuserbase() calls pwd.getpwuid() which opens ++ # /var/lib/sss/mc/passwd but then leaves the file open which makes ++ # test_close_file() to fail. ++ return (sys.executable, '-I', '-S', *args) ++ + def test_returns_pid(self): + pidfile = support.TESTFN + self.addCleanup(support.unlink, pidfile) +@@ -1515,8 +1526,8 @@ class TestPosixSpawn(unittest.TestCase): + with open({pidfile!r}, "w") as pidfile: + pidfile.write(str(os.getpid())) + """ +- pid = posix.posix_spawn(sys.executable, +- [sys.executable, '-c', script], ++ args = self.python_args('-c', script) ++ pid = posix.posix_spawn(args[0], args, + os.environ) + self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + with open(pidfile) as f: +@@ -1543,8 +1554,8 @@ class TestPosixSpawn(unittest.TestCase): + with open({envfile!r}, "w") as envfile: + envfile.write(os.environ['foo']) + """ +- pid = posix.posix_spawn(sys.executable, +- [sys.executable, '-c', script], ++ args = self.python_args('-c', script) ++ pid = posix.posix_spawn(args[0], args, + {**os.environ, 'foo': 'bar'}) + self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + with open(envfile) as f: +@@ -1552,8 +1563,8 @@ class TestPosixSpawn(unittest.TestCase): + + def test_empty_file_actions(self): + pid = posix.posix_spawn( +- sys.executable, +- [sys.executable, '-c', 'pass'], ++ self.NOOP_PROGRAM[0], ++ self.NOOP_PROGRAM, + os.environ, + [] + ) +@@ -1706,43 +1717,36 @@ class TestPosixSpawn(unittest.TestCase): + (os.POSIX_SPAWN_CLOSE, 0), + (os.POSIX_SPAWN_DUP2, 1, 4), + ] +- pid = posix.posix_spawn(sys.executable, +- [sys.executable, "-c", "pass"], ++ pid = posix.posix_spawn(self.NOOP_PROGRAM[0], ++ self.NOOP_PROGRAM, + os.environ, file_actions) + self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + + def test_bad_file_actions(self): ++ args = self.NOOP_PROGRAM + with self.assertRaises(TypeError): +- posix.posix_spawn(sys.executable, +- [sys.executable, "-c", "pass"], ++ posix.posix_spawn(args[0], args, + os.environ, [None]) + with self.assertRaises(TypeError): +- posix.posix_spawn(sys.executable, +- [sys.executable, "-c", "pass"], ++ posix.posix_spawn(args[0], args, + os.environ, [()]) + with self.assertRaises(TypeError): +- posix.posix_spawn(sys.executable, +- [sys.executable, "-c", "pass"], ++ posix.posix_spawn(args[0], args, + os.environ, [(None,)]) + with self.assertRaises(TypeError): +- posix.posix_spawn(sys.executable, +- [sys.executable, "-c", "pass"], ++ posix.posix_spawn(args[0], args, + os.environ, [(12345,)]) + with self.assertRaises(TypeError): +- posix.posix_spawn(sys.executable, +- [sys.executable, "-c", "pass"], ++ posix.posix_spawn(args[0], args, + os.environ, [(os.POSIX_SPAWN_CLOSE,)]) + with self.assertRaises(TypeError): +- posix.posix_spawn(sys.executable, +- [sys.executable, "-c", "pass"], ++ posix.posix_spawn(args[0], args, + os.environ, [(os.POSIX_SPAWN_CLOSE, 1, 2)]) + with self.assertRaises(TypeError): +- posix.posix_spawn(sys.executable, +- [sys.executable, "-c", "pass"], ++ posix.posix_spawn(args[0], args, + os.environ, [(os.POSIX_SPAWN_CLOSE, None)]) + with self.assertRaises(ValueError): +- posix.posix_spawn(sys.executable, +- [sys.executable, "-c", "pass"], ++ posix.posix_spawn(args[0], args, + os.environ, + [(os.POSIX_SPAWN_OPEN, 3, __file__ + '\0', + os.O_RDONLY, 0)]) +@@ -1759,8 +1763,8 @@ class TestPosixSpawn(unittest.TestCase): + os.O_WRONLY | os.O_CREAT | os.O_TRUNC, + stat.S_IRUSR | stat.S_IWUSR), + ] +- pid = posix.posix_spawn(sys.executable, +- [sys.executable, '-c', script], ++ args = self.python_args('-c', script) ++ pid = posix.posix_spawn(args[0], args, + os.environ, file_actions) + self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + with open(outfile) as f: +@@ -1777,8 +1781,8 @@ class TestPosixSpawn(unittest.TestCase): + with open({closefile!r}, 'w') as closefile: + closefile.write('is closed %d' % e.errno) + """ +- pid = posix.posix_spawn(sys.executable, +- [sys.executable, '-c', script], ++ args = self.python_args('-c', script) ++ pid = posix.posix_spawn(args[0], args, + os.environ, + [(os.POSIX_SPAWN_CLOSE, 0),]) + self.assertEqual(os.waitpid(pid, 0), (pid, 0)) +@@ -1796,8 +1800,8 @@ class TestPosixSpawn(unittest.TestCase): + file_actions = [ + (os.POSIX_SPAWN_DUP2, childfile.fileno(), 1), + ] +- pid = posix.posix_spawn(sys.executable, +- [sys.executable, '-c', script], ++ args = self.python_args('-c', script) ++ pid = posix.posix_spawn(args[0], args, + os.environ, file_actions) + self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + with open(dupfile) as f: +-- +2.23.0 + diff --git a/backport-subprocess-close-pipes-fds-by-using-ExitStack-GH-116.patch b/backport-subprocess-close-pipes-fds-by-using-ExitStack-GH-116.patch new file mode 100644 index 0000000000000000000000000000000000000000..223a017693b2706234a423cc600068fe9911a55b --- /dev/null +++ b/backport-subprocess-close-pipes-fds-by-using-ExitStack-GH-116.patch @@ -0,0 +1,91 @@ +From 5b3807f6215fc2c5e14963bb48665426a53580f5 Mon Sep 17 00:00:00 2001 +From: Giampaolo Rodola +Date: Tue, 29 Jan 2019 22:14:24 +0100 +Subject: [PATCH] subprocess: close pipes/fds by using ExitStack (GH-11686) + +Close pipes/fds in subprocess by using ExitStack. + +"In case of premature failure on X.Close() or os.close(X) the remaining pipes/fds will remain "open". Perhaps it makes sense to use contextlib.ExitStack." +- Rationale: https://github.com/python/cpython/pull/11575#discussion_r250288394 + +Conflict:NA +Reference:https://github.com/python/cpython/commit/bafa8487f77fa076de3a06755399daf81cb75598 + +Signed-off-by: hanxinke +--- + Lib/subprocess.py | 35 ++++++++++--------- + .../2019-01-29-17-24-52.bpo-35537.Q0ktFC.rst | 4 +++ + 2 files changed, 22 insertions(+), 17 deletions(-) + create mode 100644 Misc/NEWS.d/next/Library/2019-01-29-17-24-52.bpo-35537.Q0ktFC.rst + +diff --git a/Lib/subprocess.py b/Lib/subprocess.py +index b02b701..332c19f 100644 +--- a/Lib/subprocess.py ++++ b/Lib/subprocess.py +@@ -51,6 +51,7 @@ import signal + import builtins + import warnings + import errno ++import contextlib + from time import monotonic as _time + + # Exception classes used by this module. +@@ -1092,28 +1093,28 @@ class Popen(object): + # self._devnull is not always defined. + devnull_fd = getattr(self, '_devnull', None) + +- if _mswindows: +- if p2cread != -1: +- p2cread.Close() +- if c2pwrite != -1: +- c2pwrite.Close() +- if errwrite != -1: +- errwrite.Close() +- else: +- if p2cread != -1 and p2cwrite != -1 and p2cread != devnull_fd: +- os.close(p2cread) +- if c2pwrite != -1 and c2pread != -1 and c2pwrite != devnull_fd: +- os.close(c2pwrite) +- if errwrite != -1 and errread != -1 and errwrite != devnull_fd: +- os.close(errwrite) ++ with contextlib.ExitStack() as stack: ++ if _mswindows: ++ if p2cread != -1: ++ stack.callback(p2cread.Close) ++ if c2pwrite != -1: ++ stack.callback(c2pwrite.Close) ++ if errwrite != -1: ++ stack.callback(errwrite.Close) ++ else: ++ if p2cread != -1 and p2cwrite != -1 and p2cread != devnull_fd: ++ stack.callback(os.close, p2cread) ++ if c2pwrite != -1 and c2pread != -1 and c2pwrite != devnull_fd: ++ stack.callback(os.close, c2pwrite) ++ if errwrite != -1 and errread != -1 and errwrite != devnull_fd: ++ stack.callback(os.close, errwrite) + +- if devnull_fd is not None: +- os.close(devnull_fd) ++ if devnull_fd is not None: ++ stack.callback(os.close, devnull_fd) + + # Prevent a double close of these handles/fds from __init__ on error. + self._closed_child_pipe_fds = True + +- + if _mswindows: + # + # Windows methods +diff --git a/Misc/NEWS.d/next/Library/2019-01-29-17-24-52.bpo-35537.Q0ktFC.rst b/Misc/NEWS.d/next/Library/2019-01-29-17-24-52.bpo-35537.Q0ktFC.rst +new file mode 100644 +index 0000000..2a9588e +--- /dev/null ++++ b/Misc/NEWS.d/next/Library/2019-01-29-17-24-52.bpo-35537.Q0ktFC.rst +@@ -0,0 +1,4 @@ ++An ExitStack is now used internally within subprocess.POpen to clean up pipe ++file handles. No behavior change in normal operation. But if closing one ++handle were ever to cause an exception, the others will now be closed ++instead of leaked. (patch by Giampaolo Rodola) +-- +2.23.0 + diff --git a/python3.spec b/python3.spec index 41b10acae5484feee8720ab390a9f264c11c4ace..09431f29cdc1de4071fa13cd394194fd4c4475ca 100644 --- a/python3.spec +++ b/python3.spec @@ -3,7 +3,7 @@ Summary: Interpreter of the Python3 programming language URL: https://www.python.org/ Version: 3.7.9 -Release: 8 +Release: 9 License: Python %global branchversion 3.7 @@ -107,6 +107,39 @@ Patch320: CVE-2020-27619.patch Patch322: CVE-2021-3177.patch Patch323: backport-CVE-2021-23336.patch +Patch6003: backport-20104-Expose-posix_spawn-in-the-os-module-GH-510.patch +Patch6004: backport-20104-Fix-leaks-and-errors-in-new-os.posix_spawn.patch +Patch6005: backport-20104-Improve-error-handling-and-fix-a-reference.patch +Patch6006: backport-33630-Fix-using-of-freed-memory-in-old-versions-.patch +Patch6007: backport-33332-Add-signal.valid_signals-GH-6581.patch +Patch6008: backport-33441-Make-the-sigset_t-converter-available-in-o.patch +Patch6009: backport-20104-Add-flag-capabilities-to-posix_spawn-GH-66.patch +Patch6010: backport-33455-Pass-os.environ-in-test_posix-test_specify.patch +Patch6011: backport-Fix-TestPosixSpawn.test_close_file-GH-8992.patch +Patch6012: backport-20104-Change-the-file_actions-parameter-of-os.po.patch +Patch6013: backport-35537-subprocess-uses-os.posix_spawn-in-some-cas.patch +Patch6014: backport-35674-Add-os.posix_spawnp-GH-11554.patch +Patch6015: backport-35537-subprocess-can-use-posix_spawn-with-pipes-.patch +Patch6016: backport-subprocess-close-pipes-fds-by-using-ExitStack-GH-116.patch +Patch6017: backport-34862-Guard-definition-of-convert_sched_p.patch +Patch6018: backport-35537-Add-setsid-parameter-to-os.posix_spawn-and.patch +Patch6019: backport-35537-Skip-test_start_new_session-of-posix_spawn.patch +Patch6020: backport-35537-Fix-function-name-in-os.posix_spawnp-error.patch +Patch6021: backport-36814-ensure-os.posix_spawn-handles-None-GH-1314.patch +Patch6022: backport-35537-Rewrite-setsid-test-for-os.posix_spawn-GH-.patch +Patch6023: backport-36046-Add-user-and-group-parameters-to-subproces.patch +Patch6024: backport-36046-Fix-buildbot-failures-GH-16091.patch +Patch6025: backport-36046-posix_spawn-doesn-t-support-uid-gid-GH-163.patch +Patch6026: backport-38417-Add-umask-support-to-subprocess-GH-16726.patch +Patch6027: backport-38456-Use-bin-true-in-test_subprocess-GH-16736.patch +Patch6028: backport-38456-Handle-the-case-when-there-is-no-true-comm.patch +Patch6029: backport-39855-Fix-test_subprocess-if-nobody-user-doesn-t.patch +Patch6030: backport-35823-subprocess-Use-vfork-instead-of-fork-on-Li.patch +Patch6031: backport-35823-subprocess-Fix-handling-of-pthread_sigmask.patch +Patch6032: backport-35823-Allow-setsid-after-vfork-on-Linux.-GH-2294.patch +Patch6033: backport-42146-Fix-memory-leak-in-subprocess.Popen-in-cas.patch +Patch6034: backport-42146-Unify-cleanup-in-subprocess_fork_exec-GH-2.patch + Provides: python%{branchversion} = %{version}-%{release} Provides: python(abi) = %{branchversion} @@ -199,6 +232,39 @@ rm Lib/ensurepip/_bundled/*.whl %patch322 -p1 %patch323 -p1 +%patch6003 -p1 +%patch6004 -p1 +%patch6005 -p1 +%patch6006 -p1 +%patch6007 -p1 +%patch6008 -p1 +%patch6009 -p1 +%patch6010 -p1 +%patch6011 -p1 +%patch6012 -p1 +%patch6013 -p1 +%patch6014 -p1 +%patch6015 -p1 +%patch6016 -p1 +%patch6017 -p1 +%patch6018 -p1 +%patch6019 -p1 +%patch6020 -p1 +%patch6021 -p1 +%patch6022 -p1 +%patch6023 -p1 +%patch6024 -p1 +%patch6025 -p1 +%patch6026 -p1 +%patch6027 -p1 +%patch6028 -p1 +%patch6029 -p1 +%patch6030 -p1 +%patch6031 -p1 +%patch6032 -p1 +%patch6033 -p1 +%patch6034 -p1 + sed -i "s/generic_os/%{_vendor}/g" Lib/platform.py rm configure pyconfig.h.in @@ -799,6 +865,43 @@ export BEP_GTDLIST="$BEP_GTDLIST_TMP" %{_mandir}/*/* %changelog +* Web Jun 23 2021 hanxinke - 3.7.9-9 +- Type:enhancement +- CVE:NA +- SUG:NA +- DESC:expose posix_spawn in os module + fix leaks and errors in new os.posix_spawn + improve error handing and fix a reference + fix using of freed memory in old versions + add signal.valid_signals + make the sigset_t converter available + add flag capabilities to posix_spawn + pass os.environ in test_posix test_specify + fix TestPosixSpawn.test_close_file + change the file_actions parameter of os.posix_spawn + subprocess uses os.posix_spawn in some case + add os.posix_spawnp + subprocess can use posix_spawn with pipes + subprocess close pipes fds by using ExitStack + guard definition of convert_sched_param with POSIX_SPAWN_SETSCHEDULER + add setsid parameter to os.posix_spawn() and os.posix_spawnp() + skip test_start_new_session() of posix_spawn + fix function name in os.posix_spawnp() errors + ensure os.posix_spawn() handles None + rewrite setsid test for os.posix_spawn + add user and group parameters to subproces + fix buildbot failures + posix_spawn doesn't support uid gid + add umask support to subprocess + use bin true in test_subprocess + handle the case when there is no 'true' command + fix test_subprocess if nobody user doesn't exist + subprocess: Use vfork() instead of fork() on Linux when safe + subprocess: Fix handling of pthread_sigmask() errors + allow setsid() after vfork() on Linux + fix memory leak in subprocess.Popen() in case of uid/gid overflow + Unify cleanup in subprocess_fork_exec() + * Web Mar 03 2021 wuchaochao - 3.7.9-8 - Type:cves - ID:CVE-2021-23336