gh-157613: Prototype explicit environments for multiprocessing spawn

Doc/library/multiprocessing.rst

@@ -541,7 +541,7 @@ or creating these objects.
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 .. class:: Process(group=None, target=None, name=None, args=(), kwargs={}, \
-                   *, daemon=None)
+                   *, daemon=None, env=None)
 
    Process objects represent activity that is run in a separate process. The
    :class:`Process` class has equivalents of all the methods of
@@ -558,6 +558,26 @@ or creating these objects.
    to ``True`` or ``False``.  If ``None`` (the default), this flag will be
    inherited from the creating process.
 
+   *env* defaults to ``None``, which inherits the parent's environment.
+   Otherwise, it is a mapping of strings to strings or bytes to bytes
+   that replaces the environment used to start the child interpreter.
+   The mapping is copied during construction; do not modify it concurrently
+   with construction.  A non-``None`` value requires POSIX ``spawn``;
+   other platforms and start methods raise :exc:`ValueError` at :meth:`start`.
+   It does not override the inherited :data:`sys.path` or interpreter flags.
+
+   On POSIX, a resource tracker launched by this :meth:`start` call also
+   uses an explicit *env*.  A running tracker is reused without changing
+   its environment.  If no explicit environment is supplied for a tracker
+   launch or relaunch, it uses the launching process's own explicit
+   environment snapshot, if any; otherwise, it inherits that process's
+   current environment.  It does not retain an environment from an earlier
+   tracker launch.  Processes created with ``env=None`` have no explicit
+   snapshot.
+
+   .. versionchanged:: next
+      Added the *env* parameter.
+
    By default, no arguments are passed to *target*. The *args* argument,
    which defaults to ``()``, can be used to specify a list or tuple of the arguments
    to pass to *target*.

Lib/multiprocessing/popen_spawn_posix.py

@@ -37,9 +37,10 @@ def duplicate_for_child(self, fd):
 
     def _launch(self, process_obj):
         from . import resource_tracker
-        tracker_fd = resource_tracker.getfd()
+        tracker_fd = resource_tracker.getfd(env=process_obj._env)
         self._fds.append(tracker_fd)
         prep_data = spawn.get_preparation_data(process_obj._name)
+        prep_data['process_env'] = process_obj._env
         fp = io.BytesIO()
         set_spawning_popen(self)
         try:
@@ -56,7 +57,7 @@ def _launch(self, process_obj):
                                          pipe_handle=child_r)
             self._fds.extend([child_r, child_w])
             self.pid = util.spawnv_passfds(spawn.get_executable(),
-                                           cmd, self._fds)
+                                           cmd, self._fds, env=process_obj._env)
             os.close(child_r)
             child_r = None
             os.close(child_w)

Lib/multiprocessing/process.py

@@ -78,8 +78,14 @@ def _Popen(self):
         raise NotImplementedError
 
     def __init__(self, group=None, target=None, name=None, args=(), kwargs=None,
-                 *, daemon=None):
+                 *, daemon=None, env=None):
         assert group is None, 'group argument must be None for now'
+        # Snapshot an explicit environment at construction time.
+        if env is None:
+            self._env = None
+        else:
+            from .util import _encode_spawn_env
+            self._env = _encode_spawn_env(env)
         count = next(_process_counter)
         self._identity = _current_process._identity + (count,)
         self._config = _current_process._config.copy()
@@ -117,6 +123,11 @@ def start(self):
                'can only start a process object created by current process'
         assert not _current_process._config.get('daemon'), \
                'daemonic processes are not allowed to have children'
+        if self._env is not None:
+            from . import get_start_method
+            method = getattr(self, '_start_method', None) or get_start_method()
+            if os.name != 'posix' or method != 'spawn':
+                raise ValueError('env is only supported with POSIX spawn')
         _cleanup()
         self._popen = self._Popen(self)
         self._sentinel = self._popen.sentinel

Lib/multiprocessing/resource_tracker.py

@@ -26,6 +26,7 @@
 
 import json
 
+from . import process
 from . import spawn
 from . import util
 
@@ -203,16 +204,16 @@ def _stop_locked(
             # os.waitstatus_to_exitcode may raise an exception for invalid values
             self._exitcode = None
 
-    def getfd(self):
-        self.ensure_running()
+    def getfd(self, *, env=None):
+        self.ensure_running(env=env)
         return self._fd
 
-    def ensure_running(self):
+    def ensure_running(self, *, env=None):
         '''Make sure that resource tracker process is running.
 
         This can be run from any process.  Usually a child process will use
         the resource created by its parent.'''
-        return self._ensure_running_and_write()
+        return self._ensure_running_and_write(env=env)
 
     def _teardown_dead_process(self):
         os.close(self._fd)
@@ -233,7 +234,11 @@ def _teardown_dead_process(self):
         warnings.warn('resource_tracker: process died unexpectedly, '
                       'relaunching.  Some resources might leak.')
 
-    def _launch(self):
+    def _launch(self, env=None):
+        # Prefer this launch's explicit environment, then the launching
+        # process's snapshot.  None preserves normal environment inheritance.
+        if env is None:
+            env = getattr(process.current_process(), '_env', None)
         fds_to_pass = []
         try:
             fds_to_pass.append(sys.stderr.fileno())
@@ -260,7 +265,8 @@ def _launch(self):
             try:
                 if _HAVE_SIGMASK:
                     prev_sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS)
-                pid = util.spawnv_passfds(exe, args, fds_to_pass)
+                pid = util.spawnv_passfds(exe, args, fds_to_pass,
+                                         env=env)
             finally:
                 if prev_sigmask is not None:
                     signal.pthread_sigmask(signal.SIG_SETMASK, prev_sigmask)
@@ -286,7 +292,7 @@ def _make_probe_message(self):
             + "\n"
         ).encode("ascii")
 
-    def _ensure_running_and_write(self, msg=None):
+    def _ensure_running_and_write(self, msg=None, *, env=None):
         with self._lock:
             if self._lock._recursion_count() > 1:
                 # The code below is certainly not reentrant-safe, so bail out
@@ -304,11 +310,11 @@ def _ensure_running_and_write(self, msg=None):
                     self._write(to_send)
                 except OSError:
                     self._teardown_dead_process()
-                    self._launch()
+                    self._launch(env)
 
                 msg = None  # message was sent in probe
             else:
-                self._launch()
+                self._launch(env)
 
         while True:
             try:

Lib/multiprocessing/spawn.py

@@ -213,6 +213,11 @@ def prepare(data):
     '''
     Try to get current process ready to unpickle process object
     '''
+    if 'process_env' in data:
+        # Resource tracking can start while importing the main module or
+        # unpickling the process, before _bootstrap installs current_process.
+        process.current_process()._env = data['process_env']
+
     if 'name' in data:
         process.current_process().name = data['name']
 

Lib/multiprocessing/util.py

@@ -540,13 +540,28 @@ def _flush_std_streams():
 # Start a program with only specified fds kept open
 #
 
-def spawnv_passfds(path, args, passfds):
+def _encode_spawn_env(env):
+    """Copy a mapping to immutable exec environment entries.
+
+    The caller must not mutate the mapping while this copy is made.
+    """
+    entries = []
+    for key, value in env.items():
+        key = os.fsencode(key)
+        value = os.fsencode(value)
+        if not key or b'=' in key or b'\0' in key or b'\0' in value:
+            raise ValueError('illegal environment variable name or value')
+        entries.append(key + b'=' + value)
+    return tuple(entries)
+
+
+def spawnv_passfds(path, args, passfds, env=None):
     import _posixsubprocess
     passfds = tuple(sorted(map(int, passfds)))
     errpipe_read, errpipe_write = os.pipe()
     try:
         return _posixsubprocess.fork_exec(
-            args, [path], True, passfds, None, None,
+            args, [path], True, passfds, None, env,
             -1, -1, -1, -1, -1, -1, errpipe_read, errpipe_write,
             False, False, -1, None, None, None, -1, None)
     finally: