Package list python-procrunner / 17bf3f1
Support PEP-519 system file path objects (#20) Resolve PEP-519 path objects in command line including parameters, environment and environment override dictionary, and working directory argument. Markus Gerstel authored 2 years ago GitHub committed 2 years ago
5 changed file(s) with 142 addition(s) and 65 deletion(s). Raw diff Collapse all Expand all
00 =======
11 History
22 =======
3
4 1.0.0 (2019-03-xx)
5 ------------------
6
7 * Support file system path objects (PEP-519) in arguments
38
49 0.9.1 (2019-02-22)
510 ------------------
2323 To run with a specific environment::
2424
2525 result = procrunner.run(..., environment={ 'VARIABLE': 'value' })
26
27 To run in a specific directory::
28
29 result = procrunner.run(..., working_directory='/some/path')
5959
6060
6161 class _LineAggregator(object):
62 """Buffer that can be filled with stream data and will aggregate complete
63 lines. Lines can be printed or passed to an arbitrary callback function.
64 The lines passed to the callback function are UTF-8 decoded and do not
65 contain a trailing newline character."""
62 """
63 Buffer that can be filled with stream data and will aggregate complete
64 lines. Lines can be printed or passed to an arbitrary callback function.
65 The lines passed to the callback function are UTF-8 decoded and do not
66 contain a trailing newline character.
67 """
6668
6769 def __init__(self, print_line=False, callback=None):
6870 """Create aggregator object."""
7274 self._decoder = codecs.getincrementaldecoder("utf-8")("replace")
7375
7476 def add(self, data):
75 """Add a single character to buffer. If one or more full lines are found,
76 print them (if desired) and pass to callback function."""
77 """
78 Add a single character to buffer. If one or more full lines are found,
79 print them (if desired) and pass to callback function.
80 """
7781 data = self._decoder.decode(data)
7882 if not data:
7983 return
160164 self._thread.start()
161165
162166 def has_finished(self):
163 """Returns whether the thread reading from the stream is still alive."""
167 """
168 Returns whether the thread reading from the stream is still alive.
169 """
164170 return self._terminated
165171
166172 def get_output(self):
167 """Retrieve the stored data in full.
168 This call may block if the reading thread has not yet terminated."""
173 """
174 Retrieve the stored data in full.
175 This call may block if the reading thread has not yet terminated.
176 """
169177 self._closing = True
170178 if not self.has_finished():
171179 if self._debug:
251259 return self._buffer_len - self._buffer_pos
252260
253261
262 def _path_resolve(obj):
263 """
264 Resolve file system path (PEP-519) objects to strings.
265
266 :param obj: A file system path object or something else.
267 :return: A string representation of a file system path object or, for
268 anything that was not a file system path object, the original
269 object.
270 """
271 if obj and hasattr(obj, "__fspath__"):
272 return obj.__fspath__()
273 return obj
274
275
254276 def _windows_resolve(command):
255 """Try and find the full path and file extension of the executable to run.
256 This is so that e.g. calls to 'somescript' will point at 'somescript.cmd'
257 without the need to set shell=True in the subprocess.
258 If the executable contains periods it is a special case. Here the
259 win32api call will fail to resolve the extension automatically, and it
260 has do be done explicitly.
261
262 :param command: The command array to be run, with the first element being
263 the command with or w/o path, with or w/o extension.
264 :return: Returns the command array with the executable resolved with the
265 correct extension. If the executable cannot be resolved for any
266 reason the original command array is returned.
267 """
277 """
278 Try and find the full path and file extension of the executable to run.
279 This is so that e.g. calls to 'somescript' will point at 'somescript.cmd'
280 without the need to set shell=True in the subprocess.
281 If the executable contains periods it is a special case. Here the
282 win32api call will fail to resolve the extension automatically, and it
283 has do be done explicitly.
284
285 :param command: The command array to be run, with the first element being
286 the command with or w/o path, with or w/o extension.
287 :return: Returns the command array with the executable resolved with the
288 correct extension. If the executable cannot be resolved for any
289 reason the original command array is returned.
290 """
268291 try:
269292 import win32api
270293 except ImportError:
278301 )
279302 return command
280303
281 try:
282 # Ensure the command parameter is iterable.
283 iter(command)
284 except TypeError:
285 # If it is not iterable it could be a Mock(). Return it untouched.
304 if not command or not isinstance(command[0], six.string_types):
286305 return command
287306
288307 try:
289308 _, found_executable = win32api.FindExecutable(command[0])
290309 logger.debug("Resolved %s as %s", command[0], found_executable)
291 return [found_executable] + command[1:]
310 return (found_executable,) + tuple(command[1:])
292311 except Exception as e:
293312 if not hasattr(e, "winerror"):
294313 raise
302321 try:
303322 _, found_executable = win32api.FindExecutable(command[0] + extension)
304323 logger.debug("Resolved %s as %s", command[0], found_executable)
305 return [found_executable] + command[1:]
324 return (found_executable,) + tuple(command[1:])
306325 except Exception as e:
307326 if not hasattr(e, "winerror"):
308327 raise
325344 win32resolve=True,
326345 working_directory=None,
327346 ):
328 """Run an external process.
329
330 :param array command: Command line to be run, specified as array.
331 :param timeout: Terminate program execution after this many seconds.
332 :param boolean debug: Enable further debug messages.
333 :param stdin: Optional string that is passed to command stdin.
334 :param boolean print_stdout: Pass stdout through to sys.stdout.
335 :param boolean print_stderr: Pass stderr through to sys.stderr.
336 :param callback_stdout: Optional function which is called for each
337 stdout line.
338 :param callback_stderr: Optional function which is called for each
339 stderr line.
340 :param dict environment: The full execution environment for the command.
341 :param dict environment_override: Change environment variables from the
342 current values for command execution.
343 :param boolean win32resolve: If on Windows, find the appropriate executable
344 first. This allows running of .bat, .cmd, etc.
345 files without explicitly specifying their
346 extension.
347 :param string working_directory: If specified, run the executable from
348 within this working directory.
349 :return: A dictionary containing stdout, stderr (both as bytestrings),
350 runtime, exitcode, and more.
351 """
347 """
348 Run an external process.
349
350 File system path objects (PEP-519) are accepted in the command, environment,
351 and working directory arguments.
352
353 :param array command: Command line to be run, specified as array.
354 :param timeout: Terminate program execution after this many seconds.
355 :param boolean debug: Enable further debug messages.
356 :param stdin: Optional string that is passed to command stdin.
357 :param boolean print_stdout: Pass stdout through to sys.stdout.
358 :param boolean print_stderr: Pass stderr through to sys.stderr.
359 :param callback_stdout: Optional function which is called for each
360 stdout line.
361 :param callback_stderr: Optional function which is called for each
362 stderr line.
363 :param dict environment: The full execution environment for the command.
364 :param dict environment_override: Change environment variables from the
365 current values for command execution.
366 :param boolean win32resolve: If on Windows, find the appropriate executable
367 first. This allows running of .bat, .cmd, etc.
368 files without explicitly specifying their
369 extension.
370 :param string working_directory: If specified, run the executable from
371 within this working directory.
372 :return: A dictionary containing stdout, stderr (both as bytestrings),
373 runtime, exitcode, and more.
374 """
352375
353376 time_start = time.strftime("%Y-%m-%d %H:%M:%S GMT", time.gmtime())
354377 logger.debug("Starting external process: %s", command)
364387 max_time = start_time + timeout
365388
366389 if environment is not None:
367 env = environment
390 env = {key: _path_resolve(environment[key]) for key in environment}
368391 else:
369392 env = os.environ
370393 if environment_override:
371394 env = copy.copy(env)
372395 env.update(
373 {key: str(environment_override[key]) for key in environment_override}
396 {
397 key: str(_path_resolve(environment_override[key]))
398 for key in environment_override
399 }
374400 )
375401
402 command = tuple(_path_resolve(part) for part in command)
376403 if win32resolve and sys.platform == "win32":
377404 command = _windows_resolve(command)
378405
379406 p = subprocess.Popen(
380407 command,
381408 shell=False,
382 cwd=working_directory,
409 cwd=_path_resolve(working_directory),
383410 env=env,
384411 stdin=stdin_pipe,
385412 stdout=subprocess.PIPE,
513540
514541
515542 def run_process_dummy(command, **kwargs):
516 """A stand-in function that returns a valid result dictionary indicating a
517 successful execution. The external process is not run."""
543 """
544 A stand-in function that returns a valid result dictionary indicating a
545 successful execution. The external process is not run.
546 """
518547 warnings.warn(
519548 "procrunner.run_process_dummy() is deprecated", DeprecationWarning, stacklevel=2
520549 )
5353 "stderr": mock.sentinel.proc_stderr,
5454 "stdout": mock.sentinel.proc_stdout,
5555 "exitcode": mock_process.returncode,
56 "command": command,
56 "command": tuple(command),
5757 "runtime": mock.ANY,
5858 "timeout": False,
5959 "time_start": mock.ANY,
100100 def test_default_process_environment_is_parent_environment(mock_subprocess):
101101 mock_subprocess.Popen.side_effect = NotImplementedError() # cut calls short
102102 with pytest.raises(NotImplementedError):
103 procrunner.run(mock.Mock(), -1, False)
103 procrunner.run([mock.Mock()], -1, False)
104104 assert mock_subprocess.Popen.call_args[1]["env"] == os.environ
105105
106106
110110 mock_env = {"key": mock.sentinel.key}
111111 # Pass an environment dictionary
112112 with pytest.raises(NotImplementedError):
113 procrunner.run(mock.Mock(), -1, False, environment=copy.copy(mock_env))
113 procrunner.run([mock.Mock()], -1, False, environment=copy.copy(mock_env))
114114 assert mock_subprocess.Popen.call_args[1]["env"] == mock_env
115115
116116
122122 # Pass an environment dictionary
123123 with pytest.raises(NotImplementedError):
124124 procrunner.run(
125 mock.Mock(),
125 [mock.Mock()],
126126 -1,
127127 False,
128128 environment=copy.copy(mock_env1),
139139 mock_env2 = {"keyB": str(mock.sentinel.keyB)}
140140 with pytest.raises(NotImplementedError):
141141 procrunner.run(
142 mock.Mock(), -1, False, environment_override=copy.copy(mock_env2)
142 [mock.Mock()], -1, False, environment_override=copy.copy(mock_env2)
143143 )
144144 random_environment_variable = list(os.environ)[0]
145145 if random_environment_variable == list(mock_env2)[0]:
165165 random_environment_value = os.getenv(random_environment_variable)
166166 with pytest.raises(NotImplementedError):
167167 procrunner.run(
168 mock.Mock(),
168 [mock.Mock()],
169169 -1,
170170 False,
171171 environment_override={
22 import os
33 import sys
44
5 import mock
56 import procrunner
67 import pytest
8
9
10 def PEP519(path):
11 class MockObject:
12 @staticmethod
13 def __fspath__():
14 return path
15
16 def __repr__(self):
17 return "<path object: %s>" % path
18
19 return MockObject()
20
21
22 @pytest.mark.parametrize(
23 "obj",
24 (
25 None,
26 True,
27 False,
28 1,
29 1.0,
30 ["thing"],
31 {},
32 {1},
33 {"thing": "thing"},
34 "string",
35 b"bytes",
36 RuntimeError(),
37 ["thing", PEP519("thing")], # no recursive resolution
38 ),
39 )
40 def test_path_object_resolution_for_non_path_objs_does_not_modify_objects(obj):
41 assert procrunner._path_resolve(obj) is obj
42
43
44 def test_path_object_resolution_of_path_objects():
45 assert procrunner._path_resolve(PEP519("thing")) == "thing"
746
847
948 @pytest.mark.skipif(sys.platform != "win32", reason="windows specific test only")
2261 assert os.path.exists(resolved[0])
2362
2463 # parameters are unchanged
25 assert resolved[1:] == command[1:]
64 assert resolved[1:] == tuple(command[1:])
2665
2766
2867 @pytest.mark.skipif(sys.platform != "win32", reason="windows specific test only")