Codebase list python-procrunner / c0a9f47
Deprecate return object array access Merge pull request #60 from DiamondLightSource/deprecate-array-access Markus Gerstel authored 3 years ago GitHub committed 3 years ago
3 changed file(s) with 156 addition(s) and 72 deletion(s). Raw diff Collapse all Expand all
+5
-4
docs/usage.rst less more
88
99 To test for successful completion::
1010
11 assert not result['exitcode']
12 assert result['exitcode'] == 0 # alternatively
11 assert not result.returncode
12 assert result.returncode == 0 # alternatively
13 result.check_returncode() # raises subprocess.CalledProcessError()
1314
1415 To test for no STDERR output::
1516
16 assert not result['stderr']
17 assert result['stderr'] == b'' # alternatively
17 assert not result.stderr
18 assert result.stderr == b'' # alternatively
1819
1920 To run with a specific environment variable set::
2021
+124
-42
procrunner/__init__.py less more
77 import sys
88 import time
99 import timeit
10 import warnings
1011 from multiprocessing import Pipe
1112 from threading import Thread
1213
1617 #
1718 # - runs an external process and waits for it to finish
1819 # - does not deadlock, no matter the process stdout/stderr output behaviour
19 # - returns the exit code, stdout, stderr (separately), and the total process
20 # runtime as a dictionary
20 # - returns the exit code, stdout, stderr (separately) as a
21 # subprocess.CompletedProcess object
2122 # - process can run in a custom environment, either as a modification of
2223 # the current environment or in a new environment from scratch
2324 # - stdin can be fed to the process, the returned dictionary contains
3536 #
3637 # Returns:
3738 #
38 # {'command': ['/bin/ls', '/some/path/containing spaces'],
39 # 'exitcode': 2,
40 # 'runtime': 0.12990689277648926,
41 # 'stderr': '/bin/ls: cannot access /some/path/containing spaces: No such file or directory\n',
42 # 'stdout': '',
43 # 'time_end': '2017-11-12 19:54:49 GMT',
44 # 'time_start': '2017-11-12 19:54:49 GMT',
45 # 'timeout': False}
46 #
39 # ReturnObject(
40 # args=('/bin/ls', '/some/path/containing spaces'),
41 # returncode=2,
42 # stdout=b'',
43 # stderr=b'/bin/ls: cannot access /some/path/containing spaces: No such file or directory\n'
44 # )
45 #
46 # which also offers (albeit deprecated)
47 #
48 # result.runtime == 0.12990689277648926
49 # result.time_end == '2017-11-12 19:54:49 GMT'
50 # result.time_start == '2017-11-12 19:54:49 GMT'
51 # result.timeout == False
4752
4853 __author__ = """Markus Gerstel"""
4954 __email__ = "scientificsoftware@diamond.ac.uk"
179184 if not self.has_finished():
180185 if self._debug:
181186 logger.debug(
182 "NBSR join after %f seconds, underrun not resolved"
183 % (timeit.default_timer() - underrun_debug_timer)
187 "NBSR join after %f seconds, underrun not resolved",
188 timeit.default_timer() - underrun_debug_timer,
184189 )
185190 raise Exception("thread did not terminate")
186191 if self._debug:
187192 logger.debug(
188 "NBSR underrun resolved after %f seconds"
189 % (timeit.default_timer() - underrun_debug_timer)
193 "NBSR underrun resolved after %f seconds",
194 timeit.default_timer() - underrun_debug_timer,
190195 )
191196 if self._closed:
192197 raise Exception("streamreader double-closed")
231236 raise
232237 self._buffer_pos += len(block)
233238 if debug:
234 logger.debug("wrote %d bytes to stream" % len(block))
239 logger.debug("wrote %d bytes to stream", len(block))
235240 self._stream.close()
236241 self._terminated = True
237242 if notify:
300305 return command
301306
302307
303 class ReturnObject(dict, subprocess.CompletedProcess):
308 class ReturnObject(subprocess.CompletedProcess):
304309 """
305310 A subprocess.CompletedProcess-like object containing the executed
306311 command, stdout and stderr (both as bytestrings), and the exitcode.
310315 exited with a non-zero exit code.
311316 """
312317
313 def __init__(self, *arg, **kw):
314 super().__init__(*arg, **kw)
315 self.args = self["command"]
316 self.returncode = self["exitcode"]
317 self.stdout = self["stdout"]
318 self.stderr = self["stderr"]
318 def __init__(self, exitcode=None, command=None, stdout=None, stderr=None, **kw):
319 super().__init__(
320 args=command, returncode=exitcode, stdout=stdout, stderr=stderr
321 )
322 self._extras = {
323 "timeout": kw.get("timeout"),
324 "runtime": kw.get("runtime"),
325 "time_start": kw.get("time_start"),
326 "time_end": kw.get("time_end"),
327 }
328
329 def __getitem__(self, key):
330 warnings.warn(
331 "dictionary access to a procrunner return object is deprecated",
332 DeprecationWarning,
333 stacklevel=2,
334 )
335 if key in self._extras:
336 return self._extras[key]
337 if not hasattr(self, key):
338 raise KeyError("Unknown attribute {key}".format(key=key))
339 return getattr(self, key)
319340
320341 def __eq__(self, other):
321342 """Override equality operator to account for added fields"""
327348 """This object is not immutable, so mark it as unhashable"""
328349 return None
329350
330 def __ne__(self, other):
331 """Overrides the default implementation (unnecessary in Python 3)"""
332 return not self.__eq__(other)
351 @property
352 def cmd(self):
353 warnings.warn(
354 "procrunner return object .cmd is deprecated, use .args",
355 DeprecationWarning,
356 stacklevel=2,
357 )
358 return self.args
359
360 @property
361 def command(self):
362 warnings.warn(
363 "procrunner return object .command is deprecated, use .args",
364 DeprecationWarning,
365 stacklevel=2,
366 )
367 return self.args
368
369 @property
370 def exitcode(self):
371 warnings.warn(
372 "procrunner return object .exitcode is deprecated, use .returncode",
373 DeprecationWarning,
374 stacklevel=2,
375 )
376 return self.returncode
377
378 @property
379 def timeout(self):
380 warnings.warn(
381 "procrunner return object .timeout is deprecated",
382 DeprecationWarning,
383 stacklevel=2,
384 )
385 return self._extras["timeout"]
386
387 @property
388 def runtime(self):
389 warnings.warn(
390 "procrunner return object .runtime is deprecated",
391 DeprecationWarning,
392 stacklevel=2,
393 )
394 return self._extras["runtime"]
395
396 @property
397 def time_start(self):
398 warnings.warn(
399 "procrunner return object .time_start is deprecated",
400 DeprecationWarning,
401 stacklevel=2,
402 )
403 return self._extras["time_start"]
404
405 @property
406 def time_end(self):
407 warnings.warn(
408 "procrunner return object .time_end is deprecated",
409 DeprecationWarning,
410 stacklevel=2,
411 )
412 return self._extras["time_end"]
413
414 def update(self, dictionary):
415 self._extras.update(dictionary)
333416
334417
335418 def run(
447530 (timeout is None) or (timeit.default_timer() < max_time)
448531 ):
449532 if debug and timeout is not None:
450 logger.debug("still running (T%.2fs)" % (timeit.default_timer() - max_time))
533 logger.debug("still running (T%.2fs)", timeit.default_timer() - max_time)
451534
452535 # wait for some time or until a stream is closed
453536 try:
481564 # timeout condition
482565 timeout_encountered = True
483566 if debug:
484 logger.debug("timeout (T%.2fs)" % (timeit.default_timer() - max_time))
567 logger.debug("timeout (T%.2fs)", timeit.default_timer() - max_time)
485568
486569 # send terminate signal and wait some time for buffers to be read
487570 p.terminate()
507590 runtime = timeit.default_timer() - start_time
508591 if timeout is not None:
509592 logger.debug(
510 "Process ended after %.1f seconds with exit code %d (T%.2fs)"
511 % (runtime, p.returncode, timeit.default_timer() - max_time)
593 "Process ended after %.1f seconds with exit code %d (T%.2fs)",
594 runtime,
595 p.returncode,
596 timeit.default_timer() - max_time,
512597 )
513598 else:
514599 logger.debug(
515 "Process ended after %.1f seconds with exit code %d"
516 % (runtime, p.returncode)
600 "Process ended after %.1f seconds with exit code %d", runtime, p.returncode
517601 )
518602
519603 stdout = stdout.get_output()
521605 time_end = time.strftime("%Y-%m-%d %H:%M:%S GMT", time.gmtime())
522606
523607 result = ReturnObject(
524 {
525 "exitcode": p.returncode,
526 "command": command,
527 "stdout": stdout,
528 "stderr": stderr,
529 "timeout": timeout_encountered,
530 "runtime": runtime,
531 "time_start": time_start,
532 "time_end": time_end,
533 }
608 exitcode=p.returncode,
609 command=command,
610 stdout=stdout,
611 stderr=stderr,
612 timeout=timeout_encountered,
613 runtime=runtime,
614 time_start=time_start,
615 time_end=time_end,
534616 )
535617 if stdin is not None:
536618 result.update(
+27
-26
tests/test_procrunner.py less more
9292 assert not mock_process.terminate.called
9393 assert not mock_process.kill.called
9494 for key in expected:
95 assert actual[key] == expected[key]
95 with pytest.warns(DeprecationWarning):
96 assert actual[key] == expected[key]
9697 assert actual.args == tuple(command)
9798 assert actual.returncode == mock_process.returncode
9899 assert actual.stdout == mock.sentinel.proc_stdout
259260
260261 def test_return_object_semantics():
261262 ro = procrunner.ReturnObject(
262 {
263 "command": mock.sentinel.command,
264 "exitcode": 0,
265 "stdout": mock.sentinel.stdout,
266 "stderr": mock.sentinel.stderr,
267 }
268 )
269 assert ro["command"] == mock.sentinel.command
263 command=mock.sentinel.command,
264 exitcode=0,
265 stdout=mock.sentinel.stdout,
266 stderr=mock.sentinel.stderr,
267 )
268 with pytest.warns(DeprecationWarning):
269 assert ro["command"] == mock.sentinel.command
270270 assert ro.args == mock.sentinel.command
271 assert ro["exitcode"] == 0
271 with pytest.warns(DeprecationWarning):
272 assert ro["exitcode"] == 0
272273 assert ro.returncode == 0
273 assert ro["stdout"] == mock.sentinel.stdout
274 with pytest.warns(DeprecationWarning):
275 assert ro["stdout"] == mock.sentinel.stdout
274276 assert ro.stdout == mock.sentinel.stdout
275 assert ro["stderr"] == mock.sentinel.stderr
277 with pytest.warns(DeprecationWarning):
278 assert ro["stderr"] == mock.sentinel.stderr
276279 assert ro.stderr == mock.sentinel.stderr
277280
278281 with pytest.raises(KeyError):
279 ro["unknownkey"]
282 with pytest.warns(DeprecationWarning):
283 ro["unknownkey"]
280284 ro.update({"unknownkey": mock.sentinel.key})
281 assert ro["unknownkey"] == mock.sentinel.key
285 with pytest.warns(DeprecationWarning):
286 assert ro["unknownkey"] == mock.sentinel.key
282287
283288
284289 def test_return_object_check_function_passes_on_success():
285290 ro = procrunner.ReturnObject(
286 {
287 "command": mock.sentinel.command,
288 "exitcode": 0,
289 "stdout": mock.sentinel.stdout,
290 "stderr": mock.sentinel.stderr,
291 }
291 command=mock.sentinel.command,
292 exitcode=0,
293 stdout=mock.sentinel.stdout,
294 stderr=mock.sentinel.stderr,
292295 )
293296 ro.check_returncode()
294297
295298
296299 def test_return_object_check_function_raises_on_error():
297300 ro = procrunner.ReturnObject(
298 {
299 "command": mock.sentinel.command,
300 "exitcode": 1,
301 "stdout": mock.sentinel.stdout,
302 "stderr": mock.sentinel.stderr,
303 }
301 command=mock.sentinel.command,
302 exitcode=1,
303 stdout=mock.sentinel.stdout,
304 stderr=mock.sentinel.stderr,
304305 )
305306 with pytest.raises(Exception) as e:
306307 ro.check_returncode()