Commit e8a58c19872ee712002bae1194839d73f392b1b3 - python-procrunner

Import upstream version 2.3.1+git20221110.1.77d55b7 Debian Janitor 9 months ago

30 changed file(s) with 1371 addition(s) and 1272 deletion(s). Raw diff Collapse all Expand all

-21

~~.editorconfig~~ less more

0		# http://editorconfig.org
1
2		root = true
3
4		[*]
5		indent_style = space
6		indent_size = 4
7		trim_trailing_whitespace = true
8		insert_final_newline = true
9		charset = utf-8
10		end_of_line = lf
11
12		[*.bat]
13		indent_style = tab
14		end_of_line = crlf
15
16		[LICENSE]
17		insert_final_newline = false
18
19		[Makefile]
20		indent_style = tab

-15

~~.github/ISSUE_TEMPLATE.md~~ less more

0		* ProcRunner version:
1		* Python version:
2		* Operating System:
3
4		### Description
5
6		Describe what you were trying to get done.
7		Tell us what happened, what went wrong, and what you expected to happen.
8
9		### What I Did
10
11		```
12		Paste the command(s) you ran and the output.
13		If there was a crash, please include the traceback here.
14		```

-106

~~.gitignore~~ less more

0		# Byte-compiled / optimized / DLL files
1		__pycache__/
2		*.py[cod]
3		*$py.class
4
5		# C extensions
6		*.so
7
8		# Temporary files
9		*.sw[op]
10		*~
11
12		# Distribution / packaging
13		.Python
14		env/
15		build/
16		develop-eggs/
17		dist/
18		downloads/
19		eggs/
20		.eggs/
21		lib/
22		lib64/
23		parts/
24		sdist/
25		var/
26		wheels/
27		*.egg-info/
28		.installed.cfg
29		*.egg
30
31		# PyInstaller
32		# Usually these files are written by a python script from a template
33		# before PyInstaller builds the exe, so as to inject date/other infos into it.
34		*.manifest
35		*.spec
36
37		# Installer logs
38		pip-log.txt
39		pip-delete-this-directory.txt
40
41		# Unit test / coverage reports
42		htmlcov/
43		.tox/
44		.coverage
45		.coverage.*
46		.cache
47		nosetests.xml
48		coverage.xml
49		*.cover
50		.hypothesis/
51		.pytest_cache
52
53		# Translations
54		*.mo
55		*.pot
56
57		# Django stuff:
58		*.log
59		local_settings.py
60
61		# Flask stuff:
62		instance/
63		.webassets-cache
64
65		# Scrapy stuff:
66		.scrapy
67
68		# Sphinx documentation
69		docs/_build/
70
71		# PyBuilder
72		target/
73
74		# Jupyter Notebook
75		.ipynb_checkpoints
76
77		# pyenv
78		.python-version
79
80		# celery beat schedule file
81		celerybeat-schedule
82
83		# SageMath parsed files
84		*.sage.py
85
86		# dotenv
87		.env
88
89		# virtualenv
90		.venv
91		venv/
92		ENV/
93
94		# Spyder project settings
95		.spyderproject
96		.spyproject
97
98		# Rope project settings
99		.ropeproject
100
101		# mkdocs documentation
102		/site
103
104		# mypy
105		.mypy_cache/

-5

~~.pyup.yml~~ less more

0		# autogenerated pyup.io config file
1		# see https://pyup.io/docs/configuration/ for all available options
2
3		schedule: every month
4

-72

~~.travis.yml~~ less more

0		# Config file for automatic testing at travis-ci.org
1
2		language: python
3
4		matrix:
5		include:
6		- python: 3.8
7		dist: xenial
8		sudo: true
9		- python: 3.7
10		dist: xenial
11		sudo: true
12		- python: 3.6
13		- python: 3.5
14		- python: 2.7
15		- python: pypy
16		- os: osx
17		language: generic
18		env: CONDA=3.8 TOXENV=py38
19		- os: osx
20		language: generic
21		env: CONDA=3.7 TOXENV=py37
22		- os: osx
23		language: generic
24		env: CONDA=3.6 TOXENV=py36
25		- os: osx
26		language: generic
27		env: CONDA=3.5 TOXENV=py35
28		- os: osx
29		language: generic
30		env: CONDA=2.7 TOXENV=py27
31
32		allow_failures:
33		- env: OPTIONAL=1
34
35		fast_finish: true
36
37		before_install: \|
38		if [ ! -z "$CONDA" ]; then
39		if [ "$TRAVIS_OS_NAME" == "osx" ]; then
40		curl https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh --output miniconda.sh
41		fi
42		chmod +x miniconda.sh
43		./miniconda.sh -b
44		export PATH=$HOME/miniconda3/bin:$PATH
45		conda update --yes conda
46		conda create --yes -n travis python=$CONDA
47		source activate travis
48		# A manual check that the correct version of Python is running.
49		python --version
50		fi
51
52		# Command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors
53		install: pip install -U tox-travis
54
55		# Command to run tests, e.g. python setup.py test
56		script: tox
57
58		# Assuming you have installed the travis-ci CLI tool, after you
59		# create the Github repo and add it to Travis, run the
60		# following command to finish PyPI deployment setup:
61		# $ travis encrypt --add deploy.password
62		deploy:
63		provider: pypi
64		distributions: sdist bdist_wheel
65		user: mgerstel
66		password:
67		secure: qN3EYVlH22eLPZp5CSvc5Bz8bpP0l0wBZ8tcYMa6kBrqelYdIgqkuZESR/oCMu1YBA3AbcKJNwMuzuWf8RNGAFebD820OJThdP4czRMCv6LDbWABnv12lFHQLQSW1fMQVvb2arjnE/Ew7BFq70p0wOlIJJRwu6CoeOXW/sMeVYMivxdaHmgORq3cdMluFAy4amVb3Fc8i7mxAM0QGklO7x/UWJR/IEpUk2RlUbXL+HrzNoEjRtDeMxoCR84gKZTjVeUQ/iIQSuWwxlt7v1FNENj6ZEbE7+PS8/ylIVfPufbCr8tEEv8W58QcxQ5xPJC2g85ulsN5dM9/9FekhpyKa25B/4wKUNq5T8rKJ8WZ6hMiGffW8rmAfrGTmrBaojKBi0pb9VfXJ5KXUcunVXwQaAn2L80jLLHNsAo94ZxeoowD1eJox9Wh1NtNc+NiUv8K6spOIBsur7G5GY4JVA/yZ7w+DweEfQEp8/SEdVhK0vEYSYT4FnJHuAAmNgeedyAtoes4+a5bYYUM4qrz2OC78NWQWAnnsZhD4Y/TulkavWSexVCqfSePRoK3gcCs+rXwiM69XkMbL1Wgj1gNou+XMkntayH2ZDHkkyJi5F7ls4nqMH5RON9FfVygJMoHDRqh0p4RV25IzJ4FSYqKihHNBO31/URnU2ihpW7n8kM+mbM=
68		on:
69		tags: true
70		repo: DiamondLightSource/python-procrunner
71		python: 3.8

-4

CONTRIBUTING.rst less more

100	100	1. The pull request should include tests.
101	101	2. If the pull request adds functionality, the docs should be updated. Put
102	102	your new functionality into a function with a docstring, and add the
103		feature to the list in README.rst.
104		3. The pull request should work for Python 2.7, 3.5, 3.6, 3.7, 3.8, and for PyPy. Check
105		https://travis-ci.org/DiamondLightSource/python-procrunner/pull_requests
106		and make sure that the tests pass for all supported Python versions.
	103	feature to the list in HISTORY.rst/README.rst.
107	104
108	105	Tips
109	106	----

+65

-21

HISTORY.rst less more

0		=======
1	0	History
2		=======
	1	==================
	2
	3	3.0.0 (2022-01-??)
	4	------------------
	5	* Drop Python 3.6 support
	6	* The run() function now returns a subprocess.CompletedProcess object,
	7	which no longer allows array access operations
	8	(those were deprecated in `#60 <https://github.com/DiamondLightSource/python-procrunner/pull/60>`_)
	9	* The run() argument 'raise_timeout_exception' is now set by default,
	10	a 'False' value will lead to a UserWarning and a behavioural change.
	11	The argument is now deprecated and will be removed in a future version.
	12	(previously introduced in `#61 <https://github.com/DiamondLightSource/python-procrunner/pull/61>`_)
	13	* Calling the run() function with multiple unnamed arguments is no longer supported
	14	(previously deprecated in `#62 <https://github.com/DiamondLightSource/python-procrunner/pull/62>`_)
	15	* The run() function no longer accepts a 'debug' argument
	16	(previously deprecated in `#63 <https://github.com/DiamondLightSource/python-procrunner/pull/63>`_)
	17
	18	2.3.3 (2022-03-23)
	19	------------------
	20	* Allow specifying 'preexec_fn' and 'creationflags' keywords, which will be passed through to
	21	the subprocess call
	22
	23	2.3.2 (2022-01-28)
	24	------------------
	25	* The run() function now understands stdin=subprocess.DEVNULL to close the subprocess stdin,
	26	rather than to connect through the existing stdin, which is the current default
	27
	28	2.3.1 (2021-10-25)
	29	------------------
	30	* Add Python 3.10 support
	31
	32	2.3.0 (2020-10-29)
	33	------------------
	34	* Add Python 3.9 support, drop Python 3.5 support
	35	* Fix a file descriptor leak on subprocess execution
	36
	37	2.2.0 (2020-09-07)
	38	------------------
	39	* Calling the run() function with unnamed arguments (other than the command
	40	list as the first argument) is now deprecated. As a number of arguments
	41	will be removed in a future version the use of unnamed arguments will
	42	cause future confusion. `Use explicit keyword arguments instead (#62). <https://github.com/DiamondLightSource/python-procrunner/pull/62>`_
	43	* `The run() function debug argument has been deprecated (#63). <https://github.com/DiamondLightSource/python-procrunner/pull/63>`_
	44	This is only used to debug the NonBlockingStream* classes. Those are due
	45	to be replaced in a future release, so the argument will no longer serve
	46	a purpose. Debugging information remains available via standard logging
	47	mechanisms.
	48	* Final version supporting Python 3.5
	49
	50	2.1.0 (2020-09-05)
	51	------------------
	52	* `Deprecated array access on the return object (#60). <https://github.com/DiamondLightSource/python-procrunner/pull/60>`_
	53	The return object will become a subprocess.CompletedProcess in a future
	54	release, which no longer allows array-based access. For a translation table
	55	of array elements to attributes please see the pull request linked above.
	56	* Add a `new parameter 'raise_timeout_exception' (#61). <https://github.com/DiamondLightSource/python-procrunner/pull/61>`_
	57	When set to 'True' a subprocess.TimeoutExpired exception is raised when the
	58	process runtime exceeds the timeout threshold. This defaults to 'False' and
	59	will be set to 'True' in a future release.
	60
	61	2.0.0 (2020-06-24)
	62	------------------
	63	* Python 3.5+ only, support for Python 2.7 has been dropped
	64	* Deprecated function alias run_process() has been removed
	65	* Fixed a stability issue on Windows
3	66
4	67	1.1.0 (2019-11-04)
5	68	------------------
6
7	69	* Add Python 3.8 support, drop Python 3.4 support
8	70
9	71	1.0.2 (2019-05-20)
10	72	------------------
11
12	73	* Stop environment override variables leaking into the process environment
13	74
14	75	1.0.1 (2019-04-16)
15	76	------------------
16
17	77	* Minor fixes on the return object (implement equality,
18	78	mark as unhashable)
19	79
20	80	1.0.0 (2019-03-25)
21	81	------------------
22
23	82	* Support file system path objects (PEP-519) in arguments
24	83	* Change the return object to make it similar to
25	84	subprocess.CompletedProcess, introduced with Python 3.5+
26	85
27	86	0.9.1 (2019-02-22)
28	87	------------------
29
30	88	* Have deprecation warnings point to correct code locations
31	89
32	90	0.9.0 (2018-12-07)
33	91	------------------
34
35	92	* Trap UnicodeEncodeError when printing output. Offending characters
36	93	are replaced and a warning is logged once. Hints at incorrectly set
37	94	PYTHONIOENCODING.
38	95
39	96	0.8.1 (2018-12-04)
40	97	------------------
41
42	98	* Fix a few deprecation warnings
43	99
44	100	0.8.0 (2018-10-09)
45	101	------------------
46
47	102	* Add parameter working_directory to set the working directory
48	103	of the subprocess
49	104
50	105	0.7.2 (2018-10-05)
51	106	------------------
52
53	107	* Officially support Python 3.7
54	108
55	109	0.7.1 (2018-09-03)
56	110	------------------
57
58	111	* Accept environment variable overriding with numeric values.
59	112
60	113	0.7.0 (2018-05-13)
61	114	------------------
62
63	115	* Unicode fixes. Fix crash on invalid UTF-8 input.
64	116	* Clarify that stdout/stderr values are returned as bytestrings.
65	117	* Callbacks receive the data decoded as UTF-8 unicode strings

69	121
70	122	0.6.1 (2018-05-02)
71	123	------------------
72
73	124	* Maintenance release to add some tests for executable resolution.
74	125
75	126	0.6.0 (2018-05-02)
76	127	------------------
77
78	128	* Fix Win32 API executable resolution for commands containing a dot ('.') in
79	129	addition to a file extension (say '.bat').
80	130
81	131	0.5.1 (2018-04-27)
82	132	------------------
83
84	133	* Fix Win32API dependency installation on Windows.
85	134
86	135	0.5.0 (2018-04-26)
87	136	------------------
88
89	137	* New keyword 'win32resolve' which only takes effect on Windows and is enabled
90	138	by default. This causes procrunner to call the Win32 API FindExecutable()
91	139	function to try and lookup non-.exe files with the corresponding name. This

94	142
95	143	0.4.0 (2018-04-23)
96	144	------------------
97
98	145	* Python 2.7 support on Windows. Python3 not yet supported on Windows.
99	146
100	147	0.3.0 (2018-04-17)
101	148	------------------
102
103	149	* run_process() renamed to run()
104	150	* Python3 compatibility fixes
105	151
106	152	0.2.0 (2018-03-12)
107	153	------------------
108
109	154	* Procrunner is now Python3 3.3-3.6 compatible.
110	155
111	156	0.1.0 (2018-03-12)
112	157	------------------
113
114	158	* First release on PyPI.

-1

LICENSE less more

0		Copyright (c) 2018 Diamond Light Source.
	0	Copyright (c) 2018-2021 Diamond Light Source.
1	1	All rights reserved.
2	2
3	3	Redistribution and use in source and binary forms, with or without

-87

~~Makefile~~ less more

0		.PHONY: clean clean-test clean-pyc clean-build docs help
1		.DEFAULT_GOAL := help
2
3		define BROWSER_PYSCRIPT
4		import os, webbrowser, sys
5
6		try:
7		from urllib import pathname2url
8		except:
9		from urllib.request import pathname2url
10
11		webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1])))
12		endef
13		export BROWSER_PYSCRIPT
14
15		define PRINT_HELP_PYSCRIPT
16		import re, sys
17
18		for line in sys.stdin:
19		match = re.match(r'^([a-zA-Z_-]+):.?## (.)$$', line)
20		if match:
21		target, help = match.groups()
22		print("%-20s %s" % (target, help))
23		endef
24		export PRINT_HELP_PYSCRIPT
25
26		BROWSER := python -c "$$BROWSER_PYSCRIPT"
27
28		help:
29		@python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST)
30
31		clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts
32
33		clean-build: ## remove build artifacts
34		rm -fr build/
35		rm -fr dist/
36		rm -fr .eggs/
37		find . -name '*.egg-info' -exec rm -fr {} +
38		find . -name '*.egg' -exec rm -f {} +
39
40		clean-pyc: ## remove Python file artifacts
41		find . -name '*.pyc' -exec rm -f {} +
42		find . -name '*.pyo' -exec rm -f {} +
43		find . -name '*~' -exec rm -f {} +
44		find . -name '__pycache__' -exec rm -fr {} +
45
46		clean-test: ## remove test and coverage artifacts
47		rm -fr .tox/
48		rm -f .coverage
49		rm -fr htmlcov/
50
51		lint: ## check style with flake8
52		flake8 procrunner tests
53
54		test: ## run tests quickly with the default Python
55		py.test
56
57		test-all: ## run tests on every Python version with tox
58		tox
59
60		coverage: ## check code coverage quickly with the default Python
61		coverage run --source procrunner -m pytest
62		coverage report -m
63		coverage html
64		$(BROWSER) htmlcov/index.html
65
66		docs: ## generate Sphinx HTML documentation, including API docs
67		rm -f docs/procrunner.rst
68		rm -f docs/modules.rst
69		sphinx-apidoc -o docs/ procrunner
70		$(MAKE) -C docs clean
71		$(MAKE) -C docs html
72		$(BROWSER) docs/_build/html/index.html
73
74		servedocs: docs ## compile the docs watching for changes
75		watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D .
76
77		release: clean ## package and upload a release
78		twine upload dist/*
79
80		dist: clean ## builds source and wheel package
81		python setup.py sdist
82		python setup.py bdist_wheel
83		ls -l dist
84
85		install: clean ## install the package to the active Python's site-packages
86		python setup.py install

+250

-0

PKG-INFO less more

	0	Metadata-Version: 2.1
	1	Name: procrunner
	2	Version: 2.3.3
	3	Summary: Versatile utility function to run external processes
	4	Author: Diamond Light Source - Scientific Software et al.
	5	Author-email: scientificsoftware@diamond.ac.uk
	6	License: BSD
	7	Project-URL: Download, https://github.com/DiamondLightSource/python-procrunner/tags
	8	Project-URL: Documentation, https://procrunner.readthedocs.io/
	9	Project-URL: GitHub, https://github.com/DiamondLightSource/python-procrunner
	10	Project-URL: Bug-Tracker, https://github.com/DiamondLightSource/python-procrunner/issues
	11	Keywords: procrunner
	12	Classifier: Development Status :: 5 - Production/Stable
	13	Classifier: Intended Audience :: Developers
	14	Classifier: License :: OSI Approved :: BSD License
	15	Classifier: Natural Language :: English
	16	Classifier: Programming Language :: Python :: 3
	17	Classifier: Programming Language :: Python :: 3.7
	18	Classifier: Programming Language :: Python :: 3.8
	19	Classifier: Programming Language :: Python :: 3.9
	20	Classifier: Programming Language :: Python :: 3.10
	21	Classifier: Operating System :: OS Independent
	22	Classifier: Topic :: Software Development :: Libraries :: Python Modules
	23	Requires-Python: >=3.7
	24	License-File: LICENSE
	25
	26	==========
	27	ProcRunner
	28	==========
	29
	30
	31	.. image:: https://img.shields.io/pypi/v/procrunner.svg
	32	:target: https://pypi.python.org/pypi/procrunner
	33	:alt: PyPI release
	34
	35	.. image:: https://img.shields.io/conda/vn/conda-forge/procrunner.svg
	36	:target: https://anaconda.org/conda-forge/procrunner
	37	:alt: Conda Version
	38
	39	.. image:: https://dev.azure.com/DLS-tooling/procrunner/_apis/build/status/CI?branchName=master
	40	:target: https://github.com/DiamondLightSource/python-procrunner/commits/master
	41	:alt: Build status
	42
	43	.. image:: https://ci.appveyor.com/api/projects/status/jtq4brwri5q18d0u/branch/master
	44	:target: https://ci.appveyor.com/project/Anthchirp/python-procrunner
	45	:alt: Build status
	46
	47	.. image:: https://readthedocs.org/projects/procrunner/badge/?version=latest
	48	:target: https://procrunner.readthedocs.io/en/latest/?badge=latest
	49	:alt: Documentation Status
	50
	51	.. image:: https://img.shields.io/pypi/pyversions/procrunner.svg
	52	:target: https://pypi.python.org/pypi/procrunner
	53	:alt: Supported Python versions
	54
	55	.. image:: https://img.shields.io/badge/code%20style-black-000000.svg
	56	:target: https://github.com/ambv/black
	57	:alt: Code style: black
	58
	59	Versatile utility function to run external processes
	60
	61	* Free software: BSD license
	62	* Documentation: https://procrunner.readthedocs.io.
	63
	64
	65	Features
	66	--------
	67
	68	* runs an external process and waits for it to finish
	69	* does not deadlock, no matter the process stdout/stderr output behaviour
	70	* returns the exit code, stdout, stderr (separately, both as bytestrings),
	71	as a subprocess.CompletedProcess object
	72	* process can run in a custom environment, either as a modification of
	73	the current environment or in a new environment from scratch
	74	* stdin can be fed to the process
	75	* stdout and stderr is printed by default, can be disabled
	76	* stdout and stderr can be passed to any arbitrary function for
	77	live processing (separately, both as unicode strings)
	78	* optionally enforces a time limit on the process, raising a
	79	subprocess.TimeoutExpired exception if it is exceeded.
	80
	81
	82	Credits
	83	-------
	84
	85	This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypackage`_ project template.
	86
	87	.. _Cookiecutter: https://github.com/audreyr/cookiecutter
	88	.. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage
	89
	90
	91	History
	92	==================
	93
	94	3.0.0 (2022-01-??)
	95	------------------
	96	* Drop Python 3.6 support
	97	* The run() function now returns a subprocess.CompletedProcess object,
	98	which no longer allows array access operations
	99	(those were deprecated in `#60 <https://github.com/DiamondLightSource/python-procrunner/pull/60>`_)
	100	* The run() argument 'raise_timeout_exception' is now set by default,
	101	a 'False' value will lead to a UserWarning and a behavioural change.
	102	The argument is now deprecated and will be removed in a future version.
	103	(previously introduced in `#61 <https://github.com/DiamondLightSource/python-procrunner/pull/61>`_)
	104	* Calling the run() function with multiple unnamed arguments is no longer supported
	105	(previously deprecated in `#62 <https://github.com/DiamondLightSource/python-procrunner/pull/62>`_)
	106	* The run() function no longer accepts a 'debug' argument
	107	(previously deprecated in `#63 <https://github.com/DiamondLightSource/python-procrunner/pull/63>`_)
	108
	109	2.3.3 (2022-03-23)
	110	------------------
	111	* Allow specifying 'preexec_fn' and 'creationflags' keywords, which will be passed through to
	112	the subprocess call
	113
	114	2.3.2 (2022-01-28)
	115	------------------
	116	* The run() function now understands stdin=subprocess.DEVNULL to close the subprocess stdin,
	117	rather than to connect through the existing stdin, which is the current default
	118
	119	2.3.1 (2021-10-25)
	120	------------------
	121	* Add Python 3.10 support
	122
	123	2.3.0 (2020-10-29)
	124	------------------
	125	* Add Python 3.9 support, drop Python 3.5 support
	126	* Fix a file descriptor leak on subprocess execution
	127
	128	2.2.0 (2020-09-07)
	129	------------------
	130	* Calling the run() function with unnamed arguments (other than the command
	131	list as the first argument) is now deprecated. As a number of arguments
	132	will be removed in a future version the use of unnamed arguments will
	133	cause future confusion. `Use explicit keyword arguments instead (#62). <https://github.com/DiamondLightSource/python-procrunner/pull/62>`_
	134	* `The run() function debug argument has been deprecated (#63). <https://github.com/DiamondLightSource/python-procrunner/pull/63>`_
	135	This is only used to debug the NonBlockingStream* classes. Those are due
	136	to be replaced in a future release, so the argument will no longer serve
	137	a purpose. Debugging information remains available via standard logging
	138	mechanisms.
	139	* Final version supporting Python 3.5
	140
	141	2.1.0 (2020-09-05)
	142	------------------
	143	* `Deprecated array access on the return object (#60). <https://github.com/DiamondLightSource/python-procrunner/pull/60>`_
	144	The return object will become a subprocess.CompletedProcess in a future
	145	release, which no longer allows array-based access. For a translation table
	146	of array elements to attributes please see the pull request linked above.
	147	* Add a `new parameter 'raise_timeout_exception' (#61). <https://github.com/DiamondLightSource/python-procrunner/pull/61>`_
	148	When set to 'True' a subprocess.TimeoutExpired exception is raised when the
	149	process runtime exceeds the timeout threshold. This defaults to 'False' and
	150	will be set to 'True' in a future release.
	151
	152	2.0.0 (2020-06-24)
	153	------------------
	154	* Python 3.5+ only, support for Python 2.7 has been dropped
	155	* Deprecated function alias run_process() has been removed
	156	* Fixed a stability issue on Windows
	157
	158	1.1.0 (2019-11-04)
	159	------------------
	160	* Add Python 3.8 support, drop Python 3.4 support
	161
	162	1.0.2 (2019-05-20)
	163	------------------
	164	* Stop environment override variables leaking into the process environment
	165
	166	1.0.1 (2019-04-16)
	167	------------------
	168	* Minor fixes on the return object (implement equality,
	169	mark as unhashable)
	170
	171	1.0.0 (2019-03-25)
	172	------------------
	173	* Support file system path objects (PEP-519) in arguments
	174	* Change the return object to make it similar to
	175	subprocess.CompletedProcess, introduced with Python 3.5+
	176
	177	0.9.1 (2019-02-22)
	178	------------------
	179	* Have deprecation warnings point to correct code locations
	180
	181	0.9.0 (2018-12-07)
	182	------------------
	183	* Trap UnicodeEncodeError when printing output. Offending characters
	184	are replaced and a warning is logged once. Hints at incorrectly set
	185	PYTHONIOENCODING.
	186
	187	0.8.1 (2018-12-04)
	188	------------------
	189	* Fix a few deprecation warnings
	190
	191	0.8.0 (2018-10-09)
	192	------------------
	193	* Add parameter working_directory to set the working directory
	194	of the subprocess
	195
	196	0.7.2 (2018-10-05)
	197	------------------
	198	* Officially support Python 3.7
	199
	200	0.7.1 (2018-09-03)
	201	------------------
	202	* Accept environment variable overriding with numeric values.
	203
	204	0.7.0 (2018-05-13)
	205	------------------
	206	* Unicode fixes. Fix crash on invalid UTF-8 input.
	207	* Clarify that stdout/stderr values are returned as bytestrings.
	208	* Callbacks receive the data decoded as UTF-8 unicode strings
	209	with unknown characters replaced by \ufffd (unicode replacement
	210	character). Same applies to printing of output.
	211	* Mark stdin broken on Windows.
	212
	213	0.6.1 (2018-05-02)
	214	------------------
	215	* Maintenance release to add some tests for executable resolution.
	216
	217	0.6.0 (2018-05-02)
	218	------------------
	219	* Fix Win32 API executable resolution for commands containing a dot ('.') in
	220	addition to a file extension (say '.bat').
	221
	222	0.5.1 (2018-04-27)
	223	------------------
	224	* Fix Win32API dependency installation on Windows.
	225
	226	0.5.0 (2018-04-26)
	227	------------------
	228	* New keyword 'win32resolve' which only takes effect on Windows and is enabled
	229	by default. This causes procrunner to call the Win32 API FindExecutable()
	230	function to try and lookup non-.exe files with the corresponding name. This
	231	means .bat/.cmd/etc.. files can now be run without explicitly specifying
	232	their extension. Only supported on Python 2.7 and 3.5+.
	233
	234	0.4.0 (2018-04-23)
	235	------------------
	236	* Python 2.7 support on Windows. Python3 not yet supported on Windows.
	237
	238	0.3.0 (2018-04-17)
	239	------------------
	240	* run_process() renamed to run()
	241	* Python3 compatibility fixes
	242
	243	0.2.0 (2018-03-12)
	244	------------------
	245	* Procrunner is now Python3 3.3-3.6 compatible.
	246
	247	0.1.0 (2018-03-12)
	248	------------------
	249	* First release on PyPI.

-10

README.rst less more

10	10	:target: https://anaconda.org/conda-forge/procrunner
11	11	:alt: Conda Version
12	12
13		.. image:: https://travis-ci.org/DiamondLightSource/python-procrunner.svg?branch=master
14		:target: https://travis-ci.org/DiamondLightSource/python-procrunner
	13	.. image:: https://dev.azure.com/DLS-tooling/procrunner/_apis/build/status/CI?branchName=master
	14	:target: https://github.com/DiamondLightSource/python-procrunner/commits/master
15	15	:alt: Build status
16	16
17	17	.. image:: https://ci.appveyor.com/api/projects/status/jtq4brwri5q18d0u/branch/master

21	21	.. image:: https://readthedocs.org/projects/procrunner/badge/?version=latest
22	22	:target: https://procrunner.readthedocs.io/en/latest/?badge=latest
23	23	:alt: Documentation Status
24
25		.. image:: https://pyup.io/repos/github/DiamondLightSource/python-procrunner/shield.svg
26		:target: https://pyup.io/repos/github/DiamondLightSource/python-procrunner/
27		:alt: Updates
28	24
29	25	.. image:: https://img.shields.io/pypi/pyversions/procrunner.svg
30	26	:target: https://pypi.python.org/pypi/procrunner

46	42	* runs an external process and waits for it to finish
47	43	* does not deadlock, no matter the process stdout/stderr output behaviour
48	44	* returns the exit code, stdout, stderr (separately, both as bytestrings),
49		and the total process runtime as a dictionary
	45	as a subprocess.CompletedProcess object
50	46	* process can run in a custom environment, either as a modification of
51	47	the current environment or in a new environment from scratch
52		* stdin can be fed to the process, the returned dictionary contains
53		information how much was read by the process
	48	* stdin can be fed to the process
54	49	* stdout and stderr is printed by default, can be disabled
55	50	* stdout and stderr can be passed to any arbitrary function for
56	51	live processing (separately, both as unicode strings)
57		* optionally enforces a time limit on the process
	52	* optionally enforces a time limit on the process, raising a
	53	subprocess.TimeoutExpired exception if it is exceeded.
	54
58	55
59	56	Credits
60	57	-------

-58

~~appveyor.yml~~ less more

0		environment:
1
2		matrix:
3
4		# For Python versions available on Appveyor, see
5		# http://www.appveyor.com/docs/installed-software#python
6
7		- PYTHON: "C:\\Python27"
8		- PYTHON: "C:\\Python35"
9		UNSTABLE: 1
10		- PYTHON: "C:\\Python36"
11		UNSTABLE: 1
12		- PYTHON: "C:\\Python37"
13		UNSTABLE: 1
14		- PYTHON: "C:\\Python38"
15		UNSTABLE: 1
16		- PYTHON: "C:\\Python27-x64"
17		- PYTHON: "C:\\Python35-x64"
18		UNSTABLE: 1
19		- PYTHON: "C:\\Python36-x64"
20		UNSTABLE: 1
21		- PYTHON: "C:\\Python37-x64"
22		UNSTABLE: 1
23		- PYTHON: "C:\\Python38-x64"
24		UNSTABLE: 1
25
26		matrix:
27		allow_failures:
28		- UNSTABLE: 1
29
30		install:
31		# Upgrade to the latest pip.
32		- '%PYTHON%\\python.exe -m pip install -U pip setuptools wheel'
33		- '%PYTHON%\\python.exe -m pip install -r requirements_dev.txt'
34		# Install win32api dependency. Must use the --only-binary switch explicitly
35		# on AppVeyor
36		- "%PYTHON%\\python.exe -m pip install --only-binary pywin32 pywin32"
37
38		build: off
39
40		test_script:
41		# Note that you must use the environment variable %PYTHON% to refer to
42		# the interpreter you're using - Appveyor does not do anything special
43		# to put the Python version you want to use on PATH.
44		- "%PYTHON%\\python.exe setup.py test"
45
46		after_test:
47		# This step builds your wheels.
48		- "%PYTHON%\\python.exe setup.py bdist_wheel"
49
50		artifacts:
51		# bdist_wheel puts your built wheel in the dist directory
52		- path: dist\*
53
54		#on_success:
55		# You can use this step to upload your artifacts to a public website.
56		# See Appveyor's documentation for more details. Or you can simply
57		# access your wheels from the Appveyor "artifacts" tab for your build.

-3

~~docs/.gitignore~~ less more

0		/procrunner.rst
1		/procrunner.*.rst
2		/modules.rst

-9

docs/conf.py less more

0	0	#!/usr/bin/env python
1		# -- coding: utf-8 --
2	1	#
3	2	# procrunner documentation build configuration file, created by
4	3	# sphinx-quickstart on Fri Jun 9 13:47:02 2017.

47	46	master_doc = "index"
48	47
49	48	# General information about the project.
50		project = u"ProcRunner"
51		copyright = u"2018, Markus Gerstel"
52		author = u"Markus Gerstel"
	49	project = "ProcRunner"
	50	copyright = "2020, Diamond Light Source"
	51	author = "Diamond Light Source - Scientific Software"
53	52
54	53	# The version info for the project you're documenting, acts as replacement
55	54	# for \|version\| and \|release\|, also used in various other places throughout

128	127	(
129	128	master_doc,
130	129	"procrunner.tex",
131		u"ProcRunner Documentation",
132		u"Markus Gerstel",
	130	"procrunner Documentation",
	131	"Diamond Light Source - Scientific Software",
133	132	"manual",
134	133	)
135	134	]

139	138
140	139	# One entry per manual page. List of tuples
141	140	# (source start file, name, description, authors, manual section).
142		man_pages = [(master_doc, "procrunner", u"ProcRunner Documentation", [author], 1)]
	141	man_pages = [(master_doc, "procrunner", "procrunner Documentation", [author], 1)]
143	142
144	143
145	144	# -- Options for Texinfo output ----------------------------------------

151	150	(
152	151	master_doc,
153	152	"procrunner",
154		u"ProcRunner Documentation",
	153	"procrunner Documentation",
155	154	author,
156	155	"procrunner",
157		"One line description of project.",
	156	"Versatile utility function to run external processes",
158	157	"Miscellaneous",
159	158	)
160	159	]

-4

docs/usage.rst less more

8	8
9	9	To test for successful completion::
10	10
11		assert not result['exitcode']
12		assert result['exitcode'] == 0 # alternatively
	11	assert not result.returncode
	12	assert result.returncode == 0 # alternatively
	13	result.check_returncode() # raises subprocess.CalledProcessError()
13	14
14	15	To test for no STDERR output::
15	16
16		assert not result['stderr']
17		assert result['stderr'] == b'' # alternatively
	17	assert not result.stderr
	18	assert result.stderr == b'' # alternatively
18	19
19	20	To run with a specific environment variable set::
20	21

-629

~~procrunner/__init__.py~~ less more

0		# -- coding: utf-8 --
1
2		from __future__ import absolute_import, division, print_function
3
4		import codecs
5		import logging
6		import os
7		import select
8		import six
9		import subprocess
10		import sys
11		import time
12		import timeit
13		import warnings
14		from multiprocessing import Pipe
15		from threading import Thread
16
17		#
18		# run() - A function to synchronously run an external process, supporting
19		# the following features:
20		#
21		# - runs an external process and waits for it to finish
22		# - does not deadlock, no matter the process stdout/stderr output behaviour
23		# - returns the exit code, stdout, stderr (separately), and the total process
24		# runtime as a dictionary
25		# - process can run in a custom environment, either as a modification of
26		# the current environment or in a new environment from scratch
27		# - stdin can be fed to the process, the returned dictionary contains
28		# information how much was read by the process
29		# - stdout and stderr is printed by default, can be disabled
30		# - stdout and stderr can be passed to any arbitrary function for
31		# live processing
32		# - optionally enforces a time limit on the process
33		#
34		#
35		# Usage example:
36		#
37		# import procrunner
38		# result = procrunner.run(['/bin/ls', '/some/path/containing spaces'])
39		#
40		# Returns:
41		#
42		# {'command': ['/bin/ls', '/some/path/containing spaces'],
43		# 'exitcode': 2,
44		# 'runtime': 0.12990689277648926,
45		# 'stderr': '/bin/ls: cannot access /some/path/containing spaces: No such file or directory\n',
46		# 'stdout': '',
47		# 'time_end': '2017-11-12 19:54:49 GMT',
48		# 'time_start': '2017-11-12 19:54:49 GMT',
49		# 'timeout': False}
50		#
51
52		__author__ = """Markus Gerstel"""
53		__email__ = "scientificsoftware@diamond.ac.uk"
54		__version__ = "1.1.0"
55
56		logger = logging.getLogger("procrunner")
57		logger.addHandler(logging.NullHandler())
58
59
60		class _LineAggregator(object):
61		"""
62		Buffer that can be filled with stream data and will aggregate complete
63		lines. Lines can be printed or passed to an arbitrary callback function.
64		The lines passed to the callback function are UTF-8 decoded and do not
65		contain a trailing newline character.
66		"""
67
68		def __init__(self, print_line=False, callback=None):
69		"""Create aggregator object."""
70		self._buffer = ""
71		self._print = print_line
72		self._callback = callback
73		self._decoder = codecs.getincrementaldecoder("utf-8")("replace")
74
75		def add(self, data):
76		"""
77		Add a single character to buffer. If one or more full lines are found,
78		print them (if desired) and pass to callback function.
79		"""
80		data = self._decoder.decode(data)
81		if not data:
82		return
83		self._buffer += data
84		if "\n" in data:
85		to_print, remainder = self._buffer.rsplit("\n")
86		if self._print:
87		try:
88		print(to_print)
89		except UnicodeEncodeError:
90		print(to_print.encode(sys.getdefaultencoding(), errors="replace"))
91		if not hasattr(self, "_warned"):
92		logger.warning("output encoding error, characters replaced")
93		setattr(self, "_warned", True)
94		if self._callback:
95		self._callback(to_print)
96		self._buffer = remainder
97
98		def flush(self):
99		"""Print/send any remaining data to callback function."""
100		self._buffer += self._decoder.decode(b"", final=True)
101		if self._buffer:
102		if self._print:
103		print(self._buffer)
104		if self._callback:
105		self._callback(self._buffer)
106		self._buffer = ""
107
108
109		class _NonBlockingStreamReader(object):
110		"""Reads a stream in a thread to avoid blocking/deadlocks"""
111
112		def __init__(self, stream, output=True, debug=False, notify=None, callback=None):
113		"""Creates and starts a thread which reads from a stream."""
114		self._buffer = six.BytesIO()
115		self._closed = False
116		self._closing = False
117		self._debug = debug
118		self._stream = stream
119		self._terminated = False
120
121		def _thread_write_stream_to_buffer():
122		la = _LineAggregator(print_line=output, callback=callback)
123		char = True
124		while char:
125		if select.select([self._stream], [], [], 0.1)[0]:
126		char = self._stream.read(1)
127		if char:
128		self._buffer.write(char)
129		la.add(char)
130		else:
131		if self._closing:
132		break
133		self._terminated = True
134		la.flush()
135		if self._debug:
136		logger.debug("Stream reader terminated")
137		if notify:
138		notify()
139
140		def _thread_write_stream_to_buffer_windows():
141		line = True
142		while line:
143		line = self._stream.readline()
144		if line:
145		self._buffer.write(line)
146		if output or callback:
147		linedecode = line.decode("utf-8", "replace")
148		if output:
149		print(linedecode)
150		if callback:
151		callback(linedecode)
152		self._terminated = True
153		if self._debug:
154		logger.debug("Stream reader terminated")
155		if notify:
156		notify()
157
158		if os.name == "nt":
159		self._thread = Thread(target=_thread_write_stream_to_buffer_windows)
160		else:
161		self._thread = Thread(target=_thread_write_stream_to_buffer)
162		self._thread.daemon = True
163		self._thread.start()
164
165		def has_finished(self):
166		"""
167		Returns whether the thread reading from the stream is still alive.
168		"""
169		return self._terminated
170
171		def get_output(self):
172		"""
173		Retrieve the stored data in full.
174		This call may block if the reading thread has not yet terminated.
175		"""
176		self._closing = True
177		if not self.has_finished():
178		if self._debug:
179		# Main thread overtook stream reading thread.
180		underrun_debug_timer = timeit.default_timer()
181		logger.warning("NBSR underrun")
182		self._thread.join()
183		if not self.has_finished():
184		if self._debug:
185		logger.debug(
186		"NBSR join after %f seconds, underrun not resolved"
187		% (timeit.default_timer() - underrun_debug_timer)
188		)
189		raise Exception("thread did not terminate")
190		if self._debug:
191		logger.debug(
192		"NBSR underrun resolved after %f seconds"
193		% (timeit.default_timer() - underrun_debug_timer)
194		)
195		if self._closed:
196		raise Exception("streamreader double-closed")
197		self._closed = True
198		data = self._buffer.getvalue()
199		self._buffer.close()
200		return data
201
202
203		class _NonBlockingStreamWriter(object):
204		"""Writes to a stream in a thread to avoid blocking/deadlocks"""
205
206		def __init__(self, stream, data, debug=False, notify=None):
207		"""Creates and starts a thread which writes data to stream."""
208		self._buffer = data
209		self._buffer_len = len(data)
210		self._buffer_pos = 0
211		self._debug = debug
212		self._max_block_len = 4096
213		self._stream = stream
214		self._terminated = False
215
216		def _thread_write_buffer_to_stream():
217		while self._buffer_pos < self._buffer_len:
218		if (self._buffer_len - self._buffer_pos) > self._max_block_len:
219		block = self._buffer[
220		self._buffer_pos : (self._buffer_pos + self._max_block_len)
221		]
222		else:
223		block = self._buffer[self._buffer_pos :]
224		try:
225		self._stream.write(block)
226		except IOError as e:
227		if (
228		e.errno == 32
229		): # broken pipe, ie. process terminated without reading entire stdin
230		self._stream.close()
231		self._terminated = True
232		if notify:
233		notify()
234		return
235		raise
236		self._buffer_pos += len(block)
237		if debug:
238		logger.debug("wrote %d bytes to stream" % len(block))
239		self._stream.close()
240		self._terminated = True
241		if notify:
242		notify()
243
244		self._thread = Thread(target=_thread_write_buffer_to_stream)
245		self._thread.daemon = True
246		self._thread.start()
247
248		def has_finished(self):
249		"""Returns whether the thread writing to the stream is still alive."""
250		return self._terminated
251
252		def bytes_sent(self):
253		"""Return the number of bytes written so far."""
254		return self._buffer_pos
255
256		def bytes_remaining(self):
257		"""Return the number of bytes still to be written."""
258		return self._buffer_len - self._buffer_pos
259
260
261		def _path_resolve(obj):
262		"""
263		Resolve file system path (PEP-519) objects to strings.
264
265		:param obj: A file system path object or something else.
266		:return: A string representation of a file system path object or, for
267		anything that was not a file system path object, the original
268		object.
269		"""
270		if obj and hasattr(obj, "__fspath__"):
271		return obj.__fspath__()
272		return obj
273
274
275		def _windows_resolve(command):
276		"""
277		Try and find the full path and file extension of the executable to run.
278		This is so that e.g. calls to 'somescript' will point at 'somescript.cmd'
279		without the need to set shell=True in the subprocess.
280		If the executable contains periods it is a special case. Here the
281		win32api call will fail to resolve the extension automatically, and it
282		has do be done explicitly.
283
284		:param command: The command array to be run, with the first element being
285		the command with or w/o path, with or w/o extension.
286		:return: Returns the command array with the executable resolved with the
287		correct extension. If the executable cannot be resolved for any
288		reason the original command array is returned.
289		"""
290		try:
291		import win32api
292		except ImportError:
293		if (2, 8) < sys.version_info < (3, 5):
294		logger.info(
295		"Resolving executable names only supported on Python 2.7 and 3.5+"
296		)
297		else:
298		logger.warning(
299		"Could not resolve executable name: package win32api missing"
300		)
301		return command
302
303		if not command or not isinstance(command[0], six.string_types):
304		return command
305
306		try:
307		_, found_executable = win32api.FindExecutable(command[0])
308		logger.debug("Resolved %s as %s", command[0], found_executable)
309		return (found_executable,) + tuple(command[1:])
310		except Exception as e:
311		if not hasattr(e, "winerror"):
312		raise
313		# Keep this error message for later in case we fail to resolve the name
314		logwarning = getattr(e, "strerror", str(e))
315
316		if "." in command[0]:
317		# Special case. The win32api may not properly check file extensions, so
318		# try to resolve the executable explicitly.
319		for extension in os.getenv("PATHEXT").split(os.pathsep):
320		try:
321		_, found_executable = win32api.FindExecutable(command[0] + extension)
322		logger.debug("Resolved %s as %s", command[0], found_executable)
323		return (found_executable,) + tuple(command[1:])
324		except Exception as e:
325		if not hasattr(e, "winerror"):
326		raise
327
328		logger.warning("Error trying to resolve the executable: %s", logwarning)
329		return command
330
331
332		if sys.version_info < (3, 5):
333
334		class _ReturnObjectParent(object):
335		def check_returncode(self):
336		if self.returncode:
337		raise Exception(
338		"Call %r resulted in non-zero exit code %r"
339		% (self.args, self.returncode)
340		)
341
342
343		else:
344		_ReturnObjectParent = subprocess.CompletedProcess
345
346
347		class ReturnObject(dict, _ReturnObjectParent):
348		"""
349		A subprocess.CompletedProcess-like object containing the executed
350		command, stdout and stderr (both as bytestrings), and the exitcode.
351		Further values such as process runtime can be accessed as dictionary
352		values.
353		The check_returncode() function raises an exception if the process
354		exited with a non-zero exit code.
355		"""
356
357		def __init__(self, arg, *kw):
358		super(ReturnObject, self).__init__(arg, *kw)
359		self.args = self["command"]
360		self.returncode = self["exitcode"]
361		self.stdout = self["stdout"]
362		self.stderr = self["stderr"]
363
364		def __eq__(self, other):
365		"""Override equality operator to account for added fields"""
366		if type(other) is type(self):
367		return self.__dict__ == other.__dict__
368		return False
369
370		def __hash__(self):
371		"""This object is not immutable, so mark it as unhashable"""
372		return None
373
374		def __ne__(self, other):
375		"""Overrides the default implementation (unnecessary in Python 3)"""
376		return not self.__eq__(other)
377
378
379		def run(
380		command,
381		timeout=None,
382		debug=False,
383		stdin=None,
384		print_stdout=True,
385		print_stderr=True,
386		callback_stdout=None,
387		callback_stderr=None,
388		environment=None,
389		environment_override=None,
390		win32resolve=True,
391		working_directory=None,
392		):
393		"""
394		Run an external process.
395
396		File system path objects (PEP-519) are accepted in the command, environment,
397		and working directory arguments.
398
399		:param array command: Command line to be run, specified as array.
400		:param timeout: Terminate program execution after this many seconds.
401		:param boolean debug: Enable further debug messages.
402		:param stdin: Optional string that is passed to command stdin.
403		:param boolean print_stdout: Pass stdout through to sys.stdout.
404		:param boolean print_stderr: Pass stderr through to sys.stderr.
405		:param callback_stdout: Optional function which is called for each
406		stdout line.
407		:param callback_stderr: Optional function which is called for each
408		stderr line.
409		:param dict environment: The full execution environment for the command.
410		:param dict environment_override: Change environment variables from the
411		current values for command execution.
412		:param boolean win32resolve: If on Windows, find the appropriate executable
413		first. This allows running of .bat, .cmd, etc.
414		files without explicitly specifying their
415		extension.
416		:param string working_directory: If specified, run the executable from
417		within this working directory.
418		:return: A ReturnObject() containing the executed command, stdout and stderr
419		(both as bytestrings), and the exitcode. Further values such as
420		process runtime can be accessed as dictionary values.
421		"""
422
423		time_start = time.strftime("%Y-%m-%d %H:%M:%S GMT", time.gmtime())
424		logger.debug("Starting external process: %s", command)
425
426		if stdin is None:
427		stdin_pipe = None
428		else:
429		assert sys.platform != "win32", "stdin argument not supported on Windows"
430		stdin_pipe = subprocess.PIPE
431
432		start_time = timeit.default_timer()
433		if timeout is not None:
434		max_time = start_time + timeout
435
436		if environment is not None:
437		env = {key: _path_resolve(environment[key]) for key in environment}
438		else:
439		env = {key: value for key, value in os.environ.items()}
440		if environment_override:
441		env.update(
442		{
443		key: str(_path_resolve(environment_override[key]))
444		for key in environment_override
445		}
446		)
447
448		command = tuple(_path_resolve(part) for part in command)
449		if win32resolve and sys.platform == "win32":
450		command = _windows_resolve(command)
451
452		p = subprocess.Popen(
453		command,
454		shell=False,
455		cwd=_path_resolve(working_directory),
456		env=env,
457		stdin=stdin_pipe,
458		stdout=subprocess.PIPE,
459		stderr=subprocess.PIPE,
460		)
461
462		thread_pipe_pool = []
463		notifyee, notifier = Pipe(False)
464		thread_pipe_pool.append(notifyee)
465		stdout = _NonBlockingStreamReader(
466		p.stdout,
467		output=print_stdout,
468		debug=debug,
469		notify=notifier.close,
470		callback=callback_stdout,
471		)
472		notifyee, notifier = Pipe(False)
473		thread_pipe_pool.append(notifyee)
474		stderr = _NonBlockingStreamReader(
475		p.stderr,
476		output=print_stderr,
477		debug=debug,
478		notify=notifier.close,
479		callback=callback_stderr,
480		)
481		if stdin is not None:
482		notifyee, notifier = Pipe(False)
483		thread_pipe_pool.append(notifyee)
484		stdin = _NonBlockingStreamWriter(
485		p.stdin, data=stdin, debug=debug, notify=notifier.close
486		)
487
488		timeout_encountered = False
489
490		while (p.returncode is None) and (
491		(timeout is None) or (timeit.default_timer() < max_time)
492		):
493		if debug and timeout is not None:
494		logger.debug("still running (T%.2fs)" % (timeit.default_timer() - max_time))
495
496		# wait for some time or until a stream is closed
497		try:
498		if thread_pipe_pool:
499		# Wait for up to 0.5 seconds or for a signal on a remaining stream,
500		# which could indicate that the process has terminated.
501		try:
502		event = thread_pipe_pool[0].poll(0.5)
503		except IOError as e:
504		# on Windows this raises "IOError: [Errno 109] The pipe has been ended"
505		# which is for all intents and purposes equivalent to a True return value.
506		if e.errno != 109:
507		raise
508		event = True
509		if event:
510		# One-shot, so remove stream and watch remaining streams
511		thread_pipe_pool.pop(0)
512		if debug:
513		logger.debug("Event received from stream thread")
514		else:
515		time.sleep(0.5)
516		except KeyboardInterrupt:
517		p.kill() # if user pressed Ctrl+C we won't be able to produce a proper report anyway
518		# but at least make sure the child process dies with us
519		raise
520
521		# check if process is still running
522		p.poll()
523
524		if p.returncode is None:
525		# timeout condition
526		timeout_encountered = True
527		if debug:
528		logger.debug("timeout (T%.2fs)" % (timeit.default_timer() - max_time))
529
530		# send terminate signal and wait some time for buffers to be read
531		p.terminate()
532		if thread_pipe_pool:
533		thread_pipe_pool[0].poll(0.5)
534		if not stdout.has_finished() or not stderr.has_finished():
535		time.sleep(2)
536		p.poll()
537
538		if p.returncode is None:
539		# thread still alive
540		# send kill signal and wait some more time for buffers to be read
541		p.kill()
542		if thread_pipe_pool:
543		thread_pipe_pool[0].poll(0.5)
544		if not stdout.has_finished() or not stderr.has_finished():
545		time.sleep(5)
546		p.poll()
547
548		if p.returncode is None:
549		raise RuntimeError("Process won't terminate")
550
551		runtime = timeit.default_timer() - start_time
552		if timeout is not None:
553		logger.debug(
554		"Process ended after %.1f seconds with exit code %d (T%.2fs)"
555		% (runtime, p.returncode, timeit.default_timer() - max_time)
556		)
557		else:
558		logger.debug(
559		"Process ended after %.1f seconds with exit code %d"
560		% (runtime, p.returncode)
561		)
562
563		stdout = stdout.get_output()
564		stderr = stderr.get_output()
565		time_end = time.strftime("%Y-%m-%d %H:%M:%S GMT", time.gmtime())
566
567		result = ReturnObject(
568		{
569		"exitcode": p.returncode,
570		"command": command,
571		"stdout": stdout,
572		"stderr": stderr,
573		"timeout": timeout_encountered,
574		"runtime": runtime,
575		"time_start": time_start,
576		"time_end": time_end,
577		}
578		)
579		if stdin is not None:
580		result.update(
581		{
582		"stdin_bytes_sent": stdin.bytes_sent(),
583		"stdin_bytes_remain": stdin.bytes_remaining(),
584		}
585		)
586
587		return result
588
589
590		def run_process_dummy(command, **kwargs):
591		"""
592		A stand-in function that returns a valid result dictionary indicating a
593		successful execution. The external process is not run.
594		"""
595		warnings.warn(
596		"procrunner.run_process_dummy() is deprecated", DeprecationWarning, stacklevel=2
597		)
598
599		time_start = time.strftime("%Y-%m-%d %H:%M:%S GMT", time.gmtime())
600		logger.info("run_process is disabled. Requested command: %s", command)
601
602		result = ReturnObject(
603		{
604		"exitcode": 0,
605		"command": command,
606		"stdout": "",
607		"stderr": "",
608		"timeout": False,
609		"runtime": 0,
610		"time_start": time_start,
611		"time_end": time_start,
612		}
613		)
614		if kwargs.get("stdin") is not None:
615		result.update(
616		{"stdin_bytes_sent": len(kwargs["stdin"]), "stdin_bytes_remain": 0}
617		)
618		return result
619
620
621		def run_process(args, *kwargs):
622		"""API used up to version 0.2.0."""
623		warnings.warn(
624		"procrunner.run_process() is deprecated and has been renamed to run()",
625		DeprecationWarning,
626		stacklevel=2,
627		)
628		return run(args, *kwargs)

-0

pyproject.toml less more

	0	[build-system]
	1	requires = ["setuptools >= 40.6.0", "wheel"]
	2	build-backend = "setuptools.build_meta"
	3
	4	[tool.isort]
	5	profile="black"

-13

~~requirements_dev.txt~~ less more

0		bump2version==0.5.10
1		coverage==4.5.4
2		flake8==3.7.8
3		mock==3.0.5
4		pip==19.1.1
5		pytest==4.5.0 # pyup: <5.0 # for Python 2.7 support
6		pytest-runner==5.1
7		six==1.12.0
8		Sphinx==1.8.5 # pyup: <2.0 # for Python 2.7 support
9		tox==3.13.1
10		twine==1.13.0
11		watchdog==0.9.0
12		wheel==0.33.4

+45

-17

setup.cfg less more

0		[bumpversion]
1		current_version = 1.1.0
2		commit = True
3		tag = True
	0	[metadata]
	1	name = procrunner
	2	description = Versatile utility function to run external processes
	3	version = 2.3.3
	4	author = Diamond Light Source - Scientific Software et al.
	5	author_email = scientificsoftware@diamond.ac.uk
	6	classifiers =
	7	Development Status :: 5 - Production/Stable
	8	Intended Audience :: Developers
	9	License :: OSI Approved :: BSD License
	10	Natural Language :: English
	11	Programming Language :: Python :: 3
	12	Programming Language :: Python :: 3.7
	13	Programming Language :: Python :: 3.8
	14	Programming Language :: Python :: 3.9
	15	Programming Language :: Python :: 3.10
	16	Operating System :: OS Independent
	17	Topic :: Software Development :: Libraries :: Python Modules
	18	license = BSD
	19	license_file = LICENSE
	20	project_urls =
	21	Download = https://github.com/DiamondLightSource/python-procrunner/tags
	22	Documentation = https://procrunner.readthedocs.io/
	23	GitHub = https://github.com/DiamondLightSource/python-procrunner
	24	Bug-Tracker = https://github.com/DiamondLightSource/python-procrunner/issues
4	25
5		[bumpversion:file:setup.py]
6		search = version="{current_version}"
7		replace = version="{new_version}"
	26	[options]
	27	include_package_data = True
	28	packages = procrunner
	29	package_dir =
	30	=src
	31	python_requires = >=3.7
	32	zip_safe = False
8	33
9		[bumpversion:file:procrunner/__init__.py]
10		search = __version__ = "{current_version}"
11		replace = __version__ = "{new_version}"
12
13		[bdist_wheel]
14		universal = 1
	34	[options.packages.find]
	35	where = src
15	36
16	37	[flake8]
17		exclude = docs
18
19		[aliases]
20		test = pytest
	38	ignore = E203, E266, E501, W503
	39	max-line-length = 88
	40	select =
	41	E401,E711,E712,E713,E714,E721,E722,E901,
	42	F401,F402,F403,F405,F541,F631,F632,F633,F811,F812,F821,F822,F841,F901,
	43	W191,W291,W292,W293,W602,W603,W604,W605,W606,
	44	C4,
21	45
22	46	[tool:pytest]
23	47	collect_ignore = ['setup.py']
24	48
	49	[egg_info]
	50	tag_build =
	51	tag_date = 0
	52

-48

setup.py less more

0		#!/usr/bin/env python
1		# -- coding: utf-8 --
2
3		import sys
4		from setuptools import setup, find_packages
	0	from setuptools import setup
5	1
6	2	with open("README.rst") as readme_file:
7	3	readme = readme_file.read()

9	5	with open("HISTORY.rst") as history_file:
10	6	history = history_file.read()
11	7
12		requirements = [
13		"six",
14		'pywin32; sys_platform=="win32"',
15		]
16
17		setup_requirements = []
18		needs_pytest = {"pytest", "test", "ptr"}.intersection(sys.argv)
19		if needs_pytest:
20		setup_requirements.append("pytest-runner")
21
22		test_requirements = ["mock", "pytest"]
23
24	8	setup(
25		author="Markus Gerstel",
26		author_email="scientificsoftware@diamond.ac.uk",
27		classifiers=[
28		"Development Status :: 5 - Production/Stable",
29		"Intended Audience :: Developers",
30		"License :: OSI Approved :: BSD License",
31		"Natural Language :: English",
32		"Operating System :: OS Independent",
33		"Programming Language :: Python :: 2",
34		"Programming Language :: Python :: 2.7",
35		"Programming Language :: Python :: 3",
36		"Programming Language :: Python :: 3.5",
37		"Programming Language :: Python :: 3.6",
38		"Programming Language :: Python :: 3.7",
39		"Programming Language :: Python :: 3.8",
40		"Programming Language :: Python :: Implementation :: PyPy",
41		"Programming Language :: Python :: Implementation :: CPython",
42		"Topic :: Software Development :: Libraries :: Python Modules",
43		],
44		description="Versatile utility function to run external processes",
45		install_requires=requirements,
46		license="BSD license",
47	9	long_description=readme + "\n\n" + history,
48		include_package_data=True,
49	10	keywords="procrunner",
50		name="procrunner",
51		packages=find_packages(include=["procrunner"]),
52		setup_requires=setup_requirements,
53		test_suite="tests",
54		tests_require=test_requirements,
55		url="https://github.com/DiamondLightSource/python-procrunner",
56		version="1.1.0",
57		zip_safe=False,
58	11	)

+529

-0

src/procrunner/__init__.py less more

	0	from __future__ import annotations
	1
	2	import codecs
	3	import io
	4	import logging
	5	import os
	6	import select
	7	import shutil
	8	import subprocess
	9	import sys
	10	import time
	11	import timeit
	12	import warnings
	13	from multiprocessing import Pipe
	14	from threading import Thread
	15	from typing import Any, Callable, Optional, Union
	16
	17	#
	18	# run() - A function to synchronously run an external process, supporting
	19	# the following features:
	20	#
	21	# - runs an external process and waits for it to finish
	22	# - does not deadlock, no matter the process stdout/stderr output behaviour
	23	# - returns the exit code, stdout, stderr (separately) as a
	24	# subprocess.CompletedProcess object
	25	# - process can run in a custom environment, either as a modification of
	26	# the current environment or in a new environment from scratch
	27	# - stdin can be fed to the process
	28	# - stdout and stderr is printed by default, can be disabled
	29	# - stdout and stderr can be passed to any arbitrary function for
	30	# live processing
	31	# - optionally enforces a time limit on the process
	32	#
	33	#
	34	# Usage example:
	35	#
	36	# import procrunner
	37	# result = procrunner.run(['/bin/ls', '/some/path/containing spaces'])
	38	#
	39	# Returns:
	40	#
	41	# subprocess.CompletedProcess(
	42	# args=('/bin/ls', '/some/path/containing spaces'),
	43	# returncode=2,
	44	# stdout=b'',
	45	# stderr=b'/bin/ls: cannot access /some/path/containing spaces: No such file or directory\n'
	46	# )
	47
	48	__version__ = "2.3.3"
	49
	50	logger = logging.getLogger("procrunner")
	51	logger.addHandler(logging.NullHandler())
	52
	53
	54	class _LineAggregator:
	55	"""
	56	Buffer that can be filled with stream data and will aggregate complete
	57	lines. Lines can be printed or passed to an arbitrary callback function.
	58	The lines passed to the callback function are UTF-8 decoded and do not
	59	contain a trailing newline character.
	60	"""
	61
	62	def __init__(self, print_line=False, callback=None):
	63	"""Create aggregator object."""
	64	self._buffer = ""
	65	self._print = print_line
	66	self._callback = callback
	67	self._decoder = codecs.getincrementaldecoder("utf-8")("replace")
	68
	69	def add(self, data):
	70	"""
	71	Add a single character to buffer. If one or more full lines are found,
	72	print them (if desired) and pass to callback function.
	73	"""
	74	data = self._decoder.decode(data)
	75	if not data:
	76	return
	77	self._buffer += data
	78	if "\n" in data:
	79	to_print, remainder = self._buffer.rsplit("\n")
	80	if self._print:
	81	try:
	82	print(to_print)
	83	except UnicodeEncodeError:
	84	print(to_print.encode(sys.getdefaultencoding(), errors="replace"))
	85	if not hasattr(self, "_warned"):
	86	logger.warning("output encoding error, characters replaced")
	87	setattr(self, "_warned", True)
	88	if self._callback:
	89	self._callback(to_print)
	90	self._buffer = remainder
	91
	92	def flush(self):
	93	"""Print/send any remaining data to callback function."""
	94	self._buffer += self._decoder.decode(b"", final=True)
	95	if self._buffer:
	96	if self._print:
	97	print(self._buffer)
	98	if self._callback:
	99	self._callback(self._buffer)
	100	self._buffer = ""
	101
	102
	103	class _NonBlockingStreamReader:
	104	"""Reads a stream in a thread to avoid blocking/deadlocks"""
	105
	106	def __init__(self, stream, output=True, debug=False, notify=None, callback=None):
	107	"""Creates and starts a thread which reads from a stream."""
	108	self._buffer = io.BytesIO()
	109	self._closed = False
	110	self._closing = False
	111	self._debug = debug
	112	self._stream = stream
	113	self._terminated = False
	114
	115	def _thread_write_stream_to_buffer():
	116	la = _LineAggregator(print_line=output, callback=callback)
	117	char = True
	118	while char:
	119	if select.select([self._stream], [], [], 0.1)[0]:
	120	char = self._stream.read(1)
	121	if char:
	122	self._buffer.write(char)
	123	la.add(char)
	124	else:
	125	if self._closing:
	126	break
	127	self._stream.close()
	128	self._terminated = True
	129	la.flush()
	130	if self._debug:
	131	logger.debug("Stream reader terminated")
	132	if notify:
	133	notify()
	134
	135	def _thread_write_stream_to_buffer_windows():
	136	line = True
	137	while line:
	138	line = self._stream.readline()
	139	if line:
	140	self._buffer.write(line)
	141	if output or callback:
	142	linedecode = line.decode("utf-8", "replace")
	143	if output:
	144	print(linedecode)
	145	if callback:
	146	callback(linedecode)
	147	self._stream.close()
	148	self._terminated = True
	149	if self._debug:
	150	logger.debug("Stream reader terminated")
	151	if notify:
	152	notify()
	153
	154	if os.name == "nt":
	155	self._thread = Thread(target=_thread_write_stream_to_buffer_windows)
	156	else:
	157	self._thread = Thread(target=_thread_write_stream_to_buffer)
	158	self._thread.daemon = True
	159	self._thread.start()
	160
	161	def has_finished(self):
	162	"""
	163	Returns whether the thread reading from the stream is still alive.
	164	"""
	165	return self._terminated
	166
	167	def get_output(self):
	168	"""
	169	Retrieve the stored data in full.
	170	This call may block if the reading thread has not yet terminated.
	171	"""
	172	self._closing = True
	173	if not self.has_finished():
	174	if self._debug:
	175	# Main thread overtook stream reading thread.
	176	underrun_debug_timer = timeit.default_timer()
	177	logger.warning("NBSR underrun")
	178	self._thread.join()
	179	if not self.has_finished():
	180	if self._debug:
	181	logger.debug(
	182	"NBSR join after %f seconds, underrun not resolved",
	183	timeit.default_timer() - underrun_debug_timer,
	184	)
	185	raise Exception("thread did not terminate")
	186	if self._debug:
	187	logger.debug(
	188	"NBSR underrun resolved after %f seconds",
	189	timeit.default_timer() - underrun_debug_timer,
	190	)
	191	if self._closed:
	192	raise Exception("streamreader double-closed")
	193	self._closed = True
	194	data = self._buffer.getvalue()
	195	self._buffer.close()
	196	return data
	197
	198
	199	class _NonBlockingStreamWriter:
	200	"""Writes to a stream in a thread to avoid blocking/deadlocks"""
	201
	202	def __init__(self, stream, data, debug=False, notify=None):
	203	"""Creates and starts a thread which writes data to stream."""
	204	self._buffer = data
	205	self._buffer_len = len(data)
	206	self._buffer_pos = 0
	207	self._max_block_len = 4096
	208	self._stream = stream
	209	self._terminated = False
	210
	211	def _thread_write_buffer_to_stream():
	212	while self._buffer_pos < self._buffer_len:
	213	if (self._buffer_len - self._buffer_pos) > self._max_block_len:
	214	block = self._buffer[
	215	self._buffer_pos : (self._buffer_pos + self._max_block_len)
	216	]
	217	else:
	218	block = self._buffer[self._buffer_pos :]
	219	try:
	220	self._stream.write(block)
	221	except OSError as e:
	222	if (
	223	e.errno == 32
	224	): # broken pipe, ie. process terminated without reading entire stdin
	225	self._stream.close()
	226	self._terminated = True
	227	if notify:
	228	notify()
	229	return
	230	raise
	231	self._buffer_pos += len(block)
	232	if debug:
	233	logger.debug("wrote %d bytes to stream", len(block))
	234	self._stream.close()
	235	self._terminated = True
	236	if notify:
	237	notify()
	238
	239	self._thread = Thread(target=_thread_write_buffer_to_stream)
	240	self._thread.daemon = True
	241	self._thread.start()
	242
	243	def has_finished(self):
	244	"""Returns whether the thread writing to the stream is still alive."""
	245	return self._terminated
	246
	247	def bytes_sent(self):
	248	"""Return the number of bytes written so far."""
	249	return self._buffer_pos
	250
	251	def bytes_remaining(self):
	252	"""Return the number of bytes still to be written."""
	253	return self._buffer_len - self._buffer_pos
	254
	255
	256	def _path_resolve(obj):
	257	"""
	258	Resolve file system path (PEP-519) objects to strings.
	259
	260	:param obj: A file system path object or something else.
	261	:return: A string representation of a file system path object or, for
	262	anything that was not a file system path object, the original
	263	object.
	264	"""
	265	if obj and hasattr(obj, "__fspath__"):
	266	return obj.__fspath__()
	267	return obj
	268
	269
	270	def _windows_resolve(command, path=None):
	271	"""
	272	Try and find the full path and file extension of the executable to run.
	273	This is so that e.g. calls to 'somescript' will point at 'somescript.cmd'
	274	without the need to set shell=True in the subprocess.
	275
	276	:param command: The command array to be run, with the first element being
	277	the command with or w/o path, with or w/o extension.
	278	:return: Returns the command array with the executable resolved with the
	279	correct extension. If the executable cannot be resolved for any
	280	reason the original command array is returned.
	281	"""
	282	if not command or not isinstance(command[0], str):
	283	return command
	284
	285	found_executable = shutil.which(command[0], path=path)
	286	if found_executable:
	287	logger.debug("Resolved %s as %s", command[0], found_executable)
	288	return (found_executable, *command[1:])
	289
	290	if "\\" in command[0]:
	291	# Special case. shutil.which may not detect file extensions if a full
	292	# path is given, so try to resolve the executable explicitly
	293	for extension in os.getenv("PATHEXT").split(os.pathsep):
	294	found_executable = shutil.which(command[0] + extension, path=path)
	295	if found_executable:
	296	return (found_executable, *command[1:])
	297
	298	logger.warning("Error trying to resolve the executable: %s", command[0])
	299	return command
	300
	301
	302	def run(
	303	command,
	304	*,
	305	timeout: Optional[float] = None,
	306	callback_stderr: Optional[Callable] = None,
	307	callback_stdout: Optional[Callable] = None,
	308	creationflags: int = 0,
	309	environment: Optional[dict[str, str]] = None,
	310	environment_override: Optional[dict[str, str]] = None,
	311	preexec_fn: Optional[Callable] = None,
	312	print_stderr: bool = True,
	313	print_stdout: bool = True,
	314	raise_timeout_exception: Any = ...,
	315	stdin: Optional[Union[bytes, int]] = None,
	316	win32resolve: bool = True,
	317	working_directory: Optional[str] = None,
	318	) -> subprocess.CompletedProcess:
	319	"""
	320	Run an external process.
	321
	322	File system path objects (PEP-519) are accepted in the command, environment,
	323	and working directory arguments.
	324
	325	:param array command: Command line to be run, specified as array.
	326	:param timeout: Terminate program execution after this many seconds.
	327	:param stdin: Optional bytestring that is passed to command stdin,
	328	or subprocess.DEVNULL to disable stdin.
	329	:param boolean print_stdout: Pass stdout through to sys.stdout.
	330	:param boolean print_stderr: Pass stderr through to sys.stderr.
	331	:param callback_stdout: Optional function which is called for each
	332	stdout line.
	333	:param callback_stderr: Optional function which is called for each
	334	stderr line.
	335	:param creationflags: flags that will be passed to subprocess call
	336	:param dict environment: The full execution environment for the command.
	337	:param dict environment_override: Change environment variables from the
	338	current values for command execution.
	339	:param preexec_fn: pre-execution function, will be passed to subprocess call
	340	:param boolean win32resolve: If on Windows, find the appropriate executable
	341	first. This allows running of .bat, .cmd, etc.
	342	files without explicitly specifying their
	343	extension.
	344	:param string working_directory: If specified, run the executable from
	345	within this working directory.
	346	:param boolean raise_timeout_exception: Deprecated compatibility flag.
	347	:return: The exit code, stdout, stderr (separately, as byte strings)
	348	as a subprocess.CompletedProcess object.
	349	"""
	350
	351	logger.debug("Starting external process: %s", command)
	352
	353	if stdin is None:
	354	stdin_pipe = None
	355	elif isinstance(stdin, int):
	356	assert (
	357	stdin == subprocess.DEVNULL
	358	), "stdin argument only allows subprocess.DEVNULL as numeric argument"
	359	stdin_pipe = subprocess.DEVNULL
	360	stdin = None
	361	else:
	362	assert sys.platform != "win32", "stdin argument not supported on Windows"
	363	stdin_pipe = subprocess.PIPE
	364
	365	start_time = timeit.default_timer()
	366	if timeout is not None:
	367	max_time = start_time + timeout
	368	if not raise_timeout_exception:
	369	warnings.warn(
	370	"Using procrunner with raise_timeout_exception=False is no longer supported",
	371	UserWarning,
	372	stacklevel=3,
	373	)
	374	elif raise_timeout_exception is True:
	375	warnings.warn(
	376	"The raise_timeout_exception argument is deprecated and will be removed in a future release",
	377	DeprecationWarning,
	378	stacklevel=3,
	379	)
	380
	381	if environment is not None:
	382	env = {key: _path_resolve(environment[key]) for key in environment}
	383	else:
	384	env = {key: value for key, value in os.environ.items()}
	385	if environment_override:
	386	env.update(
	387	{
	388	key: str(_path_resolve(environment_override[key]))
	389	for key in environment_override
	390	}
	391	)
	392
	393	command = tuple(_path_resolve(part) for part in command)
	394	if win32resolve and sys.platform == "win32":
	395	command = _windows_resolve(command)
	396
	397	p = subprocess.Popen(
	398	command,
	399	shell=False,
	400	cwd=working_directory,
	401	env=env,
	402	stdin=stdin_pipe,
	403	stdout=subprocess.PIPE,
	404	stderr=subprocess.PIPE,
	405	creationflags=creationflags,
	406	preexec_fn=preexec_fn,
	407	)
	408
	409	thread_pipe_pool = []
	410	notifyee, notifier = Pipe(False)
	411	thread_pipe_pool.append(notifyee)
	412	stdout = _NonBlockingStreamReader(
	413	p.stdout,
	414	output=print_stdout,
	415	notify=notifier.close,
	416	callback=callback_stdout,
	417	)
	418	notifyee, notifier = Pipe(False)
	419	thread_pipe_pool.append(notifyee)
	420	stderr = _NonBlockingStreamReader(
	421	p.stderr,
	422	output=print_stderr,
	423	notify=notifier.close,
	424	callback=callback_stderr,
	425	)
	426	if stdin is not None:
	427	notifyee, notifier = Pipe(False)
	428	thread_pipe_pool.append(notifyee)
	429	_NonBlockingStreamWriter(p.stdin, data=stdin, notify=notifier.close)
	430
	431	timeout_encountered = False
	432
	433	while (p.returncode is None) and (
	434	(timeout is None) or (timeit.default_timer() < max_time)
	435	):
	436	# wait for some time or until a stream is closed
	437	try:
	438	if thread_pipe_pool:
	439	# Wait for up to 0.5 seconds or for a signal on a remaining stream,
	440	# which could indicate that the process has terminated.
	441	try:
	442	event = thread_pipe_pool[0].poll(0.5)
	443	except BrokenPipeError as e:
	444	# on Windows this raises "BrokenPipeError: [Errno 109] The pipe has been ended"
	445	# which is for all intents and purposes equivalent to a True return value.
	446	if e.winerror != 109:
	447	raise
	448	event = True
	449	if event:
	450	# One-shot, so remove stream and watch remaining streams
	451	thread_pipe_pool.pop(0)
	452	else:
	453	time.sleep(0.5)
	454	except KeyboardInterrupt:
	455	p.kill() # if user pressed Ctrl+C we won't be able to produce a proper report anyway
	456	# but at least make sure the child process dies with us
	457	raise
	458
	459	# check if process is still running
	460	p.poll()
	461
	462	if p.returncode is None:
	463	# timeout condition
	464	timeout_encountered = True
	465	logger.debug("timeout (T%.2fs)", timeit.default_timer() - max_time)
	466
	467	# send terminate signal and wait some time for buffers to be read
	468	p.terminate()
	469	if thread_pipe_pool:
	470	try:
	471	thread_pipe_pool[0].poll(0.5)
	472	except BrokenPipeError as e:
	473	# on Windows this raises "BrokenPipeError: [Errno 109] The pipe has been ended"
	474	# which is for all intents and purposes equivalent to a True return value.
	475	if e.winerror != 109:
	476	raise
	477	thread_pipe_pool.pop(0)
	478	if not stdout.has_finished() or not stderr.has_finished():
	479	time.sleep(2)
	480	p.poll()
	481
	482	if p.returncode is None:
	483	# thread still alive
	484	# send kill signal and wait some more time for buffers to be read
	485	p.kill()
	486	if thread_pipe_pool:
	487	try:
	488	thread_pipe_pool[0].poll(0.5)
	489	except BrokenPipeError as e:
	490	# on Windows this raises "BrokenPipeError: [Errno 109] The pipe has been ended"
	491	# which is for all intents and purposes equivalent to a True return value.
	492	if e.winerror != 109:
	493	raise
	494	thread_pipe_pool.pop(0)
	495	if not stdout.has_finished() or not stderr.has_finished():
	496	time.sleep(5)
	497	p.poll()
	498
	499	if p.returncode is None:
	500	raise RuntimeError("Process won't terminate")
	501
	502	runtime = timeit.default_timer() - start_time
	503	if timeout is not None:
	504	logger.debug(
	505	"Process ended after %.1f seconds with exit code %d (T%.2fs)",
	506	runtime,
	507	p.returncode,
	508	timeit.default_timer() - max_time,
	509	)
	510	else:
	511	logger.debug(
	512	"Process ended after %.1f seconds with exit code %d", runtime, p.returncode
	513	)
	514
	515	output_stdout = stdout.get_output()
	516	output_stderr = stderr.get_output()
	517
	518	if timeout is not None and timeout_encountered:
	519	raise subprocess.TimeoutExpired(
	520	cmd=command, timeout=timeout, output=output_stdout, stderr=output_stderr
	521	)
	522
	523	return subprocess.CompletedProcess(
	524	args=command,
	525	returncode=p.returncode,
	526	stdout=output_stdout,
	527	stderr=output_stderr,
	528	)

+250

-0

src/procrunner.egg-info/PKG-INFO less more

	0	Metadata-Version: 2.1
	1	Name: procrunner
	2	Version: 2.3.3
	3	Summary: Versatile utility function to run external processes
	4	Author: Diamond Light Source - Scientific Software et al.
	5	Author-email: scientificsoftware@diamond.ac.uk
	6	License: BSD
	7	Project-URL: Download, https://github.com/DiamondLightSource/python-procrunner/tags
	8	Project-URL: Documentation, https://procrunner.readthedocs.io/
	9	Project-URL: GitHub, https://github.com/DiamondLightSource/python-procrunner
	10	Project-URL: Bug-Tracker, https://github.com/DiamondLightSource/python-procrunner/issues
	11	Keywords: procrunner
	12	Classifier: Development Status :: 5 - Production/Stable
	13	Classifier: Intended Audience :: Developers
	14	Classifier: License :: OSI Approved :: BSD License
	15	Classifier: Natural Language :: English
	16	Classifier: Programming Language :: Python :: 3
	17	Classifier: Programming Language :: Python :: 3.7
	18	Classifier: Programming Language :: Python :: 3.8
	19	Classifier: Programming Language :: Python :: 3.9
	20	Classifier: Programming Language :: Python :: 3.10
	21	Classifier: Operating System :: OS Independent
	22	Classifier: Topic :: Software Development :: Libraries :: Python Modules
	23	Requires-Python: >=3.7
	24	License-File: LICENSE
	25
	26	==========
	27	ProcRunner
	28	==========
	29
	30
	31	.. image:: https://img.shields.io/pypi/v/procrunner.svg
	32	:target: https://pypi.python.org/pypi/procrunner
	33	:alt: PyPI release
	34
	35	.. image:: https://img.shields.io/conda/vn/conda-forge/procrunner.svg
	36	:target: https://anaconda.org/conda-forge/procrunner
	37	:alt: Conda Version
	38
	39	.. image:: https://dev.azure.com/DLS-tooling/procrunner/_apis/build/status/CI?branchName=master
	40	:target: https://github.com/DiamondLightSource/python-procrunner/commits/master
	41	:alt: Build status
	42
	43	.. image:: https://ci.appveyor.com/api/projects/status/jtq4brwri5q18d0u/branch/master
	44	:target: https://ci.appveyor.com/project/Anthchirp/python-procrunner
	45	:alt: Build status
	46
	47	.. image:: https://readthedocs.org/projects/procrunner/badge/?version=latest
	48	:target: https://procrunner.readthedocs.io/en/latest/?badge=latest
	49	:alt: Documentation Status
	50
	51	.. image:: https://img.shields.io/pypi/pyversions/procrunner.svg
	52	:target: https://pypi.python.org/pypi/procrunner
	53	:alt: Supported Python versions
	54
	55	.. image:: https://img.shields.io/badge/code%20style-black-000000.svg
	56	:target: https://github.com/ambv/black
	57	:alt: Code style: black
	58
	59	Versatile utility function to run external processes
	60
	61	* Free software: BSD license
	62	* Documentation: https://procrunner.readthedocs.io.
	63
	64
	65	Features
	66	--------
	67
	68	* runs an external process and waits for it to finish
	69	* does not deadlock, no matter the process stdout/stderr output behaviour
	70	* returns the exit code, stdout, stderr (separately, both as bytestrings),
	71	as a subprocess.CompletedProcess object
	72	* process can run in a custom environment, either as a modification of
	73	the current environment or in a new environment from scratch
	74	* stdin can be fed to the process
	75	* stdout and stderr is printed by default, can be disabled
	76	* stdout and stderr can be passed to any arbitrary function for
	77	live processing (separately, both as unicode strings)
	78	* optionally enforces a time limit on the process, raising a
	79	subprocess.TimeoutExpired exception if it is exceeded.
	80
	81
	82	Credits
	83	-------
	84
	85	This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypackage`_ project template.
	86
	87	.. _Cookiecutter: https://github.com/audreyr/cookiecutter
	88	.. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage
	89
	90
	91	History
	92	==================
	93
	94	3.0.0 (2022-01-??)
	95	------------------
	96	* Drop Python 3.6 support
	97	* The run() function now returns a subprocess.CompletedProcess object,
	98	which no longer allows array access operations
	99	(those were deprecated in `#60 <https://github.com/DiamondLightSource/python-procrunner/pull/60>`_)
	100	* The run() argument 'raise_timeout_exception' is now set by default,
	101	a 'False' value will lead to a UserWarning and a behavioural change.
	102	The argument is now deprecated and will be removed in a future version.
	103	(previously introduced in `#61 <https://github.com/DiamondLightSource/python-procrunner/pull/61>`_)
	104	* Calling the run() function with multiple unnamed arguments is no longer supported
	105	(previously deprecated in `#62 <https://github.com/DiamondLightSource/python-procrunner/pull/62>`_)
	106	* The run() function no longer accepts a 'debug' argument
	107	(previously deprecated in `#63 <https://github.com/DiamondLightSource/python-procrunner/pull/63>`_)
	108
	109	2.3.3 (2022-03-23)
	110	------------------
	111	* Allow specifying 'preexec_fn' and 'creationflags' keywords, which will be passed through to
	112	the subprocess call
	113
	114	2.3.2 (2022-01-28)
	115	------------------
	116	* The run() function now understands stdin=subprocess.DEVNULL to close the subprocess stdin,
	117	rather than to connect through the existing stdin, which is the current default
	118
	119	2.3.1 (2021-10-25)
	120	------------------
	121	* Add Python 3.10 support
	122
	123	2.3.0 (2020-10-29)
	124	------------------
	125	* Add Python 3.9 support, drop Python 3.5 support
	126	* Fix a file descriptor leak on subprocess execution
	127
	128	2.2.0 (2020-09-07)
	129	------------------
	130	* Calling the run() function with unnamed arguments (other than the command
	131	list as the first argument) is now deprecated. As a number of arguments
	132	will be removed in a future version the use of unnamed arguments will
	133	cause future confusion. `Use explicit keyword arguments instead (#62). <https://github.com/DiamondLightSource/python-procrunner/pull/62>`_
	134	* `The run() function debug argument has been deprecated (#63). <https://github.com/DiamondLightSource/python-procrunner/pull/63>`_
	135	This is only used to debug the NonBlockingStream* classes. Those are due
	136	to be replaced in a future release, so the argument will no longer serve
	137	a purpose. Debugging information remains available via standard logging
	138	mechanisms.
	139	* Final version supporting Python 3.5
	140
	141	2.1.0 (2020-09-05)
	142	------------------
	143	* `Deprecated array access on the return object (#60). <https://github.com/DiamondLightSource/python-procrunner/pull/60>`_
	144	The return object will become a subprocess.CompletedProcess in a future
	145	release, which no longer allows array-based access. For a translation table
	146	of array elements to attributes please see the pull request linked above.
	147	* Add a `new parameter 'raise_timeout_exception' (#61). <https://github.com/DiamondLightSource/python-procrunner/pull/61>`_
	148	When set to 'True' a subprocess.TimeoutExpired exception is raised when the
	149	process runtime exceeds the timeout threshold. This defaults to 'False' and
	150	will be set to 'True' in a future release.
	151
	152	2.0.0 (2020-06-24)
	153	------------------
	154	* Python 3.5+ only, support for Python 2.7 has been dropped
	155	* Deprecated function alias run_process() has been removed
	156	* Fixed a stability issue on Windows
	157
	158	1.1.0 (2019-11-04)
	159	------------------
	160	* Add Python 3.8 support, drop Python 3.4 support
	161
	162	1.0.2 (2019-05-20)
	163	------------------
	164	* Stop environment override variables leaking into the process environment
	165
	166	1.0.1 (2019-04-16)
	167	------------------
	168	* Minor fixes on the return object (implement equality,
	169	mark as unhashable)
	170
	171	1.0.0 (2019-03-25)
	172	------------------
	173	* Support file system path objects (PEP-519) in arguments
	174	* Change the return object to make it similar to
	175	subprocess.CompletedProcess, introduced with Python 3.5+
	176
	177	0.9.1 (2019-02-22)
	178	------------------
	179	* Have deprecation warnings point to correct code locations
	180
	181	0.9.0 (2018-12-07)
	182	------------------
	183	* Trap UnicodeEncodeError when printing output. Offending characters
	184	are replaced and a warning is logged once. Hints at incorrectly set
	185	PYTHONIOENCODING.
	186
	187	0.8.1 (2018-12-04)
	188	------------------
	189	* Fix a few deprecation warnings
	190
	191	0.8.0 (2018-10-09)
	192	------------------
	193	* Add parameter working_directory to set the working directory
	194	of the subprocess
	195
	196	0.7.2 (2018-10-05)
	197	------------------
	198	* Officially support Python 3.7
	199
	200	0.7.1 (2018-09-03)
	201	------------------
	202	* Accept environment variable overriding with numeric values.
	203
	204	0.7.0 (2018-05-13)
	205	------------------
	206	* Unicode fixes. Fix crash on invalid UTF-8 input.
	207	* Clarify that stdout/stderr values are returned as bytestrings.
	208	* Callbacks receive the data decoded as UTF-8 unicode strings
	209	with unknown characters replaced by \ufffd (unicode replacement
	210	character). Same applies to printing of output.
	211	* Mark stdin broken on Windows.
	212
	213	0.6.1 (2018-05-02)
	214	------------------
	215	* Maintenance release to add some tests for executable resolution.
	216
	217	0.6.0 (2018-05-02)
	218	------------------
	219	* Fix Win32 API executable resolution for commands containing a dot ('.') in
	220	addition to a file extension (say '.bat').
	221
	222	0.5.1 (2018-04-27)
	223	------------------
	224	* Fix Win32API dependency installation on Windows.
	225
	226	0.5.0 (2018-04-26)
	227	------------------
	228	* New keyword 'win32resolve' which only takes effect on Windows and is enabled
	229	by default. This causes procrunner to call the Win32 API FindExecutable()
	230	function to try and lookup non-.exe files with the corresponding name. This
	231	means .bat/.cmd/etc.. files can now be run without explicitly specifying
	232	their extension. Only supported on Python 2.7 and 3.5+.
	233
	234	0.4.0 (2018-04-23)
	235	------------------
	236	* Python 2.7 support on Windows. Python3 not yet supported on Windows.
	237
	238	0.3.0 (2018-04-17)
	239	------------------
	240	* run_process() renamed to run()
	241	* Python3 compatibility fixes
	242
	243	0.2.0 (2018-03-12)
	244	------------------
	245	* Procrunner is now Python3 3.3-3.6 compatible.
	246
	247	0.1.0 (2018-03-12)
	248	------------------
	249	* First release on PyPI.

+29

-0

src/procrunner.egg-info/SOURCES.txt less more

	0	AUTHORS.rst
	1	CONTRIBUTING.rst
	2	HISTORY.rst
	3	LICENSE
	4	MANIFEST.in
	5	README.rst
	6	pyproject.toml
	7	setup.cfg
	8	setup.py
	9	docs/Makefile
	10	docs/api.rst
	11	docs/authors.rst
	12	docs/conf.py
	13	docs/contributing.rst
	14	docs/history.rst
	15	docs/index.rst
	16	docs/installation.rst
	17	docs/make.bat
	18	docs/readme.rst
	19	docs/usage.rst
	20	src/procrunner/__init__.py
	21	src/procrunner.egg-info/PKG-INFO
	22	src/procrunner.egg-info/SOURCES.txt
	23	src/procrunner.egg-info/dependency_links.txt
	24	src/procrunner.egg-info/not-zip-safe
	25	src/procrunner.egg-info/top_level.txt
	26	tests/test_procrunner.py
	27	tests/test_procrunner_resolution.py
	28	tests/test_procrunner_system.py⏎

-0

src/procrunner.egg-info/dependency_links.txt less more

-0

src/procrunner.egg-info/not-zip-safe less more

-0

src/procrunner.egg-info/top_level.txt less more

procrunner

+55

-88

tests/test_procrunner.py less more

0		from __future__ import absolute_import, division, print_function
	0	from __future__ import annotations
1	1
2	2	import copy
3		import mock
4	3	import os
	4	import pathlib
	5	import sys
	6	from unittest import mock
	7
	8	import pytest
	9
5	10	import procrunner
6		import pytest
7		import sys
	11
	12
	13	@mock.patch("procrunner._NonBlockingStreamReader")
	14	@mock.patch("procrunner.time")
	15	@mock.patch("procrunner.subprocess")
	16	@mock.patch("procrunner.Pipe")
	17	def test_run_command_aborts_after_timeout_legacy(
	18	mock_pipe, mock_subprocess, mock_time, mock_streamreader
	19	):
	20	mock_pipe.return_value = mock.Mock(), mock.Mock()
	21	mock_process = mock.Mock()
	22	mock_process.returncode = None
	23	mock_subprocess.Popen.return_value = mock_process
	24	task = ["___"]
	25
	26	with pytest.raises(RuntimeError):
	27	with pytest.warns(DeprecationWarning, match="timeout"):
	28	procrunner.run(task, timeout=-1)
	29
	30	assert mock_subprocess.Popen.called
	31	assert mock_process.terminate.called
	32	assert mock_process.kill.called
8	33
9	34
10	35	@mock.patch("procrunner._NonBlockingStreamReader")

21	46	task = ["___"]
22	47
23	48	with pytest.raises(RuntimeError):
24		procrunner.run(task, -1, False)
	49	procrunner.run(task, timeout=-1)
25	50
26	51	assert mock_subprocess.Popen.called
27	52	assert mock_process.terminate.called

49	74	mock_streamreader.side_effect = streamreader_processing
50	75	mock_subprocess.Popen.return_value = mock_process
51	76
52		expected = {
53		"stderr": mock.sentinel.proc_stderr,
54		"stdout": mock.sentinel.proc_stdout,
55		"exitcode": mock_process.returncode,
56		"command": tuple(command),
57		"runtime": mock.ANY,
58		"timeout": False,
59		"time_start": mock.ANY,
60		"time_end": mock.ANY,
61		}
62
63	77	actual = procrunner.run(
64	78	command,
65		0.5,
66		False,
	79	timeout=0.5,
67	80	callback_stdout=mock.sentinel.callback_stdout,
68	81	callback_stderr=mock.sentinel.callback_stderr,
69		working_directory=mock.sentinel.cwd,
	82	working_directory=pathlib.Path("somecwd"),
70	83	)
71	84
72	85	assert mock_subprocess.Popen.called
73	86	assert mock_subprocess.Popen.call_args[1]["env"] == os.environ
74		assert mock_subprocess.Popen.call_args[1]["cwd"] == mock.sentinel.cwd
	87	assert mock_subprocess.Popen.call_args[1]["cwd"] in (
	88	pathlib.Path("somecwd"),
	89	"somecwd",
	90	)
75	91	mock_streamreader.assert_has_calls(
76	92	[
77	93	mock.call(
78	94	stream_stdout,
79	95	output=mock.ANY,
80		debug=mock.ANY,
81	96	notify=mock.ANY,
82	97	callback=mock.sentinel.callback_stdout,
83	98	),
84	99	mock.call(
85	100	stream_stderr,
86	101	output=mock.ANY,
87		debug=mock.ANY,
88	102	notify=mock.ANY,
89	103	callback=mock.sentinel.callback_stderr,
90	104	),

93	107	)
94	108	assert not mock_process.terminate.called
95	109	assert not mock_process.kill.called
96		for key in expected:
97		assert actual[key] == expected[key]
98		assert actual.args == tuple(command)
99		assert actual.returncode == mock_process.returncode
100		assert actual.stdout == mock.sentinel.proc_stdout
101		assert actual.stderr == mock.sentinel.proc_stderr
	110	assert actual == mock_subprocess.CompletedProcess.return_value
	111	mock_subprocess.CompletedProcess.assert_called_once_with(
	112	args=tuple(command),
	113	returncode=mock_process.returncode,
	114	stdout=mock.sentinel.proc_stdout,
	115	stderr=mock.sentinel.proc_stderr,
	116	)
102	117
103	118
104	119	@mock.patch("procrunner.subprocess")
105	120	def test_default_process_environment_is_parent_environment(mock_subprocess):
106	121	mock_subprocess.Popen.side_effect = NotImplementedError() # cut calls short
107	122	with pytest.raises(NotImplementedError):
108		procrunner.run([mock.Mock()], -1, False)
	123	procrunner.run([mock.Mock()], timeout=-1)
109	124	assert mock_subprocess.Popen.call_args[1]["env"] == os.environ
110	125
111	126

115	130	mock_env = {"key": mock.sentinel.key}
116	131	# Pass an environment dictionary
117	132	with pytest.raises(NotImplementedError):
118		procrunner.run([mock.Mock()], -1, False, environment=copy.copy(mock_env))
	133	procrunner.run(
	134	[mock.Mock()],
	135	timeout=-1,
	136	environment=copy.copy(mock_env),
	137	)
119	138	assert mock_subprocess.Popen.call_args[1]["env"] == mock_env
120	139
121	140

128	147	with pytest.raises(NotImplementedError):
129	148	procrunner.run(
130	149	[mock.Mock()],
131		-1,
132		False,
	150	timeout=-1,
133	151	environment=copy.copy(mock_env1),
134	152	environment_override=copy.copy(mock_env2),
135	153	)

144	162	mock_env2 = {"keyB": str(mock.sentinel.keyB)}
145	163	with pytest.raises(NotImplementedError):
146	164	procrunner.run(
147		[mock.Mock()], -1, False, environment_override=copy.copy(mock_env2)
	165	[mock.Mock()],
	166	timeout=-1,
	167	environment_override=copy.copy(mock_env2),
148	168	)
149	169	random_environment_variable = list(os.environ)[0]
150	170	if random_environment_variable == list(mock_env2)[0]:
151	171	random_environment_variable = list(os.environ)[1]
152		random_environment_value = os.getenv(random_environment_variable)
153	172	assert (
154	173	random_environment_variable
155	174	and random_environment_variable != list(mock_env2)[0]

171	190	with pytest.raises(NotImplementedError):
172	191	procrunner.run(
173	192	[mock.Mock()],
174		-1,
175		False,
	193	timeout=-1,
176	194	environment_override={
177	195	random_environment_variable: "X" + random_environment_value
178	196	},

191	209	def test_nonblockingstreamreader_can_read(mock_select):
192	210	import time
193	211
194		class _stream(object):
	212	class _stream:
195	213	def __init__(self):
196	214	self.data = b""
197	215	self.closed = False

258	276	callback.assert_not_called()
259	277	aggregator.flush()
260	278	callback.assert_called_once_with("morestuff")
261
262
263		def test_return_object_semantics():
264		ro = procrunner.ReturnObject(
265		{
266		"command": mock.sentinel.command,
267		"exitcode": 0,
268		"stdout": mock.sentinel.stdout,
269		"stderr": mock.sentinel.stderr,
270		}
271		)
272		assert ro["command"] == mock.sentinel.command
273		assert ro.args == mock.sentinel.command
274		assert ro["exitcode"] == 0
275		assert ro.returncode == 0
276		assert ro["stdout"] == mock.sentinel.stdout
277		assert ro.stdout == mock.sentinel.stdout
278		assert ro["stderr"] == mock.sentinel.stderr
279		assert ro.stderr == mock.sentinel.stderr
280
281		with pytest.raises(KeyError):
282		ro["unknownkey"]
283		ro.update({"unknownkey": mock.sentinel.key})
284		assert ro["unknownkey"] == mock.sentinel.key
285
286
287		def test_return_object_check_function_passes_on_success():
288		ro = procrunner.ReturnObject(
289		{
290		"command": mock.sentinel.command,
291		"exitcode": 0,
292		"stdout": mock.sentinel.stdout,
293		"stderr": mock.sentinel.stderr,
294		}
295		)
296		ro.check_returncode()
297
298
299		def test_return_object_check_function_raises_on_error():
300		ro = procrunner.ReturnObject(
301		{
302		"command": mock.sentinel.command,
303		"exitcode": 1,
304		"stdout": mock.sentinel.stdout,
305		"stderr": mock.sentinel.stderr,
306		}
307		)
308		with pytest.raises(Exception) as e:
309		ro.check_returncode()
310		assert repr(mock.sentinel.command) in str(e.value)
311		assert "1" in str(e.value)

+14

-23

tests/test_procrunner_resolution.py less more

0		from __future__ import absolute_import, division, print_function
1
2	0	import os
3	1	import sys
4	2
5		import mock
	3	import pytest
	4
6	5	import procrunner
7		import pytest
8	6
9	7
10	8	def PEP519(path):

46	44
47	45
48	46	@pytest.mark.skipif(sys.platform != "win32", reason="windows specific test only")
49		def test_pywin32_import():
50		import win32api
51
52
53		@pytest.mark.skipif(sys.platform != "win32", reason="windows specific test only")
54	47	def test_name_resolution_for_simple_exe():
55	48	command = ["cmd.exe", "/c", "echo", "hello"]
56	49

65	58
66	59
67	60	@pytest.mark.skipif(sys.platform != "win32", reason="windows specific test only")
68		def test_name_resolution_for_complex_cases(tmpdir):
69		tmpdir.chdir()
70
	61	def test_name_resolution_for_complex_cases(tmp_path):
71	62	bat = "simple_bat_extension"
72	63	cmd = "simple_cmd_extension"
73	64	exe = "simple_exe_extension"
74	65	dotshort = "more_complex_filename_with_a.dot"
75	66	dotlong = "more_complex_filename.withadot"
76	67
77		(tmpdir / bat + ".bat").ensure()
78		(tmpdir / cmd + ".cmd").ensure()
79		(tmpdir / exe + ".exe").ensure()
80		(tmpdir / dotshort + ".bat").ensure()
81		(tmpdir / dotlong + ".cmd").ensure()
	68	(tmp_path / (bat + ".bat")).touch()
	69	(tmp_path / (cmd + ".cmd")).touch()
	70	(tmp_path / (exe + ".exe")).touch()
	71	(tmp_path / (dotshort + ".bat")).touch()
	72	(tmp_path / (dotlong + ".cmd")).touch()
82	73
83	74	def is_valid(command):
84	75	assert len(command) == 1
85		assert os.path.exists(command[0])
	76	assert os.path.exists(tmp_path / command[0])
86	77
87		is_valid(procrunner._windows_resolve([bat]))
88		is_valid(procrunner._windows_resolve([cmd]))
89		is_valid(procrunner._windows_resolve([exe]))
90		is_valid(procrunner._windows_resolve([dotshort]))
91		is_valid(procrunner._windows_resolve([dotlong]))
	78	is_valid(procrunner._windows_resolve([bat], path=os.fspath(tmp_path)))
	79	is_valid(procrunner._windows_resolve([cmd], path=os.fspath(tmp_path)))
	80	is_valid(procrunner._windows_resolve([exe], path=os.fspath(tmp_path)))
	81	is_valid(procrunner._windows_resolve([dotshort], path=os.fspath(tmp_path)))
	82	is_valid(procrunner._windows_resolve([dotlong], path=os.fspath(tmp_path)))

+102

-12

tests/test_procrunner_system.py less more

0		from __future__ import absolute_import, division, print_function
	0	from __future__ import annotations
1	1
2	2	import os
	3	import subprocess
3	4	import sys
	5	import timeit
	6
	7	import pytest
4	8
5	9	import procrunner
6		import pytest
7	10
8	11
9	12	def test_simple_command_invocation():

13	16	command = ["echo", "hello"]
14	17
15	18	result = procrunner.run(command)
	19
	20	assert result.returncode == 0
	21	assert result.stdout == b"hello" + os.linesep.encode("utf-8")
	22	assert result.stderr == b""
	23
	24
	25	def test_simple_command_invocation_with_closed_stdin():
	26	if os.name == "nt":
	27	command = ["cmd.exe", "/c", "echo", "hello"]
	28	else:
	29	command = ["echo", "hello"]
	30
	31	result = procrunner.run(command, stdin=subprocess.DEVNULL)
16	32
17	33	assert result.returncode == 0
18	34	assert result.stdout == b"hello" + os.linesep.encode("utf-8")

35	51	else:
36	52	assert result.stdout == test_string
37	53	out, err = capsys.readouterr()
38		assert out == u"test\ufffdstring\n"
39		assert err == u""
	54	assert out == "test\ufffdstring\n"
	55	assert err == ""
40	56
41	57
42		def test_running_wget(tmpdir):
43		tmpdir.chdir()
	58	def test_running_wget(tmp_path):
44	59	command = ["wget", "https://www.google.com", "-O", "-"]
45	60	try:
46		result = procrunner.run(command)
	61	result = procrunner.run(command, working_directory=tmp_path)
47	62	except OSError as e:
48	63	if e.errno == 2:
49	64	pytest.skip("wget not available")

53	68	assert b"google" in result.stdout
54	69
55	70
56		def test_path_object_resolution(tmpdir):
	71	def test_path_object_resolution(tmp_path):
57	72	sentinel_value = b"sentinel"
58		tmpdir.join("tempfile").write(sentinel_value)
59		tmpdir.join("reader.py").write("print(open('tempfile').read())")
	73	tmp_path.joinpath("tempfile").write_bytes(sentinel_value)
	74	tmp_path.joinpath("reader.py").write_text(
	75	"with open('tempfile') as fh:\n print(fh.read())"
	76	)
60	77	assert "LEAK_DETECTOR" not in os.environ
61	78	result = procrunner.run(
62		[sys.executable, tmpdir.join("reader.py")],
	79	[sys.executable, tmp_path / "reader.py"],
63	80	environment_override={"PYTHONHASHSEED": "random", "LEAK_DETECTOR": "1"},
64		working_directory=tmpdir,
	81	working_directory=tmp_path,
65	82	)
66	83	assert result.returncode == 0
67	84	assert not result.stderr

69	86	assert (
70	87	"LEAK_DETECTOR" not in os.environ
71	88	), "overridden environment variable leaked into parent process"
	89
	90
	91	def test_timeout_behaviour_old_legacy(tmp_path):
	92	command = (sys.executable, "-c", "import time; time.sleep(5)")
	93	start = timeit.default_timer()
	94	try:
	95	with pytest.raises(subprocess.TimeoutExpired) as te:
	96	with pytest.warns(UserWarning, match="timeout"):
	97	procrunner.run(
	98	command,
	99	timeout=0.1,
	100	working_directory=tmp_path,
	101	raise_timeout_exception=False,
	102	)
	103	except RuntimeError:
	104	# This test sometimes fails with a RuntimeError.
	105	runtime = timeit.default_timer() - start
	106	assert runtime < 3
	107	return
	108	runtime = timeit.default_timer() - start
	109	assert runtime < 3
	110	assert te.value.stdout == b""
	111	assert te.value.stderr == b""
	112	assert te.value.timeout == 0.1
	113	assert te.value.cmd == command
	114
	115
	116	def test_timeout_behaviour_legacy(tmp_path):
	117	command = (sys.executable, "-c", "import time; time.sleep(5)")
	118	start = timeit.default_timer()
	119	try:
	120	with pytest.raises(subprocess.TimeoutExpired) as te:
	121	with pytest.warns(DeprecationWarning, match="timeout"):
	122	procrunner.run(
	123	command,
	124	timeout=0.1,
	125	working_directory=tmp_path,
	126	raise_timeout_exception=True,
	127	)
	128	except RuntimeError:
	129	# This test sometimes fails with a RuntimeError.
	130	runtime = timeit.default_timer() - start
	131	assert runtime < 3
	132	return
	133	runtime = timeit.default_timer() - start
	134	assert runtime < 3
	135	assert te.value.stdout == b""
	136	assert te.value.stderr == b""
	137	assert te.value.timeout == 0.1
	138	assert te.value.cmd == command
	139
	140
	141	def test_timeout_behaviour(tmp_path):
	142	command = (sys.executable, "-c", "import time; time.sleep(5)")
	143	start = timeit.default_timer()
	144	try:
	145	with pytest.raises(subprocess.TimeoutExpired) as te:
	146	procrunner.run(
	147	command,
	148	timeout=0.1,
	149	working_directory=tmp_path,
	150	)
	151	except RuntimeError:
	152	# This test sometimes fails with a RuntimeError.
	153	runtime = timeit.default_timer() - start
	154	assert runtime < 3
	155	return
	156	runtime = timeit.default_timer() - start
	157	assert runtime < 3
	158	assert te.value.stdout == b""
	159	assert te.value.stderr == b""
	160	assert te.value.timeout == 0.1
	161	assert te.value.cmd == command

-26

~~tox.ini~~ less more

0		[tox]
1		envlist = py27, py35, py36, py37, py38, flake8
2
3		[travis]
4		python =
5		3.8: py38
6		3.7: py37
7		3.6: py36
8		3.5: py35
9		2.7: py27
10
11		[testenv:flake8]
12		basepython = python
13		deps = flake8
14		commands = flake8 procrunner
15
16		[testenv]
17		setenv =
18		PYTHONPATH = {toxinidir}
19		deps =
20		-r{toxinidir}/requirements_dev.txt
21		; If you want to make tox run the tests with the same versions, create a
22		; requirements.txt with the pinned versions and uncomment the following line:
23		; -r{toxinidir}/requirements.txt
24		commands =
25		pytest -ra --basetemp={envtmpdir}