Commit upstream/2.3.1 - python-procrunner

Import upstream version 2.3.1 Debian Janitor 1 year, 6 months ago

27 changed file(s) with 836 addition(s) and 460 deletion(s). Raw diff Collapse all Expand all

+20

-0

.azure-pipelines/ci.yml less more

	0	steps:
	1	- task: UsePythonVersion@0
	2	inputs:
	3	versionSpec: '$(PYTHON_VERSION)'
	4	displayName: 'Use Python $(PYTHON_VERSION)'
	5
	6	- script: \|
	7	python -m pip install --upgrade pip
	8	pip install tox
	9	displayName: "Set up tox"
	10
	11	- script: \|
	12	tox -e azure
	13	displayName: "Run tests"
	14
	15	- task: PublishTestResults@2
	16	condition: succeededOrFailed()
	17	inputs:
	18	testResultsFiles: '*/test-.xml'
	19	testRunTitle: 'Publish test results for Python $(PYTHON_VERSION)'

+40

-0

.azure-pipelines/flake8-validation.py less more

	0	import os
	1	import subprocess
	2
	3	# Flake8 validation
	4	failures = 0
	5	try:
	6	flake8 = subprocess.run(
	7	[
	8	"flake8",
	9	"--exit-zero",
	10	],
	11	capture_output=True,
	12	check=True,
	13	encoding="latin-1",
	14	timeout=300,
	15	)
	16	except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
	17	print(
	18	"##vso[task.logissue type=error;]flake8 validation failed with",
	19	str(e.__class__.__name__),
	20	)
	21	print(e.stdout)
	22	print(e.stderr)
	23	print("##vso[task.complete result=Failed;]flake8 validation failed")
	24	exit()
	25	for line in flake8.stdout.split("\n"):
	26	if ":" not in line:
	27	continue
	28	filename, lineno, column, error = line.split(":", maxsplit=3)
	29	errcode, error = error.strip().split(" ", maxsplit=1)
	30	filename = os.path.normpath(filename)
	31	failures += 1
	32	print(
	33	f"##vso[task.logissue type=error;sourcepath={filename};"
	34	f"linenumber={lineno};columnnumber={column};code={errcode};]" + error
	35	)
	36
	37	if failures:
	38	print(f"##vso[task.logissue type=warning]Found {failures} flake8 violation(s)")
	39	print(f"##vso[task.complete result=Failed;]Found {failures} flake8 violation(s)")

+30

-0

.azure-pipelines/syntax-validation.py less more

	0	import ast
	1	import os
	2	import sys
	3
	4	print("Python", sys.version, "\n")
	5
	6	failures = 0
	7
	8	for base, _, files in os.walk("."):
	9	for f in files:
	10	if not f.endswith(".py"):
	11	continue
	12	filename = os.path.normpath(os.path.join(base, f))
	13	try:
	14	with open(filename) as fh:
	15	ast.parse(fh.read())
	16	except SyntaxError as se:
	17	failures += 1
	18	print(
	19	f"##vso[task.logissue type=error;sourcepath={filename};"
	20	f"linenumber={se.lineno};columnnumber={se.offset};]"
	21	f"SyntaxError: {se.msg}"
	22	)
	23	print(" " + se.text + " " * se.offset + "^")
	24	print(f"SyntaxError: {se.msg} in {filename} line {se.lineno}")
	25	print()
	26
	27	if failures:
	28	print(f"##vso[task.logissue type=warning]Found {failures} syntax error(s)")
	29	print(f"##vso[task.complete result=Failed;]Found {failures} syntax error(s)")

+134

-0

.azure-pipelines.yml less more

	0	trigger:
	1	branches:
	2	include:
	3	- '*'
	4	tags:
	5	include:
	6	- '*'
	7
	8	stages:
	9	- stage: static
	10	displayName: Static Analysis
	11	jobs:
	12	- job: checks
	13	displayName: static code analysis
	14	pool:
	15	vmImage: ubuntu-latest
	16	steps:
	17	# Run syntax validation using oldest and latest Python
	18	- task: UsePythonVersion@0
	19	displayName: Set up python
	20	inputs:
	21	versionSpec: 3.6
	22
	23	- bash: python .azure-pipelines/syntax-validation.py
	24	displayName: Syntax validation (3.6)
	25
	26	- task: UsePythonVersion@0
	27	displayName: Set up python
	28	inputs:
	29	versionSpec: 3.10
	30
	31	- bash: python .azure-pipelines/syntax-validation.py
	32	displayName: Syntax validation (3.10)
	33
	34	# Run flake8 validation
	35	- bash: \|
	36	pip install --disable-pip-version-check flake8 && \
	37	python .azure-pipelines/flake8-validation.py
	38	displayName: Flake8 validation
	39
	40	- stage: tests
	41	displayName: Run unit tests
	42	jobs:
	43	- job: linux
	44	pool:
	45	vmImage: ubuntu-latest
	46	strategy:
	47	matrix:
	48	python36:
	49	PYTHON_VERSION: 3.6
	50	python37:
	51	PYTHON_VERSION: 3.7
	52	python38:
	53	PYTHON_VERSION: 3.8
	54	python39:
	55	PYTHON_VERSION: 3.9
	56	python310:
	57	PYTHON_VERSION: 3.10
	58	steps:
	59	- template: .azure-pipelines/ci.yml
	60
	61	- job: macOS
	62	pool:
	63	vmImage: macOS-latest
	64	strategy:
	65	matrix:
	66	python37:
	67	PYTHON_VERSION: 3.7
	68	python38:
	69	PYTHON_VERSION: 3.8
	70	python39:
	71	PYTHON_VERSION: 3.9
	72	python310:
	73	PYTHON_VERSION: 3.10
	74	steps:
	75	- template: .azure-pipelines/ci.yml
	76
	77	- job: windows
	78	pool:
	79	vmImage: windows-latest
	80	strategy:
	81	matrix:
	82	python36:
	83	PYTHON_VERSION: 3.6
	84	python37:
	85	PYTHON_VERSION: 3.7
	86	python38:
	87	PYTHON_VERSION: 3.8
	88	python39:
	89	PYTHON_VERSION: 3.9
	90	python310:
	91	PYTHON_VERSION: 3.10
	92	steps:
	93	- template: .azure-pipelines/ci.yml
	94
	95	- stage: deploy
	96	displayName: Publish release
	97	dependsOn:
	98	- tests
	99	- static
	100	condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'))
	101	jobs:
	102	- job: pypi
	103	displayName: Publish pypi release
	104	pool:
	105	vmImage: ubuntu-latest
	106	steps:
	107	- task: UsePythonVersion@0
	108	displayName: Set up python
	109	inputs:
	110	versionSpec: 3.9
	111
	112	- bash: \|
	113	python -m pip install -r requirements_dev.txt
	114	displayName: Install dependencies
	115
	116	- bash: \|
	117	python setup.py sdist bdist_wheel
	118	ls -la dist
	119	displayName: Build python package
	120
	121	- task: PublishBuildArtifacts@1
	122	inputs:
	123	pathToPublish: dist/
	124	artifactName: python-release
	125
	126	- task: TwineAuthenticate@1
	127	displayName: Set up credentials
	128	inputs:
	129	pythonUploadServiceConnection: pypi-procrunner
	130
	131	- bash: \|
	132	python -m twine upload -r pypi-procrunner --config-file $(PYPIRC_PATH) dist/.tar.gz dist/.whl
	133	displayName: Publish package

+12

-0

.bumpversion.cfg less more

	0	[bumpversion]
	1	current_version = 2.3.1
	2	commit = True
	3	tag = True
	4
	5	[bumpversion:file:setup.cfg]
	6	search = version = {current_version}
	7	replace = version = {new_version}
	8
	9	[bumpversion:file:procrunner/__init__.py]
	10	search = __version__ = "{current_version}"
	11	replace = __version__ = "{new_version}"

+37

-0

.github/renovate.json less more

	0	{
	1	"extends": [
	2	"config:base"
	3	],
	4	"labels": [
	5	"dependencies"
	6	],
	7	"pip_requirements": {
	8	"fileMatch": [
	9	"^requirements.*\\.txt$"
	10	],
	11	"groupName": "all dependencies",
	12	"groupSlug": "all",
	13	"packageRules": [
	14	{
	15	"groupName": "all dependencies",
	16	"groupSlug": "all",
	17	"matchPackagePatterns": [
	18	"*"
	19	]
	20	}
	21	]
	22	},
	23	"prCreation": "not-pending",
	24	"prHourlyLimit": 2,
	25	"pre-commit": {
	26	"schedule": [
	27	"after 10am and before 4pm every 3 months on the first day of the month"
	28	],
	29	"stabilityDays": 10
	30	},
	31	"schedule": [
	32	"after 7am and before 4pm every monday"
	33	],
	34	"stabilityDays": 2,
	35	"timezone": "Europe/London"
	36	}

+32

-0

.pre-commit-config.yaml less more

	0	repos:
	1
	2	# Automatically sort imports
	3	- repo: https://github.com/PyCQA/isort
	4	rev: 5.9.3
	5	hooks:
	6	- id: isort
	7
	8	# Automatic source code formatting
	9	- repo: https://github.com/psf/black
	10	rev: 21.6b0
	11	hooks:
	12	- id: black
	13	args: [--safe, --quiet]
	14
	15	# Linting
	16	- repo: https://github.com/PyCQA/flake8
	17	rev: 3.9.2
	18	hooks:
	19	- id: flake8
	20	additional_dependencies: ['flake8-comprehensions==3.5.0']
	21
	22	# Syntax validation and some basic sanity checks
	23	- repo: https://github.com/pre-commit/pre-commit-hooks
	24	rev: v4.0.1
	25	hooks:
	26	- id: check-merge-conflict
	27	- id: check-ast
	28	- id: check-json
	29	- id: check-added-large-files
	30	args: ['--maxkb=200']
	31	- id: check-yaml

-5

~~.pyup.yml~~ less more

0		# autogenerated pyup.io config file
1		# see https://pyup.io/docs/configuration/ for all available options
2
3		schedule: every month
4

-72

~~.travis.yml~~ less more

0		# Config file for automatic testing at travis-ci.org
1
2		language: python
3
4		matrix:
5		include:
6		- python: 3.8
7		dist: xenial
8		sudo: true
9		- python: 3.7
10		dist: xenial
11		sudo: true
12		- python: 3.6
13		- python: 3.5
14		- python: 2.7
15		- python: pypy
16		- os: osx
17		language: generic
18		env: CONDA=3.8 TOXENV=py38
19		- os: osx
20		language: generic
21		env: CONDA=3.7 TOXENV=py37
22		- os: osx
23		language: generic
24		env: CONDA=3.6 TOXENV=py36
25		- os: osx
26		language: generic
27		env: CONDA=3.5 TOXENV=py35
28		- os: osx
29		language: generic
30		env: CONDA=2.7 TOXENV=py27
31
32		allow_failures:
33		- env: OPTIONAL=1
34
35		fast_finish: true
36
37		before_install: \|
38		if [ ! -z "$CONDA" ]; then
39		if [ "$TRAVIS_OS_NAME" == "osx" ]; then
40		curl https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh --output miniconda.sh
41		fi
42		chmod +x miniconda.sh
43		./miniconda.sh -b
44		export PATH=$HOME/miniconda3/bin:$PATH
45		conda update --yes conda
46		conda create --yes -n travis python=$CONDA
47		source activate travis
48		# A manual check that the correct version of Python is running.
49		python --version
50		fi
51
52		# Command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors
53		install: pip install -U tox-travis
54
55		# Command to run tests, e.g. python setup.py test
56		script: tox
57
58		# Assuming you have installed the travis-ci CLI tool, after you
59		# create the Github repo and add it to Travis, run the
60		# following command to finish PyPI deployment setup:
61		# $ travis encrypt --add deploy.password
62		deploy:
63		provider: pypi
64		distributions: sdist bdist_wheel
65		user: mgerstel
66		password:
67		secure: qN3EYVlH22eLPZp5CSvc5Bz8bpP0l0wBZ8tcYMa6kBrqelYdIgqkuZESR/oCMu1YBA3AbcKJNwMuzuWf8RNGAFebD820OJThdP4czRMCv6LDbWABnv12lFHQLQSW1fMQVvb2arjnE/Ew7BFq70p0wOlIJJRwu6CoeOXW/sMeVYMivxdaHmgORq3cdMluFAy4amVb3Fc8i7mxAM0QGklO7x/UWJR/IEpUk2RlUbXL+HrzNoEjRtDeMxoCR84gKZTjVeUQ/iIQSuWwxlt7v1FNENj6ZEbE7+PS8/ylIVfPufbCr8tEEv8W58QcxQ5xPJC2g85ulsN5dM9/9FekhpyKa25B/4wKUNq5T8rKJ8WZ6hMiGffW8rmAfrGTmrBaojKBi0pb9VfXJ5KXUcunVXwQaAn2L80jLLHNsAo94ZxeoowD1eJox9Wh1NtNc+NiUv8K6spOIBsur7G5GY4JVA/yZ7w+DweEfQEp8/SEdVhK0vEYSYT4FnJHuAAmNgeedyAtoes4+a5bYYUM4qrz2OC78NWQWAnnsZhD4Y/TulkavWSexVCqfSePRoK3gcCs+rXwiM69XkMbL1Wgj1gNou+XMkntayH2ZDHkkyJi5F7ls4nqMH5RON9FfVygJMoHDRqh0p4RV25IzJ4FSYqKihHNBO31/URnU2ihpW7n8kM+mbM=
68		on:
69		tags: true
70		repo: DiamondLightSource/python-procrunner
71		python: 3.8

-4

CONTRIBUTING.rst less more

100	100	1. The pull request should include tests.
101	101	2. If the pull request adds functionality, the docs should be updated. Put
102	102	your new functionality into a function with a docstring, and add the
103		feature to the list in README.rst.
104		3. The pull request should work for Python 2.7, 3.5, 3.6, 3.7, 3.8, and for PyPy. Check
105		https://travis-ci.org/DiamondLightSource/python-procrunner/pull_requests
106		and make sure that the tests pass for all supported Python versions.
	103	feature to the list in HISTORY.rst/README.rst.
107	104
108	105	Tips
109	106	----

+39

-19

HISTORY.rst less more

1	1	History
2	2	=======
3	3
	4	2.3.1 (2021-10-25)
	5	------------------
	6	* Add Python 3.10 support
	7
	8	2.3.0 (2020-10-29)
	9	------------------
	10	* Add Python 3.9 support, drop Python 3.5 support
	11	* Fix a file descriptor leak on subprocess execution
	12
	13	2.2.0 (2020-09-07)
	14	------------------
	15	* Calling the run() function with unnamed arguments (other than the command
	16	list as the first argument) is now deprecated. As a number of arguments
	17	will be removed in a future version the use of unnamed arguments will
	18	cause future confusion. `Use explicit keyword arguments instead (#62). <https://github.com/DiamondLightSource/python-procrunner/pull/62>`_
	19	* `The run() function debug argument has been deprecated (#63). <https://github.com/DiamondLightSource/python-procrunner/pull/63>`_
	20	This is only used to debug the NonBlockingStream* classes. Those are due
	21	to be replaced in a future release, so the argument will no longer serve
	22	a purpose. Debugging information remains available via standard logging
	23	mechanisms.
	24	* Final version supporting Python 3.5
	25
	26	2.1.0 (2020-09-05)
	27	------------------
	28	* `Deprecated array access on the return object (#60). <https://github.com/DiamondLightSource/python-procrunner/pull/60>`_
	29	The return object will become a subprocess.CompletedProcess in a future
	30	release, which no longer allows array-based access. For a translation table
	31	of array elements to attributes please see the pull request linked above.
	32	* Add a `new parameter 'raise_timeout_exception' (#61). <https://github.com/DiamondLightSource/python-procrunner/pull/61>`_
	33	When set to 'True' a subprocess.TimeoutExpired exception is raised when the
	34	process runtime exceeds the timeout threshold. This defaults to 'False' and
	35	will be set to 'True' in a future release.
	36
	37	2.0.0 (2020-06-24)
	38	------------------
	39	* Python 3.5+ only, support for Python 2.7 has been dropped
	40	* Deprecated function alias run_process() has been removed
	41	* Fixed a stability issue on Windows
	42
4	43	1.1.0 (2019-11-04)
5	44	------------------
6
7	45	* Add Python 3.8 support, drop Python 3.4 support
8	46
9	47	1.0.2 (2019-05-20)
10	48	------------------
11
12	49	* Stop environment override variables leaking into the process environment
13	50
14	51	1.0.1 (2019-04-16)
15	52	------------------
16
17	53	* Minor fixes on the return object (implement equality,
18	54	mark as unhashable)
19	55
20	56	1.0.0 (2019-03-25)
21	57	------------------
22
23	58	* Support file system path objects (PEP-519) in arguments
24	59	* Change the return object to make it similar to
25	60	subprocess.CompletedProcess, introduced with Python 3.5+
26	61
27	62	0.9.1 (2019-02-22)
28	63	------------------
29
30	64	* Have deprecation warnings point to correct code locations
31	65
32	66	0.9.0 (2018-12-07)
33	67	------------------
34
35	68	* Trap UnicodeEncodeError when printing output. Offending characters
36	69	are replaced and a warning is logged once. Hints at incorrectly set
37	70	PYTHONIOENCODING.
38	71
39	72	0.8.1 (2018-12-04)
40	73	------------------
41
42	74	* Fix a few deprecation warnings
43	75
44	76	0.8.0 (2018-10-09)
45	77	------------------
46
47	78	* Add parameter working_directory to set the working directory
48	79	of the subprocess
49	80
50	81	0.7.2 (2018-10-05)
51	82	------------------
52
53	83	* Officially support Python 3.7
54	84
55	85	0.7.1 (2018-09-03)
56	86	------------------
57
58	87	* Accept environment variable overriding with numeric values.
59	88
60	89	0.7.0 (2018-05-13)
61	90	------------------
62
63	91	* Unicode fixes. Fix crash on invalid UTF-8 input.
64	92	* Clarify that stdout/stderr values are returned as bytestrings.
65	93	* Callbacks receive the data decoded as UTF-8 unicode strings

69	97
70	98	0.6.1 (2018-05-02)
71	99	------------------
72
73	100	* Maintenance release to add some tests for executable resolution.
74	101
75	102	0.6.0 (2018-05-02)
76	103	------------------
77
78	104	* Fix Win32 API executable resolution for commands containing a dot ('.') in
79	105	addition to a file extension (say '.bat').
80	106
81	107	0.5.1 (2018-04-27)
82	108	------------------
83
84	109	* Fix Win32API dependency installation on Windows.
85	110
86	111	0.5.0 (2018-04-26)
87	112	------------------
88
89	113	* New keyword 'win32resolve' which only takes effect on Windows and is enabled
90	114	by default. This causes procrunner to call the Win32 API FindExecutable()
91	115	function to try and lookup non-.exe files with the corresponding name. This

94	118
95	119	0.4.0 (2018-04-23)
96	120	------------------
97
98	121	* Python 2.7 support on Windows. Python3 not yet supported on Windows.
99	122
100	123	0.3.0 (2018-04-17)
101	124	------------------
102
103	125	* run_process() renamed to run()
104	126	* Python3 compatibility fixes
105	127
106	128	0.2.0 (2018-03-12)
107	129	------------------
108
109	130	* Procrunner is now Python3 3.3-3.6 compatible.
110	131
111	132	0.1.0 (2018-03-12)
112	133	------------------
113
114	134	* First release on PyPI.

-1

LICENSE less more

0		Copyright (c) 2018 Diamond Light Source.
	0	Copyright (c) 2018-2021 Diamond Light Source.
1	1	All rights reserved.
2	2
3	3	Redistribution and use in source and binary forms, with or without

-10

README.rst less more

10	10	:target: https://anaconda.org/conda-forge/procrunner
11	11	:alt: Conda Version
12	12
13		.. image:: https://travis-ci.org/DiamondLightSource/python-procrunner.svg?branch=master
14		:target: https://travis-ci.org/DiamondLightSource/python-procrunner
	13	.. image:: https://dev.azure.com/DLS-tooling/procrunner/_apis/build/status/CI?branchName=master
	14	:target: https://github.com/DiamondLightSource/python-procrunner/commits/master
15	15	:alt: Build status
16	16
17	17	.. image:: https://ci.appveyor.com/api/projects/status/jtq4brwri5q18d0u/branch/master

21	21	.. image:: https://readthedocs.org/projects/procrunner/badge/?version=latest
22	22	:target: https://procrunner.readthedocs.io/en/latest/?badge=latest
23	23	:alt: Documentation Status
24
25		.. image:: https://pyup.io/repos/github/DiamondLightSource/python-procrunner/shield.svg
26		:target: https://pyup.io/repos/github/DiamondLightSource/python-procrunner/
27		:alt: Updates
28	24
29	25	.. image:: https://img.shields.io/pypi/pyversions/procrunner.svg
30	26	:target: https://pypi.python.org/pypi/procrunner

46	42	* runs an external process and waits for it to finish
47	43	* does not deadlock, no matter the process stdout/stderr output behaviour
48	44	* returns the exit code, stdout, stderr (separately, both as bytestrings),
49		and the total process runtime as a dictionary
	45	as a subprocess.CompletedProcess object
50	46	* process can run in a custom environment, either as a modification of
51	47	the current environment or in a new environment from scratch
52		* stdin can be fed to the process, the returned dictionary contains
53		information how much was read by the process
	48	* stdin can be fed to the process
54	49	* stdout and stderr is printed by default, can be disabled
55	50	* stdout and stderr can be passed to any arbitrary function for
56	51	live processing (separately, both as unicode strings)
57		* optionally enforces a time limit on the process
	52	* optionally enforces a time limit on the process, raising a
	53	subprocess.TimeoutExpired exception if it is exceeded.
	54
58	55
59	56	Credits
60	57	-------

-16

appveyor.yml less more

4	4	# For Python versions available on Appveyor, see
5	5	# http://www.appveyor.com/docs/installed-software#python
6	6
7		- PYTHON: "C:\\Python27"
8		- PYTHON: "C:\\Python35"
9		UNSTABLE: 1
10	7	- PYTHON: "C:\\Python36"
11		UNSTABLE: 1
12	8	- PYTHON: "C:\\Python37"
13		UNSTABLE: 1
14	9	- PYTHON: "C:\\Python38"
15		UNSTABLE: 1
16		- PYTHON: "C:\\Python27-x64"
17		- PYTHON: "C:\\Python35-x64"
18		UNSTABLE: 1
19	10	- PYTHON: "C:\\Python36-x64"
20		UNSTABLE: 1
21	11	- PYTHON: "C:\\Python37-x64"
22		UNSTABLE: 1
23	12	- PYTHON: "C:\\Python38-x64"
24		UNSTABLE: 1
25	13
26	14	matrix:
27	15	allow_failures:

31	19	# Upgrade to the latest pip.
32	20	- '%PYTHON%\\python.exe -m pip install -U pip setuptools wheel'
33	21	- '%PYTHON%\\python.exe -m pip install -r requirements_dev.txt'
34		# Install win32api dependency. Must use the --only-binary switch explicitly
35		# on AppVeyor
36		- "%PYTHON%\\python.exe -m pip install --only-binary pywin32 pywin32"
37	22
38	23	build: off
39	24

41	26	# Note that you must use the environment variable %PYTHON% to refer to
42	27	# the interpreter you're using - Appveyor does not do anything special
43	28	# to put the Python version you want to use on PATH.
44		- "%PYTHON%\\python.exe setup.py test"
	29	- "%PYTHON%\\python.exe -m pytest"
45	30
46	31	after_test:
47	32	# This step builds your wheels.

-9

docs/conf.py less more

0	0	#!/usr/bin/env python
1		# -- coding: utf-8 --
2	1	#
3	2	# procrunner documentation build configuration file, created by
4	3	# sphinx-quickstart on Fri Jun 9 13:47:02 2017.

47	46	master_doc = "index"
48	47
49	48	# General information about the project.
50		project = u"ProcRunner"
51		copyright = u"2018, Markus Gerstel"
52		author = u"Markus Gerstel"
	49	project = "ProcRunner"
	50	copyright = "2020, Diamond Light Source"
	51	author = "Diamond Light Source - Scientific Software"
53	52
54	53	# The version info for the project you're documenting, acts as replacement
55	54	# for \|version\| and \|release\|, also used in various other places throughout

128	127	(
129	128	master_doc,
130	129	"procrunner.tex",
131		u"ProcRunner Documentation",
132		u"Markus Gerstel",
	130	"procrunner Documentation",
	131	"Diamond Light Source - Scientific Software",
133	132	"manual",
134	133	)
135	134	]

139	138
140	139	# One entry per manual page. List of tuples
141	140	# (source start file, name, description, authors, manual section).
142		man_pages = [(master_doc, "procrunner", u"ProcRunner Documentation", [author], 1)]
	141	man_pages = [(master_doc, "procrunner", "procrunner Documentation", [author], 1)]
143	142
144	143
145	144	# -- Options for Texinfo output ----------------------------------------

151	150	(
152	151	master_doc,
153	152	"procrunner",
154		u"ProcRunner Documentation",
	153	"procrunner Documentation",
155	154	author,
156	155	"procrunner",
157		"One line description of project.",
	156	"Versatile utility function to run external processes",
158	157	"Miscellaneous",
159	158	)
160	159	]

-4

docs/usage.rst less more

8	8
9	9	To test for successful completion::
10	10
11		assert not result['exitcode']
12		assert result['exitcode'] == 0 # alternatively
	11	assert not result.returncode
	12	assert result.returncode == 0 # alternatively
	13	result.check_returncode() # raises subprocess.CalledProcessError()
13	14
14	15	To test for no STDERR output::
15	16
16		assert not result['stderr']
17		assert result['stderr'] == b'' # alternatively
	17	assert not result.stderr
	18	assert result.stderr == b'' # alternatively
18	19
19	20	To run with a specific environment variable set::
20	21

+216

-168

procrunner/__init__.py less more

0		# -- coding: utf-8 --
1
2		from __future__ import absolute_import, division, print_function
3
4	0	import codecs
	1	import functools
	2	import io
5	3	import logging
6	4	import os
7	5	import select
8		import six
	6	import shutil
9	7	import subprocess
10	8	import sys
11	9	import time

20	18	#
21	19	# - runs an external process and waits for it to finish
22	20	# - does not deadlock, no matter the process stdout/stderr output behaviour
23		# - returns the exit code, stdout, stderr (separately), and the total process
24		# runtime as a dictionary
	21	# - returns the exit code, stdout, stderr (separately) as a
	22	# subprocess.CompletedProcess object
25	23	# - process can run in a custom environment, either as a modification of
26	24	# the current environment or in a new environment from scratch
27		# - stdin can be fed to the process, the returned dictionary contains
28		# information how much was read by the process
	25	# - stdin can be fed to the process
29	26	# - stdout and stderr is printed by default, can be disabled
30	27	# - stdout and stderr can be passed to any arbitrary function for
31	28	# live processing

39	36	#
40	37	# Returns:
41	38	#
42		# {'command': ['/bin/ls', '/some/path/containing spaces'],
43		# 'exitcode': 2,
44		# 'runtime': 0.12990689277648926,
45		# 'stderr': '/bin/ls: cannot access /some/path/containing spaces: No such file or directory\n',
46		# 'stdout': '',
47		# 'time_end': '2017-11-12 19:54:49 GMT',
48		# 'time_start': '2017-11-12 19:54:49 GMT',
49		# 'timeout': False}
50		#
	39	# ReturnObject(
	40	# args=('/bin/ls', '/some/path/containing spaces'),
	41	# returncode=2,
	42	# stdout=b'',
	43	# stderr=b'/bin/ls: cannot access /some/path/containing spaces: No such file or directory\n'
	44	# )
	45	#
	46	# which also offers (albeit deprecated)
	47	#
	48	# result.runtime == 0.12990689277648926
	49	# result.time_end == '2017-11-12 19:54:49 GMT'
	50	# result.time_start == '2017-11-12 19:54:49 GMT'
	51	# result.timeout == False
51	52
52	53	__author__ = """Markus Gerstel"""
53	54	__email__ = "scientificsoftware@diamond.ac.uk"
54		__version__ = "1.1.0"
	55	__version__ = "2.3.1"
55	56
56	57	logger = logging.getLogger("procrunner")
57	58	logger.addHandler(logging.NullHandler())
58	59
59	60
60		class _LineAggregator(object):
	61	class _LineAggregator:
61	62	"""
62	63	Buffer that can be filled with stream data and will aggregate complete
63	64	lines. Lines can be printed or passed to an arbitrary callback function.

106	107	self._buffer = ""
107	108
108	109
109		class _NonBlockingStreamReader(object):
	110	class _NonBlockingStreamReader:
110	111	"""Reads a stream in a thread to avoid blocking/deadlocks"""
111	112
112	113	def __init__(self, stream, output=True, debug=False, notify=None, callback=None):
113	114	"""Creates and starts a thread which reads from a stream."""
114		self._buffer = six.BytesIO()
	115	self._buffer = io.BytesIO()
115	116	self._closed = False
116	117	self._closing = False
117	118	self._debug = debug

130	131	else:
131	132	if self._closing:
132	133	break
	134	self._stream.close()
133	135	self._terminated = True
134	136	la.flush()
135	137	if self._debug:

149	151	print(linedecode)
150	152	if callback:
151	153	callback(linedecode)
	154	self._stream.close()
152	155	self._terminated = True
153	156	if self._debug:
154	157	logger.debug("Stream reader terminated")

183	186	if not self.has_finished():
184	187	if self._debug:
185	188	logger.debug(
186		"NBSR join after %f seconds, underrun not resolved"
187		% (timeit.default_timer() - underrun_debug_timer)
	189	"NBSR join after %f seconds, underrun not resolved",
	190	timeit.default_timer() - underrun_debug_timer,
188	191	)
189	192	raise Exception("thread did not terminate")
190	193	if self._debug:
191	194	logger.debug(
192		"NBSR underrun resolved after %f seconds"
193		% (timeit.default_timer() - underrun_debug_timer)
	195	"NBSR underrun resolved after %f seconds",
	196	timeit.default_timer() - underrun_debug_timer,
194	197	)
195	198	if self._closed:
196	199	raise Exception("streamreader double-closed")

200	203	return data
201	204
202	205
203		class _NonBlockingStreamWriter(object):
	206	class _NonBlockingStreamWriter:
204	207	"""Writes to a stream in a thread to avoid blocking/deadlocks"""
205	208
206	209	def __init__(self, stream, data, debug=False, notify=None):

208	211	self._buffer = data
209	212	self._buffer_len = len(data)
210	213	self._buffer_pos = 0
211		self._debug = debug
212	214	self._max_block_len = 4096
213	215	self._stream = stream
214	216	self._terminated = False

223	225	block = self._buffer[self._buffer_pos :]
224	226	try:
225	227	self._stream.write(block)
226		except IOError as e:
	228	except OSError as e:
227	229	if (
228	230	e.errno == 32
229	231	): # broken pipe, ie. process terminated without reading entire stdin

235	237	raise
236	238	self._buffer_pos += len(block)
237	239	if debug:
238		logger.debug("wrote %d bytes to stream" % len(block))
	240	logger.debug("wrote %d bytes to stream", len(block))
239	241	self._stream.close()
240	242	self._terminated = True
241	243	if notify:

272	274	return obj
273	275
274	276
275		def _windows_resolve(command):
	277	def _windows_resolve(command, path=None):
276	278	"""
277	279	Try and find the full path and file extension of the executable to run.
278	280	This is so that e.g. calls to 'somescript' will point at 'somescript.cmd'
279	281	without the need to set shell=True in the subprocess.
280		If the executable contains periods it is a special case. Here the
281		win32api call will fail to resolve the extension automatically, and it
282		has do be done explicitly.
283	282
284	283	:param command: The command array to be run, with the first element being
285	284	the command with or w/o path, with or w/o extension.

287	286	correct extension. If the executable cannot be resolved for any
288	287	reason the original command array is returned.
289	288	"""
290		try:
291		import win32api
292		except ImportError:
293		if (2, 8) < sys.version_info < (3, 5):
294		logger.info(
295		"Resolving executable names only supported on Python 2.7 and 3.5+"
296		)
297		else:
298		logger.warning(
299		"Could not resolve executable name: package win32api missing"
300		)
	289	if not command or not isinstance(command[0], str):
301	290	return command
302	291
303		if not command or not isinstance(command[0], six.string_types):
304		return command
305
306		try:
307		_, found_executable = win32api.FindExecutable(command[0])
	292	found_executable = shutil.which(command[0], path=path)
	293	if found_executable:
308	294	logger.debug("Resolved %s as %s", command[0], found_executable)
309		return (found_executable,) + tuple(command[1:])
310		except Exception as e:
311		if not hasattr(e, "winerror"):
312		raise
313		# Keep this error message for later in case we fail to resolve the name
314		logwarning = getattr(e, "strerror", str(e))
315
316		if "." in command[0]:
317		# Special case. The win32api may not properly check file extensions, so
318		# try to resolve the executable explicitly.
	295	return (found_executable, *command[1:])
	296
	297	if "\\" in command[0]:
	298	# Special case. shutil.which may not detect file extensions if a full
	299	# path is given, so try to resolve the executable explicitly
319	300	for extension in os.getenv("PATHEXT").split(os.pathsep):
320		try:
321		_, found_executable = win32api.FindExecutable(command[0] + extension)
322		logger.debug("Resolved %s as %s", command[0], found_executable)
323		return (found_executable,) + tuple(command[1:])
324		except Exception as e:
325		if not hasattr(e, "winerror"):
326		raise
327
328		logger.warning("Error trying to resolve the executable: %s", logwarning)
	301	found_executable = shutil.which(command[0] + extension, path=path)
	302	if found_executable:
	303	return (found_executable, *command[1:])
	304
	305	logger.warning("Error trying to resolve the executable: %s", command[0])
329	306	return command
330	307
331	308
332		if sys.version_info < (3, 5):
333
334		class _ReturnObjectParent(object):
335		def check_returncode(self):
336		if self.returncode:
337		raise Exception(
338		"Call %r resulted in non-zero exit code %r"
339		% (self.args, self.returncode)
340		)
341
342
343		else:
344		_ReturnObjectParent = subprocess.CompletedProcess
345
346
347		class ReturnObject(dict, _ReturnObjectParent):
	309	class ReturnObject(subprocess.CompletedProcess):
348	310	"""
349	311	A subprocess.CompletedProcess-like object containing the executed
350	312	command, stdout and stderr (both as bytestrings), and the exitcode.
351		Further values such as process runtime can be accessed as dictionary
352		values.
353	313	The check_returncode() function raises an exception if the process
354	314	exited with a non-zero exit code.
355	315	"""
356	316
357		def __init__(self, arg, *kw):
358		super(ReturnObject, self).__init__(arg, *kw)
359		self.args = self["command"]
360		self.returncode = self["exitcode"]
361		self.stdout = self["stdout"]
362		self.stderr = self["stderr"]
	317	def __init__(self, exitcode=None, command=None, stdout=None, stderr=None, **kw):
	318	super().__init__(
	319	args=command, returncode=exitcode, stdout=stdout, stderr=stderr
	320	)
	321	self._extras = {
	322	"timeout": kw.get("timeout"),
	323	"runtime": kw.get("runtime"),
	324	"time_start": kw.get("time_start"),
	325	"time_end": kw.get("time_end"),
	326	}
	327
	328	def __getitem__(self, key):
	329	warnings.warn(
	330	"dictionary access to a procrunner return object is deprecated",
	331	DeprecationWarning,
	332	stacklevel=2,
	333	)
	334	if key in self._extras:
	335	return self._extras[key]
	336	if not hasattr(self, key):
	337	raise KeyError(f"Unknown attribute {key}")
	338	return getattr(self, key)
363	339
364	340	def __eq__(self, other):
365	341	"""Override equality operator to account for added fields"""

371	347	"""This object is not immutable, so mark it as unhashable"""
372	348	return None
373	349
374		def __ne__(self, other):
375		"""Overrides the default implementation (unnecessary in Python 3)"""
376		return not self.__eq__(other)
377
378
	350	@property
	351	def cmd(self):
	352	warnings.warn(
	353	"procrunner return object .cmd is deprecated, use .args",
	354	DeprecationWarning,
	355	stacklevel=2,
	356	)
	357	return self.args
	358
	359	@property
	360	def command(self):
	361	warnings.warn(
	362	"procrunner return object .command is deprecated, use .args",
	363	DeprecationWarning,
	364	stacklevel=2,
	365	)
	366	return self.args
	367
	368	@property
	369	def exitcode(self):
	370	warnings.warn(
	371	"procrunner return object .exitcode is deprecated, use .returncode",
	372	DeprecationWarning,
	373	stacklevel=2,
	374	)
	375	return self.returncode
	376
	377	@property
	378	def timeout(self):
	379	warnings.warn(
	380	"procrunner return object .timeout is deprecated",
	381	DeprecationWarning,
	382	stacklevel=2,
	383	)
	384	return self._extras["timeout"]
	385
	386	@property
	387	def runtime(self):
	388	warnings.warn(
	389	"procrunner return object .runtime is deprecated",
	390	DeprecationWarning,
	391	stacklevel=2,
	392	)
	393	return self._extras["runtime"]
	394
	395	@property
	396	def time_start(self):
	397	warnings.warn(
	398	"procrunner return object .time_start is deprecated",
	399	DeprecationWarning,
	400	stacklevel=2,
	401	)
	402	return self._extras["time_start"]
	403
	404	@property
	405	def time_end(self):
	406	warnings.warn(
	407	"procrunner return object .time_end is deprecated",
	408	DeprecationWarning,
	409	stacklevel=2,
	410	)
	411	return self._extras["time_end"]
	412
	413	def update(self, dictionary):
	414	self._extras.update(dictionary)
	415
	416
	417	def _deprecate_argument_calling(f):
	418	@functools.wraps(f)
	419	def wrapper(args, *kwargs):
	420	if len(args) > 1:
	421	warnings.warn(
	422	"Calling procrunner.run() with unnamed arguments (apart from "
	423	"the command) is deprecated. Use keyword arguments instead.",
	424	DeprecationWarning,
	425	stacklevel=2,
	426	)
	427	return f(args, *kwargs)
	428
	429	return wrapper
	430
	431
	432	@_deprecate_argument_calling
379	433	def run(
380	434	command,
381	435	timeout=None,
382		debug=False,
	436	debug=None,
383	437	stdin=None,
384	438	print_stdout=True,
385	439	print_stderr=True,

389	443	environment_override=None,
390	444	win32resolve=True,
391	445	working_directory=None,
	446	raise_timeout_exception=False,
392	447	):
393	448	"""
394	449	Run an external process.

398	453
399	454	:param array command: Command line to be run, specified as array.
400	455	:param timeout: Terminate program execution after this many seconds.
401		:param boolean debug: Enable further debug messages.
402		:param stdin: Optional string that is passed to command stdin.
	456	:param boolean debug: Enable further debug messages. (deprecated)
	457	:param stdin: Optional bytestring that is passed to command stdin.
403	458	:param boolean print_stdout: Pass stdout through to sys.stdout.
404	459	:param boolean print_stderr: Pass stderr through to sys.stderr.
405	460	:param callback_stdout: Optional function which is called for each

415	470	extension.
416	471	:param string working_directory: If specified, run the executable from
417	472	within this working directory.
418		:return: A ReturnObject() containing the executed command, stdout and stderr
419		(both as bytestrings), and the exitcode. Further values such as
420		process runtime can be accessed as dictionary values.
	473	:param boolean raise_timeout_exception: Forward compatibility flag. If set
	474	then a subprocess.TimeoutExpired exception is raised
	475	instead of returning an object that can be checked
	476	for a timeout condition. Defaults to False, will be
	477	changed to True in a future release.
	478	:return: The exit code, stdout, stderr (separately, as byte strings)
	479	as a subprocess.CompletedProcess object.
421	480	"""
422	481
423	482	time_start = time.strftime("%Y-%m-%d %H:%M:%S GMT", time.gmtime())

428	487	else:
429	488	assert sys.platform != "win32", "stdin argument not supported on Windows"
430	489	stdin_pipe = subprocess.PIPE
	490	if debug is not None:
	491	warnings.warn(
	492	"Use of the debug parameter is deprecated", DeprecationWarning, stacklevel=3
	493	)
431	494
432	495	start_time = timeit.default_timer()
433	496	if timeout is not None:
434	497	max_time = start_time + timeout
	498	if not raise_timeout_exception:
	499	warnings.warn(
	500	"Using procrunner with timeout and without raise_timeout_exception set is deprecated",
	501	DeprecationWarning,
	502	stacklevel=3,
	503	)
435	504
436	505	if environment is not None:
437	506	env = {key: _path_resolve(environment[key]) for key in environment}

448	517	command = tuple(_path_resolve(part) for part in command)
449	518	if win32resolve and sys.platform == "win32":
450	519	command = _windows_resolve(command)
	520	if working_directory and sys.version_info < (3, 7):
	521	working_directory = os.fspath(working_directory)
451	522
452	523	p = subprocess.Popen(
453	524	command,
454	525	shell=False,
455		cwd=_path_resolve(working_directory),
	526	cwd=working_directory,
456	527	env=env,
457	528	stdin=stdin_pipe,
458	529	stdout=subprocess.PIPE,

491	562	(timeout is None) or (timeit.default_timer() < max_time)
492	563	):
493	564	if debug and timeout is not None:
494		logger.debug("still running (T%.2fs)" % (timeit.default_timer() - max_time))
	565	logger.debug("still running (T%.2fs)", timeit.default_timer() - max_time)
495	566
496	567	# wait for some time or until a stream is closed
497	568	try:

500	571	# which could indicate that the process has terminated.
501	572	try:
502	573	event = thread_pipe_pool[0].poll(0.5)
503		except IOError as e:
504		# on Windows this raises "IOError: [Errno 109] The pipe has been ended"
	574	except BrokenPipeError as e:
	575	# on Windows this raises "BrokenPipeError: [Errno 109] The pipe has been ended"
505	576	# which is for all intents and purposes equivalent to a True return value.
506		if e.errno != 109:
	577	if e.winerror != 109:
507	578	raise
508	579	event = True
509	580	if event:

525	596	# timeout condition
526	597	timeout_encountered = True
527	598	if debug:
528		logger.debug("timeout (T%.2fs)" % (timeit.default_timer() - max_time))
	599	logger.debug("timeout (T%.2fs)", timeit.default_timer() - max_time)
529	600
530	601	# send terminate signal and wait some time for buffers to be read
531	602	p.terminate()
532	603	if thread_pipe_pool:
533		thread_pipe_pool[0].poll(0.5)
	604	try:
	605	thread_pipe_pool[0].poll(0.5)
	606	except BrokenPipeError as e:
	607	# on Windows this raises "BrokenPipeError: [Errno 109] The pipe has been ended"
	608	# which is for all intents and purposes equivalent to a True return value.
	609	if e.winerror != 109:
	610	raise
	611	thread_pipe_pool.pop(0)
534	612	if not stdout.has_finished() or not stderr.has_finished():
535	613	time.sleep(2)
536	614	p.poll()

540	618	# send kill signal and wait some more time for buffers to be read
541	619	p.kill()
542	620	if thread_pipe_pool:
543		thread_pipe_pool[0].poll(0.5)
	621	try:
	622	thread_pipe_pool[0].poll(0.5)
	623	except BrokenPipeError as e:
	624	# on Windows this raises "BrokenPipeError: [Errno 109] The pipe has been ended"
	625	# which is for all intents and purposes equivalent to a True return value.
	626	if e.winerror != 109:
	627	raise
	628	thread_pipe_pool.pop(0)
544	629	if not stdout.has_finished() or not stderr.has_finished():
545	630	time.sleep(5)
546	631	p.poll()

551	636	runtime = timeit.default_timer() - start_time
552	637	if timeout is not None:
553	638	logger.debug(
554		"Process ended after %.1f seconds with exit code %d (T%.2fs)"
555		% (runtime, p.returncode, timeit.default_timer() - max_time)
	639	"Process ended after %.1f seconds with exit code %d (T%.2fs)",
	640	runtime,
	641	p.returncode,
	642	timeit.default_timer() - max_time,
556	643	)
557	644	else:
558	645	logger.debug(
559		"Process ended after %.1f seconds with exit code %d"
560		% (runtime, p.returncode)
	646	"Process ended after %.1f seconds with exit code %d", runtime, p.returncode
561	647	)
562	648
563	649	stdout = stdout.get_output()
564	650	stderr = stderr.get_output()
	651
	652	if timeout_encountered and raise_timeout_exception:
	653	raise subprocess.TimeoutExpired(
	654	cmd=command, timeout=timeout, output=stdout, stderr=stderr
	655	)
	656
565	657	time_end = time.strftime("%Y-%m-%d %H:%M:%S GMT", time.gmtime())
566
567	658	result = ReturnObject(
568		{
569		"exitcode": p.returncode,
570		"command": command,
571		"stdout": stdout,
572		"stderr": stderr,
573		"timeout": timeout_encountered,
574		"runtime": runtime,
575		"time_start": time_start,
576		"time_end": time_end,
577		}
	659	exitcode=p.returncode,
	660	command=command,
	661	stdout=stdout,
	662	stderr=stderr,
	663	timeout=timeout_encountered,
	664	runtime=runtime,
	665	time_start=time_start,
	666	time_end=time_end,
578	667	)
579	668	if stdin is not None:
580	669	result.update(

585	674	)
586	675
587	676	return result
588
589
590		def run_process_dummy(command, **kwargs):
591		"""
592		A stand-in function that returns a valid result dictionary indicating a
593		successful execution. The external process is not run.
594		"""
595		warnings.warn(
596		"procrunner.run_process_dummy() is deprecated", DeprecationWarning, stacklevel=2
597		)
598
599		time_start = time.strftime("%Y-%m-%d %H:%M:%S GMT", time.gmtime())
600		logger.info("run_process is disabled. Requested command: %s", command)
601
602		result = ReturnObject(
603		{
604		"exitcode": 0,
605		"command": command,
606		"stdout": "",
607		"stderr": "",
608		"timeout": False,
609		"runtime": 0,
610		"time_start": time_start,
611		"time_end": time_start,
612		}
613		)
614		if kwargs.get("stdin") is not None:
615		result.update(
616		{"stdin_bytes_sent": len(kwargs["stdin"]), "stdin_bytes_remain": 0}
617		)
618		return result
619
620
621		def run_process(args, *kwargs):
622		"""API used up to version 0.2.0."""
623		warnings.warn(
624		"procrunner.run_process() is deprecated and has been renamed to run()",
625		DeprecationWarning,
626		stacklevel=2,
627		)
628		return run(args, *kwargs)

-0

pyproject.toml less more

	0	[build-system]
	1	requires = ["setuptools >= 40.6.0", "wheel"]
	2	build-backend = "setuptools.build_meta"
	3
	4	[tool.isort]
	5	profile="black"

-0

pytest.ini less more

	0	[pytest]
	1	addopts = -ra
	2	junit_family=xunit2

-0

requirements.txt less more

(New empty file)

-13

requirements_dev.txt less more

0		bump2version==0.5.10
1		coverage==4.5.4
2		flake8==3.7.8
3		mock==3.0.5
4		pip==19.1.1
5		pytest==4.5.0 # pyup: <5.0 # for Python 2.7 support
6		pytest-runner==5.1
7		six==1.12.0
8		Sphinx==1.8.5 # pyup: <2.0 # for Python 2.7 support
9		tox==3.13.1
10		twine==1.13.0
11		watchdog==0.9.0
12		wheel==0.33.4
	0	bump2version==1.0.1
	1	coverage==6.0.2
	2	pip==21.3.1
	3	pytest==6.2.5
	4	Sphinx==4.2.0
	5	tox==3.24.4
	6	twine==3.4.2
	7	wheel==0.37.0

+39

-16

setup.cfg less more

0		[bumpversion]
1		current_version = 1.1.0
2		commit = True
3		tag = True
4
5		[bumpversion:file:setup.py]
6		search = version="{current_version}"
7		replace = version="{new_version}"
8
9		[bumpversion:file:procrunner/__init__.py]
10		search = __version__ = "{current_version}"
11		replace = __version__ = "{new_version}"
12
13		[bdist_wheel]
14		universal = 1
	0	[metadata]
	1	name = procrunner
	2	description = Versatile utility function to run external processes
	3	version = 2.3.1
	4	classifiers =
	5	Development Status :: 5 - Production/Stable
	6	Intended Audience :: Developers
	7	License :: OSI Approved :: BSD License
	8	Natural Language :: English
	9	Programming Language :: Python :: 3
	10	Programming Language :: Python :: 3.6
	11	Programming Language :: Python :: 3.7
	12	Programming Language :: Python :: 3.8
	13	Programming Language :: Python :: 3.9
	14	Programming Language :: Python :: 3.10
	15	Operating System :: OS Independent
	16	Topic :: Software Development :: Libraries :: Python Modules
	17	license = BSD
	18	license_file = LICENSE
	19	project-urls =
	20	Download = https://github.com/DiamondLightSource/python-procrunner/tags
	21	Documentation = https://procrunner.readthedocs.io/
	22	GitHub = https://github.com/DiamondLightSource/python-procrunner
	23	Bug-Tracker = https://github.com/DiamondLightSource/python-procrunner/issues
15	24
16	25	[flake8]
17		exclude = docs
	26	# Black disagrees with flake8 on a few points. Ignore those.
	27	ignore = E203, E266, E501, W503
	28	# E203 whitespace before ':'
	29	# E266 too many leading '#' for block comment
	30	# E501 line too long
	31	# W503 line break before binary operator
	32
	33	max-line-length = 88
	34
	35	select =
	36	E401,E711,E712,E713,E714,E721,E722,E901,
	37	F401,F402,F403,F405,F541,F631,F632,F633,F811,F812,F821,F822,F841,F901,
	38	W191,W291,W292,W293,W602,W603,W604,W605,W606,
	39	# flake8-comprehensions, https://github.com/adamchainz/flake8-comprehensions
	40	C4,
18	41
19	42	[aliases]
20	43	test = pytest

-31

setup.py less more

0	0	#!/usr/bin/env python
1	1	# -- coding: utf-8 --
2	2
3		import sys
4		from setuptools import setup, find_packages
	3	from setuptools import find_packages, setup
5	4
6	5	with open("README.rst") as readme_file:
7	6	readme = readme_file.read()

9	8	with open("HISTORY.rst") as history_file:
10	9	history = history_file.read()
11	10
12		requirements = [
13		"six",
14		'pywin32; sys_platform=="win32"',
15		]
	11	requirements = []
16	12
17	13	setup_requirements = []
18		needs_pytest = {"pytest", "test", "ptr"}.intersection(sys.argv)
19		if needs_pytest:
20		setup_requirements.append("pytest-runner")
21	14
22		test_requirements = ["mock", "pytest"]
	15	test_requirements = ["pytest"]
23	16
24	17	setup(
25	18	author="Markus Gerstel",
26	19	author_email="scientificsoftware@diamond.ac.uk",
27		classifiers=[
28		"Development Status :: 5 - Production/Stable",
29		"Intended Audience :: Developers",
30		"License :: OSI Approved :: BSD License",
31		"Natural Language :: English",
32		"Operating System :: OS Independent",
33		"Programming Language :: Python :: 2",
34		"Programming Language :: Python :: 2.7",
35		"Programming Language :: Python :: 3",
36		"Programming Language :: Python :: 3.5",
37		"Programming Language :: Python :: 3.6",
38		"Programming Language :: Python :: 3.7",
39		"Programming Language :: Python :: 3.8",
40		"Programming Language :: Python :: Implementation :: PyPy",
41		"Programming Language :: Python :: Implementation :: CPython",
42		"Topic :: Software Development :: Libraries :: Python Modules",
43		],
44		description="Versatile utility function to run external processes",
45	20	install_requires=requirements,
46		license="BSD license",
47	21	long_description=readme + "\n\n" + history,
48	22	include_package_data=True,
49	23	keywords="procrunner",
50		name="procrunner",
51	24	packages=find_packages(include=["procrunner"]),
	25	python_requires=">=3.6",
52	26	setup_requires=setup_requirements,
53	27	test_suite="tests",
54	28	tests_require=test_requirements,
55	29	url="https://github.com/DiamondLightSource/python-procrunner",
56		version="1.1.0",
57	30	zip_safe=False,
58	31	)

+92

-47

tests/test_procrunner.py less more

0		from __future__ import absolute_import, division, print_function
1
2	0	import copy
3		import mock
4	1	import os
	2	import pathlib
	3	import sys
	4	from unittest import mock
	5
	6	import pytest
	7
5	8	import procrunner
6		import pytest
7		import sys
	9
	10
	11	@mock.patch("procrunner._NonBlockingStreamReader")
	12	@mock.patch("procrunner.time")
	13	@mock.patch("procrunner.subprocess")
	14	@mock.patch("procrunner.Pipe")
	15	def test_run_command_aborts_after_timeout_legacy(
	16	mock_pipe, mock_subprocess, mock_time, mock_streamreader
	17	):
	18	mock_pipe.return_value = mock.Mock(), mock.Mock()
	19	mock_process = mock.Mock()
	20	mock_process.returncode = None
	21	mock_subprocess.Popen.return_value = mock_process
	22	task = ["___"]
	23
	24	with pytest.raises(RuntimeError):
	25	with pytest.warns(DeprecationWarning, match="timeout"):
	26	procrunner.run(task, timeout=-1, debug=False)
	27
	28	assert mock_subprocess.Popen.called
	29	assert mock_process.terminate.called
	30	assert mock_process.kill.called
8	31
9	32
10	33	@mock.patch("procrunner._NonBlockingStreamReader")

21	44	task = ["___"]
22	45
23	46	with pytest.raises(RuntimeError):
24		procrunner.run(task, -1, False)
	47	procrunner.run(task, timeout=-1, raise_timeout_exception=True)
25	48
26	49	assert mock_subprocess.Popen.called
27	50	assert mock_process.terminate.called

62	85
63	86	actual = procrunner.run(
64	87	command,
65		0.5,
66		False,
	88	timeout=0.5,
67	89	callback_stdout=mock.sentinel.callback_stdout,
68	90	callback_stderr=mock.sentinel.callback_stderr,
69		working_directory=mock.sentinel.cwd,
	91	working_directory=pathlib.Path("somecwd"),
	92	raise_timeout_exception=True,
70	93	)
71	94
72	95	assert mock_subprocess.Popen.called
73	96	assert mock_subprocess.Popen.call_args[1]["env"] == os.environ
74		assert mock_subprocess.Popen.call_args[1]["cwd"] == mock.sentinel.cwd
	97	assert mock_subprocess.Popen.call_args[1]["cwd"] in (
	98	pathlib.Path("somecwd"),
	99	"somecwd",
	100	)
75	101	mock_streamreader.assert_has_calls(
76	102	[
77	103	mock.call(
78	104	stream_stdout,
79	105	output=mock.ANY,
80		debug=mock.ANY,
	106	debug=None,
81	107	notify=mock.ANY,
82	108	callback=mock.sentinel.callback_stdout,
83	109	),
84	110	mock.call(
85	111	stream_stderr,
86	112	output=mock.ANY,
87		debug=mock.ANY,
	113	debug=None,
88	114	notify=mock.ANY,
89	115	callback=mock.sentinel.callback_stderr,
90	116	),

94	120	assert not mock_process.terminate.called
95	121	assert not mock_process.kill.called
96	122	for key in expected:
97		assert actual[key] == expected[key]
	123	with pytest.warns(DeprecationWarning):
	124	assert actual[key] == expected[key]
98	125	assert actual.args == tuple(command)
99	126	assert actual.returncode == mock_process.returncode
100	127	assert actual.stdout == mock.sentinel.proc_stdout

105	132	def test_default_process_environment_is_parent_environment(mock_subprocess):
106	133	mock_subprocess.Popen.side_effect = NotImplementedError() # cut calls short
107	134	with pytest.raises(NotImplementedError):
108		procrunner.run([mock.Mock()], -1, False)
	135	procrunner.run([mock.Mock()], timeout=-1, raise_timeout_exception=True)
109	136	assert mock_subprocess.Popen.call_args[1]["env"] == os.environ
	137
	138
	139	@mock.patch("procrunner.subprocess")
	140	def test_using_debug_parameter_raises_warning(mock_subprocess):
	141	mock_subprocess.Popen.side_effect = NotImplementedError() # cut calls short
	142	with pytest.warns(DeprecationWarning, match="debug"):
	143	with pytest.raises(NotImplementedError):
	144	procrunner.run([mock.Mock()], debug=True)
	145	with pytest.warns(DeprecationWarning, match="debug"):
	146	with pytest.raises(NotImplementedError):
	147	procrunner.run([mock.Mock()], debug=False)
110	148
111	149
112	150	@mock.patch("procrunner.subprocess")

115	153	mock_env = {"key": mock.sentinel.key}
116	154	# Pass an environment dictionary
117	155	with pytest.raises(NotImplementedError):
118		procrunner.run([mock.Mock()], -1, False, environment=copy.copy(mock_env))
	156	procrunner.run(
	157	[mock.Mock()],
	158	timeout=-1,
	159	environment=copy.copy(mock_env),
	160	raise_timeout_exception=True,
	161	)
119	162	assert mock_subprocess.Popen.call_args[1]["env"] == mock_env
120	163
121	164

128	171	with pytest.raises(NotImplementedError):
129	172	procrunner.run(
130	173	[mock.Mock()],
131		-1,
132		False,
	174	timeout=-1,
133	175	environment=copy.copy(mock_env1),
134	176	environment_override=copy.copy(mock_env2),
	177	raise_timeout_exception=True,
135	178	)
136	179	mock_env_sum = copy.copy(mock_env1)
137	180	mock_env_sum.update({key: str(mock_env2[key]) for key in mock_env2})

144	187	mock_env2 = {"keyB": str(mock.sentinel.keyB)}
145	188	with pytest.raises(NotImplementedError):
146	189	procrunner.run(
147		[mock.Mock()], -1, False, environment_override=copy.copy(mock_env2)
	190	[mock.Mock()],
	191	timeout=-1,
	192	environment_override=copy.copy(mock_env2),
	193	raise_timeout_exception=True,
148	194	)
149	195	random_environment_variable = list(os.environ)[0]
150	196	if random_environment_variable == list(mock_env2)[0]:
151	197	random_environment_variable = list(os.environ)[1]
152		random_environment_value = os.getenv(random_environment_variable)
153	198	assert (
154	199	random_environment_variable
155	200	and random_environment_variable != list(mock_env2)[0]

171	216	with pytest.raises(NotImplementedError):
172	217	procrunner.run(
173	218	[mock.Mock()],
174		-1,
175		False,
	219	timeout=-1,
176	220	environment_override={
177	221	random_environment_variable: "X" + random_environment_value
178	222	},
	223	raise_timeout_exception=True,
179	224	)
180	225	assert (
181	226	mock_subprocess.Popen.call_args[1]["env"][random_environment_variable]

191	236	def test_nonblockingstreamreader_can_read(mock_select):
192	237	import time
193	238
194		class _stream(object):
	239	class _stream:
195	240	def __init__(self):
196	241	self.data = b""
197	242	self.closed = False

262	307
263	308	def test_return_object_semantics():
264	309	ro = procrunner.ReturnObject(
265		{
266		"command": mock.sentinel.command,
267		"exitcode": 0,
268		"stdout": mock.sentinel.stdout,
269		"stderr": mock.sentinel.stderr,
270		}
271		)
272		assert ro["command"] == mock.sentinel.command
	310	command=mock.sentinel.command,
	311	exitcode=0,
	312	stdout=mock.sentinel.stdout,
	313	stderr=mock.sentinel.stderr,
	314	)
	315	with pytest.warns(DeprecationWarning):
	316	assert ro["command"] == mock.sentinel.command
273	317	assert ro.args == mock.sentinel.command
274		assert ro["exitcode"] == 0
	318	with pytest.warns(DeprecationWarning):
	319	assert ro["exitcode"] == 0
275	320	assert ro.returncode == 0
276		assert ro["stdout"] == mock.sentinel.stdout
	321	with pytest.warns(DeprecationWarning):
	322	assert ro["stdout"] == mock.sentinel.stdout
277	323	assert ro.stdout == mock.sentinel.stdout
278		assert ro["stderr"] == mock.sentinel.stderr
	324	with pytest.warns(DeprecationWarning):
	325	assert ro["stderr"] == mock.sentinel.stderr
279	326	assert ro.stderr == mock.sentinel.stderr
280	327
281	328	with pytest.raises(KeyError):
282		ro["unknownkey"]
	329	with pytest.warns(DeprecationWarning):
	330	ro["unknownkey"]
283	331	ro.update({"unknownkey": mock.sentinel.key})
284		assert ro["unknownkey"] == mock.sentinel.key
	332	with pytest.warns(DeprecationWarning):
	333	assert ro["unknownkey"] == mock.sentinel.key
285	334
286	335
287	336	def test_return_object_check_function_passes_on_success():
288	337	ro = procrunner.ReturnObject(
289		{
290		"command": mock.sentinel.command,
291		"exitcode": 0,
292		"stdout": mock.sentinel.stdout,
293		"stderr": mock.sentinel.stderr,
294		}
	338	command=mock.sentinel.command,
	339	exitcode=0,
	340	stdout=mock.sentinel.stdout,
	341	stderr=mock.sentinel.stderr,
295	342	)
296	343	ro.check_returncode()
297	344
298	345
299	346	def test_return_object_check_function_raises_on_error():
300	347	ro = procrunner.ReturnObject(
301		{
302		"command": mock.sentinel.command,
303		"exitcode": 1,
304		"stdout": mock.sentinel.stdout,
305		"stderr": mock.sentinel.stderr,
306		}
	348	command=mock.sentinel.command,
	349	exitcode=1,
	350	stdout=mock.sentinel.stdout,
	351	stderr=mock.sentinel.stderr,
307	352	)
308	353	with pytest.raises(Exception) as e:
309	354	ro.check_returncode()

+14

-23

tests/test_procrunner_resolution.py less more

0		from __future__ import absolute_import, division, print_function
1
2	0	import os
3	1	import sys
4	2
5		import mock
	3	import pytest
	4
6	5	import procrunner
7		import pytest
8	6
9	7
10	8	def PEP519(path):

46	44
47	45
48	46	@pytest.mark.skipif(sys.platform != "win32", reason="windows specific test only")
49		def test_pywin32_import():
50		import win32api
51
52
53		@pytest.mark.skipif(sys.platform != "win32", reason="windows specific test only")
54	47	def test_name_resolution_for_simple_exe():
55	48	command = ["cmd.exe", "/c", "echo", "hello"]
56	49

65	58
66	59
67	60	@pytest.mark.skipif(sys.platform != "win32", reason="windows specific test only")
68		def test_name_resolution_for_complex_cases(tmpdir):
69		tmpdir.chdir()
70
	61	def test_name_resolution_for_complex_cases(tmp_path):
71	62	bat = "simple_bat_extension"
72	63	cmd = "simple_cmd_extension"
73	64	exe = "simple_exe_extension"
74	65	dotshort = "more_complex_filename_with_a.dot"
75	66	dotlong = "more_complex_filename.withadot"
76	67
77		(tmpdir / bat + ".bat").ensure()
78		(tmpdir / cmd + ".cmd").ensure()
79		(tmpdir / exe + ".exe").ensure()
80		(tmpdir / dotshort + ".bat").ensure()
81		(tmpdir / dotlong + ".cmd").ensure()
	68	(tmp_path / (bat + ".bat")).touch()
	69	(tmp_path / (cmd + ".cmd")).touch()
	70	(tmp_path / (exe + ".exe")).touch()
	71	(tmp_path / (dotshort + ".bat")).touch()
	72	(tmp_path / (dotlong + ".cmd")).touch()
82	73
83	74	def is_valid(command):
84	75	assert len(command) == 1
85		assert os.path.exists(command[0])
	76	assert os.path.exists(tmp_path / command[0])
86	77
87		is_valid(procrunner._windows_resolve([bat]))
88		is_valid(procrunner._windows_resolve([cmd]))
89		is_valid(procrunner._windows_resolve([exe]))
90		is_valid(procrunner._windows_resolve([dotshort]))
91		is_valid(procrunner._windows_resolve([dotlong]))
	78	is_valid(procrunner._windows_resolve([bat], path=os.fspath(tmp_path)))
	79	is_valid(procrunner._windows_resolve([cmd], path=os.fspath(tmp_path)))
	80	is_valid(procrunner._windows_resolve([exe], path=os.fspath(tmp_path)))
	81	is_valid(procrunner._windows_resolve([dotshort], path=os.fspath(tmp_path)))
	82	is_valid(procrunner._windows_resolve([dotlong], path=os.fspath(tmp_path)))

+75

-14

tests/test_procrunner_system.py less more

0		from __future__ import absolute_import, division, print_function
	0	import os
	1	import subprocess
	2	import sys
	3	import timeit
1	4
2		import os
3		import sys
	5	import pytest
4	6
5	7	import procrunner
6		import pytest
7	8
8	9
9	10	def test_simple_command_invocation():

35	36	else:
36	37	assert result.stdout == test_string
37	38	out, err = capsys.readouterr()
38		assert out == u"test\ufffdstring\n"
39		assert err == u""
	39	assert out == "test\ufffdstring\n"
	40	assert err == ""
40	41
41	42
42		def test_running_wget(tmpdir):
43		tmpdir.chdir()
	43	def test_running_wget(tmp_path):
44	44	command = ["wget", "https://www.google.com", "-O", "-"]
45	45	try:
46		result = procrunner.run(command)
	46	result = procrunner.run(command, working_directory=tmp_path)
47	47	except OSError as e:
48	48	if e.errno == 2:
49	49	pytest.skip("wget not available")

53	53	assert b"google" in result.stdout
54	54
55	55
56		def test_path_object_resolution(tmpdir):
	56	def test_path_object_resolution(tmp_path):
57	57	sentinel_value = b"sentinel"
58		tmpdir.join("tempfile").write(sentinel_value)
59		tmpdir.join("reader.py").write("print(open('tempfile').read())")
	58	tmp_path.joinpath("tempfile").write_bytes(sentinel_value)
	59	tmp_path.joinpath("reader.py").write_text(
	60	"with open('tempfile') as fh:\n print(fh.read())"
	61	)
60	62	assert "LEAK_DETECTOR" not in os.environ
61	63	result = procrunner.run(
62		[sys.executable, tmpdir.join("reader.py")],
	64	[sys.executable, tmp_path / "reader.py"],
63	65	environment_override={"PYTHONHASHSEED": "random", "LEAK_DETECTOR": "1"},
64		working_directory=tmpdir,
	66	working_directory=tmp_path,
65	67	)
66	68	assert result.returncode == 0
67	69	assert not result.stderr

69	71	assert (
70	72	"LEAK_DETECTOR" not in os.environ
71	73	), "overridden environment variable leaked into parent process"
	74
	75
	76	def test_timeout_behaviour_legacy(tmp_path):
	77	start = timeit.default_timer()
	78	try:
	79	with pytest.warns(DeprecationWarning, match="timeout"):
	80	result = procrunner.run(
	81	[sys.executable, "-c", "import time; time.sleep(5)"],
	82	timeout=0.1,
	83	working_directory=tmp_path,
	84	raise_timeout_exception=False,
	85	)
	86	except RuntimeError:
	87	# This test sometimes fails with a RuntimeError.
	88	runtime = timeit.default_timer() - start
	89	assert runtime < 3
	90	return
	91	runtime = timeit.default_timer() - start
	92	with pytest.warns(DeprecationWarning, match="\\.timeout"):
	93	assert result.timeout
	94	assert runtime < 3
	95	assert not result.stdout
	96	assert not result.stderr
	97	assert result.returncode
	98
	99
	100	def test_timeout_behaviour(tmp_path):
	101	command = (sys.executable, "-c", "import time; time.sleep(5)")
	102	start = timeit.default_timer()
	103	try:
	104	with pytest.raises(subprocess.TimeoutExpired) as te:
	105	procrunner.run(
	106	command,
	107	timeout=0.1,
	108	working_directory=tmp_path,
	109	raise_timeout_exception=True,
	110	)
	111	except RuntimeError:
	112	# This test sometimes fails with a RuntimeError.
	113	runtime = timeit.default_timer() - start
	114	assert runtime < 3
	115	return
	116	runtime = timeit.default_timer() - start
	117	assert runtime < 3
	118	assert te.value.stdout == b""
	119	assert te.value.stderr == b""
	120	assert te.value.timeout == 0.1
	121	assert te.value.cmd == command
	122
	123
	124	def test_argument_deprecation(tmp_path):
	125	with pytest.warns(DeprecationWarning, match="keyword arguments"):
	126	result = procrunner.run(
	127	[sys.executable, "-V"],
	128	None,
	129	working_directory=tmp_path,
	130	)
	131	assert not result.returncode
	132	assert result.stderr or result.stdout

+12

-8

tox.ini less more

0	0	[tox]
1		envlist = py27, py35, py36, py37, py38, flake8
	1	envlist = py36, py37, py38, flake8
2	2
3		[travis]
4		python =
5		3.8: py38
6		3.7: py37
7		3.6: py36
8		3.5: py35
9		2.7: py27
	3	[testenv:azure]
	4	basepython = python
	5	deps =
	6	pytest-azurepipelines
	7	pytest-cov
	8	-r{toxinidir}/requirements_dev.txt
	9	setenv =
	10	PYTHONDEVMODE = 1
	11	commands =
	12	pytest -ra --basetemp={envtmpdir} --cov=procrunner --cov-report=html --cov-report=xml --cov-branch
10	13
11	14	[testenv:flake8]
12	15	basepython = python

16	19	[testenv]
17	20	setenv =
18	21	PYTHONPATH = {toxinidir}
	22	PYTHONDEVMODE = 1
19	23	deps =
20	24	-r{toxinidir}/requirements_dev.txt
21	25	; If you want to make tox run the tests with the same versions, create a