diff --git a/.editorconfig b/.editorconfig deleted file mode 100644 index d4a2c44..0000000 --- a/.editorconfig +++ /dev/null @@ -1,21 +0,0 @@ -# http://editorconfig.org - -root = true - -[*] -indent_style = space -indent_size = 4 -trim_trailing_whitespace = true -insert_final_newline = true -charset = utf-8 -end_of_line = lf - -[*.bat] -indent_style = tab -end_of_line = crlf - -[LICENSE] -insert_final_newline = false - -[Makefile] -indent_style = tab diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index 190f964..0000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,15 +0,0 @@ -* ProcRunner version: -* Python version: -* Operating System: - -### Description - -Describe what you were trying to get done. -Tell us what happened, what went wrong, and what you expected to happen. - -### What I Did - -``` -Paste the command(s) you ran and the output. -If there was a crash, please include the traceback here. -``` diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 0b9ba44..0000000 --- a/.gitignore +++ /dev/null @@ -1,106 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Temporary files -*.sw[op] -*~ - -# Distribution / packaging -.Python -env/ -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -*.egg-info/ -.installed.cfg -*.egg - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -.hypothesis/ -.pytest_cache - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - -# SageMath parsed files -*.sage.py - -# dotenv -.env - -# virtualenv -.venv -venv/ -ENV/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ diff --git a/.pyup.yml b/.pyup.yml deleted file mode 100644 index 674e27f..0000000 --- a/.pyup.yml +++ /dev/null @@ -1,5 +0,0 @@ -# autogenerated pyup.io config file -# see https://pyup.io/docs/configuration/ for all available options - -schedule: every month - diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f8ff7f3..0000000 --- a/.travis.yml +++ /dev/null @@ -1,72 +0,0 @@ -# Config file for automatic testing at travis-ci.org - -language: python - -matrix: - include: - - python: 3.8 - dist: xenial - sudo: true - - python: 3.7 - dist: xenial - sudo: true - - python: 3.6 - - python: 3.5 - - python: 2.7 - - python: pypy - - os: osx - language: generic - env: CONDA=3.8 TOXENV=py38 - - os: osx - language: generic - env: CONDA=3.7 TOXENV=py37 - - os: osx - language: generic - env: CONDA=3.6 TOXENV=py36 - - os: osx - language: generic - env: CONDA=3.5 TOXENV=py35 - - os: osx - language: generic - env: CONDA=2.7 TOXENV=py27 - - allow_failures: - - env: OPTIONAL=1 - - fast_finish: true - -before_install: | - if [ ! -z "$CONDA" ]; then - if [ "$TRAVIS_OS_NAME" == "osx" ]; then - curl https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh --output miniconda.sh - fi - chmod +x miniconda.sh - ./miniconda.sh -b - export PATH=$HOME/miniconda3/bin:$PATH - conda update --yes conda - conda create --yes -n travis python=$CONDA - source activate travis - # A manual check that the correct version of Python is running. - python --version - fi - -# Command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors -install: pip install -U tox-travis - -# Command to run tests, e.g. python setup.py test -script: tox - -# Assuming you have installed the travis-ci CLI tool, after you -# create the Github repo and add it to Travis, run the -# following command to finish PyPI deployment setup: -# $ travis encrypt --add deploy.password -deploy: - provider: pypi - distributions: sdist bdist_wheel - user: mgerstel - password: - secure: qN3EYVlH22eLPZp5CSvc5Bz8bpP0l0wBZ8tcYMa6kBrqelYdIgqkuZESR/oCMu1YBA3AbcKJNwMuzuWf8RNGAFebD820OJThdP4czRMCv6LDbWABnv12lFHQLQSW1fMQVvb2arjnE/Ew7BFq70p0wOlIJJRwu6CoeOXW/sMeVYMivxdaHmgORq3cdMluFAy4amVb3Fc8i7mxAM0QGklO7x/UWJR/IEpUk2RlUbXL+HrzNoEjRtDeMxoCR84gKZTjVeUQ/iIQSuWwxlt7v1FNENj6ZEbE7+PS8/ylIVfPufbCr8tEEv8W58QcxQ5xPJC2g85ulsN5dM9/9FekhpyKa25B/4wKUNq5T8rKJ8WZ6hMiGffW8rmAfrGTmrBaojKBi0pb9VfXJ5KXUcunVXwQaAn2L80jLLHNsAo94ZxeoowD1eJox9Wh1NtNc+NiUv8K6spOIBsur7G5GY4JVA/yZ7w+DweEfQEp8/SEdVhK0vEYSYT4FnJHuAAmNgeedyAtoes4+a5bYYUM4qrz2OC78NWQWAnnsZhD4Y/TulkavWSexVCqfSePRoK3gcCs+rXwiM69XkMbL1Wgj1gNou+XMkntayH2ZDHkkyJi5F7ls4nqMH5RON9FfVygJMoHDRqh0p4RV25IzJ4FSYqKihHNBO31/URnU2ihpW7n8kM+mbM= - on: - tags: true - repo: DiamondLightSource/python-procrunner - python: 3.8 diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 341c9e7..1e191be 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -101,10 +101,7 @@ 1. The pull request should include tests. 2. If the pull request adds functionality, the docs should be updated. Put your new functionality into a function with a docstring, and add the - feature to the list in README.rst. -3. The pull request should work for Python 2.7, 3.5, 3.6, 3.7, 3.8, and for PyPy. Check - https://travis-ci.org/DiamondLightSource/python-procrunner/pull_requests - and make sure that the tests pass for all supported Python versions. + feature to the list in HISTORY.rst/README.rst. Tips ---- diff --git a/HISTORY.rst b/HISTORY.rst index 2cee455..82297b4 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,66 +1,118 @@ -======= History -======= +================== + +3.0.0 (2022-01-??) +------------------ +* Drop Python 3.6 support +* The run() function now returns a subprocess.CompletedProcess object, + which no longer allows array access operations + (those were deprecated in `#60 `_) +* The run() argument 'raise_timeout_exception' is now set by default, + a 'False' value will lead to a UserWarning and a behavioural change. + The argument is now deprecated and will be removed in a future version. + (previously introduced in `#61 `_) +* Calling the run() function with multiple unnamed arguments is no longer supported + (previously deprecated in `#62 `_) +* The run() function no longer accepts a 'debug' argument + (previously deprecated in `#63 `_) + +2.3.3 (2022-03-23) +------------------ +* Allow specifying 'preexec_fn' and 'creationflags' keywords, which will be passed through to + the subprocess call + +2.3.2 (2022-01-28) +------------------ +* The run() function now understands stdin=subprocess.DEVNULL to close the subprocess stdin, + rather than to connect through the existing stdin, which is the current default + +2.3.1 (2021-10-25) +------------------ +* Add Python 3.10 support + +2.3.0 (2020-10-29) +------------------ +* Add Python 3.9 support, drop Python 3.5 support +* Fix a file descriptor leak on subprocess execution + +2.2.0 (2020-09-07) +------------------ +* Calling the run() function with unnamed arguments (other than the command + list as the first argument) is now deprecated. As a number of arguments + will be removed in a future version the use of unnamed arguments will + cause future confusion. `Use explicit keyword arguments instead (#62). `_ +* `The run() function debug argument has been deprecated (#63). `_ + This is only used to debug the NonBlockingStream* classes. Those are due + to be replaced in a future release, so the argument will no longer serve + a purpose. Debugging information remains available via standard logging + mechanisms. +* Final version supporting Python 3.5 + +2.1.0 (2020-09-05) +------------------ +* `Deprecated array access on the return object (#60). `_ + The return object will become a subprocess.CompletedProcess in a future + release, which no longer allows array-based access. For a translation table + of array elements to attributes please see the pull request linked above. +* Add a `new parameter 'raise_timeout_exception' (#61). `_ + When set to 'True' a subprocess.TimeoutExpired exception is raised when the + process runtime exceeds the timeout threshold. This defaults to 'False' and + will be set to 'True' in a future release. + +2.0.0 (2020-06-24) +------------------ +* Python 3.5+ only, support for Python 2.7 has been dropped +* Deprecated function alias run_process() has been removed +* Fixed a stability issue on Windows 1.1.0 (2019-11-04) ------------------ - * Add Python 3.8 support, drop Python 3.4 support 1.0.2 (2019-05-20) ------------------ - * Stop environment override variables leaking into the process environment 1.0.1 (2019-04-16) ------------------ - * Minor fixes on the return object (implement equality, mark as unhashable) 1.0.0 (2019-03-25) ------------------ - * Support file system path objects (PEP-519) in arguments * Change the return object to make it similar to subprocess.CompletedProcess, introduced with Python 3.5+ 0.9.1 (2019-02-22) ------------------ - * Have deprecation warnings point to correct code locations 0.9.0 (2018-12-07) ------------------ - * Trap UnicodeEncodeError when printing output. Offending characters are replaced and a warning is logged once. Hints at incorrectly set PYTHONIOENCODING. 0.8.1 (2018-12-04) ------------------ - * Fix a few deprecation warnings 0.8.0 (2018-10-09) ------------------ - * Add parameter working_directory to set the working directory of the subprocess 0.7.2 (2018-10-05) ------------------ - * Officially support Python 3.7 0.7.1 (2018-09-03) ------------------ - * Accept environment variable overriding with numeric values. 0.7.0 (2018-05-13) ------------------ - * Unicode fixes. Fix crash on invalid UTF-8 input. * Clarify that stdout/stderr values are returned as bytestrings. * Callbacks receive the data decoded as UTF-8 unicode strings @@ -70,23 +122,19 @@ 0.6.1 (2018-05-02) ------------------ - * Maintenance release to add some tests for executable resolution. 0.6.0 (2018-05-02) ------------------ - * Fix Win32 API executable resolution for commands containing a dot ('.') in addition to a file extension (say '.bat'). 0.5.1 (2018-04-27) ------------------ - * Fix Win32API dependency installation on Windows. 0.5.0 (2018-04-26) ------------------ - * New keyword 'win32resolve' which only takes effect on Windows and is enabled by default. This causes procrunner to call the Win32 API FindExecutable() function to try and lookup non-.exe files with the corresponding name. This @@ -95,21 +143,17 @@ 0.4.0 (2018-04-23) ------------------ - * Python 2.7 support on Windows. Python3 not yet supported on Windows. 0.3.0 (2018-04-17) ------------------ - * run_process() renamed to run() * Python3 compatibility fixes 0.2.0 (2018-03-12) ------------------ - * Procrunner is now Python3 3.3-3.6 compatible. 0.1.0 (2018-03-12) ------------------ - * First release on PyPI. diff --git a/LICENSE b/LICENSE index 1e11a85..15edc93 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2018 Diamond Light Source. +Copyright (c) 2018-2021 Diamond Light Source. All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/Makefile b/Makefile deleted file mode 100644 index c919dcc..0000000 --- a/Makefile +++ /dev/null @@ -1,87 +0,0 @@ -.PHONY: clean clean-test clean-pyc clean-build docs help -.DEFAULT_GOAL := help - -define BROWSER_PYSCRIPT -import os, webbrowser, sys - -try: - from urllib import pathname2url -except: - from urllib.request import pathname2url - -webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) -endef -export BROWSER_PYSCRIPT - -define PRINT_HELP_PYSCRIPT -import re, sys - -for line in sys.stdin: - match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line) - if match: - target, help = match.groups() - print("%-20s %s" % (target, help)) -endef -export PRINT_HELP_PYSCRIPT - -BROWSER := python -c "$$BROWSER_PYSCRIPT" - -help: - @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) - -clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts - -clean-build: ## remove build artifacts - rm -fr build/ - rm -fr dist/ - rm -fr .eggs/ - find . -name '*.egg-info' -exec rm -fr {} + - find . -name '*.egg' -exec rm -f {} + - -clean-pyc: ## remove Python file artifacts - find . -name '*.pyc' -exec rm -f {} + - find . -name '*.pyo' -exec rm -f {} + - find . -name '*~' -exec rm -f {} + - find . -name '__pycache__' -exec rm -fr {} + - -clean-test: ## remove test and coverage artifacts - rm -fr .tox/ - rm -f .coverage - rm -fr htmlcov/ - -lint: ## check style with flake8 - flake8 procrunner tests - -test: ## run tests quickly with the default Python - py.test - -test-all: ## run tests on every Python version with tox - tox - -coverage: ## check code coverage quickly with the default Python - coverage run --source procrunner -m pytest - coverage report -m - coverage html - $(BROWSER) htmlcov/index.html - -docs: ## generate Sphinx HTML documentation, including API docs - rm -f docs/procrunner.rst - rm -f docs/modules.rst - sphinx-apidoc -o docs/ procrunner - $(MAKE) -C docs clean - $(MAKE) -C docs html - $(BROWSER) docs/_build/html/index.html - -servedocs: docs ## compile the docs watching for changes - watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D . - -release: clean ## package and upload a release - twine upload dist/* - -dist: clean ## builds source and wheel package - python setup.py sdist - python setup.py bdist_wheel - ls -l dist - -install: clean ## install the package to the active Python's site-packages - python setup.py install diff --git a/PKG-INFO b/PKG-INFO new file mode 100644 index 0000000..fa32950 --- /dev/null +++ b/PKG-INFO @@ -0,0 +1,250 @@ +Metadata-Version: 2.1 +Name: procrunner +Version: 2.3.3 +Summary: Versatile utility function to run external processes +Author: Diamond Light Source - Scientific Software et al. +Author-email: scientificsoftware@diamond.ac.uk +License: BSD +Project-URL: Download, https://github.com/DiamondLightSource/python-procrunner/tags +Project-URL: Documentation, https://procrunner.readthedocs.io/ +Project-URL: GitHub, https://github.com/DiamondLightSource/python-procrunner +Project-URL: Bug-Tracker, https://github.com/DiamondLightSource/python-procrunner/issues +Keywords: procrunner +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Natural Language :: English +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Operating System :: OS Independent +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Requires-Python: >=3.7 +License-File: LICENSE + +========== +ProcRunner +========== + + +.. image:: https://img.shields.io/pypi/v/procrunner.svg + :target: https://pypi.python.org/pypi/procrunner + :alt: PyPI release + +.. image:: https://img.shields.io/conda/vn/conda-forge/procrunner.svg + :target: https://anaconda.org/conda-forge/procrunner + :alt: Conda Version + +.. image:: https://dev.azure.com/DLS-tooling/procrunner/_apis/build/status/CI?branchName=master + :target: https://github.com/DiamondLightSource/python-procrunner/commits/master + :alt: Build status + +.. image:: https://ci.appveyor.com/api/projects/status/jtq4brwri5q18d0u/branch/master + :target: https://ci.appveyor.com/project/Anthchirp/python-procrunner + :alt: Build status + +.. image:: https://readthedocs.org/projects/procrunner/badge/?version=latest + :target: https://procrunner.readthedocs.io/en/latest/?badge=latest + :alt: Documentation Status + +.. image:: https://img.shields.io/pypi/pyversions/procrunner.svg + :target: https://pypi.python.org/pypi/procrunner + :alt: Supported Python versions + +.. image:: https://img.shields.io/badge/code%20style-black-000000.svg + :target: https://github.com/ambv/black + :alt: Code style: black + +Versatile utility function to run external processes + +* Free software: BSD license +* Documentation: https://procrunner.readthedocs.io. + + +Features +-------- + +* runs an external process and waits for it to finish +* does not deadlock, no matter the process stdout/stderr output behaviour +* returns the exit code, stdout, stderr (separately, both as bytestrings), + as a subprocess.CompletedProcess object +* process can run in a custom environment, either as a modification of + the current environment or in a new environment from scratch +* stdin can be fed to the process +* stdout and stderr is printed by default, can be disabled +* stdout and stderr can be passed to any arbitrary function for + live processing (separately, both as unicode strings) +* optionally enforces a time limit on the process, raising a + subprocess.TimeoutExpired exception if it is exceeded. + + +Credits +------- + +This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypackage`_ project template. + +.. _Cookiecutter: https://github.com/audreyr/cookiecutter +.. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage + + +History +================== + +3.0.0 (2022-01-??) +------------------ +* Drop Python 3.6 support +* The run() function now returns a subprocess.CompletedProcess object, + which no longer allows array access operations + (those were deprecated in `#60 `_) +* The run() argument 'raise_timeout_exception' is now set by default, + a 'False' value will lead to a UserWarning and a behavioural change. + The argument is now deprecated and will be removed in a future version. + (previously introduced in `#61 `_) +* Calling the run() function with multiple unnamed arguments is no longer supported + (previously deprecated in `#62 `_) +* The run() function no longer accepts a 'debug' argument + (previously deprecated in `#63 `_) + +2.3.3 (2022-03-23) +------------------ +* Allow specifying 'preexec_fn' and 'creationflags' keywords, which will be passed through to + the subprocess call + +2.3.2 (2022-01-28) +------------------ +* The run() function now understands stdin=subprocess.DEVNULL to close the subprocess stdin, + rather than to connect through the existing stdin, which is the current default + +2.3.1 (2021-10-25) +------------------ +* Add Python 3.10 support + +2.3.0 (2020-10-29) +------------------ +* Add Python 3.9 support, drop Python 3.5 support +* Fix a file descriptor leak on subprocess execution + +2.2.0 (2020-09-07) +------------------ +* Calling the run() function with unnamed arguments (other than the command + list as the first argument) is now deprecated. As a number of arguments + will be removed in a future version the use of unnamed arguments will + cause future confusion. `Use explicit keyword arguments instead (#62). `_ +* `The run() function debug argument has been deprecated (#63). `_ + This is only used to debug the NonBlockingStream* classes. Those are due + to be replaced in a future release, so the argument will no longer serve + a purpose. Debugging information remains available via standard logging + mechanisms. +* Final version supporting Python 3.5 + +2.1.0 (2020-09-05) +------------------ +* `Deprecated array access on the return object (#60). `_ + The return object will become a subprocess.CompletedProcess in a future + release, which no longer allows array-based access. For a translation table + of array elements to attributes please see the pull request linked above. +* Add a `new parameter 'raise_timeout_exception' (#61). `_ + When set to 'True' a subprocess.TimeoutExpired exception is raised when the + process runtime exceeds the timeout threshold. This defaults to 'False' and + will be set to 'True' in a future release. + +2.0.0 (2020-06-24) +------------------ +* Python 3.5+ only, support for Python 2.7 has been dropped +* Deprecated function alias run_process() has been removed +* Fixed a stability issue on Windows + +1.1.0 (2019-11-04) +------------------ +* Add Python 3.8 support, drop Python 3.4 support + +1.0.2 (2019-05-20) +------------------ +* Stop environment override variables leaking into the process environment + +1.0.1 (2019-04-16) +------------------ +* Minor fixes on the return object (implement equality, + mark as unhashable) + +1.0.0 (2019-03-25) +------------------ +* Support file system path objects (PEP-519) in arguments +* Change the return object to make it similar to + subprocess.CompletedProcess, introduced with Python 3.5+ + +0.9.1 (2019-02-22) +------------------ +* Have deprecation warnings point to correct code locations + +0.9.0 (2018-12-07) +------------------ +* Trap UnicodeEncodeError when printing output. Offending characters + are replaced and a warning is logged once. Hints at incorrectly set + PYTHONIOENCODING. + +0.8.1 (2018-12-04) +------------------ +* Fix a few deprecation warnings + +0.8.0 (2018-10-09) +------------------ +* Add parameter working_directory to set the working directory + of the subprocess + +0.7.2 (2018-10-05) +------------------ +* Officially support Python 3.7 + +0.7.1 (2018-09-03) +------------------ +* Accept environment variable overriding with numeric values. + +0.7.0 (2018-05-13) +------------------ +* Unicode fixes. Fix crash on invalid UTF-8 input. +* Clarify that stdout/stderr values are returned as bytestrings. +* Callbacks receive the data decoded as UTF-8 unicode strings + with unknown characters replaced by \ufffd (unicode replacement + character). Same applies to printing of output. +* Mark stdin broken on Windows. + +0.6.1 (2018-05-02) +------------------ +* Maintenance release to add some tests for executable resolution. + +0.6.0 (2018-05-02) +------------------ +* Fix Win32 API executable resolution for commands containing a dot ('.') in + addition to a file extension (say '.bat'). + +0.5.1 (2018-04-27) +------------------ +* Fix Win32API dependency installation on Windows. + +0.5.0 (2018-04-26) +------------------ +* New keyword 'win32resolve' which only takes effect on Windows and is enabled + by default. This causes procrunner to call the Win32 API FindExecutable() + function to try and lookup non-.exe files with the corresponding name. This + means .bat/.cmd/etc.. files can now be run without explicitly specifying + their extension. Only supported on Python 2.7 and 3.5+. + +0.4.0 (2018-04-23) +------------------ +* Python 2.7 support on Windows. Python3 not yet supported on Windows. + +0.3.0 (2018-04-17) +------------------ +* run_process() renamed to run() +* Python3 compatibility fixes + +0.2.0 (2018-03-12) +------------------ +* Procrunner is now Python3 3.3-3.6 compatible. + +0.1.0 (2018-03-12) +------------------ +* First release on PyPI. diff --git a/README.rst b/README.rst index fb7f5d0..01ddb3f 100644 --- a/README.rst +++ b/README.rst @@ -11,8 +11,8 @@ :target: https://anaconda.org/conda-forge/procrunner :alt: Conda Version -.. image:: https://travis-ci.org/DiamondLightSource/python-procrunner.svg?branch=master - :target: https://travis-ci.org/DiamondLightSource/python-procrunner +.. image:: https://dev.azure.com/DLS-tooling/procrunner/_apis/build/status/CI?branchName=master + :target: https://github.com/DiamondLightSource/python-procrunner/commits/master :alt: Build status .. image:: https://ci.appveyor.com/api/projects/status/jtq4brwri5q18d0u/branch/master @@ -22,10 +22,6 @@ .. image:: https://readthedocs.org/projects/procrunner/badge/?version=latest :target: https://procrunner.readthedocs.io/en/latest/?badge=latest :alt: Documentation Status - -.. image:: https://pyup.io/repos/github/DiamondLightSource/python-procrunner/shield.svg - :target: https://pyup.io/repos/github/DiamondLightSource/python-procrunner/ - :alt: Updates .. image:: https://img.shields.io/pypi/pyversions/procrunner.svg :target: https://pypi.python.org/pypi/procrunner @@ -47,15 +43,16 @@ * runs an external process and waits for it to finish * does not deadlock, no matter the process stdout/stderr output behaviour * returns the exit code, stdout, stderr (separately, both as bytestrings), - and the total process runtime as a dictionary + as a subprocess.CompletedProcess object * process can run in a custom environment, either as a modification of the current environment or in a new environment from scratch -* stdin can be fed to the process, the returned dictionary contains - information how much was read by the process +* stdin can be fed to the process * stdout and stderr is printed by default, can be disabled * stdout and stderr can be passed to any arbitrary function for live processing (separately, both as unicode strings) -* optionally enforces a time limit on the process +* optionally enforces a time limit on the process, raising a + subprocess.TimeoutExpired exception if it is exceeded. + Credits ------- diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index e6e1890..0000000 --- a/appveyor.yml +++ /dev/null @@ -1,58 +0,0 @@ -environment: - - matrix: - - # For Python versions available on Appveyor, see - # http://www.appveyor.com/docs/installed-software#python - - - PYTHON: "C:\\Python27" - - PYTHON: "C:\\Python35" - UNSTABLE: 1 - - PYTHON: "C:\\Python36" - UNSTABLE: 1 - - PYTHON: "C:\\Python37" - UNSTABLE: 1 - - PYTHON: "C:\\Python38" - UNSTABLE: 1 - - PYTHON: "C:\\Python27-x64" - - PYTHON: "C:\\Python35-x64" - UNSTABLE: 1 - - PYTHON: "C:\\Python36-x64" - UNSTABLE: 1 - - PYTHON: "C:\\Python37-x64" - UNSTABLE: 1 - - PYTHON: "C:\\Python38-x64" - UNSTABLE: 1 - -matrix: - allow_failures: - - UNSTABLE: 1 - -install: - # Upgrade to the latest pip. - - '%PYTHON%\\python.exe -m pip install -U pip setuptools wheel' - - '%PYTHON%\\python.exe -m pip install -r requirements_dev.txt' - # Install win32api dependency. Must use the --only-binary switch explicitly - # on AppVeyor - - "%PYTHON%\\python.exe -m pip install --only-binary pywin32 pywin32" - -build: off - -test_script: - # Note that you must use the environment variable %PYTHON% to refer to - # the interpreter you're using - Appveyor does not do anything special - # to put the Python version you want to use on PATH. - - "%PYTHON%\\python.exe setup.py test" - -after_test: - # This step builds your wheels. - - "%PYTHON%\\python.exe setup.py bdist_wheel" - -artifacts: - # bdist_wheel puts your built wheel in the dist directory - - path: dist\* - -#on_success: -# You can use this step to upload your artifacts to a public website. -# See Appveyor's documentation for more details. Or you can simply -# access your wheels from the Appveyor "artifacts" tab for your build. diff --git a/docs/.gitignore b/docs/.gitignore deleted file mode 100644 index 0ab3233..0000000 --- a/docs/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/procrunner.rst -/procrunner.*.rst -/modules.rst diff --git a/docs/conf.py b/docs/conf.py index e850102..abce534 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # procrunner documentation build configuration file, created by # sphinx-quickstart on Fri Jun 9 13:47:02 2017. @@ -48,9 +47,9 @@ master_doc = "index" # General information about the project. -project = u"ProcRunner" -copyright = u"2018, Markus Gerstel" -author = u"Markus Gerstel" +project = "ProcRunner" +copyright = "2020, Diamond Light Source" +author = "Diamond Light Source - Scientific Software" # The version info for the project you're documenting, acts as replacement # for |version| and |release|, also used in various other places throughout @@ -129,8 +128,8 @@ ( master_doc, "procrunner.tex", - u"ProcRunner Documentation", - u"Markus Gerstel", + "procrunner Documentation", + "Diamond Light Source - Scientific Software", "manual", ) ] @@ -140,7 +139,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [(master_doc, "procrunner", u"ProcRunner Documentation", [author], 1)] +man_pages = [(master_doc, "procrunner", "procrunner Documentation", [author], 1)] # -- Options for Texinfo output ---------------------------------------- @@ -152,10 +151,10 @@ ( master_doc, "procrunner", - u"ProcRunner Documentation", + "procrunner Documentation", author, "procrunner", - "One line description of project.", + "Versatile utility function to run external processes", "Miscellaneous", ) ] diff --git a/docs/usage.rst b/docs/usage.rst index 8ccfe60..4b87a55 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -9,13 +9,14 @@ To test for successful completion:: - assert not result['exitcode'] - assert result['exitcode'] == 0 # alternatively + assert not result.returncode + assert result.returncode == 0 # alternatively + result.check_returncode() # raises subprocess.CalledProcessError() To test for no STDERR output:: - assert not result['stderr'] - assert result['stderr'] == b'' # alternatively + assert not result.stderr + assert result.stderr == b'' # alternatively To run with a specific environment variable set:: diff --git a/procrunner/__init__.py b/procrunner/__init__.py deleted file mode 100644 index 1a1e306..0000000 --- a/procrunner/__init__.py +++ /dev/null @@ -1,629 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import, division, print_function - -import codecs -import logging -import os -import select -import six -import subprocess -import sys -import time -import timeit -import warnings -from multiprocessing import Pipe -from threading import Thread - -# -# run() - A function to synchronously run an external process, supporting -# the following features: -# -# - runs an external process and waits for it to finish -# - does not deadlock, no matter the process stdout/stderr output behaviour -# - returns the exit code, stdout, stderr (separately), and the total process -# runtime as a dictionary -# - process can run in a custom environment, either as a modification of -# the current environment or in a new environment from scratch -# - stdin can be fed to the process, the returned dictionary contains -# information how much was read by the process -# - stdout and stderr is printed by default, can be disabled -# - stdout and stderr can be passed to any arbitrary function for -# live processing -# - optionally enforces a time limit on the process -# -# -# Usage example: -# -# import procrunner -# result = procrunner.run(['/bin/ls', '/some/path/containing spaces']) -# -# Returns: -# -# {'command': ['/bin/ls', '/some/path/containing spaces'], -# 'exitcode': 2, -# 'runtime': 0.12990689277648926, -# 'stderr': '/bin/ls: cannot access /some/path/containing spaces: No such file or directory\n', -# 'stdout': '', -# 'time_end': '2017-11-12 19:54:49 GMT', -# 'time_start': '2017-11-12 19:54:49 GMT', -# 'timeout': False} -# - -__author__ = """Markus Gerstel""" -__email__ = "scientificsoftware@diamond.ac.uk" -__version__ = "1.1.0" - -logger = logging.getLogger("procrunner") -logger.addHandler(logging.NullHandler()) - - -class _LineAggregator(object): - """ - Buffer that can be filled with stream data and will aggregate complete - lines. Lines can be printed or passed to an arbitrary callback function. - The lines passed to the callback function are UTF-8 decoded and do not - contain a trailing newline character. - """ - - def __init__(self, print_line=False, callback=None): - """Create aggregator object.""" - self._buffer = "" - self._print = print_line - self._callback = callback - self._decoder = codecs.getincrementaldecoder("utf-8")("replace") - - def add(self, data): - """ - Add a single character to buffer. If one or more full lines are found, - print them (if desired) and pass to callback function. - """ - data = self._decoder.decode(data) - if not data: - return - self._buffer += data - if "\n" in data: - to_print, remainder = self._buffer.rsplit("\n") - if self._print: - try: - print(to_print) - except UnicodeEncodeError: - print(to_print.encode(sys.getdefaultencoding(), errors="replace")) - if not hasattr(self, "_warned"): - logger.warning("output encoding error, characters replaced") - setattr(self, "_warned", True) - if self._callback: - self._callback(to_print) - self._buffer = remainder - - def flush(self): - """Print/send any remaining data to callback function.""" - self._buffer += self._decoder.decode(b"", final=True) - if self._buffer: - if self._print: - print(self._buffer) - if self._callback: - self._callback(self._buffer) - self._buffer = "" - - -class _NonBlockingStreamReader(object): - """Reads a stream in a thread to avoid blocking/deadlocks""" - - def __init__(self, stream, output=True, debug=False, notify=None, callback=None): - """Creates and starts a thread which reads from a stream.""" - self._buffer = six.BytesIO() - self._closed = False - self._closing = False - self._debug = debug - self._stream = stream - self._terminated = False - - def _thread_write_stream_to_buffer(): - la = _LineAggregator(print_line=output, callback=callback) - char = True - while char: - if select.select([self._stream], [], [], 0.1)[0]: - char = self._stream.read(1) - if char: - self._buffer.write(char) - la.add(char) - else: - if self._closing: - break - self._terminated = True - la.flush() - if self._debug: - logger.debug("Stream reader terminated") - if notify: - notify() - - def _thread_write_stream_to_buffer_windows(): - line = True - while line: - line = self._stream.readline() - if line: - self._buffer.write(line) - if output or callback: - linedecode = line.decode("utf-8", "replace") - if output: - print(linedecode) - if callback: - callback(linedecode) - self._terminated = True - if self._debug: - logger.debug("Stream reader terminated") - if notify: - notify() - - if os.name == "nt": - self._thread = Thread(target=_thread_write_stream_to_buffer_windows) - else: - self._thread = Thread(target=_thread_write_stream_to_buffer) - self._thread.daemon = True - self._thread.start() - - def has_finished(self): - """ - Returns whether the thread reading from the stream is still alive. - """ - return self._terminated - - def get_output(self): - """ - Retrieve the stored data in full. - This call may block if the reading thread has not yet terminated. - """ - self._closing = True - if not self.has_finished(): - if self._debug: - # Main thread overtook stream reading thread. - underrun_debug_timer = timeit.default_timer() - logger.warning("NBSR underrun") - self._thread.join() - if not self.has_finished(): - if self._debug: - logger.debug( - "NBSR join after %f seconds, underrun not resolved" - % (timeit.default_timer() - underrun_debug_timer) - ) - raise Exception("thread did not terminate") - if self._debug: - logger.debug( - "NBSR underrun resolved after %f seconds" - % (timeit.default_timer() - underrun_debug_timer) - ) - if self._closed: - raise Exception("streamreader double-closed") - self._closed = True - data = self._buffer.getvalue() - self._buffer.close() - return data - - -class _NonBlockingStreamWriter(object): - """Writes to a stream in a thread to avoid blocking/deadlocks""" - - def __init__(self, stream, data, debug=False, notify=None): - """Creates and starts a thread which writes data to stream.""" - self._buffer = data - self._buffer_len = len(data) - self._buffer_pos = 0 - self._debug = debug - self._max_block_len = 4096 - self._stream = stream - self._terminated = False - - def _thread_write_buffer_to_stream(): - while self._buffer_pos < self._buffer_len: - if (self._buffer_len - self._buffer_pos) > self._max_block_len: - block = self._buffer[ - self._buffer_pos : (self._buffer_pos + self._max_block_len) - ] - else: - block = self._buffer[self._buffer_pos :] - try: - self._stream.write(block) - except IOError as e: - if ( - e.errno == 32 - ): # broken pipe, ie. process terminated without reading entire stdin - self._stream.close() - self._terminated = True - if notify: - notify() - return - raise - self._buffer_pos += len(block) - if debug: - logger.debug("wrote %d bytes to stream" % len(block)) - self._stream.close() - self._terminated = True - if notify: - notify() - - self._thread = Thread(target=_thread_write_buffer_to_stream) - self._thread.daemon = True - self._thread.start() - - def has_finished(self): - """Returns whether the thread writing to the stream is still alive.""" - return self._terminated - - def bytes_sent(self): - """Return the number of bytes written so far.""" - return self._buffer_pos - - def bytes_remaining(self): - """Return the number of bytes still to be written.""" - return self._buffer_len - self._buffer_pos - - -def _path_resolve(obj): - """ - Resolve file system path (PEP-519) objects to strings. - - :param obj: A file system path object or something else. - :return: A string representation of a file system path object or, for - anything that was not a file system path object, the original - object. - """ - if obj and hasattr(obj, "__fspath__"): - return obj.__fspath__() - return obj - - -def _windows_resolve(command): - """ - Try and find the full path and file extension of the executable to run. - This is so that e.g. calls to 'somescript' will point at 'somescript.cmd' - without the need to set shell=True in the subprocess. - If the executable contains periods it is a special case. Here the - win32api call will fail to resolve the extension automatically, and it - has do be done explicitly. - - :param command: The command array to be run, with the first element being - the command with or w/o path, with or w/o extension. - :return: Returns the command array with the executable resolved with the - correct extension. If the executable cannot be resolved for any - reason the original command array is returned. - """ - try: - import win32api - except ImportError: - if (2, 8) < sys.version_info < (3, 5): - logger.info( - "Resolving executable names only supported on Python 2.7 and 3.5+" - ) - else: - logger.warning( - "Could not resolve executable name: package win32api missing" - ) - return command - - if not command or not isinstance(command[0], six.string_types): - return command - - try: - _, found_executable = win32api.FindExecutable(command[0]) - logger.debug("Resolved %s as %s", command[0], found_executable) - return (found_executable,) + tuple(command[1:]) - except Exception as e: - if not hasattr(e, "winerror"): - raise - # Keep this error message for later in case we fail to resolve the name - logwarning = getattr(e, "strerror", str(e)) - - if "." in command[0]: - # Special case. The win32api may not properly check file extensions, so - # try to resolve the executable explicitly. - for extension in os.getenv("PATHEXT").split(os.pathsep): - try: - _, found_executable = win32api.FindExecutable(command[0] + extension) - logger.debug("Resolved %s as %s", command[0], found_executable) - return (found_executable,) + tuple(command[1:]) - except Exception as e: - if not hasattr(e, "winerror"): - raise - - logger.warning("Error trying to resolve the executable: %s", logwarning) - return command - - -if sys.version_info < (3, 5): - - class _ReturnObjectParent(object): - def check_returncode(self): - if self.returncode: - raise Exception( - "Call %r resulted in non-zero exit code %r" - % (self.args, self.returncode) - ) - - -else: - _ReturnObjectParent = subprocess.CompletedProcess - - -class ReturnObject(dict, _ReturnObjectParent): - """ - A subprocess.CompletedProcess-like object containing the executed - command, stdout and stderr (both as bytestrings), and the exitcode. - Further values such as process runtime can be accessed as dictionary - values. - The check_returncode() function raises an exception if the process - exited with a non-zero exit code. - """ - - def __init__(self, *arg, **kw): - super(ReturnObject, self).__init__(*arg, **kw) - self.args = self["command"] - self.returncode = self["exitcode"] - self.stdout = self["stdout"] - self.stderr = self["stderr"] - - def __eq__(self, other): - """Override equality operator to account for added fields""" - if type(other) is type(self): - return self.__dict__ == other.__dict__ - return False - - def __hash__(self): - """This object is not immutable, so mark it as unhashable""" - return None - - def __ne__(self, other): - """Overrides the default implementation (unnecessary in Python 3)""" - return not self.__eq__(other) - - -def run( - command, - timeout=None, - debug=False, - stdin=None, - print_stdout=True, - print_stderr=True, - callback_stdout=None, - callback_stderr=None, - environment=None, - environment_override=None, - win32resolve=True, - working_directory=None, -): - """ - Run an external process. - - File system path objects (PEP-519) are accepted in the command, environment, - and working directory arguments. - - :param array command: Command line to be run, specified as array. - :param timeout: Terminate program execution after this many seconds. - :param boolean debug: Enable further debug messages. - :param stdin: Optional string that is passed to command stdin. - :param boolean print_stdout: Pass stdout through to sys.stdout. - :param boolean print_stderr: Pass stderr through to sys.stderr. - :param callback_stdout: Optional function which is called for each - stdout line. - :param callback_stderr: Optional function which is called for each - stderr line. - :param dict environment: The full execution environment for the command. - :param dict environment_override: Change environment variables from the - current values for command execution. - :param boolean win32resolve: If on Windows, find the appropriate executable - first. This allows running of .bat, .cmd, etc. - files without explicitly specifying their - extension. - :param string working_directory: If specified, run the executable from - within this working directory. - :return: A ReturnObject() containing the executed command, stdout and stderr - (both as bytestrings), and the exitcode. Further values such as - process runtime can be accessed as dictionary values. - """ - - time_start = time.strftime("%Y-%m-%d %H:%M:%S GMT", time.gmtime()) - logger.debug("Starting external process: %s", command) - - if stdin is None: - stdin_pipe = None - else: - assert sys.platform != "win32", "stdin argument not supported on Windows" - stdin_pipe = subprocess.PIPE - - start_time = timeit.default_timer() - if timeout is not None: - max_time = start_time + timeout - - if environment is not None: - env = {key: _path_resolve(environment[key]) for key in environment} - else: - env = {key: value for key, value in os.environ.items()} - if environment_override: - env.update( - { - key: str(_path_resolve(environment_override[key])) - for key in environment_override - } - ) - - command = tuple(_path_resolve(part) for part in command) - if win32resolve and sys.platform == "win32": - command = _windows_resolve(command) - - p = subprocess.Popen( - command, - shell=False, - cwd=_path_resolve(working_directory), - env=env, - stdin=stdin_pipe, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - - thread_pipe_pool = [] - notifyee, notifier = Pipe(False) - thread_pipe_pool.append(notifyee) - stdout = _NonBlockingStreamReader( - p.stdout, - output=print_stdout, - debug=debug, - notify=notifier.close, - callback=callback_stdout, - ) - notifyee, notifier = Pipe(False) - thread_pipe_pool.append(notifyee) - stderr = _NonBlockingStreamReader( - p.stderr, - output=print_stderr, - debug=debug, - notify=notifier.close, - callback=callback_stderr, - ) - if stdin is not None: - notifyee, notifier = Pipe(False) - thread_pipe_pool.append(notifyee) - stdin = _NonBlockingStreamWriter( - p.stdin, data=stdin, debug=debug, notify=notifier.close - ) - - timeout_encountered = False - - while (p.returncode is None) and ( - (timeout is None) or (timeit.default_timer() < max_time) - ): - if debug and timeout is not None: - logger.debug("still running (T%.2fs)" % (timeit.default_timer() - max_time)) - - # wait for some time or until a stream is closed - try: - if thread_pipe_pool: - # Wait for up to 0.5 seconds or for a signal on a remaining stream, - # which could indicate that the process has terminated. - try: - event = thread_pipe_pool[0].poll(0.5) - except IOError as e: - # on Windows this raises "IOError: [Errno 109] The pipe has been ended" - # which is for all intents and purposes equivalent to a True return value. - if e.errno != 109: - raise - event = True - if event: - # One-shot, so remove stream and watch remaining streams - thread_pipe_pool.pop(0) - if debug: - logger.debug("Event received from stream thread") - else: - time.sleep(0.5) - except KeyboardInterrupt: - p.kill() # if user pressed Ctrl+C we won't be able to produce a proper report anyway - # but at least make sure the child process dies with us - raise - - # check if process is still running - p.poll() - - if p.returncode is None: - # timeout condition - timeout_encountered = True - if debug: - logger.debug("timeout (T%.2fs)" % (timeit.default_timer() - max_time)) - - # send terminate signal and wait some time for buffers to be read - p.terminate() - if thread_pipe_pool: - thread_pipe_pool[0].poll(0.5) - if not stdout.has_finished() or not stderr.has_finished(): - time.sleep(2) - p.poll() - - if p.returncode is None: - # thread still alive - # send kill signal and wait some more time for buffers to be read - p.kill() - if thread_pipe_pool: - thread_pipe_pool[0].poll(0.5) - if not stdout.has_finished() or not stderr.has_finished(): - time.sleep(5) - p.poll() - - if p.returncode is None: - raise RuntimeError("Process won't terminate") - - runtime = timeit.default_timer() - start_time - if timeout is not None: - logger.debug( - "Process ended after %.1f seconds with exit code %d (T%.2fs)" - % (runtime, p.returncode, timeit.default_timer() - max_time) - ) - else: - logger.debug( - "Process ended after %.1f seconds with exit code %d" - % (runtime, p.returncode) - ) - - stdout = stdout.get_output() - stderr = stderr.get_output() - time_end = time.strftime("%Y-%m-%d %H:%M:%S GMT", time.gmtime()) - - result = ReturnObject( - { - "exitcode": p.returncode, - "command": command, - "stdout": stdout, - "stderr": stderr, - "timeout": timeout_encountered, - "runtime": runtime, - "time_start": time_start, - "time_end": time_end, - } - ) - if stdin is not None: - result.update( - { - "stdin_bytes_sent": stdin.bytes_sent(), - "stdin_bytes_remain": stdin.bytes_remaining(), - } - ) - - return result - - -def run_process_dummy(command, **kwargs): - """ - A stand-in function that returns a valid result dictionary indicating a - successful execution. The external process is not run. - """ - warnings.warn( - "procrunner.run_process_dummy() is deprecated", DeprecationWarning, stacklevel=2 - ) - - time_start = time.strftime("%Y-%m-%d %H:%M:%S GMT", time.gmtime()) - logger.info("run_process is disabled. Requested command: %s", command) - - result = ReturnObject( - { - "exitcode": 0, - "command": command, - "stdout": "", - "stderr": "", - "timeout": False, - "runtime": 0, - "time_start": time_start, - "time_end": time_start, - } - ) - if kwargs.get("stdin") is not None: - result.update( - {"stdin_bytes_sent": len(kwargs["stdin"]), "stdin_bytes_remain": 0} - ) - return result - - -def run_process(*args, **kwargs): - """API used up to version 0.2.0.""" - warnings.warn( - "procrunner.run_process() is deprecated and has been renamed to run()", - DeprecationWarning, - stacklevel=2, - ) - return run(*args, **kwargs) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..7238713 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,6 @@ +[build-system] +requires = ["setuptools >= 40.6.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.isort] +profile="black" diff --git a/requirements_dev.txt b/requirements_dev.txt deleted file mode 100644 index 36a6442..0000000 --- a/requirements_dev.txt +++ /dev/null @@ -1,13 +0,0 @@ -bump2version==0.5.10 -coverage==4.5.4 -flake8==3.7.8 -mock==3.0.5 -pip==19.1.1 -pytest==4.5.0 # pyup: <5.0 # for Python 2.7 support -pytest-runner==5.1 -six==1.12.0 -Sphinx==1.8.5 # pyup: <2.0 # for Python 2.7 support -tox==3.13.1 -twine==1.13.0 -watchdog==0.9.0 -wheel==0.33.4 diff --git a/setup.cfg b/setup.cfg index 38ee3f7..b3604dd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,25 +1,53 @@ -[bumpversion] -current_version = 1.1.0 -commit = True -tag = True +[metadata] +name = procrunner +description = Versatile utility function to run external processes +version = 2.3.3 +author = Diamond Light Source - Scientific Software et al. +author_email = scientificsoftware@diamond.ac.uk +classifiers = + Development Status :: 5 - Production/Stable + Intended Audience :: Developers + License :: OSI Approved :: BSD License + Natural Language :: English + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Operating System :: OS Independent + Topic :: Software Development :: Libraries :: Python Modules +license = BSD +license_file = LICENSE +project_urls = + Download = https://github.com/DiamondLightSource/python-procrunner/tags + Documentation = https://procrunner.readthedocs.io/ + GitHub = https://github.com/DiamondLightSource/python-procrunner + Bug-Tracker = https://github.com/DiamondLightSource/python-procrunner/issues -[bumpversion:file:setup.py] -search = version="{current_version}" -replace = version="{new_version}" +[options] +include_package_data = True +packages = procrunner +package_dir = + =src +python_requires = >=3.7 +zip_safe = False -[bumpversion:file:procrunner/__init__.py] -search = __version__ = "{current_version}" -replace = __version__ = "{new_version}" - -[bdist_wheel] -universal = 1 +[options.packages.find] +where = src [flake8] -exclude = docs - -[aliases] -test = pytest +ignore = E203, E266, E501, W503 +max-line-length = 88 +select = + E401,E711,E712,E713,E714,E721,E722,E901, + F401,F402,F403,F405,F541,F631,F632,F633,F811,F812,F821,F822,F841,F901, + W191,W291,W292,W293,W602,W603,W604,W605,W606, + C4, [tool:pytest] collect_ignore = ['setup.py'] +[egg_info] +tag_build = +tag_date = 0 + diff --git a/setup.py b/setup.py index 29a129a..b181937 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,4 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import sys -from setuptools import setup, find_packages +from setuptools import setup with open("README.rst") as readme_file: readme = readme_file.read() @@ -10,50 +6,7 @@ with open("HISTORY.rst") as history_file: history = history_file.read() -requirements = [ - "six", - 'pywin32; sys_platform=="win32"', -] - -setup_requirements = [] -needs_pytest = {"pytest", "test", "ptr"}.intersection(sys.argv) -if needs_pytest: - setup_requirements.append("pytest-runner") - -test_requirements = ["mock", "pytest"] - setup( - author="Markus Gerstel", - author_email="scientificsoftware@diamond.ac.uk", - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "License :: OSI Approved :: BSD License", - "Natural Language :: English", - "Operating System :: OS Independent", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: Implementation :: PyPy", - "Programming Language :: Python :: Implementation :: CPython", - "Topic :: Software Development :: Libraries :: Python Modules", - ], - description="Versatile utility function to run external processes", - install_requires=requirements, - license="BSD license", long_description=readme + "\n\n" + history, - include_package_data=True, keywords="procrunner", - name="procrunner", - packages=find_packages(include=["procrunner"]), - setup_requires=setup_requirements, - test_suite="tests", - tests_require=test_requirements, - url="https://github.com/DiamondLightSource/python-procrunner", - version="1.1.0", - zip_safe=False, ) diff --git a/src/procrunner/__init__.py b/src/procrunner/__init__.py new file mode 100644 index 0000000..e8e1cd3 --- /dev/null +++ b/src/procrunner/__init__.py @@ -0,0 +1,529 @@ +from __future__ import annotations + +import codecs +import io +import logging +import os +import select +import shutil +import subprocess +import sys +import time +import timeit +import warnings +from multiprocessing import Pipe +from threading import Thread +from typing import Any, Callable, Optional, Union + +# +# run() - A function to synchronously run an external process, supporting +# the following features: +# +# - runs an external process and waits for it to finish +# - does not deadlock, no matter the process stdout/stderr output behaviour +# - returns the exit code, stdout, stderr (separately) as a +# subprocess.CompletedProcess object +# - process can run in a custom environment, either as a modification of +# the current environment or in a new environment from scratch +# - stdin can be fed to the process +# - stdout and stderr is printed by default, can be disabled +# - stdout and stderr can be passed to any arbitrary function for +# live processing +# - optionally enforces a time limit on the process +# +# +# Usage example: +# +# import procrunner +# result = procrunner.run(['/bin/ls', '/some/path/containing spaces']) +# +# Returns: +# +# subprocess.CompletedProcess( +# args=('/bin/ls', '/some/path/containing spaces'), +# returncode=2, +# stdout=b'', +# stderr=b'/bin/ls: cannot access /some/path/containing spaces: No such file or directory\n' +# ) + +__version__ = "2.3.3" + +logger = logging.getLogger("procrunner") +logger.addHandler(logging.NullHandler()) + + +class _LineAggregator: + """ + Buffer that can be filled with stream data and will aggregate complete + lines. Lines can be printed or passed to an arbitrary callback function. + The lines passed to the callback function are UTF-8 decoded and do not + contain a trailing newline character. + """ + + def __init__(self, print_line=False, callback=None): + """Create aggregator object.""" + self._buffer = "" + self._print = print_line + self._callback = callback + self._decoder = codecs.getincrementaldecoder("utf-8")("replace") + + def add(self, data): + """ + Add a single character to buffer. If one or more full lines are found, + print them (if desired) and pass to callback function. + """ + data = self._decoder.decode(data) + if not data: + return + self._buffer += data + if "\n" in data: + to_print, remainder = self._buffer.rsplit("\n") + if self._print: + try: + print(to_print) + except UnicodeEncodeError: + print(to_print.encode(sys.getdefaultencoding(), errors="replace")) + if not hasattr(self, "_warned"): + logger.warning("output encoding error, characters replaced") + setattr(self, "_warned", True) + if self._callback: + self._callback(to_print) + self._buffer = remainder + + def flush(self): + """Print/send any remaining data to callback function.""" + self._buffer += self._decoder.decode(b"", final=True) + if self._buffer: + if self._print: + print(self._buffer) + if self._callback: + self._callback(self._buffer) + self._buffer = "" + + +class _NonBlockingStreamReader: + """Reads a stream in a thread to avoid blocking/deadlocks""" + + def __init__(self, stream, output=True, debug=False, notify=None, callback=None): + """Creates and starts a thread which reads from a stream.""" + self._buffer = io.BytesIO() + self._closed = False + self._closing = False + self._debug = debug + self._stream = stream + self._terminated = False + + def _thread_write_stream_to_buffer(): + la = _LineAggregator(print_line=output, callback=callback) + char = True + while char: + if select.select([self._stream], [], [], 0.1)[0]: + char = self._stream.read(1) + if char: + self._buffer.write(char) + la.add(char) + else: + if self._closing: + break + self._stream.close() + self._terminated = True + la.flush() + if self._debug: + logger.debug("Stream reader terminated") + if notify: + notify() + + def _thread_write_stream_to_buffer_windows(): + line = True + while line: + line = self._stream.readline() + if line: + self._buffer.write(line) + if output or callback: + linedecode = line.decode("utf-8", "replace") + if output: + print(linedecode) + if callback: + callback(linedecode) + self._stream.close() + self._terminated = True + if self._debug: + logger.debug("Stream reader terminated") + if notify: + notify() + + if os.name == "nt": + self._thread = Thread(target=_thread_write_stream_to_buffer_windows) + else: + self._thread = Thread(target=_thread_write_stream_to_buffer) + self._thread.daemon = True + self._thread.start() + + def has_finished(self): + """ + Returns whether the thread reading from the stream is still alive. + """ + return self._terminated + + def get_output(self): + """ + Retrieve the stored data in full. + This call may block if the reading thread has not yet terminated. + """ + self._closing = True + if not self.has_finished(): + if self._debug: + # Main thread overtook stream reading thread. + underrun_debug_timer = timeit.default_timer() + logger.warning("NBSR underrun") + self._thread.join() + if not self.has_finished(): + if self._debug: + logger.debug( + "NBSR join after %f seconds, underrun not resolved", + timeit.default_timer() - underrun_debug_timer, + ) + raise Exception("thread did not terminate") + if self._debug: + logger.debug( + "NBSR underrun resolved after %f seconds", + timeit.default_timer() - underrun_debug_timer, + ) + if self._closed: + raise Exception("streamreader double-closed") + self._closed = True + data = self._buffer.getvalue() + self._buffer.close() + return data + + +class _NonBlockingStreamWriter: + """Writes to a stream in a thread to avoid blocking/deadlocks""" + + def __init__(self, stream, data, debug=False, notify=None): + """Creates and starts a thread which writes data to stream.""" + self._buffer = data + self._buffer_len = len(data) + self._buffer_pos = 0 + self._max_block_len = 4096 + self._stream = stream + self._terminated = False + + def _thread_write_buffer_to_stream(): + while self._buffer_pos < self._buffer_len: + if (self._buffer_len - self._buffer_pos) > self._max_block_len: + block = self._buffer[ + self._buffer_pos : (self._buffer_pos + self._max_block_len) + ] + else: + block = self._buffer[self._buffer_pos :] + try: + self._stream.write(block) + except OSError as e: + if ( + e.errno == 32 + ): # broken pipe, ie. process terminated without reading entire stdin + self._stream.close() + self._terminated = True + if notify: + notify() + return + raise + self._buffer_pos += len(block) + if debug: + logger.debug("wrote %d bytes to stream", len(block)) + self._stream.close() + self._terminated = True + if notify: + notify() + + self._thread = Thread(target=_thread_write_buffer_to_stream) + self._thread.daemon = True + self._thread.start() + + def has_finished(self): + """Returns whether the thread writing to the stream is still alive.""" + return self._terminated + + def bytes_sent(self): + """Return the number of bytes written so far.""" + return self._buffer_pos + + def bytes_remaining(self): + """Return the number of bytes still to be written.""" + return self._buffer_len - self._buffer_pos + + +def _path_resolve(obj): + """ + Resolve file system path (PEP-519) objects to strings. + + :param obj: A file system path object or something else. + :return: A string representation of a file system path object or, for + anything that was not a file system path object, the original + object. + """ + if obj and hasattr(obj, "__fspath__"): + return obj.__fspath__() + return obj + + +def _windows_resolve(command, path=None): + """ + Try and find the full path and file extension of the executable to run. + This is so that e.g. calls to 'somescript' will point at 'somescript.cmd' + without the need to set shell=True in the subprocess. + + :param command: The command array to be run, with the first element being + the command with or w/o path, with or w/o extension. + :return: Returns the command array with the executable resolved with the + correct extension. If the executable cannot be resolved for any + reason the original command array is returned. + """ + if not command or not isinstance(command[0], str): + return command + + found_executable = shutil.which(command[0], path=path) + if found_executable: + logger.debug("Resolved %s as %s", command[0], found_executable) + return (found_executable, *command[1:]) + + if "\\" in command[0]: + # Special case. shutil.which may not detect file extensions if a full + # path is given, so try to resolve the executable explicitly + for extension in os.getenv("PATHEXT").split(os.pathsep): + found_executable = shutil.which(command[0] + extension, path=path) + if found_executable: + return (found_executable, *command[1:]) + + logger.warning("Error trying to resolve the executable: %s", command[0]) + return command + + +def run( + command, + *, + timeout: Optional[float] = None, + callback_stderr: Optional[Callable] = None, + callback_stdout: Optional[Callable] = None, + creationflags: int = 0, + environment: Optional[dict[str, str]] = None, + environment_override: Optional[dict[str, str]] = None, + preexec_fn: Optional[Callable] = None, + print_stderr: bool = True, + print_stdout: bool = True, + raise_timeout_exception: Any = ..., + stdin: Optional[Union[bytes, int]] = None, + win32resolve: bool = True, + working_directory: Optional[str] = None, +) -> subprocess.CompletedProcess: + """ + Run an external process. + + File system path objects (PEP-519) are accepted in the command, environment, + and working directory arguments. + + :param array command: Command line to be run, specified as array. + :param timeout: Terminate program execution after this many seconds. + :param stdin: Optional bytestring that is passed to command stdin, + or subprocess.DEVNULL to disable stdin. + :param boolean print_stdout: Pass stdout through to sys.stdout. + :param boolean print_stderr: Pass stderr through to sys.stderr. + :param callback_stdout: Optional function which is called for each + stdout line. + :param callback_stderr: Optional function which is called for each + stderr line. + :param creationflags: flags that will be passed to subprocess call + :param dict environment: The full execution environment for the command. + :param dict environment_override: Change environment variables from the + current values for command execution. + :param preexec_fn: pre-execution function, will be passed to subprocess call + :param boolean win32resolve: If on Windows, find the appropriate executable + first. This allows running of .bat, .cmd, etc. + files without explicitly specifying their + extension. + :param string working_directory: If specified, run the executable from + within this working directory. + :param boolean raise_timeout_exception: Deprecated compatibility flag. + :return: The exit code, stdout, stderr (separately, as byte strings) + as a subprocess.CompletedProcess object. + """ + + logger.debug("Starting external process: %s", command) + + if stdin is None: + stdin_pipe = None + elif isinstance(stdin, int): + assert ( + stdin == subprocess.DEVNULL + ), "stdin argument only allows subprocess.DEVNULL as numeric argument" + stdin_pipe = subprocess.DEVNULL + stdin = None + else: + assert sys.platform != "win32", "stdin argument not supported on Windows" + stdin_pipe = subprocess.PIPE + + start_time = timeit.default_timer() + if timeout is not None: + max_time = start_time + timeout + if not raise_timeout_exception: + warnings.warn( + "Using procrunner with raise_timeout_exception=False is no longer supported", + UserWarning, + stacklevel=3, + ) + elif raise_timeout_exception is True: + warnings.warn( + "The raise_timeout_exception argument is deprecated and will be removed in a future release", + DeprecationWarning, + stacklevel=3, + ) + + if environment is not None: + env = {key: _path_resolve(environment[key]) for key in environment} + else: + env = {key: value for key, value in os.environ.items()} + if environment_override: + env.update( + { + key: str(_path_resolve(environment_override[key])) + for key in environment_override + } + ) + + command = tuple(_path_resolve(part) for part in command) + if win32resolve and sys.platform == "win32": + command = _windows_resolve(command) + + p = subprocess.Popen( + command, + shell=False, + cwd=working_directory, + env=env, + stdin=stdin_pipe, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + creationflags=creationflags, + preexec_fn=preexec_fn, + ) + + thread_pipe_pool = [] + notifyee, notifier = Pipe(False) + thread_pipe_pool.append(notifyee) + stdout = _NonBlockingStreamReader( + p.stdout, + output=print_stdout, + notify=notifier.close, + callback=callback_stdout, + ) + notifyee, notifier = Pipe(False) + thread_pipe_pool.append(notifyee) + stderr = _NonBlockingStreamReader( + p.stderr, + output=print_stderr, + notify=notifier.close, + callback=callback_stderr, + ) + if stdin is not None: + notifyee, notifier = Pipe(False) + thread_pipe_pool.append(notifyee) + _NonBlockingStreamWriter(p.stdin, data=stdin, notify=notifier.close) + + timeout_encountered = False + + while (p.returncode is None) and ( + (timeout is None) or (timeit.default_timer() < max_time) + ): + # wait for some time or until a stream is closed + try: + if thread_pipe_pool: + # Wait for up to 0.5 seconds or for a signal on a remaining stream, + # which could indicate that the process has terminated. + try: + event = thread_pipe_pool[0].poll(0.5) + except BrokenPipeError as e: + # on Windows this raises "BrokenPipeError: [Errno 109] The pipe has been ended" + # which is for all intents and purposes equivalent to a True return value. + if e.winerror != 109: + raise + event = True + if event: + # One-shot, so remove stream and watch remaining streams + thread_pipe_pool.pop(0) + else: + time.sleep(0.5) + except KeyboardInterrupt: + p.kill() # if user pressed Ctrl+C we won't be able to produce a proper report anyway + # but at least make sure the child process dies with us + raise + + # check if process is still running + p.poll() + + if p.returncode is None: + # timeout condition + timeout_encountered = True + logger.debug("timeout (T%.2fs)", timeit.default_timer() - max_time) + + # send terminate signal and wait some time for buffers to be read + p.terminate() + if thread_pipe_pool: + try: + thread_pipe_pool[0].poll(0.5) + except BrokenPipeError as e: + # on Windows this raises "BrokenPipeError: [Errno 109] The pipe has been ended" + # which is for all intents and purposes equivalent to a True return value. + if e.winerror != 109: + raise + thread_pipe_pool.pop(0) + if not stdout.has_finished() or not stderr.has_finished(): + time.sleep(2) + p.poll() + + if p.returncode is None: + # thread still alive + # send kill signal and wait some more time for buffers to be read + p.kill() + if thread_pipe_pool: + try: + thread_pipe_pool[0].poll(0.5) + except BrokenPipeError as e: + # on Windows this raises "BrokenPipeError: [Errno 109] The pipe has been ended" + # which is for all intents and purposes equivalent to a True return value. + if e.winerror != 109: + raise + thread_pipe_pool.pop(0) + if not stdout.has_finished() or not stderr.has_finished(): + time.sleep(5) + p.poll() + + if p.returncode is None: + raise RuntimeError("Process won't terminate") + + runtime = timeit.default_timer() - start_time + if timeout is not None: + logger.debug( + "Process ended after %.1f seconds with exit code %d (T%.2fs)", + runtime, + p.returncode, + timeit.default_timer() - max_time, + ) + else: + logger.debug( + "Process ended after %.1f seconds with exit code %d", runtime, p.returncode + ) + + output_stdout = stdout.get_output() + output_stderr = stderr.get_output() + + if timeout is not None and timeout_encountered: + raise subprocess.TimeoutExpired( + cmd=command, timeout=timeout, output=output_stdout, stderr=output_stderr + ) + + return subprocess.CompletedProcess( + args=command, + returncode=p.returncode, + stdout=output_stdout, + stderr=output_stderr, + ) diff --git a/src/procrunner.egg-info/PKG-INFO b/src/procrunner.egg-info/PKG-INFO new file mode 100644 index 0000000..fa32950 --- /dev/null +++ b/src/procrunner.egg-info/PKG-INFO @@ -0,0 +1,250 @@ +Metadata-Version: 2.1 +Name: procrunner +Version: 2.3.3 +Summary: Versatile utility function to run external processes +Author: Diamond Light Source - Scientific Software et al. +Author-email: scientificsoftware@diamond.ac.uk +License: BSD +Project-URL: Download, https://github.com/DiamondLightSource/python-procrunner/tags +Project-URL: Documentation, https://procrunner.readthedocs.io/ +Project-URL: GitHub, https://github.com/DiamondLightSource/python-procrunner +Project-URL: Bug-Tracker, https://github.com/DiamondLightSource/python-procrunner/issues +Keywords: procrunner +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Natural Language :: English +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Operating System :: OS Independent +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Requires-Python: >=3.7 +License-File: LICENSE + +========== +ProcRunner +========== + + +.. image:: https://img.shields.io/pypi/v/procrunner.svg + :target: https://pypi.python.org/pypi/procrunner + :alt: PyPI release + +.. image:: https://img.shields.io/conda/vn/conda-forge/procrunner.svg + :target: https://anaconda.org/conda-forge/procrunner + :alt: Conda Version + +.. image:: https://dev.azure.com/DLS-tooling/procrunner/_apis/build/status/CI?branchName=master + :target: https://github.com/DiamondLightSource/python-procrunner/commits/master + :alt: Build status + +.. image:: https://ci.appveyor.com/api/projects/status/jtq4brwri5q18d0u/branch/master + :target: https://ci.appveyor.com/project/Anthchirp/python-procrunner + :alt: Build status + +.. image:: https://readthedocs.org/projects/procrunner/badge/?version=latest + :target: https://procrunner.readthedocs.io/en/latest/?badge=latest + :alt: Documentation Status + +.. image:: https://img.shields.io/pypi/pyversions/procrunner.svg + :target: https://pypi.python.org/pypi/procrunner + :alt: Supported Python versions + +.. image:: https://img.shields.io/badge/code%20style-black-000000.svg + :target: https://github.com/ambv/black + :alt: Code style: black + +Versatile utility function to run external processes + +* Free software: BSD license +* Documentation: https://procrunner.readthedocs.io. + + +Features +-------- + +* runs an external process and waits for it to finish +* does not deadlock, no matter the process stdout/stderr output behaviour +* returns the exit code, stdout, stderr (separately, both as bytestrings), + as a subprocess.CompletedProcess object +* process can run in a custom environment, either as a modification of + the current environment or in a new environment from scratch +* stdin can be fed to the process +* stdout and stderr is printed by default, can be disabled +* stdout and stderr can be passed to any arbitrary function for + live processing (separately, both as unicode strings) +* optionally enforces a time limit on the process, raising a + subprocess.TimeoutExpired exception if it is exceeded. + + +Credits +------- + +This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypackage`_ project template. + +.. _Cookiecutter: https://github.com/audreyr/cookiecutter +.. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage + + +History +================== + +3.0.0 (2022-01-??) +------------------ +* Drop Python 3.6 support +* The run() function now returns a subprocess.CompletedProcess object, + which no longer allows array access operations + (those were deprecated in `#60 `_) +* The run() argument 'raise_timeout_exception' is now set by default, + a 'False' value will lead to a UserWarning and a behavioural change. + The argument is now deprecated and will be removed in a future version. + (previously introduced in `#61 `_) +* Calling the run() function with multiple unnamed arguments is no longer supported + (previously deprecated in `#62 `_) +* The run() function no longer accepts a 'debug' argument + (previously deprecated in `#63 `_) + +2.3.3 (2022-03-23) +------------------ +* Allow specifying 'preexec_fn' and 'creationflags' keywords, which will be passed through to + the subprocess call + +2.3.2 (2022-01-28) +------------------ +* The run() function now understands stdin=subprocess.DEVNULL to close the subprocess stdin, + rather than to connect through the existing stdin, which is the current default + +2.3.1 (2021-10-25) +------------------ +* Add Python 3.10 support + +2.3.0 (2020-10-29) +------------------ +* Add Python 3.9 support, drop Python 3.5 support +* Fix a file descriptor leak on subprocess execution + +2.2.0 (2020-09-07) +------------------ +* Calling the run() function with unnamed arguments (other than the command + list as the first argument) is now deprecated. As a number of arguments + will be removed in a future version the use of unnamed arguments will + cause future confusion. `Use explicit keyword arguments instead (#62). `_ +* `The run() function debug argument has been deprecated (#63). `_ + This is only used to debug the NonBlockingStream* classes. Those are due + to be replaced in a future release, so the argument will no longer serve + a purpose. Debugging information remains available via standard logging + mechanisms. +* Final version supporting Python 3.5 + +2.1.0 (2020-09-05) +------------------ +* `Deprecated array access on the return object (#60). `_ + The return object will become a subprocess.CompletedProcess in a future + release, which no longer allows array-based access. For a translation table + of array elements to attributes please see the pull request linked above. +* Add a `new parameter 'raise_timeout_exception' (#61). `_ + When set to 'True' a subprocess.TimeoutExpired exception is raised when the + process runtime exceeds the timeout threshold. This defaults to 'False' and + will be set to 'True' in a future release. + +2.0.0 (2020-06-24) +------------------ +* Python 3.5+ only, support for Python 2.7 has been dropped +* Deprecated function alias run_process() has been removed +* Fixed a stability issue on Windows + +1.1.0 (2019-11-04) +------------------ +* Add Python 3.8 support, drop Python 3.4 support + +1.0.2 (2019-05-20) +------------------ +* Stop environment override variables leaking into the process environment + +1.0.1 (2019-04-16) +------------------ +* Minor fixes on the return object (implement equality, + mark as unhashable) + +1.0.0 (2019-03-25) +------------------ +* Support file system path objects (PEP-519) in arguments +* Change the return object to make it similar to + subprocess.CompletedProcess, introduced with Python 3.5+ + +0.9.1 (2019-02-22) +------------------ +* Have deprecation warnings point to correct code locations + +0.9.0 (2018-12-07) +------------------ +* Trap UnicodeEncodeError when printing output. Offending characters + are replaced and a warning is logged once. Hints at incorrectly set + PYTHONIOENCODING. + +0.8.1 (2018-12-04) +------------------ +* Fix a few deprecation warnings + +0.8.0 (2018-10-09) +------------------ +* Add parameter working_directory to set the working directory + of the subprocess + +0.7.2 (2018-10-05) +------------------ +* Officially support Python 3.7 + +0.7.1 (2018-09-03) +------------------ +* Accept environment variable overriding with numeric values. + +0.7.0 (2018-05-13) +------------------ +* Unicode fixes. Fix crash on invalid UTF-8 input. +* Clarify that stdout/stderr values are returned as bytestrings. +* Callbacks receive the data decoded as UTF-8 unicode strings + with unknown characters replaced by \ufffd (unicode replacement + character). Same applies to printing of output. +* Mark stdin broken on Windows. + +0.6.1 (2018-05-02) +------------------ +* Maintenance release to add some tests for executable resolution. + +0.6.0 (2018-05-02) +------------------ +* Fix Win32 API executable resolution for commands containing a dot ('.') in + addition to a file extension (say '.bat'). + +0.5.1 (2018-04-27) +------------------ +* Fix Win32API dependency installation on Windows. + +0.5.0 (2018-04-26) +------------------ +* New keyword 'win32resolve' which only takes effect on Windows and is enabled + by default. This causes procrunner to call the Win32 API FindExecutable() + function to try and lookup non-.exe files with the corresponding name. This + means .bat/.cmd/etc.. files can now be run without explicitly specifying + their extension. Only supported on Python 2.7 and 3.5+. + +0.4.0 (2018-04-23) +------------------ +* Python 2.7 support on Windows. Python3 not yet supported on Windows. + +0.3.0 (2018-04-17) +------------------ +* run_process() renamed to run() +* Python3 compatibility fixes + +0.2.0 (2018-03-12) +------------------ +* Procrunner is now Python3 3.3-3.6 compatible. + +0.1.0 (2018-03-12) +------------------ +* First release on PyPI. diff --git a/src/procrunner.egg-info/SOURCES.txt b/src/procrunner.egg-info/SOURCES.txt new file mode 100644 index 0000000..6df22a6 --- /dev/null +++ b/src/procrunner.egg-info/SOURCES.txt @@ -0,0 +1,29 @@ +AUTHORS.rst +CONTRIBUTING.rst +HISTORY.rst +LICENSE +MANIFEST.in +README.rst +pyproject.toml +setup.cfg +setup.py +docs/Makefile +docs/api.rst +docs/authors.rst +docs/conf.py +docs/contributing.rst +docs/history.rst +docs/index.rst +docs/installation.rst +docs/make.bat +docs/readme.rst +docs/usage.rst +src/procrunner/__init__.py +src/procrunner.egg-info/PKG-INFO +src/procrunner.egg-info/SOURCES.txt +src/procrunner.egg-info/dependency_links.txt +src/procrunner.egg-info/not-zip-safe +src/procrunner.egg-info/top_level.txt +tests/test_procrunner.py +tests/test_procrunner_resolution.py +tests/test_procrunner_system.py \ No newline at end of file diff --git a/src/procrunner.egg-info/dependency_links.txt b/src/procrunner.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/procrunner.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/src/procrunner.egg-info/not-zip-safe b/src/procrunner.egg-info/not-zip-safe new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/procrunner.egg-info/not-zip-safe @@ -0,0 +1 @@ + diff --git a/src/procrunner.egg-info/top_level.txt b/src/procrunner.egg-info/top_level.txt new file mode 100644 index 0000000..b2dc074 --- /dev/null +++ b/src/procrunner.egg-info/top_level.txt @@ -0,0 +1 @@ +procrunner diff --git a/tests/test_procrunner.py b/tests/test_procrunner.py index c645968..d645fd7 100644 --- a/tests/test_procrunner.py +++ b/tests/test_procrunner.py @@ -1,11 +1,36 @@ -from __future__ import absolute_import, division, print_function +from __future__ import annotations import copy -import mock import os +import pathlib +import sys +from unittest import mock + +import pytest + import procrunner -import pytest -import sys + + +@mock.patch("procrunner._NonBlockingStreamReader") +@mock.patch("procrunner.time") +@mock.patch("procrunner.subprocess") +@mock.patch("procrunner.Pipe") +def test_run_command_aborts_after_timeout_legacy( + mock_pipe, mock_subprocess, mock_time, mock_streamreader +): + mock_pipe.return_value = mock.Mock(), mock.Mock() + mock_process = mock.Mock() + mock_process.returncode = None + mock_subprocess.Popen.return_value = mock_process + task = ["___"] + + with pytest.raises(RuntimeError): + with pytest.warns(DeprecationWarning, match="timeout"): + procrunner.run(task, timeout=-1) + + assert mock_subprocess.Popen.called + assert mock_process.terminate.called + assert mock_process.kill.called @mock.patch("procrunner._NonBlockingStreamReader") @@ -22,7 +47,7 @@ task = ["___"] with pytest.raises(RuntimeError): - procrunner.run(task, -1, False) + procrunner.run(task, timeout=-1) assert mock_subprocess.Popen.called assert mock_process.terminate.called @@ -50,42 +75,31 @@ mock_streamreader.side_effect = streamreader_processing mock_subprocess.Popen.return_value = mock_process - expected = { - "stderr": mock.sentinel.proc_stderr, - "stdout": mock.sentinel.proc_stdout, - "exitcode": mock_process.returncode, - "command": tuple(command), - "runtime": mock.ANY, - "timeout": False, - "time_start": mock.ANY, - "time_end": mock.ANY, - } - actual = procrunner.run( command, - 0.5, - False, + timeout=0.5, callback_stdout=mock.sentinel.callback_stdout, callback_stderr=mock.sentinel.callback_stderr, - working_directory=mock.sentinel.cwd, + working_directory=pathlib.Path("somecwd"), ) assert mock_subprocess.Popen.called assert mock_subprocess.Popen.call_args[1]["env"] == os.environ - assert mock_subprocess.Popen.call_args[1]["cwd"] == mock.sentinel.cwd + assert mock_subprocess.Popen.call_args[1]["cwd"] in ( + pathlib.Path("somecwd"), + "somecwd", + ) mock_streamreader.assert_has_calls( [ mock.call( stream_stdout, output=mock.ANY, - debug=mock.ANY, notify=mock.ANY, callback=mock.sentinel.callback_stdout, ), mock.call( stream_stderr, output=mock.ANY, - debug=mock.ANY, notify=mock.ANY, callback=mock.sentinel.callback_stderr, ), @@ -94,19 +108,20 @@ ) assert not mock_process.terminate.called assert not mock_process.kill.called - for key in expected: - assert actual[key] == expected[key] - assert actual.args == tuple(command) - assert actual.returncode == mock_process.returncode - assert actual.stdout == mock.sentinel.proc_stdout - assert actual.stderr == mock.sentinel.proc_stderr + assert actual == mock_subprocess.CompletedProcess.return_value + mock_subprocess.CompletedProcess.assert_called_once_with( + args=tuple(command), + returncode=mock_process.returncode, + stdout=mock.sentinel.proc_stdout, + stderr=mock.sentinel.proc_stderr, + ) @mock.patch("procrunner.subprocess") def test_default_process_environment_is_parent_environment(mock_subprocess): mock_subprocess.Popen.side_effect = NotImplementedError() # cut calls short with pytest.raises(NotImplementedError): - procrunner.run([mock.Mock()], -1, False) + procrunner.run([mock.Mock()], timeout=-1) assert mock_subprocess.Popen.call_args[1]["env"] == os.environ @@ -116,7 +131,11 @@ mock_env = {"key": mock.sentinel.key} # Pass an environment dictionary with pytest.raises(NotImplementedError): - procrunner.run([mock.Mock()], -1, False, environment=copy.copy(mock_env)) + procrunner.run( + [mock.Mock()], + timeout=-1, + environment=copy.copy(mock_env), + ) assert mock_subprocess.Popen.call_args[1]["env"] == mock_env @@ -129,8 +148,7 @@ with pytest.raises(NotImplementedError): procrunner.run( [mock.Mock()], - -1, - False, + timeout=-1, environment=copy.copy(mock_env1), environment_override=copy.copy(mock_env2), ) @@ -145,12 +163,13 @@ mock_env2 = {"keyB": str(mock.sentinel.keyB)} with pytest.raises(NotImplementedError): procrunner.run( - [mock.Mock()], -1, False, environment_override=copy.copy(mock_env2) + [mock.Mock()], + timeout=-1, + environment_override=copy.copy(mock_env2), ) random_environment_variable = list(os.environ)[0] if random_environment_variable == list(mock_env2)[0]: random_environment_variable = list(os.environ)[1] - random_environment_value = os.getenv(random_environment_variable) assert ( random_environment_variable and random_environment_variable != list(mock_env2)[0] @@ -172,8 +191,7 @@ with pytest.raises(NotImplementedError): procrunner.run( [mock.Mock()], - -1, - False, + timeout=-1, environment_override={ random_environment_variable: "X" + random_environment_value }, @@ -192,7 +210,7 @@ def test_nonblockingstreamreader_can_read(mock_select): import time - class _stream(object): + class _stream: def __init__(self): self.data = b"" self.closed = False @@ -259,54 +277,3 @@ callback.assert_not_called() aggregator.flush() callback.assert_called_once_with("morestuff") - - -def test_return_object_semantics(): - ro = procrunner.ReturnObject( - { - "command": mock.sentinel.command, - "exitcode": 0, - "stdout": mock.sentinel.stdout, - "stderr": mock.sentinel.stderr, - } - ) - assert ro["command"] == mock.sentinel.command - assert ro.args == mock.sentinel.command - assert ro["exitcode"] == 0 - assert ro.returncode == 0 - assert ro["stdout"] == mock.sentinel.stdout - assert ro.stdout == mock.sentinel.stdout - assert ro["stderr"] == mock.sentinel.stderr - assert ro.stderr == mock.sentinel.stderr - - with pytest.raises(KeyError): - ro["unknownkey"] - ro.update({"unknownkey": mock.sentinel.key}) - assert ro["unknownkey"] == mock.sentinel.key - - -def test_return_object_check_function_passes_on_success(): - ro = procrunner.ReturnObject( - { - "command": mock.sentinel.command, - "exitcode": 0, - "stdout": mock.sentinel.stdout, - "stderr": mock.sentinel.stderr, - } - ) - ro.check_returncode() - - -def test_return_object_check_function_raises_on_error(): - ro = procrunner.ReturnObject( - { - "command": mock.sentinel.command, - "exitcode": 1, - "stdout": mock.sentinel.stdout, - "stderr": mock.sentinel.stderr, - } - ) - with pytest.raises(Exception) as e: - ro.check_returncode() - assert repr(mock.sentinel.command) in str(e.value) - assert "1" in str(e.value) diff --git a/tests/test_procrunner_resolution.py b/tests/test_procrunner_resolution.py index 33738f8..4c557dd 100644 --- a/tests/test_procrunner_resolution.py +++ b/tests/test_procrunner_resolution.py @@ -1,11 +1,9 @@ -from __future__ import absolute_import, division, print_function - import os import sys -import mock +import pytest + import procrunner -import pytest def PEP519(path): @@ -47,11 +45,6 @@ @pytest.mark.skipif(sys.platform != "win32", reason="windows specific test only") -def test_pywin32_import(): - import win32api - - -@pytest.mark.skipif(sys.platform != "win32", reason="windows specific test only") def test_name_resolution_for_simple_exe(): command = ["cmd.exe", "/c", "echo", "hello"] @@ -66,27 +59,25 @@ @pytest.mark.skipif(sys.platform != "win32", reason="windows specific test only") -def test_name_resolution_for_complex_cases(tmpdir): - tmpdir.chdir() - +def test_name_resolution_for_complex_cases(tmp_path): bat = "simple_bat_extension" cmd = "simple_cmd_extension" exe = "simple_exe_extension" dotshort = "more_complex_filename_with_a.dot" dotlong = "more_complex_filename.withadot" - (tmpdir / bat + ".bat").ensure() - (tmpdir / cmd + ".cmd").ensure() - (tmpdir / exe + ".exe").ensure() - (tmpdir / dotshort + ".bat").ensure() - (tmpdir / dotlong + ".cmd").ensure() + (tmp_path / (bat + ".bat")).touch() + (tmp_path / (cmd + ".cmd")).touch() + (tmp_path / (exe + ".exe")).touch() + (tmp_path / (dotshort + ".bat")).touch() + (tmp_path / (dotlong + ".cmd")).touch() def is_valid(command): assert len(command) == 1 - assert os.path.exists(command[0]) + assert os.path.exists(tmp_path / command[0]) - is_valid(procrunner._windows_resolve([bat])) - is_valid(procrunner._windows_resolve([cmd])) - is_valid(procrunner._windows_resolve([exe])) - is_valid(procrunner._windows_resolve([dotshort])) - is_valid(procrunner._windows_resolve([dotlong])) + is_valid(procrunner._windows_resolve([bat], path=os.fspath(tmp_path))) + is_valid(procrunner._windows_resolve([cmd], path=os.fspath(tmp_path))) + is_valid(procrunner._windows_resolve([exe], path=os.fspath(tmp_path))) + is_valid(procrunner._windows_resolve([dotshort], path=os.fspath(tmp_path))) + is_valid(procrunner._windows_resolve([dotlong], path=os.fspath(tmp_path))) diff --git a/tests/test_procrunner_system.py b/tests/test_procrunner_system.py index 5955463..84bd420 100644 --- a/tests/test_procrunner_system.py +++ b/tests/test_procrunner_system.py @@ -1,10 +1,13 @@ -from __future__ import absolute_import, division, print_function +from __future__ import annotations import os +import subprocess import sys +import timeit + +import pytest import procrunner -import pytest def test_simple_command_invocation(): @@ -14,6 +17,19 @@ command = ["echo", "hello"] result = procrunner.run(command) + + assert result.returncode == 0 + assert result.stdout == b"hello" + os.linesep.encode("utf-8") + assert result.stderr == b"" + + +def test_simple_command_invocation_with_closed_stdin(): + if os.name == "nt": + command = ["cmd.exe", "/c", "echo", "hello"] + else: + command = ["echo", "hello"] + + result = procrunner.run(command, stdin=subprocess.DEVNULL) assert result.returncode == 0 assert result.stdout == b"hello" + os.linesep.encode("utf-8") @@ -36,15 +52,14 @@ else: assert result.stdout == test_string out, err = capsys.readouterr() - assert out == u"test\ufffdstring\n" - assert err == u"" + assert out == "test\ufffdstring\n" + assert err == "" -def test_running_wget(tmpdir): - tmpdir.chdir() +def test_running_wget(tmp_path): command = ["wget", "https://www.google.com", "-O", "-"] try: - result = procrunner.run(command) + result = procrunner.run(command, working_directory=tmp_path) except OSError as e: if e.errno == 2: pytest.skip("wget not available") @@ -54,15 +69,17 @@ assert b"google" in result.stdout -def test_path_object_resolution(tmpdir): +def test_path_object_resolution(tmp_path): sentinel_value = b"sentinel" - tmpdir.join("tempfile").write(sentinel_value) - tmpdir.join("reader.py").write("print(open('tempfile').read())") + tmp_path.joinpath("tempfile").write_bytes(sentinel_value) + tmp_path.joinpath("reader.py").write_text( + "with open('tempfile') as fh:\n print(fh.read())" + ) assert "LEAK_DETECTOR" not in os.environ result = procrunner.run( - [sys.executable, tmpdir.join("reader.py")], + [sys.executable, tmp_path / "reader.py"], environment_override={"PYTHONHASHSEED": "random", "LEAK_DETECTOR": "1"}, - working_directory=tmpdir, + working_directory=tmp_path, ) assert result.returncode == 0 assert not result.stderr @@ -70,3 +87,76 @@ assert ( "LEAK_DETECTOR" not in os.environ ), "overridden environment variable leaked into parent process" + + +def test_timeout_behaviour_old_legacy(tmp_path): + command = (sys.executable, "-c", "import time; time.sleep(5)") + start = timeit.default_timer() + try: + with pytest.raises(subprocess.TimeoutExpired) as te: + with pytest.warns(UserWarning, match="timeout"): + procrunner.run( + command, + timeout=0.1, + working_directory=tmp_path, + raise_timeout_exception=False, + ) + except RuntimeError: + # This test sometimes fails with a RuntimeError. + runtime = timeit.default_timer() - start + assert runtime < 3 + return + runtime = timeit.default_timer() - start + assert runtime < 3 + assert te.value.stdout == b"" + assert te.value.stderr == b"" + assert te.value.timeout == 0.1 + assert te.value.cmd == command + + +def test_timeout_behaviour_legacy(tmp_path): + command = (sys.executable, "-c", "import time; time.sleep(5)") + start = timeit.default_timer() + try: + with pytest.raises(subprocess.TimeoutExpired) as te: + with pytest.warns(DeprecationWarning, match="timeout"): + procrunner.run( + command, + timeout=0.1, + working_directory=tmp_path, + raise_timeout_exception=True, + ) + except RuntimeError: + # This test sometimes fails with a RuntimeError. + runtime = timeit.default_timer() - start + assert runtime < 3 + return + runtime = timeit.default_timer() - start + assert runtime < 3 + assert te.value.stdout == b"" + assert te.value.stderr == b"" + assert te.value.timeout == 0.1 + assert te.value.cmd == command + + +def test_timeout_behaviour(tmp_path): + command = (sys.executable, "-c", "import time; time.sleep(5)") + start = timeit.default_timer() + try: + with pytest.raises(subprocess.TimeoutExpired) as te: + procrunner.run( + command, + timeout=0.1, + working_directory=tmp_path, + ) + except RuntimeError: + # This test sometimes fails with a RuntimeError. + runtime = timeit.default_timer() - start + assert runtime < 3 + return + runtime = timeit.default_timer() - start + assert runtime < 3 + assert te.value.stdout == b"" + assert te.value.stderr == b"" + assert te.value.timeout == 0.1 + assert te.value.cmd == command diff --git a/tox.ini b/tox.ini deleted file mode 100644 index e6887db..0000000 --- a/tox.ini +++ /dev/null @@ -1,26 +0,0 @@ -[tox] -envlist = py27, py35, py36, py37, py38, flake8 - -[travis] -python = - 3.8: py38 - 3.7: py37 - 3.6: py36 - 3.5: py35 - 2.7: py27 - -[testenv:flake8] -basepython = python -deps = flake8 -commands = flake8 procrunner - -[testenv] -setenv = - PYTHONPATH = {toxinidir} -deps = - -r{toxinidir}/requirements_dev.txt -; If you want to make tox run the tests with the same versions, create a -; requirements.txt with the pinned versions and uncomment the following line: -; -r{toxinidir}/requirements.txt -commands = - pytest -ra --basetemp={envtmpdir}