Codebase list python-procrunner / 1a4134e
New upstream release. Debian Janitor 2 years ago
28 changed file(s) with 842 addition(s) and 460 deletion(s). Raw diff Collapse all Expand all
0 steps:
1 - task: UsePythonVersion@0
2 inputs:
3 versionSpec: '$(PYTHON_VERSION)'
4 displayName: 'Use Python $(PYTHON_VERSION)'
5
6 - script: |
7 python -m pip install --upgrade pip
8 pip install tox
9 displayName: "Set up tox"
10
11 - script: |
12 tox -e azure
13 displayName: "Run tests"
14
15 - task: PublishTestResults@2
16 condition: succeededOrFailed()
17 inputs:
18 testResultsFiles: '**/test-*.xml'
19 testRunTitle: 'Publish test results for Python $(PYTHON_VERSION)'
0 import os
1 import subprocess
2
3 # Flake8 validation
4 failures = 0
5 try:
6 flake8 = subprocess.run(
7 [
8 "flake8",
9 "--exit-zero",
10 ],
11 capture_output=True,
12 check=True,
13 encoding="latin-1",
14 timeout=300,
15 )
16 except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
17 print(
18 "##vso[task.logissue type=error;]flake8 validation failed with",
19 str(e.__class__.__name__),
20 )
21 print(e.stdout)
22 print(e.stderr)
23 print("##vso[task.complete result=Failed;]flake8 validation failed")
24 exit()
25 for line in flake8.stdout.split("\n"):
26 if ":" not in line:
27 continue
28 filename, lineno, column, error = line.split(":", maxsplit=3)
29 errcode, error = error.strip().split(" ", maxsplit=1)
30 filename = os.path.normpath(filename)
31 failures += 1
32 print(
33 f"##vso[task.logissue type=error;sourcepath={filename};"
34 f"linenumber={lineno};columnnumber={column};code={errcode};]" + error
35 )
36
37 if failures:
38 print(f"##vso[task.logissue type=warning]Found {failures} flake8 violation(s)")
39 print(f"##vso[task.complete result=Failed;]Found {failures} flake8 violation(s)")
0 import ast
1 import os
2 import sys
3
4 print("Python", sys.version, "\n")
5
6 failures = 0
7
8 for base, _, files in os.walk("."):
9 for f in files:
10 if not f.endswith(".py"):
11 continue
12 filename = os.path.normpath(os.path.join(base, f))
13 try:
14 with open(filename) as fh:
15 ast.parse(fh.read())
16 except SyntaxError as se:
17 failures += 1
18 print(
19 f"##vso[task.logissue type=error;sourcepath={filename};"
20 f"linenumber={se.lineno};columnnumber={se.offset};]"
21 f"SyntaxError: {se.msg}"
22 )
23 print(" " + se.text + " " * se.offset + "^")
24 print(f"SyntaxError: {se.msg} in {filename} line {se.lineno}")
25 print()
26
27 if failures:
28 print(f"##vso[task.logissue type=warning]Found {failures} syntax error(s)")
29 print(f"##vso[task.complete result=Failed;]Found {failures} syntax error(s)")
0 trigger:
1 branches:
2 include:
3 - '*'
4 tags:
5 include:
6 - '*'
7
8 stages:
9 - stage: static
10 displayName: Static Analysis
11 jobs:
12 - job: checks
13 displayName: static code analysis
14 pool:
15 vmImage: ubuntu-latest
16 steps:
17 # Run syntax validation using oldest and latest Python
18 - task: UsePythonVersion@0
19 displayName: Set up python
20 inputs:
21 versionSpec: 3.6
22
23 - bash: python .azure-pipelines/syntax-validation.py
24 displayName: Syntax validation (3.6)
25
26 - task: UsePythonVersion@0
27 displayName: Set up python
28 inputs:
29 versionSpec: 3.10
30
31 - bash: python .azure-pipelines/syntax-validation.py
32 displayName: Syntax validation (3.10)
33
34 # Run flake8 validation
35 - bash: |
36 pip install --disable-pip-version-check flake8 && \
37 python .azure-pipelines/flake8-validation.py
38 displayName: Flake8 validation
39
40 - stage: tests
41 displayName: Run unit tests
42 jobs:
43 - job: linux
44 pool:
45 vmImage: ubuntu-latest
46 strategy:
47 matrix:
48 python36:
49 PYTHON_VERSION: 3.6
50 python37:
51 PYTHON_VERSION: 3.7
52 python38:
53 PYTHON_VERSION: 3.8
54 python39:
55 PYTHON_VERSION: 3.9
56 python310:
57 PYTHON_VERSION: 3.10
58 steps:
59 - template: .azure-pipelines/ci.yml
60
61 - job: macOS
62 pool:
63 vmImage: macOS-latest
64 strategy:
65 matrix:
66 python37:
67 PYTHON_VERSION: 3.7
68 python38:
69 PYTHON_VERSION: 3.8
70 python39:
71 PYTHON_VERSION: 3.9
72 python310:
73 PYTHON_VERSION: 3.10
74 steps:
75 - template: .azure-pipelines/ci.yml
76
77 - job: windows
78 pool:
79 vmImage: windows-latest
80 strategy:
81 matrix:
82 python36:
83 PYTHON_VERSION: 3.6
84 python37:
85 PYTHON_VERSION: 3.7
86 python38:
87 PYTHON_VERSION: 3.8
88 python39:
89 PYTHON_VERSION: 3.9
90 python310:
91 PYTHON_VERSION: 3.10
92 steps:
93 - template: .azure-pipelines/ci.yml
94
95 - stage: deploy
96 displayName: Publish release
97 dependsOn:
98 - tests
99 - static
100 condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'))
101 jobs:
102 - job: pypi
103 displayName: Publish pypi release
104 pool:
105 vmImage: ubuntu-latest
106 steps:
107 - task: UsePythonVersion@0
108 displayName: Set up python
109 inputs:
110 versionSpec: 3.9
111
112 - bash: |
113 python -m pip install -r requirements_dev.txt
114 displayName: Install dependencies
115
116 - bash: |
117 python setup.py sdist bdist_wheel
118 ls -la dist
119 displayName: Build python package
120
121 - task: PublishBuildArtifacts@1
122 inputs:
123 pathToPublish: dist/
124 artifactName: python-release
125
126 - task: TwineAuthenticate@1
127 displayName: Set up credentials
128 inputs:
129 pythonUploadServiceConnection: pypi-procrunner
130
131 - bash: |
132 python -m twine upload -r pypi-procrunner --config-file $(PYPIRC_PATH) dist/*.tar.gz dist/*.whl
133 displayName: Publish package
0 [bumpversion]
1 current_version = 2.3.1
2 commit = True
3 tag = True
4
5 [bumpversion:file:setup.cfg]
6 search = version = {current_version}
7 replace = version = {new_version}
8
9 [bumpversion:file:procrunner/__init__.py]
10 search = __version__ = "{current_version}"
11 replace = __version__ = "{new_version}"
0 {
1 "extends": [
2 "config:base"
3 ],
4 "labels": [
5 "dependencies"
6 ],
7 "pip_requirements": {
8 "fileMatch": [
9 "^requirements.*\\.txt$"
10 ],
11 "groupName": "all dependencies",
12 "groupSlug": "all",
13 "packageRules": [
14 {
15 "groupName": "all dependencies",
16 "groupSlug": "all",
17 "matchPackagePatterns": [
18 "*"
19 ]
20 }
21 ]
22 },
23 "prCreation": "not-pending",
24 "prHourlyLimit": 2,
25 "pre-commit": {
26 "schedule": [
27 "after 10am and before 4pm every 3 months on the first day of the month"
28 ],
29 "stabilityDays": 10
30 },
31 "schedule": [
32 "after 7am and before 4pm every monday"
33 ],
34 "stabilityDays": 2,
35 "timezone": "Europe/London"
36 }
0 repos:
1
2 # Automatically sort imports
3 - repo: https://github.com/PyCQA/isort
4 rev: 5.9.3
5 hooks:
6 - id: isort
7
8 # Automatic source code formatting
9 - repo: https://github.com/psf/black
10 rev: 21.6b0
11 hooks:
12 - id: black
13 args: [--safe, --quiet]
14
15 # Linting
16 - repo: https://github.com/PyCQA/flake8
17 rev: 3.9.2
18 hooks:
19 - id: flake8
20 additional_dependencies: ['flake8-comprehensions==3.5.0']
21
22 # Syntax validation and some basic sanity checks
23 - repo: https://github.com/pre-commit/pre-commit-hooks
24 rev: v4.0.1
25 hooks:
26 - id: check-merge-conflict
27 - id: check-ast
28 - id: check-json
29 - id: check-added-large-files
30 args: ['--maxkb=200']
31 - id: check-yaml
+0
-5
.pyup.yml less more
0 # autogenerated pyup.io config file
1 # see https://pyup.io/docs/configuration/ for all available options
2
3 schedule: every month
4
+0
-72
.travis.yml less more
0 # Config file for automatic testing at travis-ci.org
1
2 language: python
3
4 matrix:
5 include:
6 - python: 3.8
7 dist: xenial
8 sudo: true
9 - python: 3.7
10 dist: xenial
11 sudo: true
12 - python: 3.6
13 - python: 3.5
14 - python: 2.7
15 - python: pypy
16 - os: osx
17 language: generic
18 env: CONDA=3.8 TOXENV=py38
19 - os: osx
20 language: generic
21 env: CONDA=3.7 TOXENV=py37
22 - os: osx
23 language: generic
24 env: CONDA=3.6 TOXENV=py36
25 - os: osx
26 language: generic
27 env: CONDA=3.5 TOXENV=py35
28 - os: osx
29 language: generic
30 env: CONDA=2.7 TOXENV=py27
31
32 allow_failures:
33 - env: OPTIONAL=1
34
35 fast_finish: true
36
37 before_install: |
38 if [ ! -z "$CONDA" ]; then
39 if [ "$TRAVIS_OS_NAME" == "osx" ]; then
40 curl https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh --output miniconda.sh
41 fi
42 chmod +x miniconda.sh
43 ./miniconda.sh -b
44 export PATH=$HOME/miniconda3/bin:$PATH
45 conda update --yes conda
46 conda create --yes -n travis python=$CONDA
47 source activate travis
48 # A manual check that the correct version of Python is running.
49 python --version
50 fi
51
52 # Command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors
53 install: pip install -U tox-travis
54
55 # Command to run tests, e.g. python setup.py test
56 script: tox
57
58 # Assuming you have installed the travis-ci CLI tool, after you
59 # create the Github repo and add it to Travis, run the
60 # following command to finish PyPI deployment setup:
61 # $ travis encrypt --add deploy.password
62 deploy:
63 provider: pypi
64 distributions: sdist bdist_wheel
65 user: mgerstel
66 password:
67 secure: qN3EYVlH22eLPZp5CSvc5Bz8bpP0l0wBZ8tcYMa6kBrqelYdIgqkuZESR/oCMu1YBA3AbcKJNwMuzuWf8RNGAFebD820OJThdP4czRMCv6LDbWABnv12lFHQLQSW1fMQVvb2arjnE/Ew7BFq70p0wOlIJJRwu6CoeOXW/sMeVYMivxdaHmgORq3cdMluFAy4amVb3Fc8i7mxAM0QGklO7x/UWJR/IEpUk2RlUbXL+HrzNoEjRtDeMxoCR84gKZTjVeUQ/iIQSuWwxlt7v1FNENj6ZEbE7+PS8/ylIVfPufbCr8tEEv8W58QcxQ5xPJC2g85ulsN5dM9/9FekhpyKa25B/4wKUNq5T8rKJ8WZ6hMiGffW8rmAfrGTmrBaojKBi0pb9VfXJ5KXUcunVXwQaAn2L80jLLHNsAo94ZxeoowD1eJox9Wh1NtNc+NiUv8K6spOIBsur7G5GY4JVA/yZ7w+DweEfQEp8/SEdVhK0vEYSYT4FnJHuAAmNgeedyAtoes4+a5bYYUM4qrz2OC78NWQWAnnsZhD4Y/TulkavWSexVCqfSePRoK3gcCs+rXwiM69XkMbL1Wgj1gNou+XMkntayH2ZDHkkyJi5F7ls4nqMH5RON9FfVygJMoHDRqh0p4RV25IzJ4FSYqKihHNBO31/URnU2ihpW7n8kM+mbM=
68 on:
69 tags: true
70 repo: DiamondLightSource/python-procrunner
71 python: 3.8
100100 1. The pull request should include tests.
101101 2. If the pull request adds functionality, the docs should be updated. Put
102102 your new functionality into a function with a docstring, and add the
103 feature to the list in README.rst.
104 3. The pull request should work for Python 2.7, 3.5, 3.6, 3.7, 3.8, and for PyPy. Check
105 https://travis-ci.org/DiamondLightSource/python-procrunner/pull_requests
106 and make sure that the tests pass for all supported Python versions.
103 feature to the list in HISTORY.rst/README.rst.
107104
108105 Tips
109106 ----
11 History
22 =======
33
4 2.3.1 (2021-10-25)
5 ------------------
6 * Add Python 3.10 support
7
8 2.3.0 (2020-10-29)
9 ------------------
10 * Add Python 3.9 support, drop Python 3.5 support
11 * Fix a file descriptor leak on subprocess execution
12
13 2.2.0 (2020-09-07)
14 ------------------
15 * Calling the run() function with unnamed arguments (other than the command
16 list as the first argument) is now deprecated. As a number of arguments
17 will be removed in a future version the use of unnamed arguments will
18 cause future confusion. `Use explicit keyword arguments instead (#62). <https://github.com/DiamondLightSource/python-procrunner/pull/62>`_
19 * `The run() function debug argument has been deprecated (#63). <https://github.com/DiamondLightSource/python-procrunner/pull/63>`_
20 This is only used to debug the NonBlockingStream* classes. Those are due
21 to be replaced in a future release, so the argument will no longer serve
22 a purpose. Debugging information remains available via standard logging
23 mechanisms.
24 * Final version supporting Python 3.5
25
26 2.1.0 (2020-09-05)
27 ------------------
28 * `Deprecated array access on the return object (#60). <https://github.com/DiamondLightSource/python-procrunner/pull/60>`_
29 The return object will become a subprocess.CompletedProcess in a future
30 release, which no longer allows array-based access. For a translation table
31 of array elements to attributes please see the pull request linked above.
32 * Add a `new parameter 'raise_timeout_exception' (#61). <https://github.com/DiamondLightSource/python-procrunner/pull/61>`_
33 When set to 'True' a subprocess.TimeoutExpired exception is raised when the
34 process runtime exceeds the timeout threshold. This defaults to 'False' and
35 will be set to 'True' in a future release.
36
37 2.0.0 (2020-06-24)
38 ------------------
39 * Python 3.5+ only, support for Python 2.7 has been dropped
40 * Deprecated function alias run_process() has been removed
41 * Fixed a stability issue on Windows
42
443 1.1.0 (2019-11-04)
544 ------------------
6
745 * Add Python 3.8 support, drop Python 3.4 support
846
947 1.0.2 (2019-05-20)
1048 ------------------
11
1249 * Stop environment override variables leaking into the process environment
1350
1451 1.0.1 (2019-04-16)
1552 ------------------
16
1753 * Minor fixes on the return object (implement equality,
1854 mark as unhashable)
1955
2056 1.0.0 (2019-03-25)
2157 ------------------
22
2358 * Support file system path objects (PEP-519) in arguments
2459 * Change the return object to make it similar to
2560 subprocess.CompletedProcess, introduced with Python 3.5+
2661
2762 0.9.1 (2019-02-22)
2863 ------------------
29
3064 * Have deprecation warnings point to correct code locations
3165
3266 0.9.0 (2018-12-07)
3367 ------------------
34
3568 * Trap UnicodeEncodeError when printing output. Offending characters
3669 are replaced and a warning is logged once. Hints at incorrectly set
3770 PYTHONIOENCODING.
3871
3972 0.8.1 (2018-12-04)
4073 ------------------
41
4274 * Fix a few deprecation warnings
4375
4476 0.8.0 (2018-10-09)
4577 ------------------
46
4778 * Add parameter working_directory to set the working directory
4879 of the subprocess
4980
5081 0.7.2 (2018-10-05)
5182 ------------------
52
5383 * Officially support Python 3.7
5484
5585 0.7.1 (2018-09-03)
5686 ------------------
57
5887 * Accept environment variable overriding with numeric values.
5988
6089 0.7.0 (2018-05-13)
6190 ------------------
62
6391 * Unicode fixes. Fix crash on invalid UTF-8 input.
6492 * Clarify that stdout/stderr values are returned as bytestrings.
6593 * Callbacks receive the data decoded as UTF-8 unicode strings
6997
7098 0.6.1 (2018-05-02)
7199 ------------------
72
73100 * Maintenance release to add some tests for executable resolution.
74101
75102 0.6.0 (2018-05-02)
76103 ------------------
77
78104 * Fix Win32 API executable resolution for commands containing a dot ('.') in
79105 addition to a file extension (say '.bat').
80106
81107 0.5.1 (2018-04-27)
82108 ------------------
83
84109 * Fix Win32API dependency installation on Windows.
85110
86111 0.5.0 (2018-04-26)
87112 ------------------
88
89113 * New keyword 'win32resolve' which only takes effect on Windows and is enabled
90114 by default. This causes procrunner to call the Win32 API FindExecutable()
91115 function to try and lookup non-.exe files with the corresponding name. This
94118
95119 0.4.0 (2018-04-23)
96120 ------------------
97
98121 * Python 2.7 support on Windows. Python3 not yet supported on Windows.
99122
100123 0.3.0 (2018-04-17)
101124 ------------------
102
103125 * run_process() renamed to run()
104126 * Python3 compatibility fixes
105127
106128 0.2.0 (2018-03-12)
107129 ------------------
108
109130 * Procrunner is now Python3 3.3-3.6 compatible.
110131
111132 0.1.0 (2018-03-12)
112133 ------------------
113
114134 * First release on PyPI.
0 Copyright (c) 2018 Diamond Light Source.
0 Copyright (c) 2018-2021 Diamond Light Source.
11 All rights reserved.
22
33 Redistribution and use in source and binary forms, with or without
1010 :target: https://anaconda.org/conda-forge/procrunner
1111 :alt: Conda Version
1212
13 .. image:: https://travis-ci.org/DiamondLightSource/python-procrunner.svg?branch=master
14 :target: https://travis-ci.org/DiamondLightSource/python-procrunner
13 .. image:: https://dev.azure.com/DLS-tooling/procrunner/_apis/build/status/CI?branchName=master
14 :target: https://github.com/DiamondLightSource/python-procrunner/commits/master
1515 :alt: Build status
1616
1717 .. image:: https://ci.appveyor.com/api/projects/status/jtq4brwri5q18d0u/branch/master
2121 .. image:: https://readthedocs.org/projects/procrunner/badge/?version=latest
2222 :target: https://procrunner.readthedocs.io/en/latest/?badge=latest
2323 :alt: Documentation Status
24
25 .. image:: https://pyup.io/repos/github/DiamondLightSource/python-procrunner/shield.svg
26 :target: https://pyup.io/repos/github/DiamondLightSource/python-procrunner/
27 :alt: Updates
2824
2925 .. image:: https://img.shields.io/pypi/pyversions/procrunner.svg
3026 :target: https://pypi.python.org/pypi/procrunner
4642 * runs an external process and waits for it to finish
4743 * does not deadlock, no matter the process stdout/stderr output behaviour
4844 * returns the exit code, stdout, stderr (separately, both as bytestrings),
49 and the total process runtime as a dictionary
45 as a subprocess.CompletedProcess object
5046 * process can run in a custom environment, either as a modification of
5147 the current environment or in a new environment from scratch
52 * stdin can be fed to the process, the returned dictionary contains
53 information how much was read by the process
48 * stdin can be fed to the process
5449 * stdout and stderr is printed by default, can be disabled
5550 * stdout and stderr can be passed to any arbitrary function for
5651 live processing (separately, both as unicode strings)
57 * optionally enforces a time limit on the process
52 * optionally enforces a time limit on the process, raising a
53 subprocess.TimeoutExpired exception if it is exceeded.
54
5855
5956 Credits
6057 -------
44 # For Python versions available on Appveyor, see
55 # http://www.appveyor.com/docs/installed-software#python
66
7 - PYTHON: "C:\\Python27"
8 - PYTHON: "C:\\Python35"
9 UNSTABLE: 1
107 - PYTHON: "C:\\Python36"
11 UNSTABLE: 1
128 - PYTHON: "C:\\Python37"
13 UNSTABLE: 1
149 - PYTHON: "C:\\Python38"
15 UNSTABLE: 1
16 - PYTHON: "C:\\Python27-x64"
17 - PYTHON: "C:\\Python35-x64"
18 UNSTABLE: 1
1910 - PYTHON: "C:\\Python36-x64"
20 UNSTABLE: 1
2111 - PYTHON: "C:\\Python37-x64"
22 UNSTABLE: 1
2312 - PYTHON: "C:\\Python38-x64"
24 UNSTABLE: 1
2513
2614 matrix:
2715 allow_failures:
3119 # Upgrade to the latest pip.
3220 - '%PYTHON%\\python.exe -m pip install -U pip setuptools wheel'
3321 - '%PYTHON%\\python.exe -m pip install -r requirements_dev.txt'
34 # Install win32api dependency. Must use the --only-binary switch explicitly
35 # on AppVeyor
36 - "%PYTHON%\\python.exe -m pip install --only-binary pywin32 pywin32"
3722
3823 build: off
3924
4126 # Note that you must use the environment variable %PYTHON% to refer to
4227 # the interpreter you're using - Appveyor does not do anything special
4328 # to put the Python version you want to use on PATH.
44 - "%PYTHON%\\python.exe setup.py test"
29 - "%PYTHON%\\python.exe -m pytest"
4530
4631 after_test:
4732 # This step builds your wheels.
0 python-procrunner (2.3.1-1) UNRELEASED; urgency=low
1
2 * New upstream release.
3
4 -- Debian Janitor <janitor@jelmer.uk> Wed, 17 Nov 2021 20:42:32 -0000
5
06 python-procrunner (1.1.0-1) unstable; urgency=medium
17
28 * First release (Closes: #962456)
00 #!/usr/bin/env python
1 # -*- coding: utf-8 -*-
21 #
32 # procrunner documentation build configuration file, created by
43 # sphinx-quickstart on Fri Jun 9 13:47:02 2017.
4746 master_doc = "index"
4847
4948 # General information about the project.
50 project = u"ProcRunner"
51 copyright = u"2018, Markus Gerstel"
52 author = u"Markus Gerstel"
49 project = "ProcRunner"
50 copyright = "2020, Diamond Light Source"
51 author = "Diamond Light Source - Scientific Software"
5352
5453 # The version info for the project you're documenting, acts as replacement
5554 # for |version| and |release|, also used in various other places throughout
128127 (
129128 master_doc,
130129 "procrunner.tex",
131 u"ProcRunner Documentation",
132 u"Markus Gerstel",
130 "procrunner Documentation",
131 "Diamond Light Source - Scientific Software",
133132 "manual",
134133 )
135134 ]
139138
140139 # One entry per manual page. List of tuples
141140 # (source start file, name, description, authors, manual section).
142 man_pages = [(master_doc, "procrunner", u"ProcRunner Documentation", [author], 1)]
141 man_pages = [(master_doc, "procrunner", "procrunner Documentation", [author], 1)]
143142
144143
145144 # -- Options for Texinfo output ----------------------------------------
151150 (
152151 master_doc,
153152 "procrunner",
154 u"ProcRunner Documentation",
153 "procrunner Documentation",
155154 author,
156155 "procrunner",
157 "One line description of project.",
156 "Versatile utility function to run external processes",
158157 "Miscellaneous",
159158 )
160159 ]
88
99 To test for successful completion::
1010
11 assert not result['exitcode']
12 assert result['exitcode'] == 0 # alternatively
11 assert not result.returncode
12 assert result.returncode == 0 # alternatively
13 result.check_returncode() # raises subprocess.CalledProcessError()
1314
1415 To test for no STDERR output::
1516
16 assert not result['stderr']
17 assert result['stderr'] == b'' # alternatively
17 assert not result.stderr
18 assert result.stderr == b'' # alternatively
1819
1920 To run with a specific environment variable set::
2021
0 # -*- coding: utf-8 -*-
1
2 from __future__ import absolute_import, division, print_function
3
40 import codecs
1 import functools
2 import io
53 import logging
64 import os
75 import select
8 import six
6 import shutil
97 import subprocess
108 import sys
119 import time
2018 #
2119 # - runs an external process and waits for it to finish
2220 # - does not deadlock, no matter the process stdout/stderr output behaviour
23 # - returns the exit code, stdout, stderr (separately), and the total process
24 # runtime as a dictionary
21 # - returns the exit code, stdout, stderr (separately) as a
22 # subprocess.CompletedProcess object
2523 # - process can run in a custom environment, either as a modification of
2624 # the current environment or in a new environment from scratch
27 # - stdin can be fed to the process, the returned dictionary contains
28 # information how much was read by the process
25 # - stdin can be fed to the process
2926 # - stdout and stderr is printed by default, can be disabled
3027 # - stdout and stderr can be passed to any arbitrary function for
3128 # live processing
3936 #
4037 # Returns:
4138 #
42 # {'command': ['/bin/ls', '/some/path/containing spaces'],
43 # 'exitcode': 2,
44 # 'runtime': 0.12990689277648926,
45 # 'stderr': '/bin/ls: cannot access /some/path/containing spaces: No such file or directory\n',
46 # 'stdout': '',
47 # 'time_end': '2017-11-12 19:54:49 GMT',
48 # 'time_start': '2017-11-12 19:54:49 GMT',
49 # 'timeout': False}
50 #
39 # ReturnObject(
40 # args=('/bin/ls', '/some/path/containing spaces'),
41 # returncode=2,
42 # stdout=b'',
43 # stderr=b'/bin/ls: cannot access /some/path/containing spaces: No such file or directory\n'
44 # )
45 #
46 # which also offers (albeit deprecated)
47 #
48 # result.runtime == 0.12990689277648926
49 # result.time_end == '2017-11-12 19:54:49 GMT'
50 # result.time_start == '2017-11-12 19:54:49 GMT'
51 # result.timeout == False
5152
5253 __author__ = """Markus Gerstel"""
5354 __email__ = "scientificsoftware@diamond.ac.uk"
54 __version__ = "1.1.0"
55 __version__ = "2.3.1"
5556
5657 logger = logging.getLogger("procrunner")
5758 logger.addHandler(logging.NullHandler())
5859
5960
60 class _LineAggregator(object):
61 class _LineAggregator:
6162 """
6263 Buffer that can be filled with stream data and will aggregate complete
6364 lines. Lines can be printed or passed to an arbitrary callback function.
106107 self._buffer = ""
107108
108109
109 class _NonBlockingStreamReader(object):
110 class _NonBlockingStreamReader:
110111 """Reads a stream in a thread to avoid blocking/deadlocks"""
111112
112113 def __init__(self, stream, output=True, debug=False, notify=None, callback=None):
113114 """Creates and starts a thread which reads from a stream."""
114 self._buffer = six.BytesIO()
115 self._buffer = io.BytesIO()
115116 self._closed = False
116117 self._closing = False
117118 self._debug = debug
130131 else:
131132 if self._closing:
132133 break
134 self._stream.close()
133135 self._terminated = True
134136 la.flush()
135137 if self._debug:
149151 print(linedecode)
150152 if callback:
151153 callback(linedecode)
154 self._stream.close()
152155 self._terminated = True
153156 if self._debug:
154157 logger.debug("Stream reader terminated")
183186 if not self.has_finished():
184187 if self._debug:
185188 logger.debug(
186 "NBSR join after %f seconds, underrun not resolved"
187 % (timeit.default_timer() - underrun_debug_timer)
189 "NBSR join after %f seconds, underrun not resolved",
190 timeit.default_timer() - underrun_debug_timer,
188191 )
189192 raise Exception("thread did not terminate")
190193 if self._debug:
191194 logger.debug(
192 "NBSR underrun resolved after %f seconds"
193 % (timeit.default_timer() - underrun_debug_timer)
195 "NBSR underrun resolved after %f seconds",
196 timeit.default_timer() - underrun_debug_timer,
194197 )
195198 if self._closed:
196199 raise Exception("streamreader double-closed")
200203 return data
201204
202205
203 class _NonBlockingStreamWriter(object):
206 class _NonBlockingStreamWriter:
204207 """Writes to a stream in a thread to avoid blocking/deadlocks"""
205208
206209 def __init__(self, stream, data, debug=False, notify=None):
208211 self._buffer = data
209212 self._buffer_len = len(data)
210213 self._buffer_pos = 0
211 self._debug = debug
212214 self._max_block_len = 4096
213215 self._stream = stream
214216 self._terminated = False
223225 block = self._buffer[self._buffer_pos :]
224226 try:
225227 self._stream.write(block)
226 except IOError as e:
228 except OSError as e:
227229 if (
228230 e.errno == 32
229231 ): # broken pipe, ie. process terminated without reading entire stdin
235237 raise
236238 self._buffer_pos += len(block)
237239 if debug:
238 logger.debug("wrote %d bytes to stream" % len(block))
240 logger.debug("wrote %d bytes to stream", len(block))
239241 self._stream.close()
240242 self._terminated = True
241243 if notify:
272274 return obj
273275
274276
275 def _windows_resolve(command):
277 def _windows_resolve(command, path=None):
276278 """
277279 Try and find the full path and file extension of the executable to run.
278280 This is so that e.g. calls to 'somescript' will point at 'somescript.cmd'
279281 without the need to set shell=True in the subprocess.
280 If the executable contains periods it is a special case. Here the
281 win32api call will fail to resolve the extension automatically, and it
282 has do be done explicitly.
283282
284283 :param command: The command array to be run, with the first element being
285284 the command with or w/o path, with or w/o extension.
287286 correct extension. If the executable cannot be resolved for any
288287 reason the original command array is returned.
289288 """
290 try:
291 import win32api
292 except ImportError:
293 if (2, 8) < sys.version_info < (3, 5):
294 logger.info(
295 "Resolving executable names only supported on Python 2.7 and 3.5+"
296 )
297 else:
298 logger.warning(
299 "Could not resolve executable name: package win32api missing"
300 )
289 if not command or not isinstance(command[0], str):
301290 return command
302291
303 if not command or not isinstance(command[0], six.string_types):
304 return command
305
306 try:
307 _, found_executable = win32api.FindExecutable(command[0])
292 found_executable = shutil.which(command[0], path=path)
293 if found_executable:
308294 logger.debug("Resolved %s as %s", command[0], found_executable)
309 return (found_executable,) + tuple(command[1:])
310 except Exception as e:
311 if not hasattr(e, "winerror"):
312 raise
313 # Keep this error message for later in case we fail to resolve the name
314 logwarning = getattr(e, "strerror", str(e))
315
316 if "." in command[0]:
317 # Special case. The win32api may not properly check file extensions, so
318 # try to resolve the executable explicitly.
295 return (found_executable, *command[1:])
296
297 if "\\" in command[0]:
298 # Special case. shutil.which may not detect file extensions if a full
299 # path is given, so try to resolve the executable explicitly
319300 for extension in os.getenv("PATHEXT").split(os.pathsep):
320 try:
321 _, found_executable = win32api.FindExecutable(command[0] + extension)
322 logger.debug("Resolved %s as %s", command[0], found_executable)
323 return (found_executable,) + tuple(command[1:])
324 except Exception as e:
325 if not hasattr(e, "winerror"):
326 raise
327
328 logger.warning("Error trying to resolve the executable: %s", logwarning)
301 found_executable = shutil.which(command[0] + extension, path=path)
302 if found_executable:
303 return (found_executable, *command[1:])
304
305 logger.warning("Error trying to resolve the executable: %s", command[0])
329306 return command
330307
331308
332 if sys.version_info < (3, 5):
333
334 class _ReturnObjectParent(object):
335 def check_returncode(self):
336 if self.returncode:
337 raise Exception(
338 "Call %r resulted in non-zero exit code %r"
339 % (self.args, self.returncode)
340 )
341
342
343 else:
344 _ReturnObjectParent = subprocess.CompletedProcess
345
346
347 class ReturnObject(dict, _ReturnObjectParent):
309 class ReturnObject(subprocess.CompletedProcess):
348310 """
349311 A subprocess.CompletedProcess-like object containing the executed
350312 command, stdout and stderr (both as bytestrings), and the exitcode.
351 Further values such as process runtime can be accessed as dictionary
352 values.
353313 The check_returncode() function raises an exception if the process
354314 exited with a non-zero exit code.
355315 """
356316
357 def __init__(self, *arg, **kw):
358 super(ReturnObject, self).__init__(*arg, **kw)
359 self.args = self["command"]
360 self.returncode = self["exitcode"]
361 self.stdout = self["stdout"]
362 self.stderr = self["stderr"]
317 def __init__(self, exitcode=None, command=None, stdout=None, stderr=None, **kw):
318 super().__init__(
319 args=command, returncode=exitcode, stdout=stdout, stderr=stderr
320 )
321 self._extras = {
322 "timeout": kw.get("timeout"),
323 "runtime": kw.get("runtime"),
324 "time_start": kw.get("time_start"),
325 "time_end": kw.get("time_end"),
326 }
327
328 def __getitem__(self, key):
329 warnings.warn(
330 "dictionary access to a procrunner return object is deprecated",
331 DeprecationWarning,
332 stacklevel=2,
333 )
334 if key in self._extras:
335 return self._extras[key]
336 if not hasattr(self, key):
337 raise KeyError(f"Unknown attribute {key}")
338 return getattr(self, key)
363339
364340 def __eq__(self, other):
365341 """Override equality operator to account for added fields"""
371347 """This object is not immutable, so mark it as unhashable"""
372348 return None
373349
374 def __ne__(self, other):
375 """Overrides the default implementation (unnecessary in Python 3)"""
376 return not self.__eq__(other)
377
378
350 @property
351 def cmd(self):
352 warnings.warn(
353 "procrunner return object .cmd is deprecated, use .args",
354 DeprecationWarning,
355 stacklevel=2,
356 )
357 return self.args
358
359 @property
360 def command(self):
361 warnings.warn(
362 "procrunner return object .command is deprecated, use .args",
363 DeprecationWarning,
364 stacklevel=2,
365 )
366 return self.args
367
368 @property
369 def exitcode(self):
370 warnings.warn(
371 "procrunner return object .exitcode is deprecated, use .returncode",
372 DeprecationWarning,
373 stacklevel=2,
374 )
375 return self.returncode
376
377 @property
378 def timeout(self):
379 warnings.warn(
380 "procrunner return object .timeout is deprecated",
381 DeprecationWarning,
382 stacklevel=2,
383 )
384 return self._extras["timeout"]
385
386 @property
387 def runtime(self):
388 warnings.warn(
389 "procrunner return object .runtime is deprecated",
390 DeprecationWarning,
391 stacklevel=2,
392 )
393 return self._extras["runtime"]
394
395 @property
396 def time_start(self):
397 warnings.warn(
398 "procrunner return object .time_start is deprecated",
399 DeprecationWarning,
400 stacklevel=2,
401 )
402 return self._extras["time_start"]
403
404 @property
405 def time_end(self):
406 warnings.warn(
407 "procrunner return object .time_end is deprecated",
408 DeprecationWarning,
409 stacklevel=2,
410 )
411 return self._extras["time_end"]
412
413 def update(self, dictionary):
414 self._extras.update(dictionary)
415
416
417 def _deprecate_argument_calling(f):
418 @functools.wraps(f)
419 def wrapper(*args, **kwargs):
420 if len(args) > 1:
421 warnings.warn(
422 "Calling procrunner.run() with unnamed arguments (apart from "
423 "the command) is deprecated. Use keyword arguments instead.",
424 DeprecationWarning,
425 stacklevel=2,
426 )
427 return f(*args, **kwargs)
428
429 return wrapper
430
431
432 @_deprecate_argument_calling
379433 def run(
380434 command,
381435 timeout=None,
382 debug=False,
436 debug=None,
383437 stdin=None,
384438 print_stdout=True,
385439 print_stderr=True,
389443 environment_override=None,
390444 win32resolve=True,
391445 working_directory=None,
446 raise_timeout_exception=False,
392447 ):
393448 """
394449 Run an external process.
398453
399454 :param array command: Command line to be run, specified as array.
400455 :param timeout: Terminate program execution after this many seconds.
401 :param boolean debug: Enable further debug messages.
402 :param stdin: Optional string that is passed to command stdin.
456 :param boolean debug: Enable further debug messages. (deprecated)
457 :param stdin: Optional bytestring that is passed to command stdin.
403458 :param boolean print_stdout: Pass stdout through to sys.stdout.
404459 :param boolean print_stderr: Pass stderr through to sys.stderr.
405460 :param callback_stdout: Optional function which is called for each
415470 extension.
416471 :param string working_directory: If specified, run the executable from
417472 within this working directory.
418 :return: A ReturnObject() containing the executed command, stdout and stderr
419 (both as bytestrings), and the exitcode. Further values such as
420 process runtime can be accessed as dictionary values.
473 :param boolean raise_timeout_exception: Forward compatibility flag. If set
474 then a subprocess.TimeoutExpired exception is raised
475 instead of returning an object that can be checked
476 for a timeout condition. Defaults to False, will be
477 changed to True in a future release.
478 :return: The exit code, stdout, stderr (separately, as byte strings)
479 as a subprocess.CompletedProcess object.
421480 """
422481
423482 time_start = time.strftime("%Y-%m-%d %H:%M:%S GMT", time.gmtime())
428487 else:
429488 assert sys.platform != "win32", "stdin argument not supported on Windows"
430489 stdin_pipe = subprocess.PIPE
490 if debug is not None:
491 warnings.warn(
492 "Use of the debug parameter is deprecated", DeprecationWarning, stacklevel=3
493 )
431494
432495 start_time = timeit.default_timer()
433496 if timeout is not None:
434497 max_time = start_time + timeout
498 if not raise_timeout_exception:
499 warnings.warn(
500 "Using procrunner with timeout and without raise_timeout_exception set is deprecated",
501 DeprecationWarning,
502 stacklevel=3,
503 )
435504
436505 if environment is not None:
437506 env = {key: _path_resolve(environment[key]) for key in environment}
448517 command = tuple(_path_resolve(part) for part in command)
449518 if win32resolve and sys.platform == "win32":
450519 command = _windows_resolve(command)
520 if working_directory and sys.version_info < (3, 7):
521 working_directory = os.fspath(working_directory)
451522
452523 p = subprocess.Popen(
453524 command,
454525 shell=False,
455 cwd=_path_resolve(working_directory),
526 cwd=working_directory,
456527 env=env,
457528 stdin=stdin_pipe,
458529 stdout=subprocess.PIPE,
491562 (timeout is None) or (timeit.default_timer() < max_time)
492563 ):
493564 if debug and timeout is not None:
494 logger.debug("still running (T%.2fs)" % (timeit.default_timer() - max_time))
565 logger.debug("still running (T%.2fs)", timeit.default_timer() - max_time)
495566
496567 # wait for some time or until a stream is closed
497568 try:
500571 # which could indicate that the process has terminated.
501572 try:
502573 event = thread_pipe_pool[0].poll(0.5)
503 except IOError as e:
504 # on Windows this raises "IOError: [Errno 109] The pipe has been ended"
574 except BrokenPipeError as e:
575 # on Windows this raises "BrokenPipeError: [Errno 109] The pipe has been ended"
505576 # which is for all intents and purposes equivalent to a True return value.
506 if e.errno != 109:
577 if e.winerror != 109:
507578 raise
508579 event = True
509580 if event:
525596 # timeout condition
526597 timeout_encountered = True
527598 if debug:
528 logger.debug("timeout (T%.2fs)" % (timeit.default_timer() - max_time))
599 logger.debug("timeout (T%.2fs)", timeit.default_timer() - max_time)
529600
530601 # send terminate signal and wait some time for buffers to be read
531602 p.terminate()
532603 if thread_pipe_pool:
533 thread_pipe_pool[0].poll(0.5)
604 try:
605 thread_pipe_pool[0].poll(0.5)
606 except BrokenPipeError as e:
607 # on Windows this raises "BrokenPipeError: [Errno 109] The pipe has been ended"
608 # which is for all intents and purposes equivalent to a True return value.
609 if e.winerror != 109:
610 raise
611 thread_pipe_pool.pop(0)
534612 if not stdout.has_finished() or not stderr.has_finished():
535613 time.sleep(2)
536614 p.poll()
540618 # send kill signal and wait some more time for buffers to be read
541619 p.kill()
542620 if thread_pipe_pool:
543 thread_pipe_pool[0].poll(0.5)
621 try:
622 thread_pipe_pool[0].poll(0.5)
623 except BrokenPipeError as e:
624 # on Windows this raises "BrokenPipeError: [Errno 109] The pipe has been ended"
625 # which is for all intents and purposes equivalent to a True return value.
626 if e.winerror != 109:
627 raise
628 thread_pipe_pool.pop(0)
544629 if not stdout.has_finished() or not stderr.has_finished():
545630 time.sleep(5)
546631 p.poll()
551636 runtime = timeit.default_timer() - start_time
552637 if timeout is not None:
553638 logger.debug(
554 "Process ended after %.1f seconds with exit code %d (T%.2fs)"
555 % (runtime, p.returncode, timeit.default_timer() - max_time)
639 "Process ended after %.1f seconds with exit code %d (T%.2fs)",
640 runtime,
641 p.returncode,
642 timeit.default_timer() - max_time,
556643 )
557644 else:
558645 logger.debug(
559 "Process ended after %.1f seconds with exit code %d"
560 % (runtime, p.returncode)
646 "Process ended after %.1f seconds with exit code %d", runtime, p.returncode
561647 )
562648
563649 stdout = stdout.get_output()
564650 stderr = stderr.get_output()
651
652 if timeout_encountered and raise_timeout_exception:
653 raise subprocess.TimeoutExpired(
654 cmd=command, timeout=timeout, output=stdout, stderr=stderr
655 )
656
565657 time_end = time.strftime("%Y-%m-%d %H:%M:%S GMT", time.gmtime())
566
567658 result = ReturnObject(
568 {
569 "exitcode": p.returncode,
570 "command": command,
571 "stdout": stdout,
572 "stderr": stderr,
573 "timeout": timeout_encountered,
574 "runtime": runtime,
575 "time_start": time_start,
576 "time_end": time_end,
577 }
659 exitcode=p.returncode,
660 command=command,
661 stdout=stdout,
662 stderr=stderr,
663 timeout=timeout_encountered,
664 runtime=runtime,
665 time_start=time_start,
666 time_end=time_end,
578667 )
579668 if stdin is not None:
580669 result.update(
585674 )
586675
587676 return result
588
589
590 def run_process_dummy(command, **kwargs):
591 """
592 A stand-in function that returns a valid result dictionary indicating a
593 successful execution. The external process is not run.
594 """
595 warnings.warn(
596 "procrunner.run_process_dummy() is deprecated", DeprecationWarning, stacklevel=2
597 )
598
599 time_start = time.strftime("%Y-%m-%d %H:%M:%S GMT", time.gmtime())
600 logger.info("run_process is disabled. Requested command: %s", command)
601
602 result = ReturnObject(
603 {
604 "exitcode": 0,
605 "command": command,
606 "stdout": "",
607 "stderr": "",
608 "timeout": False,
609 "runtime": 0,
610 "time_start": time_start,
611 "time_end": time_start,
612 }
613 )
614 if kwargs.get("stdin") is not None:
615 result.update(
616 {"stdin_bytes_sent": len(kwargs["stdin"]), "stdin_bytes_remain": 0}
617 )
618 return result
619
620
621 def run_process(*args, **kwargs):
622 """API used up to version 0.2.0."""
623 warnings.warn(
624 "procrunner.run_process() is deprecated and has been renamed to run()",
625 DeprecationWarning,
626 stacklevel=2,
627 )
628 return run(*args, **kwargs)
0 [build-system]
1 requires = ["setuptools >= 40.6.0", "wheel"]
2 build-backend = "setuptools.build_meta"
3
4 [tool.isort]
5 profile="black"
0 [pytest]
1 addopts = -ra
2 junit_family=xunit2
(New empty file)
0 bump2version==0.5.10
1 coverage==4.5.4
2 flake8==3.7.8
3 mock==3.0.5
4 pip==19.1.1
5 pytest==4.5.0 # pyup: <5.0 # for Python 2.7 support
6 pytest-runner==5.1
7 six==1.12.0
8 Sphinx==1.8.5 # pyup: <2.0 # for Python 2.7 support
9 tox==3.13.1
10 twine==1.13.0
11 watchdog==0.9.0
12 wheel==0.33.4
0 bump2version==1.0.1
1 coverage==6.0.2
2 pip==21.3.1
3 pytest==6.2.5
4 Sphinx==4.2.0
5 tox==3.24.4
6 twine==3.4.2
7 wheel==0.37.0
0 [bumpversion]
1 current_version = 1.1.0
2 commit = True
3 tag = True
4
5 [bumpversion:file:setup.py]
6 search = version="{current_version}"
7 replace = version="{new_version}"
8
9 [bumpversion:file:procrunner/__init__.py]
10 search = __version__ = "{current_version}"
11 replace = __version__ = "{new_version}"
12
13 [bdist_wheel]
14 universal = 1
0 [metadata]
1 name = procrunner
2 description = Versatile utility function to run external processes
3 version = 2.3.1
4 classifiers =
5 Development Status :: 5 - Production/Stable
6 Intended Audience :: Developers
7 License :: OSI Approved :: BSD License
8 Natural Language :: English
9 Programming Language :: Python :: 3
10 Programming Language :: Python :: 3.6
11 Programming Language :: Python :: 3.7
12 Programming Language :: Python :: 3.8
13 Programming Language :: Python :: 3.9
14 Programming Language :: Python :: 3.10
15 Operating System :: OS Independent
16 Topic :: Software Development :: Libraries :: Python Modules
17 license = BSD
18 license_file = LICENSE
19 project-urls =
20 Download = https://github.com/DiamondLightSource/python-procrunner/tags
21 Documentation = https://procrunner.readthedocs.io/
22 GitHub = https://github.com/DiamondLightSource/python-procrunner
23 Bug-Tracker = https://github.com/DiamondLightSource/python-procrunner/issues
1524
1625 [flake8]
17 exclude = docs
26 # Black disagrees with flake8 on a few points. Ignore those.
27 ignore = E203, E266, E501, W503
28 # E203 whitespace before ':'
29 # E266 too many leading '#' for block comment
30 # E501 line too long
31 # W503 line break before binary operator
32
33 max-line-length = 88
34
35 select =
36 E401,E711,E712,E713,E714,E721,E722,E901,
37 F401,F402,F403,F405,F541,F631,F632,F633,F811,F812,F821,F822,F841,F901,
38 W191,W291,W292,W293,W602,W603,W604,W605,W606,
39 # flake8-comprehensions, https://github.com/adamchainz/flake8-comprehensions
40 C4,
1841
1942 [aliases]
2043 test = pytest
00 #!/usr/bin/env python
11 # -*- coding: utf-8 -*-
22
3 import sys
4 from setuptools import setup, find_packages
3 from setuptools import find_packages, setup
54
65 with open("README.rst") as readme_file:
76 readme = readme_file.read()
98 with open("HISTORY.rst") as history_file:
109 history = history_file.read()
1110
12 requirements = [
13 "six",
14 'pywin32; sys_platform=="win32"',
15 ]
11 requirements = []
1612
1713 setup_requirements = []
18 needs_pytest = {"pytest", "test", "ptr"}.intersection(sys.argv)
19 if needs_pytest:
20 setup_requirements.append("pytest-runner")
2114
22 test_requirements = ["mock", "pytest"]
15 test_requirements = ["pytest"]
2316
2417 setup(
2518 author="Markus Gerstel",
2619 author_email="scientificsoftware@diamond.ac.uk",
27 classifiers=[
28 "Development Status :: 5 - Production/Stable",
29 "Intended Audience :: Developers",
30 "License :: OSI Approved :: BSD License",
31 "Natural Language :: English",
32 "Operating System :: OS Independent",
33 "Programming Language :: Python :: 2",
34 "Programming Language :: Python :: 2.7",
35 "Programming Language :: Python :: 3",
36 "Programming Language :: Python :: 3.5",
37 "Programming Language :: Python :: 3.6",
38 "Programming Language :: Python :: 3.7",
39 "Programming Language :: Python :: 3.8",
40 "Programming Language :: Python :: Implementation :: PyPy",
41 "Programming Language :: Python :: Implementation :: CPython",
42 "Topic :: Software Development :: Libraries :: Python Modules",
43 ],
44 description="Versatile utility function to run external processes",
4520 install_requires=requirements,
46 license="BSD license",
4721 long_description=readme + "\n\n" + history,
4822 include_package_data=True,
4923 keywords="procrunner",
50 name="procrunner",
5124 packages=find_packages(include=["procrunner"]),
25 python_requires=">=3.6",
5226 setup_requires=setup_requirements,
5327 test_suite="tests",
5428 tests_require=test_requirements,
5529 url="https://github.com/DiamondLightSource/python-procrunner",
56 version="1.1.0",
5730 zip_safe=False,
5831 )
0 from __future__ import absolute_import, division, print_function
1
20 import copy
3 import mock
41 import os
2 import pathlib
3 import sys
4 from unittest import mock
5
6 import pytest
7
58 import procrunner
6 import pytest
7 import sys
9
10
11 @mock.patch("procrunner._NonBlockingStreamReader")
12 @mock.patch("procrunner.time")
13 @mock.patch("procrunner.subprocess")
14 @mock.patch("procrunner.Pipe")
15 def test_run_command_aborts_after_timeout_legacy(
16 mock_pipe, mock_subprocess, mock_time, mock_streamreader
17 ):
18 mock_pipe.return_value = mock.Mock(), mock.Mock()
19 mock_process = mock.Mock()
20 mock_process.returncode = None
21 mock_subprocess.Popen.return_value = mock_process
22 task = ["___"]
23
24 with pytest.raises(RuntimeError):
25 with pytest.warns(DeprecationWarning, match="timeout"):
26 procrunner.run(task, timeout=-1, debug=False)
27
28 assert mock_subprocess.Popen.called
29 assert mock_process.terminate.called
30 assert mock_process.kill.called
831
932
1033 @mock.patch("procrunner._NonBlockingStreamReader")
2144 task = ["___"]
2245
2346 with pytest.raises(RuntimeError):
24 procrunner.run(task, -1, False)
47 procrunner.run(task, timeout=-1, raise_timeout_exception=True)
2548
2649 assert mock_subprocess.Popen.called
2750 assert mock_process.terminate.called
6285
6386 actual = procrunner.run(
6487 command,
65 0.5,
66 False,
88 timeout=0.5,
6789 callback_stdout=mock.sentinel.callback_stdout,
6890 callback_stderr=mock.sentinel.callback_stderr,
69 working_directory=mock.sentinel.cwd,
91 working_directory=pathlib.Path("somecwd"),
92 raise_timeout_exception=True,
7093 )
7194
7295 assert mock_subprocess.Popen.called
7396 assert mock_subprocess.Popen.call_args[1]["env"] == os.environ
74 assert mock_subprocess.Popen.call_args[1]["cwd"] == mock.sentinel.cwd
97 assert mock_subprocess.Popen.call_args[1]["cwd"] in (
98 pathlib.Path("somecwd"),
99 "somecwd",
100 )
75101 mock_streamreader.assert_has_calls(
76102 [
77103 mock.call(
78104 stream_stdout,
79105 output=mock.ANY,
80 debug=mock.ANY,
106 debug=None,
81107 notify=mock.ANY,
82108 callback=mock.sentinel.callback_stdout,
83109 ),
84110 mock.call(
85111 stream_stderr,
86112 output=mock.ANY,
87 debug=mock.ANY,
113 debug=None,
88114 notify=mock.ANY,
89115 callback=mock.sentinel.callback_stderr,
90116 ),
94120 assert not mock_process.terminate.called
95121 assert not mock_process.kill.called
96122 for key in expected:
97 assert actual[key] == expected[key]
123 with pytest.warns(DeprecationWarning):
124 assert actual[key] == expected[key]
98125 assert actual.args == tuple(command)
99126 assert actual.returncode == mock_process.returncode
100127 assert actual.stdout == mock.sentinel.proc_stdout
105132 def test_default_process_environment_is_parent_environment(mock_subprocess):
106133 mock_subprocess.Popen.side_effect = NotImplementedError() # cut calls short
107134 with pytest.raises(NotImplementedError):
108 procrunner.run([mock.Mock()], -1, False)
135 procrunner.run([mock.Mock()], timeout=-1, raise_timeout_exception=True)
109136 assert mock_subprocess.Popen.call_args[1]["env"] == os.environ
137
138
139 @mock.patch("procrunner.subprocess")
140 def test_using_debug_parameter_raises_warning(mock_subprocess):
141 mock_subprocess.Popen.side_effect = NotImplementedError() # cut calls short
142 with pytest.warns(DeprecationWarning, match="debug"):
143 with pytest.raises(NotImplementedError):
144 procrunner.run([mock.Mock()], debug=True)
145 with pytest.warns(DeprecationWarning, match="debug"):
146 with pytest.raises(NotImplementedError):
147 procrunner.run([mock.Mock()], debug=False)
110148
111149
112150 @mock.patch("procrunner.subprocess")
115153 mock_env = {"key": mock.sentinel.key}
116154 # Pass an environment dictionary
117155 with pytest.raises(NotImplementedError):
118 procrunner.run([mock.Mock()], -1, False, environment=copy.copy(mock_env))
156 procrunner.run(
157 [mock.Mock()],
158 timeout=-1,
159 environment=copy.copy(mock_env),
160 raise_timeout_exception=True,
161 )
119162 assert mock_subprocess.Popen.call_args[1]["env"] == mock_env
120163
121164
128171 with pytest.raises(NotImplementedError):
129172 procrunner.run(
130173 [mock.Mock()],
131 -1,
132 False,
174 timeout=-1,
133175 environment=copy.copy(mock_env1),
134176 environment_override=copy.copy(mock_env2),
177 raise_timeout_exception=True,
135178 )
136179 mock_env_sum = copy.copy(mock_env1)
137180 mock_env_sum.update({key: str(mock_env2[key]) for key in mock_env2})
144187 mock_env2 = {"keyB": str(mock.sentinel.keyB)}
145188 with pytest.raises(NotImplementedError):
146189 procrunner.run(
147 [mock.Mock()], -1, False, environment_override=copy.copy(mock_env2)
190 [mock.Mock()],
191 timeout=-1,
192 environment_override=copy.copy(mock_env2),
193 raise_timeout_exception=True,
148194 )
149195 random_environment_variable = list(os.environ)[0]
150196 if random_environment_variable == list(mock_env2)[0]:
151197 random_environment_variable = list(os.environ)[1]
152 random_environment_value = os.getenv(random_environment_variable)
153198 assert (
154199 random_environment_variable
155200 and random_environment_variable != list(mock_env2)[0]
171216 with pytest.raises(NotImplementedError):
172217 procrunner.run(
173218 [mock.Mock()],
174 -1,
175 False,
219 timeout=-1,
176220 environment_override={
177221 random_environment_variable: "X" + random_environment_value
178222 },
223 raise_timeout_exception=True,
179224 )
180225 assert (
181226 mock_subprocess.Popen.call_args[1]["env"][random_environment_variable]
191236 def test_nonblockingstreamreader_can_read(mock_select):
192237 import time
193238
194 class _stream(object):
239 class _stream:
195240 def __init__(self):
196241 self.data = b""
197242 self.closed = False
262307
263308 def test_return_object_semantics():
264309 ro = procrunner.ReturnObject(
265 {
266 "command": mock.sentinel.command,
267 "exitcode": 0,
268 "stdout": mock.sentinel.stdout,
269 "stderr": mock.sentinel.stderr,
270 }
271 )
272 assert ro["command"] == mock.sentinel.command
310 command=mock.sentinel.command,
311 exitcode=0,
312 stdout=mock.sentinel.stdout,
313 stderr=mock.sentinel.stderr,
314 )
315 with pytest.warns(DeprecationWarning):
316 assert ro["command"] == mock.sentinel.command
273317 assert ro.args == mock.sentinel.command
274 assert ro["exitcode"] == 0
318 with pytest.warns(DeprecationWarning):
319 assert ro["exitcode"] == 0
275320 assert ro.returncode == 0
276 assert ro["stdout"] == mock.sentinel.stdout
321 with pytest.warns(DeprecationWarning):
322 assert ro["stdout"] == mock.sentinel.stdout
277323 assert ro.stdout == mock.sentinel.stdout
278 assert ro["stderr"] == mock.sentinel.stderr
324 with pytest.warns(DeprecationWarning):
325 assert ro["stderr"] == mock.sentinel.stderr
279326 assert ro.stderr == mock.sentinel.stderr
280327
281328 with pytest.raises(KeyError):
282 ro["unknownkey"]
329 with pytest.warns(DeprecationWarning):
330 ro["unknownkey"]
283331 ro.update({"unknownkey": mock.sentinel.key})
284 assert ro["unknownkey"] == mock.sentinel.key
332 with pytest.warns(DeprecationWarning):
333 assert ro["unknownkey"] == mock.sentinel.key
285334
286335
287336 def test_return_object_check_function_passes_on_success():
288337 ro = procrunner.ReturnObject(
289 {
290 "command": mock.sentinel.command,
291 "exitcode": 0,
292 "stdout": mock.sentinel.stdout,
293 "stderr": mock.sentinel.stderr,
294 }
338 command=mock.sentinel.command,
339 exitcode=0,
340 stdout=mock.sentinel.stdout,
341 stderr=mock.sentinel.stderr,
295342 )
296343 ro.check_returncode()
297344
298345
299346 def test_return_object_check_function_raises_on_error():
300347 ro = procrunner.ReturnObject(
301 {
302 "command": mock.sentinel.command,
303 "exitcode": 1,
304 "stdout": mock.sentinel.stdout,
305 "stderr": mock.sentinel.stderr,
306 }
348 command=mock.sentinel.command,
349 exitcode=1,
350 stdout=mock.sentinel.stdout,
351 stderr=mock.sentinel.stderr,
307352 )
308353 with pytest.raises(Exception) as e:
309354 ro.check_returncode()
0 from __future__ import absolute_import, division, print_function
1
20 import os
31 import sys
42
5 import mock
3 import pytest
4
65 import procrunner
7 import pytest
86
97
108 def PEP519(path):
4644
4745
4846 @pytest.mark.skipif(sys.platform != "win32", reason="windows specific test only")
49 def test_pywin32_import():
50 import win32api
51
52
53 @pytest.mark.skipif(sys.platform != "win32", reason="windows specific test only")
5447 def test_name_resolution_for_simple_exe():
5548 command = ["cmd.exe", "/c", "echo", "hello"]
5649
6558
6659
6760 @pytest.mark.skipif(sys.platform != "win32", reason="windows specific test only")
68 def test_name_resolution_for_complex_cases(tmpdir):
69 tmpdir.chdir()
70
61 def test_name_resolution_for_complex_cases(tmp_path):
7162 bat = "simple_bat_extension"
7263 cmd = "simple_cmd_extension"
7364 exe = "simple_exe_extension"
7465 dotshort = "more_complex_filename_with_a.dot"
7566 dotlong = "more_complex_filename.withadot"
7667
77 (tmpdir / bat + ".bat").ensure()
78 (tmpdir / cmd + ".cmd").ensure()
79 (tmpdir / exe + ".exe").ensure()
80 (tmpdir / dotshort + ".bat").ensure()
81 (tmpdir / dotlong + ".cmd").ensure()
68 (tmp_path / (bat + ".bat")).touch()
69 (tmp_path / (cmd + ".cmd")).touch()
70 (tmp_path / (exe + ".exe")).touch()
71 (tmp_path / (dotshort + ".bat")).touch()
72 (tmp_path / (dotlong + ".cmd")).touch()
8273
8374 def is_valid(command):
8475 assert len(command) == 1
85 assert os.path.exists(command[0])
76 assert os.path.exists(tmp_path / command[0])
8677
87 is_valid(procrunner._windows_resolve([bat]))
88 is_valid(procrunner._windows_resolve([cmd]))
89 is_valid(procrunner._windows_resolve([exe]))
90 is_valid(procrunner._windows_resolve([dotshort]))
91 is_valid(procrunner._windows_resolve([dotlong]))
78 is_valid(procrunner._windows_resolve([bat], path=os.fspath(tmp_path)))
79 is_valid(procrunner._windows_resolve([cmd], path=os.fspath(tmp_path)))
80 is_valid(procrunner._windows_resolve([exe], path=os.fspath(tmp_path)))
81 is_valid(procrunner._windows_resolve([dotshort], path=os.fspath(tmp_path)))
82 is_valid(procrunner._windows_resolve([dotlong], path=os.fspath(tmp_path)))
0 from __future__ import absolute_import, division, print_function
0 import os
1 import subprocess
2 import sys
3 import timeit
14
2 import os
3 import sys
5 import pytest
46
57 import procrunner
6 import pytest
78
89
910 def test_simple_command_invocation():
3536 else:
3637 assert result.stdout == test_string
3738 out, err = capsys.readouterr()
38 assert out == u"test\ufffdstring\n"
39 assert err == u""
39 assert out == "test\ufffdstring\n"
40 assert err == ""
4041
4142
42 def test_running_wget(tmpdir):
43 tmpdir.chdir()
43 def test_running_wget(tmp_path):
4444 command = ["wget", "https://www.google.com", "-O", "-"]
4545 try:
46 result = procrunner.run(command)
46 result = procrunner.run(command, working_directory=tmp_path)
4747 except OSError as e:
4848 if e.errno == 2:
4949 pytest.skip("wget not available")
5353 assert b"google" in result.stdout
5454
5555
56 def test_path_object_resolution(tmpdir):
56 def test_path_object_resolution(tmp_path):
5757 sentinel_value = b"sentinel"
58 tmpdir.join("tempfile").write(sentinel_value)
59 tmpdir.join("reader.py").write("print(open('tempfile').read())")
58 tmp_path.joinpath("tempfile").write_bytes(sentinel_value)
59 tmp_path.joinpath("reader.py").write_text(
60 "with open('tempfile') as fh:\n print(fh.read())"
61 )
6062 assert "LEAK_DETECTOR" not in os.environ
6163 result = procrunner.run(
62 [sys.executable, tmpdir.join("reader.py")],
64 [sys.executable, tmp_path / "reader.py"],
6365 environment_override={"PYTHONHASHSEED": "random", "LEAK_DETECTOR": "1"},
64 working_directory=tmpdir,
66 working_directory=tmp_path,
6567 )
6668 assert result.returncode == 0
6769 assert not result.stderr
6971 assert (
7072 "LEAK_DETECTOR" not in os.environ
7173 ), "overridden environment variable leaked into parent process"
74
75
76 def test_timeout_behaviour_legacy(tmp_path):
77 start = timeit.default_timer()
78 try:
79 with pytest.warns(DeprecationWarning, match="timeout"):
80 result = procrunner.run(
81 [sys.executable, "-c", "import time; time.sleep(5)"],
82 timeout=0.1,
83 working_directory=tmp_path,
84 raise_timeout_exception=False,
85 )
86 except RuntimeError:
87 # This test sometimes fails with a RuntimeError.
88 runtime = timeit.default_timer() - start
89 assert runtime < 3
90 return
91 runtime = timeit.default_timer() - start
92 with pytest.warns(DeprecationWarning, match="\\.timeout"):
93 assert result.timeout
94 assert runtime < 3
95 assert not result.stdout
96 assert not result.stderr
97 assert result.returncode
98
99
100 def test_timeout_behaviour(tmp_path):
101 command = (sys.executable, "-c", "import time; time.sleep(5)")
102 start = timeit.default_timer()
103 try:
104 with pytest.raises(subprocess.TimeoutExpired) as te:
105 procrunner.run(
106 command,
107 timeout=0.1,
108 working_directory=tmp_path,
109 raise_timeout_exception=True,
110 )
111 except RuntimeError:
112 # This test sometimes fails with a RuntimeError.
113 runtime = timeit.default_timer() - start
114 assert runtime < 3
115 return
116 runtime = timeit.default_timer() - start
117 assert runtime < 3
118 assert te.value.stdout == b""
119 assert te.value.stderr == b""
120 assert te.value.timeout == 0.1
121 assert te.value.cmd == command
122
123
124 def test_argument_deprecation(tmp_path):
125 with pytest.warns(DeprecationWarning, match="keyword arguments"):
126 result = procrunner.run(
127 [sys.executable, "-V"],
128 None,
129 working_directory=tmp_path,
130 )
131 assert not result.returncode
132 assert result.stderr or result.stdout
00 [tox]
1 envlist = py27, py35, py36, py37, py38, flake8
1 envlist = py36, py37, py38, flake8
22
3 [travis]
4 python =
5 3.8: py38
6 3.7: py37
7 3.6: py36
8 3.5: py35
9 2.7: py27
3 [testenv:azure]
4 basepython = python
5 deps =
6 pytest-azurepipelines
7 pytest-cov
8 -r{toxinidir}/requirements_dev.txt
9 setenv =
10 PYTHONDEVMODE = 1
11 commands =
12 pytest -ra --basetemp={envtmpdir} --cov=procrunner --cov-report=html --cov-report=xml --cov-branch
1013
1114 [testenv:flake8]
1215 basepython = python
1619 [testenv]
1720 setenv =
1821 PYTHONPATH = {toxinidir}
22 PYTHONDEVMODE = 1
1923 deps =
2024 -r{toxinidir}/requirements_dev.txt
2125 ; If you want to make tox run the tests with the same versions, create a