diff --git a/procrunner/__init__.py b/procrunner/__init__.py index 711b4b6..896f599 100644 --- a/procrunner/__init__.py +++ b/procrunner/__init__.py @@ -2,6 +2,7 @@ from __future__ import absolute_import, division, print_function +import codecs import copy import logging import os @@ -67,12 +68,14 @@ self._buffer = '' self._print = print_line self._callback = callback + self._decoder = codecs.getincrementaldecoder('utf-8')('replace') def add(self, data): '''Add a single character to buffer. If one or more full lines are found, print them (if desired) and pass to callback function.''' - data = data.decode('utf-8') + data = self._decoder.decode(data) + if not data: return self._buffer += data - if "\n" in data: + if '\n' in data: to_print, remainder = self._buffer.rsplit('\n') if self._print: print(to_print) @@ -81,6 +84,7 @@ self._buffer = remainder def flush(self): '''Print/send any remaining data to callback function.''' + self._buffer += self._decoder.decode(b'', final=True) if self._buffer: if self._print: print(self._buffer) diff --git a/tests/test_procrunner_system.py b/tests/test_procrunner_system.py index 174a52f..ba1b7ea 100644 --- a/tests/test_procrunner_system.py +++ b/tests/test_procrunner_system.py @@ -4,6 +4,7 @@ import sys import procrunner +import pytest def test_simple_command_invocation(): if os.name == 'nt': @@ -17,10 +18,27 @@ assert result['stdout'] == b'hello' + os.linesep.encode('utf-8') assert result['stderr'] == b'' -def test_input_encoding(): +def test_decode_invalid_utf8_input(capsys): command = [sys.executable, '-c', 'import sys; sys.stdout.write("".join(chr(x) for x in ' - '(0x74,0x65,0x73,0x74,0xa0,0x50,0x73,0x74,0x72,0x69,0x6e,0x67,0x0a)' + '(0x74,0x65,0x73,0x74,0xa0,0x73,0x74,0x72,0x69,0x6e,0x67,0x0a)' '))'] result = procrunner.run(command) + assert result['exitcode'] == 0 + assert not result['stderr'] + assert result['stdout'] == b'test\xa0string\n' + out, err = capsys.readouterr() + assert out == u'test\ufffdstring\n' + assert err == u'' +def test_running_wget(tmpdir): + tmpdir.chdir() + command = ['wget', 'https://www.google.com', '-O', '-'] + try: + result = procrunner.run(command) + except OSError as e: + if e.errno == 2: + pytest.skip('wget not available') + raise assert result['exitcode'] == 0 + assert b'http' in result['stderr'] + assert b'google' in result['stdout']