Make sure output_stream can handle non-utf8 bytes

This is needed to safely output raw subunit v2 streams.

Author: rbtcollins

NEWS

@@ -17,6 +17,12 @@ CHANGES
 * When list-tests encounters an error, a much clearer response will
   now be shown. (Robert Collins, #1271133)
 
+INTERNALS
+---------
+
+* ``UI.output_stream`` is now tested for handling of non-utf8 bytestreams.
+  (Robert Collins)
+
 0.0.18
 ++++++
 

testrepository/tests/test_setup.py

@@ -35,7 +35,7 @@ def test_bdist(self):
         proc = subprocess.Popen([sys.executable, path, 'bdist'],
             stdin=subprocess.PIPE, stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT, universal_newlines=True)
-        output, _ = proc.communicate()
+        output, err = proc.communicate()
         self.assertThat(output, MatchesAny(
             # win32
             DocTestMatches("""...
@@ -48,4 +48,5 @@ def test_bdist(self):
 ...bin/testr ...
 """, doctest.ELLIPSIS)
             ))
-        self.assertEqual(0, proc.returncode)
+        self.assertEqual(0, proc.returncode,
+            "Setup failed out=%r err=%r" % (output, err))

testrepository/tests/test_ui.py

@@ -119,6 +119,11 @@ def test_output_stream(self):
         ui = self.get_test_ui()
         ui.output_stream(BytesIO())
 
+    def test_output_stream_non_utf8(self):
+        # When the stream has non-utf8 bytes it still outputs correctly.
+        ui = self.get_test_ui()
+        ui.output_stream(BytesIO(_b('\xfa')))
+
     def test_output_table(self):
         # output_table shows a table.
         ui = self.get_test_ui()

testrepository/tests/ui/test_cli.py

@@ -109,7 +109,8 @@ def test_outputs_error_string(self):
         except Exception:
             err_tuple = sys.exc_info()
         expected = str(err_tuple[1]) + '\n'
-        stdout = StringIO()
+        bytestream = BytesIO()
+        stdout = TextIOWrapper(bytestream, 'utf8', line_buffering=True)
         stdin = StringIO()
         stderr = StringIO()
         ui = cli.UI([], stdin, stdout, stderr)
@@ -128,6 +129,9 @@ def test_error_enters_pdb_when_TESTR_PDB_set(self):
             <BLANKLINE>
             fooo
             """)
+        # This should be a BytesIO + Textwrapper, but pdb on 2.7 writes bytes
+        # - this code is the most pragmatic to test on 2.6 and up, and 3.2 and
+        # up.
         stdout = StringIO()
         stdin = StringIO(_u('c\n'))
         stderr = StringIO()
@@ -196,7 +200,8 @@ def test_outputs_summary_to_stdout(self):
             ui._stdout.buffer.getvalue())
 
     def test_parse_error_goes_to_stderr(self):
-        stdout = StringIO()
+        bytestream = BytesIO()
+        stdout = TextIOWrapper(bytestream, 'utf8', line_buffering=True)
         stdin = StringIO()
         stderr = StringIO()
         ui = cli.UI(['one'], stdin, stdout, stderr)
@@ -206,7 +211,8 @@ def test_parse_error_goes_to_stderr(self):
         self.assertEqual("Could not find command 'one'.\n", stderr.getvalue())
 
     def test_parse_excess_goes_to_stderr(self):
-        stdout = StringIO()
+        bytestream = BytesIO()
+        stdout = TextIOWrapper(bytestream, 'utf8', line_buffering=True)
         stdin = StringIO()
         stderr = StringIO()
         ui = cli.UI(['one'], stdin, stdout, stderr)
@@ -248,7 +254,8 @@ def test_run_subunit_option(self):
         self.assertEqual(True, ui.options.subunit)
 
     def test_dash_dash_help_shows_help(self):
-        stdout = StringIO()
+        bytestream = BytesIO()
+        stdout = TextIOWrapper(bytestream, 'utf8', line_buffering=True)
         stdin = StringIO()
         stderr = StringIO()
         ui = cli.UI(['--help'], stdin, stdout, stderr)
@@ -263,7 +270,7 @@ def test_dash_dash_help_shows_help(self):
             self.assertThat(exc_info, MatchesException(SystemExit(0)))
         else:
             self.fail('ui.set_command did not raise')
-        self.assertThat(stdout.getvalue(),
+        self.assertThat(bytestream.getvalue().decode('utf8'),
             DocTestMatches("""Usage: run.py bar [options] foo
 ...
 A command that can be run...
@@ -352,9 +359,11 @@ def make_result(self, stream=None, argv=None, filter_tags=None):
     def test_initial_stream(self):
         # CLITestResult.__init__ does not do anything to the stream it is
         # given.
-        stream = StringIO()
-        cli.CLITestResult(cli.UI(None, None, None, None), stream, lambda: None)
-        self.assertEqual('', stream.getvalue())
+        bytestream = BytesIO()
+        stream = TextIOWrapper(bytestream, 'utf8', line_buffering=True)
+        ui = cli.UI(None, None, None, None)
+        cli.CLITestResult(ui, stream, lambda: None)
+        self.assertEqual(_b(''), bytestream.getvalue())
 
     def test_format_error(self):
         # CLITestResult formats errors by giving them a big fat line, a title
@@ -376,7 +385,8 @@ def test_format_error_includes_tags(self):
     def test_addFail_outputs_error(self):
         # CLITestResult.status test_status='fail' outputs the given error
         # immediately to the stream.
-        stream = StringIO()
+        bytestream = BytesIO()
+        stream = TextIOWrapper(bytestream, 'utf8', line_buffering=True)
         result = self.make_result(stream)[0]
         error = self.make_exc_info()
         error_text = 'foo\nbar\n'
@@ -385,7 +395,7 @@ def test_addFail_outputs_error(self):
             file_name='traceback', mime_type='text/plain;charset=utf8',
             file_bytes=error_text.encode('utf8'))
         self.assertThat(
-            stream.getvalue(),
+            bytestream.getvalue().decode('utf8'),
             DocTestMatches(result._format_error('FAIL', self, error_text)))
 
     def test_addFailure_handles_string_encoding(self):
@@ -412,7 +422,8 @@ def test_subunit_output(self):
         self.assertEqual(b'', bytestream.getvalue())
 
     def test_make_result_tag_filter(self):
-        stream = StringIO()
+        bytestream = BytesIO()
+        stream = TextIOWrapper(bytestream, 'utf8', line_buffering=True)
         result, summary = self.make_result(
             stream, filter_tags=set(['worker-0']))
         # Generate a bunch of results with tags in the same events that
@@ -438,5 +449,5 @@ def test_make_result_tag_filter(self):
 ----------------------------------------------------------------------
 Ran 1 tests
 FAILED (id=None, failures=1, skips=1)
-""", stream.getvalue())
+""", bytestream.getvalue().decode('utf8'))
 

testrepository/ui/cli.py

@@ -93,6 +93,7 @@ def __init__(self, argv, stdin, stdout, stderr):
         self._stdin = stdin
         self._stdout = stdout
         self._stderr = stderr
+        self._binary_stdout = None
 
     def _iter_streams(self, stream_type):
         # Only the first stream declared in a command can be accepted at the
@@ -155,13 +156,17 @@ def output_rest(self, rest_string):
             self._stdout.write(_u('\n'))
 
     def output_stream(self, stream):
+        if not self._binary_stdout:
+            self._binary_stdout = subunit.make_stream_binary(self._stdout)
         contents = stream.read(65536)
         assert type(contents) is bytes, \
             "Bad stream contents %r" % type(contents)
-        # Outputs bytes, treat them as utf8. Probably needs fixing.
+        # If there are unflushed bytes in the text wrapper, we need to sync..
+        self._stdout.flush()
         while contents:
-            self._stdout.write(contents.decode('utf8'))
+            self._binary_stdout.write(contents)
             contents = stream.read(65536)
+        self._binary_stdout.flush()
 
     def output_table(self, table):
         # stringify