#1185 kojiwrapper: Decode output of runroot as UTF-8
Closed 4 years ago by lsedlar. Opened 4 years ago by lsedlar.
lsedlar/pungi ostree-decode-error  into  master

file modified
+11 -2
@@ -19,6 +19,7 @@ 

  import time

  import threading

  import contextlib

+ import sys

  

  import koji

  from kobo.shortcuts import run
@@ -146,8 +147,16 @@ 

          """

          task_id = None

          with self.get_koji_cmd_env() as env:

-             retcode, output = run(command, can_fail=True, logfile=log_file,

-                                   show_cmd=True, env=env, universal_newlines=True)

+             extra_kwarg = {"encoding": "utf-8"} if sys.version_info >= (3, 6) else {}

+             retcode, output = run(

+                 command,

+                 can_fail=True,

+                 logfile=log_file,

+                 show_cmd=True,

+                 env=env,

+                 universal_newlines=True,

+                 **extra_kwarg

+             )

          if "--task-id" in command:

              first_line = output.splitlines()[0]

              if re.match(r'^\d+$', first_line):

file modified
+24 -22
@@ -405,6 +405,21 @@ 

  

  

  class RunrootKojiWrapperTest(KojiWrapperBaseTestCase):

+ 

+     def kwargs(self, **kwargs):

+         defaults= {

+             "can_fail": True,

+             "env": None,

+             "logfile": None,

+             "show_cmd": True,

+             "universal_newlines": True,

+         }

+         if sys.version_info >= (3, 6):

+             defaults["encoding"] = "utf-8"

+ 

+         defaults.update(kwargs)

+         return defaults

+ 

      def test_get_cmd_minimal(self):

          cmd = self.koji.get_runroot_cmd('tgt', 's390x', 'date', use_shell=False, task_id=False)

          self.assertEqual(len(cmd), 7)
@@ -457,11 +472,7 @@ 

  

          result = self.koji.run_runroot_cmd(cmd)

          self.assertDictEqual(result, {'retcode': 0, 'output': output, 'task_id': None})

-         self.assertEqual(

-             run.call_args_list,

-             [mock.call(cmd, can_fail=True, env=None, logfile=None, show_cmd=True,

-                        universal_newlines=True)]

-         )

+         self.assertEqual(run.call_args_list, [mock.call(cmd, **self.kwargs())])

  

      @mock.patch('pungi.wrappers.kojiwrapper.run')

      def test_run_runroot_cmd_with_task_id(self, run):
@@ -471,11 +482,7 @@ 

  

          result = self.koji.run_runroot_cmd(cmd)

          self.assertDictEqual(result, {'retcode': 0, 'output': output, 'task_id': 1234})

-         self.assertEqual(

-             run.call_args_list,

-             [mock.call(cmd, can_fail=True, env=None, logfile=None, show_cmd=True,

-                        universal_newlines=True)]

-         )

+         self.assertEqual(run.call_args_list, [mock.call(cmd, **self.kwargs())])

  

      @mock.patch('pungi.wrappers.kojiwrapper.run')

      def test_run_runroot_cmd_with_task_id_and_fail(self, run):
@@ -485,11 +492,7 @@ 

  

          result = self.koji.run_runroot_cmd(cmd)

          self.assertDictEqual(result, {'retcode': 1, 'output': output, 'task_id': None})

-         self.assertEqual(

-             run.call_args_list,

-             [mock.call(cmd, can_fail=True, env=None, logfile=None, show_cmd=True,

-                        universal_newlines=True)]

-         )

+         self.assertEqual(run.call_args_list, [mock.call(cmd, **self.kwargs())])

  

      @mock.patch('pungi.wrappers.kojiwrapper.run')

      def test_run_runroot_cmd_with_task_id_and_fail_but_emit_id(self, run):
@@ -499,11 +502,7 @@ 

  

          result = self.koji.run_runroot_cmd(cmd)

          self.assertDictEqual(result, {'retcode': 1, 'output': output, 'task_id': 12345})

-         self.assertEqual(

-             run.call_args_list,

-             [mock.call(cmd, can_fail=True, env=None, logfile=None, show_cmd=True,

-                        universal_newlines=True)]

-         )

+         self.assertEqual(run.call_args_list, [mock.call(cmd, **self.kwargs())])

  

      @mock.patch.dict('os.environ', {'FOO': 'BAR'}, clear=True)

      @mock.patch('shutil.rmtree')
@@ -521,8 +520,11 @@ 

          self.assertDictEqual(result, {'retcode': 0, 'output': output, 'task_id': None})

          self.assertEqual(

              run.call_args_list,

-             [mock.call(cmd, can_fail=True, env={'KRB5CCNAME': 'DIR:/tmp/foo', 'FOO': 'BAR'},

-                        logfile=None, show_cmd=True, universal_newlines=True)]

+             [

+                 mock.call(

+                     cmd, **self.kwargs(env={'KRB5CCNAME': 'DIR:/tmp/foo', 'FOO': 'BAR'})

+                 ),

+             ],

          )

  

  

I'm not able to reproduce the issue. This is really a guess at what might be a possible fix.


Starting with Python 3.6, it is possible to specify encoding for output of a subprocess.Popen. Try to use it to decode as UTF-8. Particularly output of ostree runroot tasks is using unicode characters.

This should be safe: anything that is valid ASCII is also valid UTF-8.

Relates: https://pagure.io/dusty/failed-composes/issue/1642

The code looks reasonable.

Looks sane to me!

I'm not able to reproduce the issue.

Hmm, you should be able to reproduce this by just trying to run any command that outputs non-ASCII chars, no? E.g. echo 🐶.

Hmm, you should be able to reproduce this by just trying to run any command that outputs non-ASCII chars, no? E.g. echo 🐶.

Well, yes as long as locale is set. I didn't realize it was set, and it works in UTF-8 locale just fine.

I don't think this patch is a sufficient fix. It will work for decoding the output, but since kobo immediately writes it log file, that write will still use default encoding (ascii) and fail.

Fixing the locale is a better way to go: #1186.

Pull-Request has been closed by lsedlar

4 years ago