pkg_deb: Create deb pkgs with unicode descriptions

pkg_deb would fail with an error when attempting to create debian
packages with descriptions containing unicode characters or with
utf-8 encoded description_files. Python's bytes objects can only
contain ascii literal characters and so need decoding when they
are sent to strings and non-ascii strings need encoding when sent
to bytes objects.

Resolves: #7370

Testing Done:
- bazel test tools/build_defs/pkg:build_test

Closes #7484.

PiperOrigin-RevId: 240218536

tools/build_defs/pkg/BUILD

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 licenses(["notice"])  # Apache 2.0
 
 filegroup(
@@ -213,9 +214,9 @@ pkg_deb(
         "dep1",
         "dep2",
     ],
-    description = "toto",
+    description = "toto ®, Й, ק ,م, ๗, あ, 叶, 葉, 말, ü and é",
     distribution = "trusty",
-    maintainer = "someone@somewhere.com",
+    maintainer = "soméone@somewhere.com",
     make_deb = ":make_deb",
     package = "titi",
     urgency = "low",

tools/build_defs/pkg/build_test.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# -*- coding: utf-8 -*-
 
 # Copyright 2015 The Bazel Authors. All rights reserved.
 #
@@ -181,8 +182,9 @@ function test_deb() {
   check_eq "-rwxr-xr-x" "$(get_deb_permission test-deb.deb ./usr/titi)"
   check_eq "-rw-r--r--" "$(get_deb_permission test-deb.deb ./etc/nsswitch.conf)"
   get_deb_description test-deb.deb >$TEST_log
-  expect_log "Description: toto"
+  expect_log "Description: toto ®, Й, ק ,م, ๗, あ, 叶, 葉, 말, ü and é"
   expect_log "Package: titi"
+  expect_log "somé[email protected]"
   expect_log "Depends: dep1, dep2"
 
   get_changes titi_test_all.changes >$TEST_log

tools/build_defs/pkg/make_deb.py

@@ -80,11 +80,12 @@
 
 
 def MakeGflags():
+  """Creates a flag for each of the control file fields."""
   for field in DEBIAN_FIELDS:
     fieldname = field[0].replace('-', '_').lower()
     msg = 'The value for the %s content header entry.' % field[0]
     if len(field) > 3:
-      if type(field[3]) is list:
+      if isinstance(field[3], list):
         gflags.DEFINE_multistring(fieldname, field[3], msg)
       else:
         gflags.DEFINE_string(fieldname, field[3], msg)
@@ -133,16 +134,18 @@ def AddArFileEntry(fileobj, filename,
 def MakeDebianControlField(name, value, wrap=False):
   """Add a field to a debian control file."""
   result = name + ': '
-  if type(value) is list:
-    value = ', '.join(value)
+  if isinstance(value, str):
+    value = value.decode('utf-8')
+  if isinstance(value, list):
+    value = u', '.join(value)
   if wrap:
-    result += ' '.join(value.split('\n'))
+    result += u' '.join(value.split('\n'))
     result = textwrap.fill(result,
                            break_on_hyphens=False,
                            break_long_words=False)
   else:
     result += value
-  return result.replace('\n', '\n ') + '\n'
+  return result.replace(u'\n', u'\n ') + u'\n'
 
 
 def CreateDebControl(extrafiles=None, **kwargs):
@@ -159,7 +162,8 @@ def CreateDebControl(extrafiles=None, **kwargs):
   with gzip.GzipFile('control.tar.gz', mode='w', fileobj=tar, mtime=0) as gz:
     with tarfile.open('control.tar.gz', mode='w', fileobj=gz) as f:
       tarinfo = tarfile.TarInfo('control')
-      tarinfo.size = len(controlfile)
+      # Don't discard unicode characters when computing the size
+      tarinfo.size = len(controlfile.encode('utf-8'))
       f.addfile(tarinfo, fileobj=BytesIO(controlfile.encode('utf-8')))
       if extrafiles:
         for name, (data, mode) in extrafiles.items():
@@ -261,34 +265,43 @@ def CreateChanges(output,
   debsize = str(os.path.getsize(deb_file))
   deb_basename = os.path.basename(deb_file)
 
-  changesdata = ''.join(
-      MakeDebianControlField(*x)
-      for x in [('Format', '1.8'), ('Date', time.ctime(timestamp)), (
-          'Source', package
-      ), ('Binary', package
-         ), ('Architecture', architecture), ('Version', version), (
-             'Distribution', distribution
-         ), ('Urgency', urgency), ('Maintainer', maintainer), (
-             'Changed-By', maintainer
-         ), ('Description', '\n%s - %s' % (package, short_description)
-            ), ('Changes', ('\n%s (%s) %s; urgency=%s'
-                            '\nChanges are tracked in revision control.'
-                           ) % (package, version, distribution, urgency)
-               ), ('Files', '\n' + ' '.join(
-                   [checksums['md5'], debsize, section, priority, deb_basename])
-                  ), ('Checksums-Sha1', '\n' + ' '.join(
-                      [checksums['sha1'], debsize, deb_basename])
-                     ), ('Checksums-Sha256', '\n' + ' '.join(
-                         [checksums['sha256'], debsize, deb_basename]))])
+  changesdata = ''.join([
+      MakeDebianControlField('Format', '1.8'),
+      MakeDebianControlField('Date', time.ctime(timestamp)),
+      MakeDebianControlField('Source', package),
+      MakeDebianControlField('Binary', package),
+      MakeDebianControlField('Architecture', architecture),
+      MakeDebianControlField('Version', version),
+      MakeDebianControlField('Distribution', distribution),
+      MakeDebianControlField('Urgency', urgency),
+      MakeDebianControlField('Maintainer', maintainer),
+      MakeDebianControlField('Changed-By', maintainer),
+      MakeDebianControlField('Description',
+                             '\n%s - %s' % (package, short_description)),
+      MakeDebianControlField('Changes',
+                             ('\n%s (%s) %s; urgency=%s'
+                              '\nChanges are tracked in revision control.') %
+                             (package, version, distribution, urgency)),
+      MakeDebianControlField(
+          'Files', '\n' + ' '.join(
+              [checksums['md5'], debsize, section, priority, deb_basename])),
+      MakeDebianControlField(
+          'Checksums-Sha1',
+          '\n' + ' '.join([checksums['sha1'], debsize, deb_basename])),
+      MakeDebianControlField(
+          'Checksums-Sha256',
+          '\n' + ' '.join([checksums['sha256'], debsize, deb_basename]))
+  ])
   with open(output, 'w') as changes_fh:
-    changes_fh.write(changesdata)
+    changes_fh.write(changesdata.encode('utf-8'))
 
 
 def GetFlagValue(flagvalue, strip=True):
   if flagvalue:
+    flagvalue = flagvalue.decode('utf-8')
     if flagvalue[0] == '@':
       with open(flagvalue[1:], 'r') as f:
-        flagvalue = f.read()
+        flagvalue = f.read().decode('utf-8')
     if strip:
       return flagvalue.strip()
   return flagvalue