Enforce utf8 when opening files in python scripts

dev-tools/generate_notice.py

@@ -15,13 +15,8 @@ def read_file(filename):
         print("File not found {}".format(filename))
         return ""
 
-    try:
-        with open(filename, 'r') as f:
-            return f.read()
-    except UnicodeDecodeError:
-        # try latin-1
-        with open(filename, 'r', encoding="ISO-8859-1") as f:
-            return f.read()
+    with open(filename, 'r', encoding='utf_8') as f:
+        return f.read()
 
 
 def get_library_path(license):
@@ -37,7 +32,7 @@ def get_library_path(license):
 
 def read_versions(vendor):
     libs = []
-    with open(os.path.join(vendor, "vendor.json")) as f:
+    with open(os.path.join(vendor, "vendor.json"), encoding='utf_8') as f:
         govendor = json.load(f)
         for package in govendor["package"]:
             libs.append(package)
@@ -222,11 +217,11 @@ def get_url(repo):
 def create_notice(filename, beat, copyright, vendor_dirs, csvfile, overrides=None):
     dependencies = gather_dependencies(vendor_dirs, overrides=overrides)
     if not csvfile:
-        with open(filename, "w+") as f:
+        with open(filename, "w+", encoding='utf_8') as f:
             write_notice_file(f, beat, copyright, dependencies)
             print("Available at {}".format(filename))
     else:
-        with open(csvfile, "wb") as f:
+        with open(csvfile, "wb", encoding='utf_8') as f:
             csvwriter = csv.writer(f)
             write_csv_file(csvwriter, dependencies)
             print("Available at {}".format(csvfile))

filebeat/scripts/docs_collector.py

@@ -38,13 +38,13 @@ def collect(beat_name):
         module_file = generated_note
         module_file += "[[filebeat-module-" + module + "]]\n"
 
-        with open(module_doc) as f:
+        with open(module_doc, encoding='utf_8') as f:
             module_file += f.read()
 
         beat_path = os.path.join(module_dir, "_meta")
 
         # Load title from fields.yml
-        with open(beat_path + "/fields.yml") as f:
+        with open(beat_path + "/fields.yml", encoding='utf_8') as f:
             fields = yaml.load(f.read())
             title = fields[0]["title"]
 
@@ -61,8 +61,8 @@ def collect(beat_name):
 """
 
         # Write module docs
-        with open(os.path.abspath("docs") + "/modules/" +
-                  module + ".asciidoc", 'w') as f:
+        docs_path = os.path.join(os.path.abspath("docs"), "modules",  module + ".asciidoc")
+        with open(docs_path, 'w', encoding='utf_8') as f:
             f.write(module_file)
 
     module_list_output = generated_note
@@ -76,7 +76,7 @@ def collect(beat_name):
         module_list_output += "include::modules/" + m + ".asciidoc[]\n"
 
     # Write module link list
-    with open(os.path.abspath("docs") + "/modules_list.asciidoc", 'w') as f:
+    with open(os.path.abspath("docs") + "/modules_list.asciidoc", 'w', encoding='utf_8') as f:
         f.write(module_list_output)
 
 

libbeat/scripts/unpack_dashboards.py

@@ -27,7 +27,7 @@ def transform_data(data, method):
 
 
 def transform_file(path, method):
-    with open(path) as f:
+    with open(path, encoding='utf_8') as f:
         data = json.load(f)
 
     transform_data(data, method)
@@ -52,5 +52,5 @@ def transform_file(path, method):
         data = transform_file(path, method)
         new_data = json.dumps(data, sort_keys=True, indent=4)
 
-        with open(path, 'w') as f:
+        with open(path, 'w', encoding='utf_8') as f:
             f.write(new_data)