Merge remote-tracking branch 'origin/dev' into master

pyro/Application.py

@@ -55,7 +55,7 @@ def _try_fix_input_path(input_path: str) -> str:
         return input_path
 
     @staticmethod
-    def _validate_project_file(ppj: PapyrusProject):
+    def _validate_project_file(ppj: PapyrusProject) -> None:
         if ppj.imports_node is None and \
                 (ppj.scripts_node is not None or ppj.folders_node is not None):
             Application.log.error('Cannot proceed without imports defined in project')

pyro/PackageManager.py

@@ -19,7 +19,6 @@
 from pyro.CaseInsensitiveList import CaseInsensitiveList
 from pyro.Constants import (GameType,
                             XmlAttributeName)
-from pyro.Enums.ZipCompression import ZipCompression
 from pyro.PapyrusProject import PapyrusProject
 from pyro.ProcessManager import ProcessManager
 from pyro.ProjectOptions import ProjectOptions
@@ -36,6 +35,8 @@ class PackageManager:
     DEFAULT_GLFLAGS = glob.NODIR | glob.MATCHBASE | glob.SPLIT | glob.REALPATH | glob.FOLLOW | glob.IGNORECASE | glob.MINUSNEGATE
     DEFAULT_WCFLAGS = wcmatch.SYMLINKS | wcmatch.IGNORECASE | wcmatch.MINUSNEGATE
 
+    COMPRESS_TYPE = {'store': 0, 'deflate': 8}
+
     includes: int = 0
 
     def __init__(self, ppj: PapyrusProject) -> None:
@@ -46,7 +47,7 @@ def __init__(self, ppj: PapyrusProject) -> None:
         self.zip_extension = '.zip'
 
     @staticmethod
-    def _can_compress_package(containing_folder: str):
+    def _can_compress_package(containing_folder: str) -> bool:
         flags = wcmatch.RECURSIVE | wcmatch.IGNORECASE
 
         # voices bad because bethesda no likey
@@ -147,9 +148,9 @@ def _generate_include_paths(includes_node: etree.ElementBase, root_path: str, zi
 
         for match_node in filter(is_match_node, includes_node):
             attr_in: str = match_node.get(XmlAttributeName.IN).strip()
-            attr_no_recurse: bool = match_node.get(XmlAttributeName.NO_RECURSE) == 'True'
+            attr_no_recurse: bool = match_node.get(XmlAttributeName.NO_RECURSE) == 'True'  # type: ignore
             attr_exclude: str = match_node.get(XmlAttributeName.EXCLUDE).strip()
-            attr_path: str = match_node.get(XmlAttributeName.PATH).strip()
+            attr_path: str = match_node.get(XmlAttributeName.PATH).strip()  # type: ignore
 
             in_path: str = os.path.normpath(attr_in)
 
@@ -275,7 +276,7 @@ def create_packages(self) -> None:
                 if os.path.isabs(source_path):
                     relpath: str = os.path.relpath(source_path, root_dir)
                 else:
-                    relpath: str = source_path
+                    relpath: str = source_path  # type: ignore
                     source_path = os.path.join(self.ppj.project_path, source_path)
 
                 adj_relpath = os.path.normpath(os.path.join(attr_path, relpath))
@@ -324,11 +325,13 @@ def create_zip(self) -> None:
 
             self._check_write_permission(file_path)
 
-            if self.options.zip_compression in ('store', 'deflate'):
-                compress_type = ZipCompression.get(self.options.zip_compression)
-            else:
-                compress_str = zip_node.get(XmlAttributeName.COMPRESSION)
-                compress_type = ZipCompression.get(compress_str)
+            compress_str: str = self.options.zip_compression or zip_node.get(XmlAttributeName.COMPRESSION)
+
+            try:
+                compress_type = self.COMPRESS_TYPE[compress_str.casefold()]
+            except KeyError:
+                PackageManager.log.error(f'"{compress_str}" is not a valid compression type, defaulting to STORE')
+                compress_type = 0
 
             root_dir: str = self.ppj._get_path(zip_node.get(XmlAttributeName.ROOT_DIR),
                                                relative_root_path=self.ppj.project_path,

pyro/PapyrusProject.py

@@ -3,6 +3,7 @@
 import io
 import os
 import sys
+import time
 import typing
 from copy import deepcopy
 
@@ -102,7 +103,8 @@ def __init__(self, options: ProjectOptions) -> None:
         if self.options.output_path and not os.path.isabs(self.options.output_path):
             self.options.output_path = self.get_output_path()
 
-        bool_attr = lambda element, attr_name: element is not None and element.get(attr_name) == 'True'
+        def bool_attr(element: etree.Element, attr_name: str) -> bool:
+            return element is not None and element.get(attr_name) == 'True'
 
         self.optimize = bool_attr(self.ppj_root, XmlAttributeName.OPTIMIZE)
         self.release = bool_attr(self.ppj_root, XmlAttributeName.RELEASE)
@@ -287,6 +289,10 @@ def _parse_variables(self, variables_node: etree.ElementBase) -> None:
             value = self.variables[key]
             self.variables.update({key: self.parse(value)})
 
+        self.variables.update({
+            'UNIXTIME': str(int(time.time()))
+        })
+
     def _update_attributes(self, parent_node: etree.ElementBase) -> None:
         """Updates attributes of element tree with missing attributes and default values"""
         ppj_bool_keys = [
@@ -433,8 +439,9 @@ def _get_implicit_folder_imports(self) -> list:
         if self.folders_node is None:
             return []
 
-        try_append_path = lambda path: implicit_paths.append(path) \
-            if os.path.isdir(path) and path not in self.import_paths else None
+        def try_append_path(path: str) -> None:
+            if os.path.isdir(path) and path not in self.import_paths:
+                implicit_paths.append(path)
 
         for folder_node in filter(is_folder_node, self.folders_node):
             folder_path: str = os.path.normpath(folder_node.text)
@@ -482,7 +489,7 @@ def _get_psc_paths(self) -> dict:
         """Returns script paths from Folders and Scripts nodes"""
         object_names: dict = {}
 
-        def add_object_name(p):
+        def add_object_name(p: str) -> None:
             object_names[p if not os.path.isabs(p) else self._calculate_object_name(p)] = p
 
         # try to populate paths with scripts from Folders and Scripts nodes
@@ -554,7 +561,7 @@ def _get_remote_path(self, node: etree.ElementBase) -> str:
         return local_path
 
     @staticmethod
-    def try_fix_namespace_path(node: etree.ElementBase):
+    def try_fix_namespace_path(node: etree.ElementBase) -> None:
         if is_namespace_path(node):
             node.text = node.text.replace(':', os.sep)
 

pyro/Performance/CompileData.py

@@ -11,14 +11,14 @@ class CompileData:
     success_count: int = field(init=False, default_factory=int)
     command_count: int = field(init=False, default_factory=int)
 
-    def __post_init__(self):
+    def __post_init__(self) -> None:
         self.time = TimeElapsed()
 
     @property
     def failed_count(self) -> int:
         return self.command_count - self.success_count
 
-    def to_string(self):
+    def to_string(self) -> str:
         raw_time, avg_time = ('{0:.3f}s'.format(t)
                               for t in (self.time.value(), self.time.average(self.success_count)))
 

pyro/Performance/PackageData.py

@@ -9,10 +9,10 @@ class PackageData:
     time: TimeElapsed = field(init=False, default_factory=TimeElapsed)
     file_count: int = field(init=False, default_factory=int)
 
-    def __post_init__(self):
+    def __post_init__(self) -> None:
         self.time = TimeElapsed()
 
-    def to_string(self):
+    def to_string(self) -> str:
         raw_time, avg_time = ('{0:.3f}s'.format(t)
                               for t in (self.time.value(), self.time.average(self.file_count)))
 

pyro/Performance/ZippingData.py

@@ -9,10 +9,10 @@ class ZippingData:
     time: TimeElapsed = field(init=False, default_factory=TimeElapsed)
     file_count: int = field(init=False, default_factory=int)
 
-    def __post_init__(self):
+    def __post_init__(self) -> None:
         self.time = TimeElapsed()
 
-    def to_string(self):
+    def to_string(self) -> str:
         raw_time, avg_time = ('{0:.3f}s'.format(t)
                               for t in (self.time.value(), self.time.average(self.file_count)))
 

pyro/PexHeader.py

@@ -1,5 +1,6 @@
 from dataclasses import dataclass, field
-from typing import IO
+from typing import (IO,  # type: ignore
+                    Literal)
 
 from pyro.PexTypes import PexInt
 from pyro.PexTypes import PexStr
@@ -8,7 +9,7 @@
 @dataclass
 class PexHeader:
     size: int = field(init=False, default=0)
-    endianness: str = field(init=False, default='little')
+    endianness: Literal['little', 'big'] = field(init=False, default='little')
 
     magic: PexInt = field(init=False, default_factory=PexInt)
     major_version: PexInt = field(init=False, default_factory=PexInt)
@@ -35,4 +36,7 @@ def read(self, f: IO, name: str, length: int) -> None:
             if isinstance(obj, PexInt):
                 obj.value = int.from_bytes(obj.data, self.endianness, signed=False)
             elif isinstance(obj, PexStr):
-                obj.value = obj.data.decode('ascii')
+                try:
+                    obj.value = obj.data.decode('ascii')
+                except UnicodeDecodeError:
+                    obj.value = obj.data.decode('utf-8')

pyro/ProjectBase.py

@@ -57,6 +57,7 @@ def _get_path(path: str, *, relative_root_path: str, fallback_path: Union[str, l
         :param fallback_path: Absolute path to return if path empty or unset
         """
         if path or startswith(path, (os.curdir, os.pardir)):
+            path = os.path.expanduser(os.path.expandvars(path))
             return path if os.path.isabs(path) else os.path.normpath(os.path.join(relative_root_path, path))
         if isinstance(fallback_path, list):
             return os.path.abspath(os.path.join(*fallback_path))

pyro/Remotes/BitbucketRemote.py

@@ -26,7 +26,7 @@ def _fetch_payloads(self, request_url: str) -> Generator:
             sys.exit(1)
 
         if response.status != 200:
-            status: HTTPStatus = HTTPStatus(response.status)
+            status: HTTPStatus = HTTPStatus(response.status)  # type: ignore
             yield 'Failed to load remote: "%s" (%s %s)' % (request_url, response.status, status.phrase)
             sys.exit(1)
 

pyro/Remotes/GenericRemote.py

@@ -15,10 +15,10 @@ def fetch_contents(self, url: str, output_path: str) -> Generator:
         """
         parsed_url = urlparse(url)
 
-        schemeless_url = url.removeprefix(f'{parsed_url.scheme}://')
+        schemeless_url = url.removeprefix(f'{parsed_url.scheme}://')  # type: ignore
 
         if endswith(parsed_url.netloc, 'github.com', ignorecase=True):
-            if self.config or not self.access_token:
+            if self.config or not self.access_token:  # type: ignore
                 self.access_token = self.find_access_token(schemeless_url)
                 if not self.access_token:
                     raise PermissionError('Cannot download from GitHub remote without access token')

pyro/Remotes/GitHubRemote.py

@@ -49,15 +49,15 @@ def fetch_contents(self, url: str, output_path: str) -> Generator:
             sys.exit(1)
 
         if response.status != 200:
-            status: HTTPStatus = HTTPStatus(response.status)
+            status: HTTPStatus = HTTPStatus(response.status)  # type: ignore
             yield 'Failed to load remote: "%s" (%s %s)' % (request_url.url, response.status, status.phrase)
             sys.exit(1)
 
         payload_objects: Union[dict, list] = json.loads(response.read().decode('utf-8'))
 
         if 'contents_url' in payload_objects:
-            branch = payload_objects['default_branch']
-            contents_url = payload_objects['contents_url'].replace('{+path}', f'?ref={branch}')
+            branch = payload_objects['default_branch']  # type: ignore
+            contents_url = payload_objects['contents_url'].replace('{+path}', f'?ref={branch}')  # type: ignore
             yield from self.fetch_contents(contents_url, output_path)
             return
 
@@ -69,7 +69,7 @@ def fetch_contents(self, url: str, output_path: str) -> Generator:
             if not self.force_overwrite and os.path.isfile(target_path):
                 with open(target_path, mode='rb') as f:
                     data = f.read()
-                    sha1 = hashlib.sha1(b'blob %s\x00%s' % (len(data), data.decode()))
+                    sha1 = hashlib.sha1(b'blob %s\x00%s' % (len(data), data.decode()))  # type: ignore
 
                     if sha1.hexdigest() == payload_object['sha']:
                         continue

pyro/Remotes/GiteaRemote.py

@@ -49,7 +49,7 @@ def fetch_contents(self, url: str, output_path: str) -> Generator:
             sys.exit(1)
 
         if response.status != 200:
-            status: HTTPStatus = HTTPStatus(response.status)
+            status: HTTPStatus = HTTPStatus(response.status)  # type: ignore
             yield 'Failed to load remote: "%s" (%s %s)' % (request_url.url, response.status, status.phrase)
             sys.exit(1)
 
@@ -63,7 +63,7 @@ def fetch_contents(self, url: str, output_path: str) -> Generator:
             if not self.force_overwrite and os.path.isfile(target_path):
                 with open(target_path, mode='rb') as f:
                     data = f.read()
-                    sha1 = hashlib.sha1(b'blob %s\x00%s' % (len(data), data.decode()))
+                    sha1 = hashlib.sha1(b'blob %s\x00%s' % (len(data), data.decode()))  # type: ignore
 
                     if sha1.hexdigest() == payload_object['sha']:
                         continue

pyro/Remotes/RemoteUri.py

@@ -6,5 +6,5 @@ class RemoteUri:
     owner: str = field(init=False, default_factory=str)
     repo: str = field(init=False, default_factory=str)
     branch: str = field(init=False, default_factory=str)
-    data: dict = field(init=False, default=dict)
+    data: dict = field(init=False, default=dict)  # type: ignore
     url: str = field(init=False, default_factory=str)

pyro/XmlRoot.py

@@ -17,6 +17,5 @@ def find(self, key: str) -> etree.ElementBase:
         path = key if not self.ns else f'ns:{key}', {'ns': self.ns}
         return self.node.find(*path)
 
-    # noinspection Mypy
-    def get(self, key: str, default: Any = None) -> Any:
+    def get(self, key: str, default: Any = None) -> Any:  # type: ignore
         return self.node.get(key, default)