diff --git a/docs/changelog.txt b/docs/changelog.txt index c8cf5e8..2921fd6 100644 --- a/docs/changelog.txt +++ b/docs/changelog.txt @@ -1,3 +1,4 @@ +- Supported ucas/utoc assets for UE5 - Added support for UE5.4. - Fixed typos. diff --git a/src/unreal/archive.py b/src/unreal/archive.py index b8a416e..e108863 100644 --- a/src/unreal/archive.py +++ b/src/unreal/archive.py @@ -22,6 +22,7 @@ class ArchiveBase: io: IOBase is_reading = False is_writing = False + is_ucas = False def __init__(self, io: IOBase, endian="little", context: dict = {}): self.io = io @@ -65,13 +66,35 @@ def write(self, obj): def close(self): self.io.close() + def raise_error(self, msg="Parse failed. Make sure you specified UE version correctly."): + if (hasattr(self, "uasset")): + msg += " (" + self.uasset.file_name + ")" + raise RuntimeError(msg) + def check(self, actual, expected, msg="Parse failed. Make sure you specified UE version correctly."): if actual == expected: return print(f"offset: {self.tell()}") print(f"actual: {actual}") print(f"expected: {expected}") - raise RuntimeError(msg) + self.raise_error(msg) + + def check_buffer_size(self, size): + if self.tell() + size > self.size: + raise RuntimeError( + "There is no buffer that has specified size." + f" (Offset: {self.tell()}, Size: {size})" + ) + + def update_with_current_offset(self, obj, attr_name): + if self.is_reading: + # Checks obj.attr_name is the same as the current offset + current_offs = self.tell() + serialized_offs = getattr(obj, attr_name) + self.check(serialized_offs, current_offs) + else: + # Update obj.attr_name with the current offset + setattr(obj, attr_name, self.tell()) class ArchiveRead(ArchiveBase): @@ -121,11 +144,7 @@ class Buffer(Bytes): @staticmethod def read(ar: ArchiveBase) -> bytes: size = ar.args[0] - if ar.tell() + size > ar.size: - raise RuntimeError( - "There is no buffer that has specified size." - f" (Offset: {ar.tell()}, Size: {size})" - ) + ar.check_buffer_size(size) return ar.read(size) @@ -234,6 +253,26 @@ def write(ar: ArchiveBase, val: str): ar.write(str_byte + b"\x00" * (1 + utf16)) +class StringWithLen: + @staticmethod + def get_args(ar: ArchiveBase): + num = ar.args[0] + utf16 = ar.args[1] + encode = "utf-16-le" if utf16 else "ascii" + return num, utf16, encode + + @staticmethod + def read(ar: ArchiveBase) -> str: + num, utf16, encode = StringWithLen.get_args(ar) + string = ar.read(num * (1 + utf16)).decode(encode) + return string + + @staticmethod + def write(ar: ArchiveBase, val: str): + _, utf16, encode = StringWithLen.get_args(ar) + ar.write(val.encode(encode)) + + class SerializableBase: def serialize(self, ar: ArchiveBase): # pragma: no cover pass diff --git a/src/unreal/city_hash.py b/src/unreal/city_hash.py new file mode 100644 index 0000000..17c8759 --- /dev/null +++ b/src/unreal/city_hash.py @@ -0,0 +1,167 @@ +# Converted UE4's codes (CityHash.cpp, etc.) to python. +# https://github.com/EpicGames/UnrealEngine + +# Bit mask to use uint64 and uint32 on python +MASK_64 = 0xFFFFFFFFFFFFFFFF +MASK_32 = 0xFFFFFFFF + +# Some primes between 2^63 and 2^64 for various uses. +k0 = 0xc3a5c85c97cb3127 +k1 = 0xb492b66fbe98f273 +k2 = 0x9ae16a3b2f90404f + + +def to_uint(binary: bytes) -> int: + return int.from_bytes(binary, "little") + + +# use char* as uint64 pointer +def fetch64(binary: bytes) -> int: + return to_uint(binary[:8]) + + +# use char* as uint32 pointer +def fetch32(binary: bytes) -> int: + return to_uint(binary[:4]) + + +def bswap_64(i: int) -> int: + i &= MASK_64 + b = i.to_bytes(8, byteorder="little") + return int.from_bytes(b, "big") + + +def rotate(val: int, shift: int) -> int: + val &= MASK_64 + return val if shift == 0 else ((val >> shift) | (val << (64 - shift))) & MASK_64 + + +def shift_mix(val: int) -> int: + val &= MASK_64 + return (val ^ (val >> 47)) & MASK_64 + + +def hash_len_16(u: int, v: int, mul: int) -> int: + a = ((u ^ v) * mul) & MASK_64 + a ^= (a >> 47) + b = ((v ^ a) * mul) & MASK_64 + b ^= (b >> 47) + b *= mul + return b & MASK_64 + + +def hash_len_16_2(u: int, v: int) -> int: + kMul = 0x9ddfea08eb382d69 + return hash_len_16(u, v, kMul) + + +def hash_len_0to16(binary: bytes) -> int: + length = len(binary) + if length >= 8: + mul = k2 + length * 2 + a = fetch64(binary) + k2 + b = fetch64(binary[-8:]) + c = rotate(b, 37) * mul + a + d = (rotate(a, 25) + b) * mul + return hash_len_16(c, d, mul) + if length >= 4: + mul = k2 + length * 2 + a = fetch32(binary) + return hash_len_16(length + (a << 3), fetch32(binary[-4:]), mul) + if length > 0: + a = binary[0] + b = binary[length >> 1] + c = binary[:-1] + y = (a + (b << 8)) & MASK_32 + z = (length + (c << 2)) & MASK_32 + return (shift_mix(y * k2 ^ z * k0) * k2) & MASK_64 + return k2 + + +def hash_len_17to32(binary: bytes) -> int: + length = len(binary) + mul = k2 + length * 2 + a = fetch64(binary) * k1 + b = fetch64(binary[8:]) + c = fetch64(binary[-8:]) * mul + d = fetch64(binary[-16:]) * k2 + return (hash_len_16( + rotate(a + b, 43) + rotate(c, 30) + d, + a + rotate(b + k2, 18) + c, + mul) + ) & MASK_64 + + +def hash_len_33to64(binary: bytes) -> int: + length = len(binary) + mul = k2 + length * 2 + a = fetch64(binary) * k2 + b = fetch64(binary[8:]) + c = fetch64(binary[-24:]) + d = fetch64(binary[-32:]) + e = fetch64(binary[16:]) * k2 + f = fetch64(binary[24:]) * 9 + g = fetch64(binary[-8:]) + h = fetch64(binary[-16:]) * mul + u = rotate(a + g, 43) + (rotate(b, 30) + c) * 9 + v = ((a + g) ^ d) + f + 1 + w = bswap_64((u + v) * mul) + h + x = rotate(e + f, 42) + c + y = (bswap_64((v + w) * mul) + g) * mul + z = e + f + c + a = (bswap_64((x + z) * mul + y) + b) + b = shift_mix((z + a) * mul + d + h) * mul + return (b + x) & MASK_64 + + +def weak_hash_len32_with_seeds(binary: bytes, a: int, b: int) -> int: + return weak_hash_len32_with_seeds2( + fetch64(binary), + fetch64(binary[8:]), + fetch64(binary[16:]), + fetch64(binary[24:]), + a, + b) + + +def weak_hash_len32_with_seeds2(w: int, x: int, y: int, z: int, a: int, b: int) -> int: + a += w + b = rotate(b + a + z, 21) + c = a + a += x + a += y + b += rotate(a, 44) + return (a + z) & MASK_64, (b + c) & MASK_64 + + +def city_hash_64(binary: bytes) -> int: + length = len(binary) + if length <= 32: + if length <= 16: + return hash_len_0to16(binary) + else: + return hash_len_17to32(binary) + elif length <= 64: + return hash_len_33to64(binary) + + x = fetch64(binary[-40:]) + y = fetch64(binary[-16:]) + fetch64(binary[-56:]) + z = hash_len_16_2(fetch64(binary[-48:]) + length, fetch64(binary[-24:])) + v_lo, v_hi = weak_hash_len32_with_seeds(binary[-64:], length, z) + w_lo, w_hi = weak_hash_len32_with_seeds(binary[-32:], y + k1, x) + x = x * k1 + fetch64(binary) + length = (length - 1) & (~63) + binary = binary[:length] + + while (len(binary) > 0): + x = rotate(x + y + v_lo + fetch64(binary[8:]), 37) * k1 + y = rotate(y + v_hi + fetch64(binary[48:]), 42) * k1 + x ^= w_hi + y += v_lo + fetch64(binary[40:]) + z = rotate(z + w_lo, 33) * k1 + v_lo, v_hi = weak_hash_len32_with_seeds(binary, v_hi * k1, x + w_lo) + w_lo, w_hi = weak_hash_len32_with_seeds(binary[32:], z + w_hi, y + fetch64(binary[16:])) + z, x = x, z + binary = binary[64:] + return hash_len_16_2(hash_len_16_2(v_lo, w_lo) + shift_mix(y) * k1 + z, + hash_len_16_2(v_hi, w_hi) + x) diff --git a/src/unreal/crc.py b/src/unreal/crc.py index 4dc5bdd..33ad1c0 100644 --- a/src/unreal/crc.py +++ b/src/unreal/crc.py @@ -168,7 +168,7 @@ def memcrc(string): return ~crc & 0xFFFFFFFF -def generate_hash(string): +def strcrc(string): """Generate hash from a string. Args: diff --git a/src/unreal/data_resource.py b/src/unreal/data_resource.py index 5798f7d..e1ee9b1 100644 --- a/src/unreal/data_resource.py +++ b/src/unreal/data_resource.py @@ -196,3 +196,46 @@ def print(self, padding=2): print(pad + f" data size: {self.data_size}") print(pad + f" outer index: {self.outer_index}") print(pad + f" legacy bulk data flags: {self.bulk_flags}") + + +class BulkDataMapEntry(SerializableBase, DataResourceBase): + """data resource for ucas assets. (FBulkDataMapEntry) + + Notes: + UnrealEngine/Engine/Source/Runtime/CoreUObject/Public/Serialization/AsyncLoading2.h + The latest UE version will write the meta data in .uasset. + """ + def __init__(self): + super().__init__() + self.flags = 0 + self.duplicated_offset = -1 + + def serialize(self, ar: ArchiveBase): + if ar.is_writing: + if not ar.valid: + self.update_bulk_flags(ar) + + ar << (Int64, self, "offset") + ar << (Int64, self, "duplicated_offset") + ar << (Int64, self, "data_size") + ar << (Uint32, self, "bulk_flags") + ar == (Uint32, 0, "pad") + + if ar.is_reading: + self.unpack_bulk_flags(ar) + + def update(self, data_size: int, has_uexp_bulk: bool): + super().update(data_size, has_uexp_bulk) + self.has_64bit_size = True + + def print(self, padding=2): + pad = " " * padding + print(pad + "DataResource") + print(pad + f" serial offset: {self.offset}") + print(pad + f" duplicated serial offset: {self.duplicated_offset}") + print(pad + f" data size: {self.data_size}") + print(pad + f" flags: {self.bulk_flags}") + + @staticmethod + def get_struct_size(ar: ArchiveBase) -> int: + return 32 diff --git a/src/unreal/file_summary.py b/src/unreal/file_summary.py new file mode 100644 index 0000000..f5315ee --- /dev/null +++ b/src/unreal/file_summary.py @@ -0,0 +1,408 @@ +from enum import IntEnum +import os + +from .crc import strcrc_deprecated +from .archive import (ArchiveBase, + Uint32, Uint64, Int32, Int64, Int32Array, + Bytes, Buffer, String, SerializableBase) +from .import_export import (NameBase, ImportBase, ExportBase, + UassetName, UassetImport, UassetExport, + ZenName, ZenImport, ZenExport) +from .data_resource import DataResourceBase, UassetDataResource, BulkDataMapEntry + + +class PackageFlags(IntEnum): + PKG_UnversionedProperties = 0x2000 # Uses unversioned property serialization + PKG_FilterEditorOnly = 0x80000000 # Package has editor-only data filtered out + + +class FileSummaryBase(SerializableBase): + """Info for .uasset file (FPackageFileSummary) + + Notes: + UnrealEngine/Engine/Source/Runtime/CoreUObject/Private/UObject/PackageFileSummary.cpp + """ + + def serialize(self, ar: ArchiveBase): + self.file_name = ar.name + + def serialize_version_info(self, ar: ArchiveBase): + """ + Version info. But most assets have zeros for these variables. (unversioning) + So, we can't get UE version from them. + - LegacyUE3Version + - FileVersionUE.FileVersionUE4 + - FileVersionUE.FileVersionUE5 (Added at 5.0) + - FileVersionLicenseeUE + - CustomVersionContainer + """ + ar << (Bytes, self, "version_info", 16 + 4 * (ar.version >= "5.0")) + + def serialize_name_map(self, ar: ArchiveBase, name_list: list[NameBase]) -> list[NameBase]: + pass + + def serialize_imports(self, ar: ArchiveBase, imports: list[ImportBase]) -> list[ImportBase]: + pass + + def serialize_exports(self, ar: ArchiveBase, exports: list[ExportBase]) -> list[ExportBase]: + pass + + def skip_exports(self, ar: ArchiveBase, count: int): + pass + + def serialize_data_resources(self, ar: ArchiveBase, imports: list[DataResourceBase]) -> list[DataResourceBase]: + pass + + def print(self): + pass + + def is_unversioned(self): + return (self.pkg_flags & PackageFlags.PKG_UnversionedProperties) > 0 + + def update_package_source(self, file_name=None, is_official=True): + pass + + +class UassetFileSummary(FileSummaryBase): + """Info for .uasset file (FPackageFileSummary) + + Notes: + UnrealEngine/Engine/Source/Runtime/CoreUObject/Private/UObject/PackageFileSummary.cpp + """ + + def serialize(self, ar: ArchiveBase): + super().serialize(ar) + + ar << (Bytes, self, "tag", 4) + + """ + File version + positive: 3.x + -3: 4.0 ~ 4.6 + -5: 4.7 ~ 4.9 + -6: 4.10 ~ 4.13 + -7: 4.14 ~ 4.27 + -8: 5.0 ~ + """ + expected_version = ( + -8 + (ar.version <= "4.6") * 2 + (ar.version <= "4.9") + + (ar.version <= "4.13") + (ar.version <= "4.27") + ) + ar == (Int32, expected_version, "header.file_version") + + self.serialize_version_info(ar) + + ar << (Int32, self, "uasset_size") # TotalHeaderSize + ar << (String, self, "package_name") + + # PackageFlags + ar << (Uint32, self, "pkg_flags") + if ar.is_reading: + ar.check(self.pkg_flags & PackageFlags.PKG_FilterEditorOnly > 0, True, + msg="Unsupported file format detected. (PKG_FilterEditorOnlyitorOnly is false.)") + + # Name table + ar << (Int32, self, "name_count") + ar << (Int32, self, "name_offset") + + if ar.version >= "5.1": + # SoftObjectPaths + ar == (Int32, 0, "soft_object_count") + if ar.is_writing: + self.soft_object_offset = self.import_offset + ar << (Int32, self, "soft_object_offset") + + if ar.version >= "4.9": + # GatherableTextData + ar == (Int32, 0, "gatherable_text_count") + ar == (Int32, 0, "gatherable_text_offset") + + # Exports + ar << (Int32, self, "export_count") + ar << (Int32, self, "export_offset") + + # Imports + ar << (Int32, self, "import_count") + ar << (Int32, self, "import_offset") + + # DependsOffset + ar << (Int32, self, "depends_offset") + + if ar.version >= "4.4" and ar.version <= "4.14": + # StringAssetReferencesCount + ar == (Int32, 0, "string_asset_count") + if ar.is_writing: + self.string_asset_offset = self.asset_registry_data_offset + ar << (Int32, self, "string_asset_offset") + elif ar.version >= "4.15": + # SoftPackageReferencesCount + ar == (Int32, 0, "soft_package_count") + ar == (Int32, 0, "soft_package_offset") + + # SearchableNamesOffset + ar == (Int32, 0, "searchable_name_offset") + + # ThumbnailTableOffset + ar == (Int32, 0, "thumbnail_table_offset") + + ar << (Bytes, self, "guid", 16) # GUID + + # Generations: Export count and name count for previous versions of this package + ar << (Int32, self, "generation_count") + if self.generation_count <= 0 or self.generation_count >= 10: + raise RuntimeError(f"Unexpected value. (generation_count: {self.generation_count})") + ar << (Int32Array, self, "generation_data", self.generation_count * 2) + + """ + - SavedByEngineVersion (14 bytes) + - CompatibleWithEngineVersion (14 bytes) (4.8 ~ ) + """ + ar << (Bytes, self, "empty_engine_version", 14 * (1 + (ar.version >= "4.8"))) + + # CompressionFlags, CompressedChunks + ar << (Bytes, self, "compression_info", 8) + + """ + PackageSource: + Value that is used to determine if the package was saved by developer or not. + CRC hash for shipping builds. Others for user created files. + """ + ar << (Uint32, self, "package_source") + + # AdditionalPackagesToCook (zero length array) + ar == (Int32, 0, "additional_packages_to_cook") + + if ar.version <= "4.13": + ar == (Int32, 0, "num_texture_allocations") + ar << (Int32, self, "asset_registry_data_offset") + ar << (Int32, self, "bulk_offset") # .uasset + .uexp - 4 (BulkDataStartOffset) + + # WorldTileInfoDataOffset + ar == (Int32, 0, "world_tile_info_offset") + + # ChunkIDs (zero length array), ChunkID + ar == (Int32Array, [0, 0], "ChunkID", 2) + + if ar.version <= "4.13": + return + + # PreloadDependency + ar << (Int32, self, "preload_dependency_count") + ar << (Int32, self, "preload_dependency_offset") + + if ar.version <= "4.27": + return + + # Number of names that are referenced from serialized export data + ar << (Int32, self, "referenced_names_count") + + # Location into the file on disk for the payload table of contents data + ar << (Int64, self, "payload_toc_offset") + + if ar.version <= "5.1": + return + + # Location into the file of the data resource + ar << (Int32, self, "data_resource_offset") + + def serialize_name_map(self, ar: ArchiveBase, name_list: list[UassetName]) -> list[UassetName]: + if ar.is_reading: + name_list = [UassetName() for i in range(self.name_count)] + else: + self.name_count = len(name_list) + list(map(lambda x: x.serialize(ar), name_list)) + return name_list + + def serialize_imports(self, ar: ArchiveBase, imports: list[UassetImport]) -> list[UassetImport]: + ar.update_with_current_offset(self, "import_offset") + if ar.is_reading: + imports = [UassetImport() for i in range(self.import_count)] + else: + self.import_count = len(imports) + list(map(lambda x: x.serialize(ar), imports)) + return imports + + def serialize_exports(self, ar: ArchiveBase, exports: list[UassetExport]) -> list[UassetExport]: + ar.update_with_current_offset(self, "export_offset") + if ar.is_reading: + exports = [UassetExport() for i in range(self.export_count)] + else: + self.export_count = len(exports) + list(map(lambda x: x.serialize(ar), exports)) + return exports + + def skip_exports(self, ar: ArchiveBase, count: int): + ar.seek(UassetExport.get_struct_size(ar.version) * count, 1) + + def serialize_data_resources(self, ar: ArchiveBase, + data_resources: list[UassetDataResource]) -> list[UassetDataResource]: + ar.update_with_current_offset(self, "data_resource_offset") + if ar.is_writing: + self.data_resource_count = len(data_resources) + ar == (Int32, 1, "deta_resource_version") + ar << (Int32, self, "data_resource_count") + + if ar.is_reading: + data_resources = [UassetDataResource() for i in range(self.data_resource_count)] + list(map(lambda x: x.serialize(ar), data_resources)) + return data_resources + + def print(self): + print("File Summary") + print(f" file size: {self.uasset_size}") + print(f" number of names: {self.name_count}") + print(" name directory offset: 193") + print(f" number of exports: {self.export_count}") + print(f" export directory offset: {self.export_offset}") + print(f" number of imports: {self.import_count}") + print(f" import directory offset: {self.import_offset}") + print(f" depends offset: {self.depends_offset}") + print(f" file length (uasset+uexp-4): {self.bulk_offset}") + print(f" official asset: {self.is_official()}") + print(f" unversioned: {self.is_unversioned()}") + + def update_package_source(self, file_name=None, is_official=True): + if file_name is not None: + self.file_name = file_name + if is_official: + crc = strcrc_deprecated("".join(os.path.basename(self.file_name).split(".")[:-1])) + else: + # UE doesn't care this value. So, we can embed any four bytes here. + crc = int.from_bytes(b"MOD ", "little") + self.package_source = crc + + def is_official(self): + crc = strcrc_deprecated("".join(os.path.basename(self.file_name).split(".")[:-1])) + return self.package_source == crc + + +class ZenPackageSummary(FileSummaryBase): + """Info for ucas assets (FZenPackageSummary) + + Notes: + UnrealEngine/Engine/Source/Runtime/CoreUObject/Public/Serialization/AsyncLoading2.h + """ + + def serialize(self, ar: ArchiveBase): + super().serialize(ar) + + # TODO: update offsets and cooked_header_size when writing + ar << (Uint32, self, "has_version_info") + ar << (Uint32, self, "uasset_size") + ar << (Uint32, self, "package_name_id") + ar << (Uint32, self, "package_name_number") + ar << (Uint32, self, "pkg_flags") + ar << (Uint32, self, "cooked_header_size") # uasset_size when using UassetFileSummary + ar << (Int32, self, "export_hashes_offset") + ar << (Int32, self, "import_offset") + ar << (Int32, self, "export_offset") + ar << (Int32, self, "export_bundle_entries_offset") + if ar.version >= "5.3": + ar << (Int32, self, "dependency_bundle_headers_offset") + ar << (Int32, self, "dependency_bundle_entries_offset") + ar << (Int32, self, "imported_package_names_offset") + else: + ar << (Int32, self, "graph_data_offset") + if self.has_version_info: + self.serialize_version_info(ar) + self.name_offset = ar.tell() + + def print(self): + print("File Summary") + print(f" file size: {self.uasset_size}") + print(f" cooked header size: {self.cooked_header_size}") + print(f" name directory offset: {self.name_offset}") + print(f" import directory offset: {self.import_offset}") + print(f" export directory offset: {self.export_offset}") + print(f" unversioned: {self.is_unversioned()}") + + def serialize_name_map(self, ar: ArchiveBase, name_list: list[ZenName]) -> list[ZenName]: + if ar.is_writing: + self.name_count = len(name_list) + new_name_buffer_size = sum(len(str(n)) for n in name_list) + # update cooked_header_size when changing pixel format + self.cooked_header_size += new_name_buffer_size - self.name_buffer_size + self.name_buffer_size = new_name_buffer_size + ar << (Uint32, self, "name_count") + ar << (Uint32, self, "name_buffer_size") + ar == (Uint64, 0xC1640000, "hash_version") + if ar.is_reading: + ar.check_buffer_size(self.name_buffer_size) + name_list = [ZenName() for i in range(self.name_count)] + list(map(lambda x: x.serialize_hash(ar), name_list)) + list(map(lambda x: x.serialize_head(ar), name_list)) + list(map(lambda x: x.serialize_string(ar), name_list)) + if ar.is_writing: + self.pad_size = (8 - (ar.tell() % 8)) % 8 + if ar.version >= "5.2": + ar << (Uint64, self, "pad_size") + ar == (Buffer, b"\x00" * self.pad_size, "pad", self.pad_size) + return name_list + + def serialize_data_resources(self, ar: ArchiveBase, + data_resources: list[BulkDataMapEntry]) -> list[BulkDataMapEntry]: + struct_size = BulkDataMapEntry.get_struct_size(ar) + + if ar.is_writing: + self.bulk_data_map_size = len(data_resources) * struct_size + + ar << (Int64, self, "bulk_data_map_size") + + if ar.is_reading: + ar.check_buffer_size(self.bulk_data_map_size) + data_resource_count = self.bulk_data_map_size // struct_size + data_resources = [BulkDataMapEntry() for i in range(data_resource_count)] + + list(map(lambda x: x.serialize(ar), data_resources)) + return data_resources + + def serialize_export_hashes(self, ar: ArchiveBase): + size = self.import_offset - self.export_hashes_offset + ar.update_with_current_offset(self, "export_hashes_offset") + ar << (Buffer, self, "export_hashes", size) + + def serialize_imports(self, ar: ArchiveBase, imports: list[ZenImport]) -> list[ZenImport]: + ar.update_with_current_offset(self, "import_offset") + if ar.is_reading: + struct_size = ZenImport.get_struct_size(ar) + import_count = (self.export_offset - self.import_offset) // struct_size + imports = [ZenImport() for i in range(import_count)] + list(map(lambda x: x.serialize(ar), imports)) + return imports + + def serialize_exports(self, ar: ArchiveBase, exports: list[ZenExport]) -> list[ZenExport]: + ar.update_with_current_offset(self, "export_offset") + if ar.is_reading: + struct_size = ZenExport.get_struct_size(ar) + export_count = (self.export_bundle_entries_offset - self.export_offset) // struct_size + exports = [ZenExport() for i in range(export_count)] + list(map(lambda x: x.serialize(ar), exports)) + return exports + + def skip_exports(self, ar: ArchiveBase, count: int): + ar.seek(ZenExport.get_struct_size(ar.version) * count, 1) + + def serialize_others(self, ar: ArchiveBase): + if ar.version >= "5.3": + export_bundle_entries_size = (self.dependency_bundle_headers_offset + - self.export_bundle_entries_offset) + dependency_bundle_headers_size = (self.dependency_bundle_entries_offset + - self.dependency_bundle_headers_offset) + dependency_bundle_entries_size = (self.imported_package_names_offset + - self.dependency_bundle_entries_offset) + imported_package_names_size = self.uasset_size - self.imported_package_names_offset + ar.update_with_current_offset(self, "export_bundle_entries_offset") + ar << (Buffer, self, "export_bundle_entries", export_bundle_entries_size) + ar.update_with_current_offset(self, "dependency_bundle_headers_offset") + ar << (Buffer, self, "dependency_bundle_headers", dependency_bundle_headers_size) + ar.update_with_current_offset(self, "dependency_bundle_entries_offset") + ar << (Buffer, self, "dependency_bundle_entries", dependency_bundle_entries_size) + ar.update_with_current_offset(self, "imported_package_names_offset") + ar << (Buffer, self, "imported_package_names", imported_package_names_size) + else: + export_bundle_entries_size = self.graph_data_offset - self.export_bundle_entries_offset + graph_data_size = self.uasset_size - self.graph_data_offset + ar.update_with_current_offset(self, "export_bundle_entries_offset") + ar << (Buffer, self, "export_bundle_entries", export_bundle_entries_size) + ar.update_with_current_offset(self, "graph_data_offset") + ar << (Buffer, self, "graph_data", graph_data_size) diff --git a/src/unreal/import_export.py b/src/unreal/import_export.py new file mode 100644 index 0000000..678075c --- /dev/null +++ b/src/unreal/import_export.py @@ -0,0 +1,378 @@ +from enum import IntEnum +from .crc import strcrc +from .city_hash import city_hash_64 +from .version import VersionInfo +from .archive import (ArchiveBase, + Uint8, Uint32, Uint64, Int32, Bytes, + String, StringWithLen, + SerializableBase) + + +class ObjectFlags(IntEnum): + RF_Public = 1 + RF_Standalone = 2 # Main object in the asset + RF_Transactional = 8 + RF_ClassDefaultObject = 0x10 # Default object + RF_ArchetypeObject = 0x20 # Template for another object + + +class NameBase(SerializableBase): + def __init__(self): + self.hash = None + + def serialize(self, ar: ArchiveBase): + pass + + def __str__(self): + return self.name + + def update(self, new_name, update_hash=False): + pass + + +class ImportBase(SerializableBase): + def serialize(self, ar: ArchiveBase): + pass + + def name_import(self, imports: list, name_list: list[NameBase]) -> str: + pass + + def print(self, padding=2): + pass + + +class ExportBase(SerializableBase): + TEXTURE_CLASSES = [ + "Texture2D", "TextureCube", "LightMapTexture2D", "ShadowMapTexture2D", + "Texture2DArray", "TextureCubeArray", "VolumeTexture" + ] + + def __init__(self): + self.object = None # The actual data will be stored here + self.meta_size = 0 # binary size of meta data + + def serialize(self, ar: ArchiveBase): + pass + + def name_export(self, exports: list, imports: list[ImportBase], name_list: list[NameBase]): + pass + + def is_base(self): + return (self.object_flags & ObjectFlags.RF_ArchetypeObject > 0 + or self.object_flags & ObjectFlags.RF_ClassDefaultObject > 0) + + def is_standalone(self): + return self.object_flags & ObjectFlags.RF_Standalone > 0 + + def is_public(self): + return self.object_flags & ObjectFlags.RF_Public > 0 + + def is_texture(self): + return self.class_name in ExportBase.TEXTURE_CLASSES + + def update(self, size, offset): + self.size = size + self.offset = offset + + @staticmethod + def get_meta_size(version: VersionInfo): + pass + + +class UassetName(NameBase): + def serialize(self, ar: ArchiveBase): + ar << (String, self, "name") + if ar.version <= "4.11": + return + ar << (Bytes, self, "hash", 4) + + def update(self, new_name, update_hash=False): + self.name = new_name + if update_hash: + self.hash = strcrc(new_name) + + +class UassetImport(ImportBase): + """Meta data for an object that is contained within another file. (FObjectImport) + + Notes: + UnrealEngine/Engine/Source/Runtime/CoreUObject/Private/UObject/ObjectResource.cpp + """ + + def serialize(self, ar: ArchiveBase): + ar << (Int32, self, "class_package_name_id") + ar << (Int32, self, "class_package_name_number") + ar << (Int32, self, "class_name_id") + ar << (Int32, self, "class_name_number") + ar << (Int32, self, "class_package_import_id") + ar << (Int32, self, "name_id") + ar << (Int32, self, "name_number") + if ar.version >= "5.0": + ar << (Uint32, self, "optional") + + def name_import(self, imports: list[ImportBase], name_list: list[NameBase]) -> str: + self.name = str(name_list[self.name_id]) + self.class_name = str(name_list[self.class_name_id]) + if self.class_package_import_id == 0: + self.package_name = "None" + else: + self.package_name = name_list[imports[-self.class_package_import_id - 1].name_id] + return self.name + + def print(self, padding=2): + pad = " " * padding + print(f"{pad}{self.name}") + print(f"{pad} class: {self.class_name}") + print(f"{pad} package_name: {self.package_name}") + + +class UassetExport(ExportBase): + """Meta data for an object that is contained within this file. (FObjectExport) + + Notes: + UnrealEngine/Engine/Source/Runtime/CoreUObject/Private/UObject/ObjectResource.cpp + """ + + def serialize(self, ar: ArchiveBase): + ar << (Int32, self, "class_index") # -: import id, +: export id + ar << (Int32, self, "super_index") + if ar.version >= "4.14": + ar << (Int32, self, "template_index") + else: + self.template_index = 0 + ar << (Int32, self, "outer_index") # 0: main object, 1: not main + ar << (Int32, self, "name_id") + ar << (Int32, self, "name_number") + ar << (Uint32, self, "object_flags") # & 8: main object + if ar.version <= "4.15": + ar << (Uint32, self, "size") + else: + ar << (Uint64, self, "size") + ar << (Uint32, self, "offset") + + # packageguid and other flags. + remain_size = self.get_remainings_size(ar.version) + ar << (Bytes, self, "remainings", remain_size) + + @staticmethod + def get_remainings_size(version: VersionInfo) -> int: + sizes = [ + ["4.2", 32], + ["4.10", 36], + ["4.13", 40], + ["4.15", 60], + ["4.27", 64], + ["5.0", 68] + ] + for ver, size in sizes: + if version <= ver: + return size + # 5.1 ~ + return 56 + + @staticmethod + def get_struct_size(version: VersionInfo): + meta_size = 32 + if version >= "4.14": + meta_size += 4 + if version >= "4.16": + meta_size += 4 + meta_size += UassetExport.get_remainings_size(version) + return meta_size + + def name_export(self, exports: list[ExportBase], imports: list[ImportBase], name_list: list[NameBase]): + self.name = str(name_list[self.name_id]) + + def index_to_name(index: int): + if index == 0: + return "None" + elif -index - 1 < 0: + return exports[index].name + return imports[-index - 1].name + + self.class_name = index_to_name(self.class_index) + self.super_name = index_to_name(self.super_index) + self.template_name = index_to_name(self.template_index) + + def print(self, padding=2): + pad = " " * padding + print(pad + f"{self.name}") + print(pad + f" class: {self.class_name}") + print(pad + f" super: {self.super_name}") + if (self.template_name): + print(pad + f" template: {self.template_name}") + print(pad + f" size: {self.size}") + print(pad + f" offset: {self.offset}") + print(pad + f" is public: {self.is_public()}") + print(pad + f" is standalone: {self.is_standalone()}") + print(pad + f" is base: {self.is_base()}") + print(pad + f" object flags: {self.object_flags}") + + +class ZenName(NameBase): + def serialize_hash(self, ar: ArchiveBase): + ar << (Uint64, self, "hash") + + def serialize_head(self, ar: ArchiveBase): + ar << (Bytes, self, "head", 2) + + def serialize_string(self, ar: ArchiveBase): + length = self.head[1] + ((self.head[0] & 0x7F) << 8) + is_utf16 = (self.head[0] & 0x80) > 0 + ar << (StringWithLen, self, "name", length, is_utf16) + + def update(self, new_name, update_hash=False): + length = len(new_name) + is_utf16 = not new_name.isascii() + self.head = bytes([(length >> 8) + is_utf16 * 0x80, length & 0xFF]) + self.name = new_name + if update_hash: + string = new_name.lower() + if string.isascii(): + binary = string.encode("ascii") + else: + binary = string.encode("utf-16-le") + self.hash = city_hash_64(binary) + + +class ImportType(IntEnum): + Export = 0 + ScriptImport = 1 + PackageImport = 2 + + +# Ucas assets don't have class names. +# So, we need to determine them from hashes. +# The object hashes can be generated by generate_hash_from_object_path() +SCRIPT_OBJECTS = { + # hash: ("class", "super class", "package") + 0x11acced3dc7c0922: ("/Script/Engine", "Package", "None"), + 0x1b93bca796d1fa6f: ("Texture2D", "Class", "/Script/Engine"), + 0x2bfad34ac8b1f6d0: ("Default__Texture2D", "Texture2D", "/Script/Engine"), + 0x21ff31428abdc8ae: ("TextureCube", "Class", "/Script/Engine"), + 0x3712d23e90fd5fe5: ("Default__TextureCube", "TextureCube", "/Script/Engine"), + 0x2461c85f4ba3d161: ("VolumeTexture", "Class", "/Script/Engine"), + 0x015b0407da6ae563: ("Default__VolumeTexture", "VolumeTexture", "/Script/Engine"), + 0x2b74936cc124e6fb: ("Texture2DArray", "Class", "/Script/Engine"), + 0x250cd2505b93e715: ("Default__Texture2DArray", "Texture2DArray", "/Script/Engine"), + 0x22ebbf4da0c22e82: ("TextureCubeArray", "Class", "/Script/Engine"), + 0x14dba7ea9c83a397: ("Default__TextureCubeArray", "Texture2DArray", "/Script/Engine"), + 0x2fe6ca4e48506419: ("LightMapTexture2D", "Class", "/Script/Engine"), + 0x029e125411d1912f: ("Default__LightMapTexture2D", "LightMapTexture2D", "/Script/Engine"), + 0x1e90a76c6b6d37bf: ("ShadowMapTexture2D", "Class", "/Script/Engine"), + 0x01bb4bc588d632f7: ("Default__ShadowMapTexture2D", "ShadowMapTexture2D", "/Script/Engine"), +} + + +class ZenImport(ImportBase): + """Import exntries for ucas assets (FPackageObjectIndex) + + Notes: + UnrealEngine/Engine/Source/Runtime/CoreUObject/Public/Serialization/AsyncLoading2.h + """ + INDEX_BITS = 62 + INDEX_MASK = (1 << INDEX_BITS) - 1 + + def serialize(self, ar: ArchiveBase): + if ar.is_writing: + self.type_and_id = self.type << ZenImport.INDEX_BITS | self.id + ar << (Uint64, self, "type_and_id") + self.id = self.type_and_id & ZenImport.INDEX_MASK + self.type = self.type_and_id >> ZenImport.INDEX_BITS + + def name_import(self, imports: list[ImportBase], name_list: list[ZenName]) -> str: + if (self.is_script_object() and self.id in SCRIPT_OBJECTS): + # self.id is a city hash generated from a object path + self.name, self.class_name, self.package_name = SCRIPT_OBJECTS[self.id] + else: + self.name = "???" + self.class_name = "???" + self.package_name = "???" + return self.name + + def is_invalid(self) -> bool: + return self.type_and_id == 0xFFFFFFFFFFFFFFFF + + def is_script_object(self) -> bool: + return (self.type & ImportType.ScriptImport) > 0 + + def is_export(self) -> bool: + return (self.type & ImportType.Export) > 0 + + @staticmethod + def get_struct_size(ar: ArchiveBase) -> int: + return 8 + + def generate_hash_from_object_path(self): + if (self.package_name == "None"): + object_path = self.name + else: + object_path = self.package_name + "." + self.name + object_path = object_path.replace(".", "/") + object_path = object_path.replace(":", "/") + object_path = object_path.lower() + binary = object_path.encode("utf-16-le") + self.id = city_hash_64(binary) & ~(3 << ZenImport.INDEX_BITS) + + def print(self, padding=2): + pad = " " * padding + print(f"{pad}{self.name}") + print(f"{pad} type: {self.type}") + print(f"{pad} id: {hex(self.id)}") + print(f"{pad} class: {self.class_name}") + print(f"{pad} package_name: {self.package_name}") + + +class ZenExport(ExportBase): + """Export entries for ucas assets. (FExportMapEntry) + + Notes: + UnrealEngine/Engine/Source/Runtime/CoreUObject/Public/Serialization/AsyncLoading2.h + """ + + def serialize(self, ar: ArchiveBase): + ar << (Uint64, self, "offset") + ar << (Uint64, self, "size") + ar << (Uint32, self, "name_id") + ar << (Uint32, self, "name_number") + ar << (ZenImport, self, "outer_index") + ar << (ZenImport, self, "class_index") + ar << (ZenImport, self, "super_index") + ar << (ZenImport, self, "template_index") + ar << (Uint64, self, "public_export_hash") + ar << (Uint32, self, "object_flags") + ar << (Uint8, self, "filter_flags") + ar == (Bytes, b"\x00\x00\x00", "pad", 3) + + def name_export(self, exports: list[ExportBase], imports: list[ZenImport], name_list: list[ZenName]): + self.name = str(name_list[self.name_id]) + + def index_to_name(index: ZenImport) -> str: + if index.is_invalid(): + return "None" + elif index.is_export(): + return exports[index.id].name + elif (index.is_script_object() and index.id in SCRIPT_OBJECTS): + return SCRIPT_OBJECTS[index.id][0] + return "???" + + self.class_name = index_to_name(self.class_index) + self.super_name = index_to_name(self.super_index) + self.template_name = index_to_name(self.template_index) + + @staticmethod + def get_struct_size(version: VersionInfo): + return 72 + + def print(self, padding=2): + pad = " " * padding + print(pad + f"{self.name}") + print(pad + f" class: {self.class_name}") + print(pad + f" super: {self.super_name}") + print(pad + f" template: {self.template_name}") + print(pad + f" size: {self.size}") + print(pad + f" offset: {self.offset}") + print(pad + f" is public: {self.is_public()}") + print(pad + f" is standalone: {self.is_standalone()}") + print(pad + f" is base: {self.is_base()}") + print(pad + f" object flags: {self.object_flags}") diff --git a/src/unreal/uasset.py b/src/unreal/uasset.py index 24a16e7..0d62bbd 100644 --- a/src/unreal/uasset.py +++ b/src/unreal/uasset.py @@ -1,281 +1,21 @@ """Classes for .uasset""" -from enum import IntEnum import io from io import IOBase import os from util import mkdir -from .crc import generate_hash, strcrc_deprecated -from .data_resource import UassetDataResource +from .import_export import ExportBase from .utexture import Utexture from .version import VersionInfo from .archive import (ArchiveBase, ArchiveRead, ArchiveWrite, - Uint32, Uint64, Int32, Int64, Bytes, String, - Int32Array, StructArray, Buffer, + Int32, Int32Array, Bytes, Buffer, SerializableBase) +from .file_summary import (UassetFileSummary, ZenPackageSummary) UASSET_EXT = ["uasset", "uexp", "ubulk", "uptnl"] -class PackageFlags(IntEnum): - PKG_UnversionedProperties = 0x2000 # Uses unversioned property serialization - PKG_FilterEditorOnly = 0x80000000 # Package has editor-only data filtered out - - -class ObjectFlags(IntEnum): - RF_Public = 1 - RF_Standalone = 2 # Main object in the asset - RF_Transactional = 8 - RF_ClassDefaultObject = 0x10 # Default object - RF_ArchetypeObject = 0x20 # Template for another object - - -class UassetFileSummary(SerializableBase): - """Info for .uasset file (FPackageFileSummary) - - Notes: - UnrealEngine/Engine/Source/Runtime/CoreUObject/Private/UObject/PackageFileSummary.cpp - """ - TAG = b"\xC1\x83\x2A\x9E" # Magic for uasset files - TAG_SWAPPED = b"\x9E\x2A\x83\xC1" # for big endian files - TAG_UCAS = b"\x00\x00\x00\x00" # ucas assets don't have tag and file version. - - def serialize(self, ar: ArchiveBase): - self.file_name = ar.name - - ar << (Bytes, self, "tag", 4) - ar.endian = self.get_endian() - - """ - File version - positive: 3.x - -3: 4.0 ~ 4.6 - -5: 4.7 ~ 4.9 - -6: 4.10 ~ 4.13 - -7: 4.14 ~ 4.27 - -8: 5.0 ~ - """ - expected_version = ( - -8 + (ar.version <= "4.6") * 2 + (ar.version <= "4.9") - + (ar.version <= "4.13") + (ar.version <= "4.27") - ) - ar == (Int32, expected_version, "header.file_version") - - """ - Version info. But most assets have zeros for these variables. (unversioning) - So, we can't get UE version from them. - - LegacyUE3Version - - FileVersionUE.FileVersionUE4 - - FileVersionUE.FileVersionUE5 (Added at 5.0) - - FileVersionLicenseeUE - - CustomVersionContainer - """ - ar << (Bytes, self, "version_info", 16 + 4 * (ar.version >= "5.0")) - - ar << (Int32, self, "uasset_size") # TotalHeaderSize - ar << (String, self, "package_name") - - # PackageFlags - ar << (Uint32, self, "pkg_flags") - if ar.is_reading: - ar.check(self.pkg_flags & PackageFlags.PKG_FilterEditorOnly > 0, True, - msg="Unsupported file format detected. (PKG_FilterEditorOnlyitorOnly is false.)") - - # Name table - ar << (Int32, self, "name_count") - ar << (Int32, self, "name_offset") - - if ar.version >= "5.1": - # SoftObjectPaths - ar == (Int32, 0, "soft_object_count") - if ar.is_writing: - self.soft_object_offset = self.import_offset - ar << (Int32, self, "soft_object_offset") - - if ar.version >= "4.9": - # GatherableTextData - ar == (Int32, 0, "gatherable_text_count") - ar == (Int32, 0, "gatherable_text_offset") - - # Exports - ar << (Int32, self, "export_count") - ar << (Int32, self, "export_offset") - - # Imports - ar << (Int32, self, "import_count") - ar << (Int32, self, "import_offset") - - # DependsOffset - ar << (Int32, self, "depends_offset") - - if ar.version >= "4.4" and ar.version <= "4.14": - # StringAssetReferencesCount - ar == (Int32, 0, "string_asset_count") - if ar.is_writing: - self.string_asset_offset = self.asset_registry_data_offset - ar << (Int32, self, "string_asset_offset") - elif ar.version >= "4.15": - # SoftPackageReferencesCount - ar == (Int32, 0, "soft_package_count") - ar == (Int32, 0, "soft_package_offset") - - # SearchableNamesOffset - ar == (Int32, 0, "searchable_name_offset") - - # ThumbnailTableOffset - ar == (Int32, 0, "thumbnail_table_offset") - - ar << (Bytes, self, "guid", 16) # GUID - - # Generations: Export count and name count for previous versions of this package - ar << (Int32, self, "generation_count") - if self.generation_count <= 0 or self.generation_count >= 10: - raise RuntimeError(f"Unexpected value. (generation_count: {self.generation_count})") - ar << (Int32Array, self, "generation_data", self.generation_count * 2) - - """ - - SavedByEngineVersion (14 bytes) - - CompatibleWithEngineVersion (14 bytes) (4.8 ~ ) - """ - ar << (Bytes, self, "empty_engine_version", 14 * (1 + (ar.version >= "4.8"))) - - # CompressionFlags, CompressedChunks - ar << (Bytes, self, "compression_info", 8) - - """ - PackageSource: - Value that is used to determine if the package was saved by developer or not. - CRC hash for shipping builds. Others for user created files. - """ - ar << (Uint32, self, "package_source") - - # AdditionalPackagesToCook (zero length array) - ar == (Int32, 0, "additional_packages_to_cook") - - if ar.version <= "4.13": - ar == (Int32, 0, "num_texture_allocations") - ar << (Int32, self, "asset_registry_data_offset") - ar << (Int32, self, "bulk_offset") # .uasset + .uexp - 4 (BulkDataStartOffset) - - # WorldTileInfoDataOffset - ar == (Int32, 0, "world_tile_info_offset") - - # ChunkIDs (zero length array), ChunkID - ar == (Int32Array, [0, 0], "ChunkID", 2) - - if ar.version <= "4.13": - return - - # PreloadDependency - ar << (Int32, self, "preload_dependency_count") - ar << (Int32, self, "preload_dependency_offset") - - if ar.version <= "4.27": - return - - # Number of names that are referenced from serialized export data - ar << (Int32, self, "referenced_names_count") - - # Location into the file on disk for the payload table of contents data - ar << (Int64, self, "payload_toc_offset") - - if ar.version <= "5.1": - return - - # Location into the file of the data resource - ar << (Int32, self, "data_resource_offset") - - def print(self): - print("File Summary") - print(f" file size: {self.uasset_size}") - print(f" number of names: {self.name_count}") - print(" name directory offset: 193") - print(f" number of exports: {self.export_count}") - print(f" export directory offset: {self.export_offset}") - print(f" number of imports: {self.import_count}") - print(f" import directory offset: {self.import_offset}") - print(f" depends offset: {self.depends_offset}") - print(f" file length (uasset+uexp-4): {self.bulk_offset}") - print(f" official asset: {self.is_official()}") - print(f" unversioned: {self.is_unversioned()}") - - def is_unversioned(self): - return (self.pkg_flags & PackageFlags.PKG_UnversionedProperties) > 0 - - def is_official(self): - crc = strcrc_deprecated("".join(os.path.basename(self.file_name).split(".")[:-1])) - return self.package_source == crc - - def update_package_source(self, file_name=None, is_official=True): - if file_name is not None: - self.file_name = file_name - if is_official: - crc = strcrc_deprecated("".join(os.path.basename(self.file_name).split(".")[:-1])) - else: - crc = int.from_bytes(b"MOD ", "little") - self.package_source = crc - - def get_endian(self): - if self.tag == UassetFileSummary.TAG: - return "little" - elif self.tag == UassetFileSummary.TAG_SWAPPED: - return "big" - elif self.tag == UassetFileSummary.TAG_UCAS: - raise RuntimeError("Assets should be from *.pak. *.ucas is not supported yet.") - raise RuntimeError(f"Invalid tag detected. ({self.tag})") - - -class Name(SerializableBase): - def __init__(self): - self.hash = None - - def serialize(self, ar: ArchiveBase): - ar << (String, self, "name") - if ar.version <= "4.11": - return - ar << (Bytes, self, "hash", 4) - - def __str__(self): - return self.name - - def update(self, new_name, update_hash=False): - self.name = new_name - if update_hash: - self.hash = generate_hash(new_name) - - -class UassetImport(SerializableBase): - """Meta data for an object that is contained within another file. (FObjectImport) - - Notes: - UnrealEngine/Engine/Source/Runtime/CoreUObject/Private/UObject/ObjectResource.cpp - """ - - def serialize(self, ar: ArchiveBase): - ar << (Int32, self, "class_package_name_id") - ar << (Int32, self, "class_package_name_number") - ar << (Int32, self, "class_name_id") - ar << (Int32, self, "class_name_number") - ar << (Int32, self, "class_package_import_id") - ar << (Int32, self, "name_id") - ar << (Int32, self, "name_number") - if ar.version >= "5.0": - ar << (Uint32, self, "optional") - - def name_import(self, name_list: list[Name]) -> str: - self.name = str(name_list[self.name_id]) - self.class_name = str(name_list[self.class_name_id]) - self.class_package_name = str(name_list[self.class_package_name_id]) - return self.name - - def print(self, padding=2): - pad = " " * padding - print(pad + self.name) - print(pad + " class: " + self.class_name) - print(pad + " class_file: " + self.class_package_name) - - class Uunknown(SerializableBase): """Unknown Uobject.""" def __init__(self, uasset, size): @@ -289,109 +29,19 @@ def serialize(self, uexp_io: ArchiveBase): self.uexp_size = len(self.bin) -class UassetExport(SerializableBase): - """Meta data for an object that is contained within this file. (FObjectExport) - - Notes: - UnrealEngine/Engine/Source/Runtime/CoreUObject/Private/UObject/ObjectResource.cpp - """ - TEXTURE_CLASSES = [ - "Texture2D", "TextureCube", "LightMapTexture2D", "ShadowMapTexture2D", - "Texture2DArray", "TextureCubeArray", "VolumeTexture" - ] - - def __init__(self): - self.object = None # The actual data will be stored here - self.meta_size = 0 # binary size of meta data - - def serialize(self, ar: ArchiveBase): - ar << (Int32, self, "class_import_id") # -: import id, +: export id - if ar.version >= "4.14": - ar << (Int32, self, "template_index") - ar << (Int32, self, "super_import_id") - ar << (Int32, self, "outer_index") # 0: main object, 1: not main - ar << (Int32, self, "name_id") - ar << (Int32, self, "name_number") - ar << (Uint32, self, "object_flags") # & 8: main object - if ar.version <= "4.15": - ar << (Uint32, self, "size") - else: - ar << (Uint64, self, "size") - ar << (Uint32, self, "offset") - - # packageguid and other flags. - remain_size = self.get_remainings_size(ar.version) - ar << (Bytes, self, "remainings", remain_size) - - @staticmethod - def get_remainings_size(version: VersionInfo) -> int: - sizes = [ - ["4.2", 32], - ["4.10", 36], - ["4.13", 40], - ["4.15", 60], - ["4.27", 64], - ["5.0", 68] - ] - for ver, size in sizes: - if version <= ver: - return size - # 5.1 ~ - return 56 - - @staticmethod - def get_meta_size(version: VersionInfo): - meta_size = 32 - if version >= "4.14": - meta_size += 4 - if version >= "4.16": - meta_size += 4 - meta_size += UassetExport.get_remainings_size(version) - return meta_size - - def update(self, size, offset): - self.size = size - self.offset = offset - - def name_export(self, exports: list["UassetExport"], imports: list[UassetImport], name_list: list[Name]): - self.name = str(name_list[self.name_id]) - if -self.class_import_id - 1 < 0: - self.class_name = exports[self.class_import_id].name - else: - self.class_name = imports[-self.class_import_id - 1].name - self.super_name = imports[-self.super_import_id - 1].name - - def is_base(self): - return (self.object_flags & ObjectFlags.RF_ArchetypeObject > 0 - or self.object_flags & ObjectFlags.RF_ClassDefaultObject > 0) - - def is_standalone(self): - return self.object_flags & ObjectFlags.RF_Standalone > 0 - - def is_public(self): - return self.object_flags & ObjectFlags.RF_Public > 0 - - def is_texture(self): - return self.class_name in UassetExport.TEXTURE_CLASSES - - def print(self, padding=2): - pad = " " * padding - print(pad + f"{self.name}") - print(pad + f" class: {self.class_name}") - print(pad + f" super: {self.super_name}") - print(pad + f" size: {self.size}") - print(pad + f" offset: {self.offset}") - print(pad + f" is public: {self.is_public()}") - print(pad + f" is standalone: {self.is_standalone()}") - print(pad + f" is base: {self.is_base()}") - print(pad + f" object flags: {self.object_flags}") - - class Uasset: + TAG = b"\xC1\x83\x2A\x9E" # Magic for uasset files + TAG_SWAPPED = b"\x9E\x2A\x83\xC1" # for big endian files + TAG_UCAS = b"\x00\x00\x00\x00" # ucas assets don't have tag and file version. + def __init__(self, file_path: str, version: str = "ff7r", verbose=False): if not os.path.isfile(file_path): raise RuntimeError(f"Not File. ({file_path})") + self.name_list = None + self.imports = None + self.exports = None + self.data_resources = [] self.texture = None self.io_dict = {} self.bin_dict = {} @@ -399,50 +49,59 @@ def __init__(self, file_path: str, version: str = "ff7r", verbose=False): self.io_dict[k] = None self.bin_dict[k] = None self.file_name, ext = os.path.splitext(file_path) - if ext[1:] not in UASSET_EXT: + print(ext) + if not ext or ext[1:] not in UASSET_EXT: raise RuntimeError(f"Not Uasset. ({file_path})") uasset_file = self.file_name + ".uasset" print("load: " + uasset_file) self.version = VersionInfo(version) - self.context = {"version": self.version, "verbose": verbose, "valid": False} + self.context = {"version": self.version, "verbose": verbose, "valid": False, "uasset": self} ar = ArchiveRead(open(uasset_file, "rb"), context=self.context) self.serialize(ar) ar.close() self.read_export_objects(verbose=verbose) + def print_name_map(self): + print("Names") + for i, name in zip(range(len(self.name_list)), self.name_list): + print(" {}: {}".format(i, name)) + def serialize(self, ar: ArchiveBase): # read header + self.check_tag(ar) if ar.is_reading: - ar << (UassetFileSummary, self, "header") + if ar.is_ucas: + ar << (ZenPackageSummary, self, "header") + else: + ar << (UassetFileSummary, self, "header") if ar.verbose: self.header.print() else: ar.seek(self.header.name_offset) # read name map - ar << (StructArray, self, "name_list", Name, self.header.name_count) + self.name_list = self.header.serialize_name_map(ar, self.name_list) if ar.verbose: - print("Names") - for i, name in zip(range(len(self.name_list)), self.name_list): - print(" {}: {}".format(i, name)) + self.print_name_map() + + if ar.is_ucas: + if ar.version >= "5.2": + # read bulk data map entries + self.data_resources = self.header.serialize_data_resources(ar, self.data_resources) + self.header.serialize_export_hashes(ar) # read imports + self.imports = self.header.serialize_imports(ar, self.imports) if ar.is_reading: - ar.check(self.header.import_offset, ar.tell()) - else: - self.header.import_offset = ar.tell() - ar << (StructArray, self, "imports", UassetImport, self.header.import_count) - if ar.is_reading: - list(map(lambda x: x.name_import(self.name_list), self.imports)) + list(map(lambda x: x.name_import(self.imports, self.name_list), self.imports)) if ar.verbose: print("Imports") list(map(lambda x: x.print(), self.imports)) if ar.is_reading: # read exports - ar.check(self.header.export_offset, ar.tell()) - ar << (StructArray, self, "exports", UassetExport, self.header.export_count) + self.exports = self.header.serialize_exports(ar, self.exports) list(map(lambda x: x.name_export(self.exports, self.imports, self.name_list), self.exports)) if ar.verbose: print("Exports") @@ -451,43 +110,34 @@ def serialize(self, ar: ArchiveBase): else: # skip exports part self.header.export_offset = ar.tell() - ar.seek(UassetExport.get_meta_size(ar.version) * self.header.export_count, 1) + self.header.skip_exports(ar, len(self.exports)) if self.version not in ["4.15", "4.14"]: self.header.depends_offset = ar.tell() - # read depends map - ar << (Int32Array, self, "depends_map", self.header.export_count) - - # write asset registry data - if ar.is_reading: - ar.check(self.header.asset_registry_data_offset, ar.tell()) + if ar.is_ucas: + self.header.serialize_others(ar) else: - self.header.asset_registry_data_offset = ar.tell() - ar == (Int32, 0, "asset_registry_data") + # read depends map + ar << (Int32Array, self, "depends_map", self.header.export_count) - # Preload dependencies (import and export ids that must be serialized before other exports) - if self.has_uexp(): - if ar.is_reading: - ar.check(self.header.preload_dependency_offset, ar.tell()) - else: - self.header.preload_dependency_offset = ar.tell() - ar << (Int32Array, self, "preload_dependency_ids", self.header.preload_dependency_count) + # write asset registry data + ar.update_with_current_offset(self.header, "asset_registry_data_offset") + ar == (Int32, 0, "asset_registry_data") - if ar.version >= "5.2": - if ar.is_reading: - ar.check(self.header.data_resource_offset, ar.tell()) - else: - self.header.data_resource_offset = ar.tell() - self.data_resource_count = len(self.data_resources) - ar == (Int32, 1, "deta_resource_version") - ar << (Int32, self, "data_resource_count") - ar << (StructArray, self, "data_resources", UassetDataResource, self.data_resource_count) - else: - self.data_resources = [] + # Preload dependencies (import and export ids that must be serialized before other exports) + if self.has_uexp(): + ar.update_with_current_offset(self.header, "preload_dependency_offset") + ar << (Int32Array, self, "preload_dependency_ids", self.header.preload_dependency_count) + + if ar.version >= "5.2": + self.data_resources = self.header.serialize_data_resources(ar, self.data_resources) if ar.is_reading: ar.check(ar.tell(), self.get_size()) - if not self.has_uexp(): + if self.is_ucas: + self.uexp_size = ar.size - self.header.uasset_size + self.bin_dict["uexp"] = ar.read(self.uexp_size) + elif not self.has_uexp(): self.uexp_size = self.header.bulk_offset - self.header.uasset_size self.bin_dict["uexp"] = ar.read(self.uexp_size) self.bin_dict["ubulk"] = ar.read(ar.size - ar.tell() - 4) @@ -502,18 +152,53 @@ def serialize(self, ar: ArchiveBase): # write exports ar.seek(self.header.export_offset) - offset = self.header.uasset_size + if self.is_ucas: + if ar.version >= "5.2": + offset = 0 + else: + offset = self.header.cooked_header_size + else: + offset = self.header.uasset_size for export in self.exports: export.update(export.size, offset) offset += export.size - ar << (StructArray, self, "exports", UassetExport, self.header.export_count) + self.exports = self.header.serialize_exports(ar, self.exports) - if not self.has_uexp(): + if self.is_ucas: + ar.seek(self.header.uasset_size) + ar.write(self.bin_dict["uexp"]) + elif not self.has_uexp(): ar.seek(self.header.uasset_size) ar.write(self.bin_dict["uexp"]) ar.write(self.bin_dict["ubulk"]) ar.write(self.header.tag) + def check_tag(self, ar: ArchiveBase): + if ar.is_reading: + self.tag = ar.read(4) + ar.seek(0) + if self.tag == Uasset.TAG: + ar.endian = "little" + ar.is_ucas = False + self.context["is_ucas"] = False + self.is_ucas = False + elif self.tag == Uasset.TAG_SWAPPED: + ar.endian = "big" + ar.is_ucas = False + self.context["is_ucas"] = False + self.is_ucas = False + elif self.tag == b"\x00\x00\x00\x00" or self.tag == b"\x01\x00\x00\x00": + if ar.version <= "4.24": + raise RuntimeError(f"Invalid tag detected. ({self.tag})") + if ar.version <= "4.27": + raise RuntimeError(f"Ucas assets for this UE version are not supported yet. ({ar.version})") + ar.endian = "little" + ar.is_ucas = True + self.context["is_ucas"] = True + self.is_ucas = True + else: + raise RuntimeError(f"Invalid tag detected. ({self.tag})") + def read_export_objects(self, verbose=False): uexp_io = self.get_io(ext="uexp", rb=True) for exp in self.exports: @@ -552,7 +237,8 @@ def save(self, file: str, valid=False): uasset_file = self.file_name + ".uasset" print("save :" + uasset_file) - self.context = {"version": self.version, "verbose": False, "valid": valid} + self.context["verbose"] = False + self.context["valid"] = valid self.write_export_objects() ar = ArchiveWrite(open(uasset_file, "wb"), context=self.context) @@ -573,7 +259,7 @@ def get_size(string): # Update file size self.header.uasset_size += get_size(new_name) - get_size(old_name) - def get_main_export(self) -> UassetExport: + def get_main_export(self) -> ExportBase: main_list = [exp for exp in self.exports if (exp.is_public() and not exp.is_base())] standalone_list = [exp for exp in main_list if exp.is_standalone()] if len(standalone_list) > 0: @@ -590,7 +276,7 @@ def get_main_class_name(self): return main_obj.class_name def has_uexp(self): - return self.version >= "4.16" and (self.header.preload_dependency_count >= 0) + return self.is_ucas or self.version >= "4.16" and (self.header.preload_dependency_count >= 0) def has_ubulk(self): for exp in self.exports: @@ -612,10 +298,10 @@ def get_texture_list(self) -> list[Utexture]: return textures def __get_io_base(self, file: str, bin: bytes, rb: bool) -> IOBase: - if self.has_uexp(): - opened_io = open(file, "rb" if rb else "wb") - else: + if (self.is_ucas and file.endswith("uexp")) or not self.has_uexp(): opened_io = io.BytesIO(bin if rb else b"") + else: + opened_io = open(file, "rb" if rb else "wb") if rb: return ArchiveRead(opened_io, context=self.context) @@ -633,15 +319,15 @@ def __close_io(self, ext="uexp", rb=True): ar = self.io_dict[ext] if ext == "uexp": self.uexp_size = ar.tell() - if self.has_uexp(): + if self.has_uexp() and not self.is_ucas: if rb: - ar.check(ar.read(4), UassetFileSummary.TAG) + ar.check(ar.read(4), Uasset.TAG) else: - ar.write(UassetFileSummary.TAG) + ar.write(Uasset.TAG) else: if rb: ar.check(ar.tell(), ar.size) - if not self.has_uexp() and not rb: + if not rb and ((self.is_ucas and ext == "uexp") or not self.has_uexp()): ar.seek(0) self.bin_dict[ext] = ar.read() ar.close() diff --git a/src/unreal/umipmap.py b/src/unreal/umipmap.py index f8bf23a..b265b30 100644 --- a/src/unreal/umipmap.py +++ b/src/unreal/umipmap.py @@ -1,5 +1,5 @@ """Mipmap class for texture asset""" -from .data_resource import LegacyDataResource, UassetDataResource +from .data_resource import LegacyDataResource, UassetDataResource, BulkDataMapEntry from .archive import (ArchiveBase, Int32, Uint32, Uint16, Buffer, SerializableBase) @@ -17,9 +17,12 @@ def __init__(self): self.depth = 1 self.data_resource = None - def init_data_resource(self, version): - if version >= "5.2": - self.data_resource = UassetDataResource() + def init_data_resource(self, uasset): + if uasset.version >= "5.2": + if uasset.is_ucas: + self.data_resource = BulkDataMapEntry() + else: + self.data_resource = UassetDataResource() else: self.data_resource = LegacyDataResource() diff --git a/src/unreal/utexture.py b/src/unreal/utexture.py index 03197f7..87b2aeb 100644 --- a/src/unreal/utexture.py +++ b/src/unreal/utexture.py @@ -141,7 +141,7 @@ def __calculate_prop_size(self, ar: ArchiveBase): """ while (ar.read(1)[0] != search_bin[0]): if (ar.tell() >= err_offset): - raise RuntimeError("Parse Failed. Make sure you specified UE4 version correctly.") + ar.raise_error() if ar.read(7) == search_bin[1:]: # Found search_bin @@ -154,7 +154,10 @@ def __calculate_prop_size(self, ar: ArchiveBase): def __serialize_uexp(self, ar: ArchiveBase, ubulk_start_offset: int = 0, uptnl_start_offset: int = 0): start_offset = ar.tell() - uasset_size = self.uasset.get_size() + if ar.is_ucas and (ar.version <= "5.1"): + uasset_size = self.uasset.header.cooked_header_size + else: + uasset_size = self.uasset.get_size() if ar.is_reading: prop_size = self.__calculate_prop_size(ar) else: @@ -396,7 +399,7 @@ def inject_dds(self, dds: DDS): self.mipmaps = [Umipmap() for i in range(len(dds.mipmap_size_list))] offset = 0 for size, mip, i in zip(dds.mipmap_size_list, self.mipmaps, range(len(self.mipmaps))): - mip.init_data_resource(self.version) + mip.init_data_resource(self.uasset) # get a mip data from slices data = b"" bin_size = int(size[0] * size[1] * self.byte_per_pixel) diff --git a/tests/test_ue_utils.py b/tests/test_ue_utils.py index cefee68..73d0775 100644 --- a/tests/test_ue_utils.py +++ b/tests/test_ue_utils.py @@ -1,6 +1,6 @@ """Tests for crc.py and version.py""" import pytest -from unreal.crc import generate_hash, strcrc_deprecated +from unreal.crc import strcrc, strcrc_deprecated from unreal.version import VersionInfo test_cases = { @@ -19,7 +19,7 @@ @pytest.mark.parametrize("name, true_hash", test_cases["name_hash"]) def test_name_hash(name, true_hash): - name_hash = generate_hash(name) + name_hash = strcrc(name) assert name_hash == true_hash