Skip to content

Commit

Permalink
Add functionality for serializing depot manifests
Browse files Browse the repository at this point in the history
  • Loading branch information
@NicknineTheEagle
NicknineTheEagle committed Jul 18, 2022
1 parent 2479712 commit 79fea9f
Showing 1 changed file with 171 additions and 3 deletions.
174 changes: 171 additions & 3 deletions SteamKit2/SteamKit2/Types/DepotManifest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Linq;

namespace SteamKit2
{
Expand Down Expand Up @@ -81,6 +82,10 @@ public class FileData
/// </summary>
public string FileName { get; internal set; }
/// <summary>
/// Gets the name hash of the file.
/// </summary>
public byte[] FileNameHash { get; internal set; }
/// <summary>
/// Gets the chunks that this file is composed of.
/// </summary>
public List<ChunkData> Chunks { get; private set; }
Expand All @@ -98,9 +103,13 @@ public class FileData
/// Gets the hash of this file.
/// </summary>
public byte[] FileHash { get; private set; }
/// <summary>
/// Gets symlink target of this file.
/// </summary>
public string LinkTarget { get; private set; }


internal FileData(string filename, EDepotFileFlag flag, ulong size, byte[] hash, bool encrypted, int numChunks)
internal FileData(string filename, byte[] filenameHash, EDepotFileFlag flag, ulong size, byte[] hash, string linkTarget, bool encrypted, int numChunks)
{
if (encrypted)
{
Expand All @@ -111,10 +120,12 @@ internal FileData(string filename, EDepotFileFlag flag, ulong size, byte[] hash,
this.FileName = filename.Replace(altDirChar, Path.DirectorySeparatorChar);
}

this.FileNameHash = filenameHash;
this.Flags = flag;
this.TotalSize = size;
this.FileHash = hash;
this.Chunks = new List<ChunkData>( numChunks );
this.LinkTarget = linkTarget;
}
}

Expand Down Expand Up @@ -149,6 +160,10 @@ internal FileData(string filename, EDepotFileFlag flag, ulong size, byte[] hash,
/// Gets the total compressed size of all files in this depot.
/// </summary>
public ulong TotalCompressedSize { get; private set; }
/// <summary>
/// Gets CRC-32 checksum of encrypted manifest payload.
/// </summary>
public uint EncryptedCRC { get; private set; }


internal DepotManifest(byte[] data)
Expand Down Expand Up @@ -194,10 +209,53 @@ public bool DecryptFilenames(byte[] encryptionKey)
file.FileName = Encoding.UTF8.GetString( filename ).TrimEnd( new char[] { '\0' } ).Replace(altDirChar, Path.DirectorySeparatorChar);
}

// Sort file entries alphabetically because that's what Steam does
// TODO: Doesn't match Steam sorting if there are non-ASCII names present
Files.Sort( ( f1, f2 ) => StringComparer.OrdinalIgnoreCase.Compare( f1.FileName, f2.FileName ) );

FilenamesEncrypted = false;
return true;
}

/// <summary>
/// Serializes depot manifest and saves the output to a file.
/// </summary>
/// <param name="filename">Output file name.</param>
/// <returns><c>true</c> if serialization was successful; otherwise, <c>false</c>.</returns>
public bool SaveToFile( string filename )
{
using ( var fs = File.Open( filename, FileMode.Create ) )
using ( var bw = new BinaryWriter( fs ) )
{
var data = Serialize();
if ( data != null )
{
bw.Write( data );
return true;
}
}

return false;
}

/// <summary>
/// Loads binary manifest from a file and deserializes it.
/// </summary>
/// <param name="filename">Input file name.</param>
/// <returns><c>DepotManifest</c> object if deserialization was successful; otherwise, <c>null</c>.</returns>
public static DepotManifest? LoadFromFile( string filename )
{
if ( !File.Exists( filename ) )
return null;

using ( var fs = File.Open( filename, FileMode.Open ) )
using ( var ms = new MemoryStream() )
{
fs.CopyTo( ms );
return Deserialize( ms.ToArray() );
}
}

void InternalDeserialize(byte[] data)
{
ContentManifestPayload? payload = null;
Expand Down Expand Up @@ -276,7 +334,7 @@ void ParseBinaryManifest(Steam3Manifest manifest)

foreach (var file_mapping in manifest.Mapping)
{
FileData filedata = new FileData(file_mapping.FileName!, file_mapping.Flags, file_mapping.TotalSize, file_mapping.HashContent!, FilenamesEncrypted, file_mapping.Chunks!.Length);
FileData filedata = new FileData(file_mapping.FileName!, file_mapping.HashFileName!, file_mapping.Flags, file_mapping.TotalSize, file_mapping.HashContent!, "", FilenamesEncrypted, file_mapping.Chunks!.Length);

foreach (var chunk in file_mapping.Chunks)
{
Expand All @@ -293,7 +351,7 @@ void ParseProtobufManifestPayload(ContentManifestPayload payload)

foreach (var file_mapping in payload.mappings)
{
FileData filedata = new FileData(file_mapping.filename, (EDepotFileFlag)file_mapping.flags, file_mapping.size, file_mapping.sha_content, FilenamesEncrypted, file_mapping.chunks.Count);
FileData filedata = new FileData(file_mapping.filename, file_mapping.sha_filename, (EDepotFileFlag)file_mapping.flags, file_mapping.size, file_mapping.sha_content, file_mapping.linktarget, FilenamesEncrypted, file_mapping.chunks.Count);

foreach (var chunk in file_mapping.chunks)
{
Expand All @@ -312,6 +370,116 @@ void ParseProtobufManifestMetadata(ContentManifestMetadata metadata)
CreationTime = DateUtils.DateTimeFromUnixTime( metadata.creation_time );
TotalUncompressedSize = metadata.cb_disk_original;
TotalCompressedSize = metadata.cb_disk_compressed;
EncryptedCRC = metadata.crc_encrypted;
}

byte[]? Serialize()
{
DebugLog.Assert( Files != null, nameof( DepotManifest ), "Files was null when attempting to serialize manifest." );

var payload = new ContentManifestPayload();
var uniqueChunks = new List<byte[]>();

foreach ( var file in Files )
{
var protofile = new ContentManifestPayload.FileMapping();
protofile.filename = file.FileName.Replace( '/', '\\' );
protofile.size = file.TotalSize;
protofile.flags = (uint)file.Flags;
if ( FilenamesEncrypted )
{
// Assume the name is unmodified
protofile.sha_filename = file.FileNameHash;
}
else
{
protofile.sha_filename = CryptoHelper.SHAHash( Encoding.UTF8.GetBytes( file.FileName.Replace( '/', '\\' ).ToLower() ) );
}
protofile.sha_content = file.FileHash;
if ( !string.IsNullOrWhiteSpace( file.LinkTarget ) )
{
protofile.linktarget = file.LinkTarget;
}

foreach ( var chunk in file.Chunks )
{
var protochunk = new ContentManifestPayload.FileMapping.ChunkData();
protochunk.sha = chunk.ChunkID;
protochunk.crc = BitConverter.ToUInt32( chunk.Checksum!, 0 );
protochunk.offset = chunk.Offset;
protochunk.cb_original = chunk.UncompressedLength;
protochunk.cb_compressed = chunk.CompressedLength;

protofile.chunks.Add( protochunk );
if ( !uniqueChunks.Exists( x => x.SequenceEqual( chunk.ChunkID! ) ) )
{
uniqueChunks.Add( chunk.ChunkID! );
}
}

payload.mappings.Add( protofile );
}

var metadata = new ContentManifestMetadata();
metadata.depot_id = DepotID;
metadata.gid_manifest = ManifestGID;
metadata.creation_time = (uint)DateUtils.DateTimeToUnixTime( CreationTime );
metadata.filenames_encrypted = FilenamesEncrypted;
metadata.cb_disk_original = TotalUncompressedSize;
metadata.cb_disk_compressed = TotalCompressedSize;
metadata.unique_chunks = (uint)uniqueChunks.Count;

// Calculate payload CRC
using ( var ms_payload = new MemoryStream() )
{
Serializer.Serialize<ContentManifestPayload>( ms_payload, payload );

int len = ( int )ms_payload.Length;
byte[] data = new byte[ 4 + len ];
Buffer.BlockCopy( BitConverter.GetBytes( len ), 0, data, 0, 4 );
Buffer.BlockCopy( ms_payload.ToArray(), 0, data, 4, len );
uint crc32 = Crc32.Compute( data );

if ( FilenamesEncrypted )
{
metadata.crc_encrypted = crc32;
}
else
{
metadata.crc_encrypted = EncryptedCRC;
metadata.crc_clear = crc32;
}
}

using var ms = new MemoryStream();
using var bw = new BinaryWriter( ms );

// Write Protobuf payload
using ( var ms_payload = new MemoryStream() )
{
Serializer.Serialize<ContentManifestPayload>( ms_payload, payload );
bw.Write( DepotManifest.PROTOBUF_PAYLOAD_MAGIC );
bw.Write( ( int )ms_payload.Length );
bw.Write( ms_payload.ToArray() );
}

// Write Protobuf metadata
using ( var ms_metadata = new MemoryStream() )
{
Serializer.Serialize<ContentManifestMetadata>( ms_metadata, metadata );
bw.Write( DepotManifest.PROTOBUF_METADATA_MAGIC );
bw.Write( ( int )ms_metadata.Length );
bw.Write( ms_metadata.ToArray() );
}

// Write empty signature section
bw.Write( DepotManifest.PROTOBUF_SIGNATURE_MAGIC );
bw.Write( 0 );

// Write EOF marker
bw.Write( DepotManifest.PROTOBUF_ENDOFMANIFEST_MAGIC );

return ms.ToArray();
}
}
}

0 comments on commit 79fea9f

Please sign in to comment.