Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add functionality for serializing depot manifests #1113

Merged
merged 1 commit into from
Sep 13, 2022
Merged

Add functionality for serializing depot manifests #1113

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
177 changes: 173 additions & 4 deletions SteamKit2/SteamKit2/Types/DepotManifest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Linq;

namespace SteamKit2
{
Expand Down Expand Up @@ -81,6 +82,10 @@ public class FileData
/// </summary>
public string FileName { get; internal set; }
/// <summary>
/// Gets SHA-1 hash of this file's name.
/// </summary>
public byte[] FileNameHash { get; internal set; }
/// <summary>
/// Gets the chunks that this file is composed of.
/// </summary>
public List<ChunkData> Chunks { get; private set; }
Expand All @@ -95,12 +100,16 @@ public class FileData
/// </summary>
public ulong TotalSize { get; private set; }
/// <summary>
/// Gets the hash of this file.
/// Gets SHA-1 hash of this file.
/// </summary>
public byte[] FileHash { get; private set; }
/// <summary>
/// Gets symlink target of this file.
/// </summary>
public string LinkTarget { get; private set; }


internal FileData(string filename, EDepotFileFlag flag, ulong size, byte[] hash, bool encrypted, int numChunks)
internal FileData(string filename, byte[] filenameHash, EDepotFileFlag flag, ulong size, byte[] hash, string linkTarget, bool encrypted, int numChunks)
{
if (encrypted)
{
Expand All @@ -111,10 +120,12 @@ internal FileData(string filename, EDepotFileFlag flag, ulong size, byte[] hash,
this.FileName = filename.Replace(altDirChar, Path.DirectorySeparatorChar);
}

this.FileNameHash = filenameHash;
this.Flags = flag;
this.TotalSize = size;
this.FileHash = hash;
this.Chunks = new List<ChunkData>( numChunks );
this.LinkTarget = linkTarget;
}
}

Expand Down Expand Up @@ -149,6 +160,10 @@ internal FileData(string filename, EDepotFileFlag flag, ulong size, byte[] hash,
/// Gets the total compressed size of all files in this depot.
/// </summary>
public ulong TotalCompressedSize { get; private set; }
/// <summary>
/// Gets CRC-32 checksum of encrypted manifest payload.
/// </summary>
public uint EncryptedCRC { get; private set; }


internal DepotManifest(byte[] data)
Expand Down Expand Up @@ -194,10 +209,53 @@ public bool DecryptFilenames(byte[] encryptionKey)
file.FileName = Encoding.UTF8.GetString( filename ).TrimEnd( new char[] { '\0' } ).Replace(altDirChar, Path.DirectorySeparatorChar);
}

// Sort file entries alphabetically because that's what Steam does
// TODO: Doesn't match Steam sorting if there are non-ASCII names present
Files.Sort( ( f1, f2 ) => StringComparer.OrdinalIgnoreCase.Compare( f1.FileName, f2.FileName ) );

FilenamesEncrypted = false;
return true;
}

/// <summary>
/// Serializes depot manifest and saves the output to a file.
/// </summary>
/// <param name="filename">Output file name.</param>
/// <returns><c>true</c> if serialization was successful; otherwise, <c>false</c>.</returns>
public bool SaveToFile( string filename )
{
using ( var fs = File.Open( filename, FileMode.Create ) )
using ( var bw = new BinaryWriter( fs ) )
{
var data = Serialize();
if ( data != null )
{
bw.Write( data );
return true;
}
}

return false;
}

/// <summary>
/// Loads binary manifest from a file and deserializes it.
/// </summary>
/// <param name="filename">Input file name.</param>
/// <returns><c>DepotManifest</c> object if deserialization was successful; otherwise, <c>null</c>.</returns>
public static DepotManifest? LoadFromFile( string filename )
{
if ( !File.Exists( filename ) )
return null;

using ( var fs = File.Open( filename, FileMode.Open ) )
using ( var ms = new MemoryStream() )
{
fs.CopyTo( ms );
return Deserialize( ms.ToArray() );
}
}

void InternalDeserialize(byte[] data)
{
ContentManifestPayload? payload = null;
Expand Down Expand Up @@ -276,7 +334,7 @@ void ParseBinaryManifest(Steam3Manifest manifest)

foreach (var file_mapping in manifest.Mapping)
{
FileData filedata = new FileData(file_mapping.FileName!, file_mapping.Flags, file_mapping.TotalSize, file_mapping.HashContent!, FilenamesEncrypted, file_mapping.Chunks!.Length);
FileData filedata = new FileData(file_mapping.FileName!, file_mapping.HashFileName!, file_mapping.Flags, file_mapping.TotalSize, file_mapping.HashContent!, "", FilenamesEncrypted, file_mapping.Chunks!.Length);

foreach (var chunk in file_mapping.Chunks)
{
Expand All @@ -293,7 +351,7 @@ void ParseProtobufManifestPayload(ContentManifestPayload payload)

foreach (var file_mapping in payload.mappings)
{
FileData filedata = new FileData(file_mapping.filename, (EDepotFileFlag)file_mapping.flags, file_mapping.size, file_mapping.sha_content, FilenamesEncrypted, file_mapping.chunks.Count);
FileData filedata = new FileData(file_mapping.filename, file_mapping.sha_filename, (EDepotFileFlag)file_mapping.flags, file_mapping.size, file_mapping.sha_content, file_mapping.linktarget, FilenamesEncrypted, file_mapping.chunks.Count);

foreach (var chunk in file_mapping.chunks)
{
Expand All @@ -312,6 +370,117 @@ void ParseProtobufManifestMetadata(ContentManifestMetadata metadata)
CreationTime = DateUtils.DateTimeFromUnixTime( metadata.creation_time );
TotalUncompressedSize = metadata.cb_disk_original;
TotalCompressedSize = metadata.cb_disk_compressed;
EncryptedCRC = metadata.crc_encrypted;
}

byte[]? Serialize()
{
DebugLog.Assert( Files != null, nameof( DepotManifest ), "Files was null when attempting to serialize manifest." );

var payload = new ContentManifestPayload();
var uniqueChunks = new List<byte[]>();

foreach ( var file in Files )
{
var protofile = new ContentManifestPayload.FileMapping();
protofile.filename = file.FileName.Replace( '/', '\\' );
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This was incorrectly replacing slashes if the filename was not decrypted.

Discovered while adding a roundtrip test: 7559904

protofile.size = file.TotalSize;
protofile.flags = ( uint )file.Flags;
if ( FilenamesEncrypted )
{
// Assume the name is unmodified
protofile.sha_filename = file.FileNameHash;
}
else
{
protofile.sha_filename = CryptoHelper.SHAHash( Encoding.UTF8.GetBytes( file.FileName.Replace( '/', '\\' ).ToLower() ) );
}
protofile.sha_content = file.FileHash;
if ( !string.IsNullOrWhiteSpace( file.LinkTarget ) )
{
protofile.linktarget = file.LinkTarget;
}

foreach ( var chunk in file.Chunks )
{
var protochunk = new ContentManifestPayload.FileMapping.ChunkData();
protochunk.sha = chunk.ChunkID;
protochunk.crc = BitConverter.ToUInt32( chunk.Checksum!, 0 );
protochunk.offset = chunk.Offset;
protochunk.cb_original = chunk.UncompressedLength;
protochunk.cb_compressed = chunk.CompressedLength;

protofile.chunks.Add( protochunk );
if ( !uniqueChunks.Exists( x => x.SequenceEqual( chunk.ChunkID! ) ) )
{
uniqueChunks.Add( chunk.ChunkID! );
}
}

payload.mappings.Add( protofile );
}

var metadata = new ContentManifestMetadata();
metadata.depot_id = DepotID;
metadata.gid_manifest = ManifestGID;
metadata.creation_time = ( uint )DateUtils.DateTimeToUnixTime( CreationTime );
metadata.filenames_encrypted = FilenamesEncrypted;
metadata.cb_disk_original = TotalUncompressedSize;
metadata.cb_disk_compressed = TotalCompressedSize;
metadata.unique_chunks = ( uint )uniqueChunks.Count;

// Calculate payload CRC
using ( var ms_payload = new MemoryStream() )
{
Serializer.Serialize<ContentManifestPayload>( ms_payload, payload );

int len = ( int )ms_payload.Length;
byte[] data = new byte[ 4 + len ];
Buffer.BlockCopy( BitConverter.GetBytes( len ), 0, data, 0, 4 );
Buffer.BlockCopy( ms_payload.ToArray(), 0, data, 4, len );
uint crc32 = Crc32.Compute( data );

if ( FilenamesEncrypted )
{
metadata.crc_encrypted = crc32;
metadata.crc_clear = 0;
}
else
{
metadata.crc_encrypted = EncryptedCRC;
metadata.crc_clear = crc32;
}
}

using var ms = new MemoryStream();
using var bw = new BinaryWriter( ms );

// Write Protobuf payload
using ( var ms_payload = new MemoryStream() )
{
Serializer.Serialize<ContentManifestPayload>( ms_payload, payload );
bw.Write( DepotManifest.PROTOBUF_PAYLOAD_MAGIC );
bw.Write( ( int )ms_payload.Length );
bw.Write( ms_payload.ToArray() );
}

// Write Protobuf metadata
using ( var ms_metadata = new MemoryStream() )
{
Serializer.Serialize<ContentManifestMetadata>( ms_metadata, metadata );
bw.Write( DepotManifest.PROTOBUF_METADATA_MAGIC );
bw.Write( ( int )ms_metadata.Length );
bw.Write( ms_metadata.ToArray() );
}

// Write empty signature section
bw.Write( DepotManifest.PROTOBUF_SIGNATURE_MAGIC );
bw.Write( 0 );

// Write EOF marker
bw.Write( DepotManifest.PROTOBUF_ENDOFMANIFEST_MAGIC );

return ms.ToArray();
}
}
}