From 1c97b421242e15e599802da4341e6967db144c00 Mon Sep 17 00:00:00 2001 From: NicknineTheEagle Date: Mon, 21 Oct 2024 16:36:14 +0300 Subject: [PATCH] Use Steam manifest format --- DepotDownloader/ContentDownloader.cs | 180 ++++++++++----------------- DepotDownloader/ProtoManifest.cs | 35 ++++++ DepotDownloader/Util.cs | 101 ++++++++++++++- 3 files changed, 199 insertions(+), 117 deletions(-) diff --git a/DepotDownloader/ContentDownloader.cs b/DepotDownloader/ContentDownloader.cs index f69804487..1d1cab092 100644 --- a/DepotDownloader/ContentDownloader.cs +++ b/DepotDownloader/ContentDownloader.cs @@ -598,10 +598,10 @@ static async Task GetDepotInfo(uint depotId, uint appId, ulon return new DepotDownloadInfo(depotId, appId, manifestId, branch, installDir, depotKey); } - private class ChunkMatch(ProtoManifest.ChunkData oldChunk, ProtoManifest.ChunkData newChunk) + private class ChunkMatch(DepotManifest.ChunkData oldChunk, DepotManifest.ChunkData newChunk) { - public ProtoManifest.ChunkData OldChunk { get; } = oldChunk; - public ProtoManifest.ChunkData NewChunk { get; } = newChunk; + public DepotManifest.ChunkData OldChunk { get; } = oldChunk; + public DepotManifest.ChunkData NewChunk { get; } = newChunk; } private class DepotFilesData @@ -609,9 +609,9 @@ private class DepotFilesData public DepotDownloadInfo depotDownloadInfo; public DepotDownloadCounter depotCounter; public string stagingDir; - public ProtoManifest manifest; - public ProtoManifest previousManifest; - public List filteredFiles; + public DepotManifest manifest; + public DepotManifest previousManifest; + public List filteredFiles; public HashSet allFileNames; } @@ -694,8 +694,8 @@ private static async Task ProcessDepotManifestAndFiles(Cancellat Console.WriteLine("Processing depot {0}", depot.DepotId); - ProtoManifest oldProtoManifest = null; - ProtoManifest newProtoManifest = null; + DepotManifest oldManifest = null; + DepotManifest newManifest = null; var configDir = Path.Combine(depot.InstallDir, CONFIG_DIR); var lastManifestId = INVALID_MANIFEST_ID; @@ -707,64 +707,21 @@ private static async Task ProcessDepotManifestAndFiles(Cancellat if (lastManifestId != INVALID_MANIFEST_ID) { - var oldManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.DepotId, lastManifestId)); - - if (File.Exists(oldManifestFileName)) - { - byte[] expectedChecksum; - - try - { - expectedChecksum = File.ReadAllBytes(oldManifestFileName + ".sha"); - } - catch (IOException) - { - expectedChecksum = null; - } - - oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName, out var currentChecksum); - - if (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)) - { - // We only have to show this warning if the old manifest ID was different - if (lastManifestId != depot.ManifestId) - Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", lastManifestId); - oldProtoManifest = null; - } - } + // We only have to show this warning if the old manifest ID was different + var badHashWarning = (lastManifestId != depot.ManifestId); + oldManifest = Util.LoadManifestFromFile(configDir, depot.DepotId, lastManifestId, badHashWarning); } - if (lastManifestId == depot.ManifestId && oldProtoManifest != null) + if (lastManifestId == depot.ManifestId && oldManifest != null) { - newProtoManifest = oldProtoManifest; + newManifest = oldManifest; Console.WriteLine("Already have manifest {0} for depot {1}.", depot.ManifestId, depot.DepotId); } else { - var newManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.DepotId, depot.ManifestId)); - if (newManifestFileName != null) - { - byte[] expectedChecksum; - - try - { - expectedChecksum = File.ReadAllBytes(newManifestFileName + ".sha"); - } - catch (IOException) - { - expectedChecksum = null; - } - - newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName, out var currentChecksum); - - if (newProtoManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))) - { - Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", depot.ManifestId); - newProtoManifest = null; - } - } + newManifest = Util.LoadManifestFromFile(configDir, depot.DepotId, depot.ManifestId, true); - if (newProtoManifest != null) + if (newManifest != null) { Console.WriteLine("Already have manifest {0} for depot {1}.", depot.ManifestId, depot.DepotId); } @@ -772,7 +729,6 @@ private static async Task ProcessDepotManifestAndFiles(Cancellat { Console.Write("Downloading depot manifest... "); - DepotManifest depotManifest = null; ulong manifestRequestCode = 0; var manifestRequestCodeExpiration = DateTime.MinValue; @@ -820,7 +776,7 @@ private static async Task ProcessDepotManifestAndFiles(Cancellat depot.ManifestId, connection, cdnPool.ProxyServer != null ? cdnPool.ProxyServer : "no proxy"); - depotManifest = await cdnPool.CDNClient.DownloadManifestAsync( + newManifest = await cdnPool.CDNClient.DownloadManifestAsync( depot.DepotId, depot.ManifestId, manifestRequestCode, @@ -872,9 +828,9 @@ private static async Task ProcessDepotManifestAndFiles(Cancellat cdnPool.ReturnBrokenConnection(connection); Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.DepotId, depot.ManifestId, e.Message); } - } while (depotManifest == null); + } while (newManifest == null); - if (depotManifest == null) + if (newManifest == null) { Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.ManifestId, depot.DepotId); cts.Cancel(); @@ -883,28 +839,22 @@ private static async Task ProcessDepotManifestAndFiles(Cancellat // Throw the cancellation exception if requested so that this task is marked failed cts.Token.ThrowIfCancellationRequested(); - - newProtoManifest = new ProtoManifest(depotManifest, depot.ManifestId); - newProtoManifest.SaveToFile(newManifestFileName, out var checksum); - File.WriteAllBytes(newManifestFileName + ".sha", checksum); - + Util.SaveManifestToFile(configDir, newManifest); Console.WriteLine(" Done!"); } } - newProtoManifest.Files.Sort((x, y) => string.Compare(x.FileName, y.FileName, StringComparison.Ordinal)); - - Console.WriteLine("Manifest {0} ({1})", depot.ManifestId, newProtoManifest.CreationTime); + Console.WriteLine("Manifest {0} ({1})", depot.ManifestId, newManifest.CreationTime); if (Config.DownloadManifestOnly) { - DumpManifestToTextFile(depot, newProtoManifest); + DumpManifestToTextFile(depot, newManifest); return null; } var stagingDir = Path.Combine(depot.InstallDir, STAGING_DIR); - var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList(); + var filesAfterExclusions = newManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList(); var allFileNames = new HashSet(filesAfterExclusions.Count); // Pre-process @@ -936,8 +886,8 @@ private static async Task ProcessDepotManifestAndFiles(Cancellat depotDownloadInfo = depot, depotCounter = depotCounter, stagingDir = stagingDir, - manifest = newProtoManifest, - previousManifest = oldProtoManifest, + manifest = newManifest, + previousManifest = oldManifest, filteredFiles = filesAfterExclusions, allFileNames = allFileNames }; @@ -952,7 +902,7 @@ private static async Task DownloadSteam3AsyncDepotFiles(CancellationTokenSource Console.WriteLine("Downloading depot {0}", depot.DepotId); var files = depotFilesData.filteredFiles.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray(); - var networkChunkQueue = new ConcurrentQueue<(FileStreamData fileStreamData, ProtoManifest.FileData fileData, ProtoManifest.ChunkData chunk)>(); + var networkChunkQueue = new ConcurrentQueue<(FileStreamData fileStreamData, DepotManifest.FileData fileData, DepotManifest.ChunkData chunk)>(); await Util.InvokeAsync( files.Select(file => new Func(async () => @@ -1006,8 +956,8 @@ private static void DownloadSteam3AsyncDepotFile( CancellationTokenSource cts, GlobalDownloadCounter downloadCounter, DepotFilesData depotFilesData, - ProtoManifest.FileData file, - ConcurrentQueue<(FileStreamData, ProtoManifest.FileData, ProtoManifest.ChunkData)> networkChunkQueue) + DepotManifest.FileData file, + ConcurrentQueue<(FileStreamData, DepotManifest.FileData, DepotManifest.ChunkData)> networkChunkQueue) { cts.Token.ThrowIfCancellationRequested(); @@ -1015,7 +965,7 @@ private static void DownloadSteam3AsyncDepotFile( var stagingDir = depotFilesData.stagingDir; var depotDownloadCounter = depotFilesData.depotCounter; var oldProtoManifest = depotFilesData.previousManifest; - ProtoManifest.FileData oldManifestFile = null; + DepotManifest.FileData oldManifestFile = null; if (oldProtoManifest != null) { oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName); @@ -1030,7 +980,7 @@ private static void DownloadSteam3AsyncDepotFile( File.Delete(fileStagingPath); } - List neededChunks; + List neededChunks; var fi = new FileInfo(fileFinalPath); var fileDidExist = fi.Exists; if (!fileDidExist) @@ -1048,7 +998,7 @@ private static void DownloadSteam3AsyncDepotFile( throw new ContentDownloaderException(string.Format("Failed to allocate file {0}: {1}", fileFinalPath, ex.Message)); } - neededChunks = new List(file.Chunks); + neededChunks = new List(file.Chunks); } else { @@ -1092,7 +1042,7 @@ private static void DownloadSteam3AsyncDepotFile( fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin); var adler = Util.AdlerHash(fsOld, (int)match.OldChunk.UncompressedLength); - if (!adler.SequenceEqual(match.OldChunk.Checksum)) + if (!adler.SequenceEqual(BitConverter.GetBytes(match.OldChunk.Checksum))) { neededChunks.Add(match.NewChunk); } @@ -1211,9 +1161,9 @@ private static async Task DownloadSteam3AsyncDepotFileChunk( CancellationTokenSource cts, GlobalDownloadCounter downloadCounter, DepotFilesData depotFilesData, - ProtoManifest.FileData file, + DepotManifest.FileData file, FileStreamData fileStreamData, - ProtoManifest.ChunkData chunk) + DepotManifest.ChunkData chunk) { cts.Token.ThrowIfCancellationRequested(); @@ -1222,17 +1172,8 @@ private static async Task DownloadSteam3AsyncDepotFileChunk( var chunkID = Convert.ToHexString(chunk.ChunkID).ToLowerInvariant(); - var data = new DepotManifest.ChunkData - { - ChunkID = chunk.ChunkID, - Checksum = BitConverter.ToUInt32(chunk.Checksum), - Offset = chunk.Offset, - CompressedLength = chunk.CompressedLength, - UncompressedLength = chunk.UncompressedLength - }; - var written = 0; - var chunkBuffer = ArrayPool.Shared.Rent((int)data.UncompressedLength); + var chunkBuffer = ArrayPool.Shared.Rent((int)chunk.UncompressedLength); try { @@ -1256,7 +1197,7 @@ private static async Task DownloadSteam3AsyncDepotFileChunk( DebugLog.WriteLine("ContentDownloader", "Downloading chunk {0} from {1} with {2}", chunkID, connection, cdnPool.ProxyServer != null ? cdnPool.ProxyServer : "no proxy"); written = await cdnPool.CDNClient.DownloadDepotChunkAsync( depot.DepotId, - data, + chunk, connection, chunkBuffer, depot.DepotKey, @@ -1325,7 +1266,7 @@ private static async Task DownloadSteam3AsyncDepotFileChunk( fileStreamData.fileStream = File.Open(fileFinalPath, FileMode.Open); } - fileStreamData.fileStream.Seek((long)data.Offset, SeekOrigin.Begin); + fileStreamData.fileStream.Seek((long)chunk.Offset, SeekOrigin.Begin); await fileStreamData.fileStream.WriteAsync(chunkBuffer.AsMemory(0, written), cts.Token); } finally @@ -1369,44 +1310,55 @@ private static async Task DownloadSteam3AsyncDepotFileChunk( } } - static void DumpManifestToTextFile(DepotDownloadInfo depot, ProtoManifest manifest) + class ChunkIdComparer : IEqualityComparer + { + public bool Equals(byte[] x, byte[] y) + { + if (ReferenceEquals(x, y)) return true; + if (x == null || y == null) return false; + return x.SequenceEqual(y); + } + + public int GetHashCode(byte[] obj) + { + ArgumentNullException.ThrowIfNull(obj); + + // ChunkID is SHA-1, so we can just use the first 4 bytes + return BitConverter.ToInt32(obj, 0); + } + } + + static void DumpManifestToTextFile(DepotDownloadInfo depot, DepotManifest manifest) { var txtManifest = Path.Combine(depot.InstallDir, $"manifest_{depot.DepotId}_{depot.ManifestId}.txt"); using var sw = new StreamWriter(txtManifest); - sw.WriteLine($"Content Manifest for Depot {depot.DepotId}"); + sw.WriteLine($"Content Manifest for Depot {depot.DepotId} "); sw.WriteLine(); - sw.WriteLine($"Manifest ID / date : {depot.ManifestId} / {manifest.CreationTime}"); + sw.WriteLine($"Manifest ID / date : {depot.ManifestId} / {manifest.CreationTime} "); - int numFiles = 0, numChunks = 0; - ulong uncompressedSize = 0, compressedSize = 0; + var uniqueChunks = new HashSet(new ChunkIdComparer()); foreach (var file in manifest.Files) { - if (file.Flags.HasFlag(EDepotFileFlag.Directory)) - continue; - - numFiles++; - numChunks += file.Chunks.Count; - foreach (var chunk in file.Chunks) { - uncompressedSize += chunk.UncompressedLength; - compressedSize += chunk.CompressedLength; + uniqueChunks.Add(chunk.ChunkID); } } - sw.WriteLine($"Total number of files : {numFiles}"); - sw.WriteLine($"Total number of chunks : {numChunks}"); - sw.WriteLine($"Total bytes on disk : {uncompressedSize}"); - sw.WriteLine($"Total bytes compressed : {compressedSize}"); + sw.WriteLine($"Total number of files : {manifest.Files.Count} "); + sw.WriteLine($"Total number of chunks : {uniqueChunks.Count} "); + sw.WriteLine($"Total bytes on disk : {manifest.TotalUncompressedSize} "); + sw.WriteLine($"Total bytes compressed : {manifest.TotalCompressedSize} "); + sw.WriteLine(); sw.WriteLine(); sw.WriteLine(" Size Chunks File SHA Flags Name"); foreach (var file in manifest.Files) { - var sha1Hash = Convert.ToHexString(file.FileHash); - sw.WriteLine($"{file.TotalSize,14} {file.Chunks.Count,6} {sha1Hash} {file.Flags,5:D} {file.FileName}"); + var sha1Hash = Convert.ToHexString(file.FileHash).ToLower(); + sw.WriteLine($"{file.TotalSize,14:d} {file.Chunks.Count,6:d} {sha1Hash} {(int)file.Flags,5:x} {file.FileName}"); } } } diff --git a/DepotDownloader/ProtoManifest.cs b/DepotDownloader/ProtoManifest.cs index b6eb01001..de9046ae6 100644 --- a/DepotDownloader/ProtoManifest.cs +++ b/DepotDownloader/ProtoManifest.cs @@ -6,6 +6,7 @@ using System.IO; using System.IO.Compression; using System.Security.Cryptography; +using System.Text; using ProtoBuf; using SteamKit2; @@ -157,5 +158,39 @@ public void SaveToFile(string filename, out byte[] checksum) using var ds = new DeflateStream(fs, CompressionMode.Compress); ms.CopyTo(ds); } + + public DepotManifest ConvertToSteamManifest(uint depotId) + { + ulong uncompressedSize = 0, compressedSize = 0; + var newManifest = new DepotManifest(); + newManifest.Files = new List(Files.Count); + + foreach (var file in Files) + { + var fileNameHash = SHA1.HashData(Encoding.UTF8.GetBytes(file.FileName.Replace('/', '\\').ToLowerInvariant())); + var newFile = new DepotManifest.FileData(file.FileName, fileNameHash, file.Flags, file.TotalSize, file.FileHash, null, false, file.Chunks.Count); + + foreach (var chunk in file.Chunks) + { + var newChunk = new DepotManifest.ChunkData(chunk.ChunkID, BitConverter.ToUInt32(chunk.Checksum, 0), chunk.Offset, chunk.CompressedLength, chunk.UncompressedLength); + newFile.Chunks.Add(newChunk); + + uncompressedSize += chunk.UncompressedLength; + compressedSize += chunk.CompressedLength; + } + + newManifest.Files.Add(newFile); + } + + newManifest.FilenamesEncrypted = false; + newManifest.DepotID = depotId; + newManifest.ManifestGID = ID; + newManifest.CreationTime = CreationTime; + newManifest.TotalUncompressedSize = uncompressedSize; + newManifest.TotalCompressedSize = compressedSize; + newManifest.EncryptedCRC = 0; + + return newManifest; + } } } diff --git a/DepotDownloader/Util.cs b/DepotDownloader/Util.cs index 46045ae46..b23eb8569 100644 --- a/DepotDownloader/Util.cs +++ b/DepotDownloader/Util.cs @@ -9,6 +9,7 @@ using System.Security.Cryptography; using System.Text; using System.Threading.Tasks; +using SteamKit2; namespace DepotDownloader { @@ -78,16 +79,16 @@ public static string ReadPassword() } // Validate a file against Steam3 Chunk data - public static List ValidateSteam3FileChecksums(FileStream fs, ProtoManifest.ChunkData[] chunkdata) + public static List ValidateSteam3FileChecksums(FileStream fs, DepotManifest.ChunkData[] chunkdata) { - var neededChunks = new List(); + var neededChunks = new List(); foreach (var data in chunkdata) { fs.Seek((long)data.Offset, SeekOrigin.Begin); var adler = AdlerHash(fs, (int)data.UncompressedLength); - if (!adler.SequenceEqual(data.Checksum)) + if (!adler.SequenceEqual(BitConverter.GetBytes(data.Checksum))) { neededChunks.Add(data); } @@ -110,6 +111,100 @@ public static byte[] AdlerHash(Stream stream, int length) return BitConverter.GetBytes(a | (b << 16)); } + public static byte[] FileSHAHash(string filename) + { + using (var fs = File.Open(filename, FileMode.Open)) + using (var sha = SHA1.Create()) + { + var output = sha.ComputeHash(fs); + + return output; + } + } + + public static DepotManifest LoadManifestFromFile(string directory, uint depotId, ulong manifestId, bool badHashWarning) + { + // Try loading Steam format manifest first. + var filename = Path.Combine(directory, string.Format("{0}_{1}.manifest", depotId, manifestId)); + + if (File.Exists(filename)) + { + byte[] expectedChecksum; + + try + { + expectedChecksum = File.ReadAllBytes(filename + ".sha"); + } + catch (IOException) + { + expectedChecksum = null; + } + + var currentChecksum = FileSHAHash(filename); + + if (expectedChecksum != null && expectedChecksum.SequenceEqual(currentChecksum)) + { + return DepotManifest.LoadFromFile(filename); + } + else if (badHashWarning) + { + Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", manifestId); + } + } + + // Try converting legacy manifest format. + filename = Path.Combine(directory, string.Format("{0}_{1}.bin", depotId, manifestId)); + + if (File.Exists(filename)) + { + byte[] expectedChecksum; + + try + { + expectedChecksum = File.ReadAllBytes(filename + ".sha"); + } + catch (IOException) + { + expectedChecksum = null; + } + + byte[] currentChecksum; + var oldManifest = ProtoManifest.LoadFromFile(filename, out currentChecksum); + + if (oldManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))) + { + oldManifest = null; + + if (badHashWarning) + { + Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", manifestId); + } + } + + if (oldManifest != null) + { + return oldManifest.ConvertToSteamManifest(depotId); + } + } + + return null; + } + + public static bool SaveManifestToFile(string directory, DepotManifest manifest) + { + try + { + var filename = Path.Combine(directory, string.Format("{0}_{1}.manifest", manifest.DepotID, manifest.ManifestGID)); + manifest.SaveToFile(filename); + File.WriteAllBytes(filename + ".sha", FileSHAHash(filename)); + return true; // If serialization completes without throwing an exception, return true + } + catch (Exception) + { + return false; // Return false if an error occurs + } + } + public static byte[] DecodeHexString(string hex) { if (hex == null)