From 32cb38bb166f6f3a9c98b303776a8a45fd27e482 Mon Sep 17 00:00:00 2001 From: lipengzha Date: Fri, 30 Dec 2022 10:36:09 +0800 Subject: [PATCH] Release v79.0 (#71) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. 优化基础实现、关闭不必要的Log输出 2. 优化对依赖加载材质的额外Shader编译， 支持`NoPostLoadCacheDDC`参数控制（UE4中需要修改引擎才能使用该优化） 3. 优化CookCluster的分配策略 4. 优化Pak文件列表收集 5. 优化资源分析性能 6. 优化模块的可扩展性 7. 修复跨引擎版本的兼容性问题 8. 修复Shipping的C4172 ERROR 9. 修复不支持WP导致的基础包打包错误（`Found mor than one redistered Cook Package Splitter`） 10. 支持`PakSaveDirRegular`，自定义Pak的输出路径 11. 支持PakPreset/CookAndPak以统一的方式执行 12. 支持全局的AllowCookPlatforms --- .../Classes/Commandlets/CommandletHelper.cpp | 4 +- .../Commandlets/HotPatcherCommandlet.cpp | 2 +- .../HotPatcherCore/HotPatcherCore.Build.cs | 2 +- .../Cooker/MultiCooker/SingleCookerProxy.cpp | 114 +++---- .../HotWorldPartitionCookPackageSplitter.cpp | 2 +- .../Private/CreatePatch/PatcherProxy.cpp | 25 +- .../Private/FlibHotPatcherCoreHelper.cpp | 85 +++++- .../FlibShaderCodeLibraryHelper.cpp | 22 +- .../Public/CommandletBase/CommandletHelper.h | 4 +- .../Public/FlibHotPatcherCoreHelper.h | 22 +- .../ShaderPatch/FlibShaderCodeLibraryHelper.h | 3 +- .../AssetTypeActions_PrimaryAssetLabel.cpp | 2 +- .../CreatePatch/SHotPatcherPatchWidget.cpp | 23 +- .../Private/FlibHotPatcherEditorHelper.cpp | 45 +++ .../Private/HotPatcherEditor.cpp | 77 ++--- .../Private/SHotPatcherWidgetBase.cpp | 23 +- .../Public/FlibHotPatcherEditorHelper.h | 4 +- .../Public/HotPatcherEditor.h | 6 +- .../Private/BaseTypes/FChunkInfo.cpp | 1 + .../CreatePatch/FExportPatchSettings.cpp | 9 + .../CreatePatch/HotPatcherSettingBase.cpp | 13 +- .../FDefaultAssetDependenciesParser.cpp | 2 +- .../Private/FlibAssetManageHelper.cpp | 37 ++- .../Private/FlibPatchParserHelper.cpp | 286 +++++++++++------- .../Public/BaseTypes/FHotPatcherVersion.h | 3 +- .../Public/BaseTypes/FPackageTracker.h | 33 ++ .../Public/CreatePatch/FExportPatchSettings.h | 13 +- .../CreatePatch/HotPatcherSettingBase.h | 10 +- .../Public/CreatePatch/TimeRecorder.h | 3 +- .../Public/FlibAssetManageHelper.h | 6 +- .../Public/FlibPatchParserHelper.h | 5 +- Mods/HotChunker | 2 +- Mods/HotMultiCooker | 2 +- 33 files changed, 593 insertions(+), 297 deletions(-) diff --git a/HotPatcher/Source/HotPatcherCore/Classes/Commandlets/CommandletHelper.cpp b/HotPatcher/Source/HotPatcherCore/Classes/Commandlets/CommandletHelper.cpp index 481a429b..e2140761 100644 --- a/HotPatcher/Source/HotPatcherCore/Classes/Commandlets/CommandletHelper.cpp +++ b/HotPatcher/Source/HotPatcherCore/Classes/Commandlets/CommandletHelper.cpp @@ -88,7 +88,9 @@ void CommandletHelper::MainTick(TFunction IsRequestExit) // main loop FDateTime LastConnectionTime = FDateTime::UtcNow(); - while (GIsRunning && !IsRequestingExit() && !IsRequestExit()) + while (GIsRunning && + // !IsRequestingExit() && + !IsRequestExit()) { GEngine->UpdateTimeAndHandleMaxTickRate(); GEngine->Tick(FApp::GetDeltaTime(), false); diff --git a/HotPatcher/Source/HotPatcherCore/Classes/Commandlets/HotPatcherCommandlet.cpp b/HotPatcher/Source/HotPatcherCore/Classes/Commandlets/HotPatcherCommandlet.cpp index 98be16b0..df4bb57f 100644 --- a/HotPatcher/Source/HotPatcherCore/Classes/Commandlets/HotPatcherCommandlet.cpp +++ b/HotPatcher/Source/HotPatcherCore/Classes/Commandlets/HotPatcherCommandlet.cpp @@ -68,7 +68,7 @@ int32 UHotPatcherCommandlet::Main(const FString& Params) FString FinalConfig; THotPatcherTemplateHelper::TSerializeStructAsJsonString(*ExportPatchSetting,FinalConfig); - UE_LOG(LogHotPatcherCommandlet, Display, TEXT("%s"), *FinalConfig); + // UE_LOG(LogHotPatcherCommandlet, Display, TEXT("%s"), *FinalConfig); UPatcherProxy* PatcherProxy = NewObject(); diff --git a/HotPatcher/Source/HotPatcherCore/HotPatcherCore.Build.cs b/HotPatcher/Source/HotPatcherCore/HotPatcherCore.Build.cs index 44c04de0..edfe51f9 100644 --- a/HotPatcher/Source/HotPatcherCore/HotPatcherCore.Build.cs +++ b/HotPatcher/Source/HotPatcherCore/HotPatcherCore.Build.cs @@ -190,7 +190,7 @@ public HotPatcherCore(ReadOnlyTargetRules Target) : base(Target) PublicDefinitions.AddRange(new string[] { "TOOL_NAME=\"HotPatcher\"", - "CURRENT_VERSION_ID=78", + "CURRENT_VERSION_ID=79", "CURRENT_PATCH_ID=0", "REMOTE_VERSION_FILE=\"https://imzlp.com/opensource/version.json\"" }); diff --git a/HotPatcher/Source/HotPatcherCore/Private/Cooker/MultiCooker/SingleCookerProxy.cpp b/HotPatcher/Source/HotPatcherCore/Private/Cooker/MultiCooker/SingleCookerProxy.cpp index 31a3af14..168370fb 100644 --- a/HotPatcher/Source/HotPatcherCore/Private/Cooker/MultiCooker/SingleCookerProxy.cpp +++ b/HotPatcher/Source/HotPatcherCore/Private/Cooker/MultiCooker/SingleCookerProxy.cpp @@ -49,14 +49,21 @@ void FFreezePackageTracker::NotifyUObjectCreated(const UObjectBase* Object, int3 } } } - +#define NO_POSTLOAD_CACHE_DDC_OPTION TEXT("-NoPostLoadCacheDDC") void USingleCookerProxy::Init(FPatcherEntitySettingBase* InSetting) { SCOPED_NAMED_EVENT_TEXT("Init",FColor::Red); Super::Init(InSetting); // IConsoleVariable* StreamableDelegateDelayFrames = IConsoleManager::Get().FindConsoleVariable(TEXT("s.StreamableDelegateDelayFrames")); // StreamableDelegateDelayFrames->Set(0); - + UFlibHotPatcherCoreHelper::DumpActiveTargetPlatforms(); + + FString Cmdline = FCommandLine::Get(); + if(!Cmdline.Contains(NO_POSTLOAD_CACHE_DDC_OPTION,ESearchCase::IgnoreCase)) + { + FCommandLine::Append(*FString::Printf(TEXT(" %s"),NO_POSTLOAD_CACHE_DDC_OPTION)); + UE_LOG(LogHotPatcher,Display,TEXT("Append %s to Cmdline."),NO_POSTLOAD_CACHE_DDC_OPTION); + } #if WITH_PACKAGE_CONTEXT if(GetSettingObject()->bOverrideSavePackageContext) { @@ -112,62 +119,52 @@ int32 USingleCookerProxy::MakeCookQueue(FCookCluster& InCluser) const int32 NumberOfAssetsPerFrame = GetSettingObject()->GetNumberOfAssetsPerFrame(); - if(GetSettingObject()->bForceCookInOneFrame) + for(auto Class:GetPreCacheClasses()) { - CookCluserQueue.Enqueue(InCluser); - ++MakeClusterCount; - return MakeClusterCount; - } - else - { - for(auto Class:GetPreCacheClasses()) + TArray ObjectAssets = UFlibAssetManageHelper::GetAssetDetailsByClass(InCluser.AssetDetails,Class,true); + if(ObjectAssets.Num()) { - TArray ObjectAssets = UFlibAssetManageHelper::GetAssetDetailsByClass(InCluser.AssetDetails,Class,true); - if(ObjectAssets.Num()) - { - DumpCookerInfo.Append(FString::Printf(TEXT("\t%s -- %d\n"),*Class->GetName(),ObjectAssets.Num())); - } - int32 ClassesNumberOfAssetsPerFrame = GetClassAssetNumOfPerCluster(Class); + DumpCookerInfo.Append(FString::Printf(TEXT("\t%s -- %d\n"),*Class->GetName(),ObjectAssets.Num())); + } + int32 ClassesNumberOfAssetsPerFrame = GetClassAssetNumOfPerCluster(Class); - while(ObjectAssets.Num()) - { - int32 ClusterAssetNum = ClassesNumberOfAssetsPerFrame < 1 ? ObjectAssets.Num() : ClassesNumberOfAssetsPerFrame; - int32 NewClusterAssetNum = FMath::Min(ClusterAssetNum,ObjectAssets.Num()); + while(ObjectAssets.Num()) + { + int32 ClusterAssetNum = ClassesNumberOfAssetsPerFrame < 1 ? ObjectAssets.Num() : ClassesNumberOfAssetsPerFrame; + int32 NewClusterAssetNum = FMath::Min(ClusterAssetNum,ObjectAssets.Num()); - TArray CulsterObjectAssets(ObjectAssets.GetData(),NewClusterAssetNum); - FCookCluster NewCluster; - NewCluster.AssetDetails = std::move(CulsterObjectAssets); - ObjectAssets.RemoveAt(0,NewClusterAssetNum); - NewCluster.Platforms = GetSettingObject()->CookTargetPlatforms; - NewCluster.bPreGeneratePlatformData = GetSettingObject()->bPreGeneratePlatformData; - - NewCluster.CookActionCallback.OnAssetCooked = GetOnPackageSavedCallback(); - NewCluster.CookActionCallback.OnCookBegin = GetOnCookAssetBeginCallback(); - CookCluserQueue.Enqueue(NewCluster); - ++MakeClusterCount; - } + TArray CulsterObjectAssets(ObjectAssets.GetData(),NewClusterAssetNum); + FCookCluster NewCluster; + NewCluster.AssetDetails = std::move(CulsterObjectAssets); + ObjectAssets.RemoveAt(0,NewClusterAssetNum); + NewCluster.Platforms = GetSettingObject()->CookTargetPlatforms; + NewCluster.bPreGeneratePlatformData = GetSettingObject()->bPreGeneratePlatformData; + NewCluster.CookActionCallback.OnAssetCooked = GetOnPackageSavedCallback(); + NewCluster.CookActionCallback.OnCookBegin = GetOnCookAssetBeginCallback(); + CookCluserQueue.Enqueue(NewCluster); + ++MakeClusterCount; } - - if(InCluser.AssetDetails.Num()) + } + if(InCluser.AssetDetails.Num()) + { + int32 OtherAssetNumPerFrame = NumberOfAssetsPerFrame < 1 ? InCluser.AssetDetails.Num() : NumberOfAssetsPerFrame; + int32 SplitNum = (InCluser.AssetDetails.Num() / OtherAssetNumPerFrame) + 1; + + const TArray> SplitedAssets= THotPatcherTemplateHelper::SplitArray(InCluser.AssetDetails,SplitNum); + for(const auto& AssetDetails:SplitedAssets) { - int32 OtherAssetNumPerFrame = NumberOfAssetsPerFrame < 1 ? InCluser.AssetDetails.Num() : NumberOfAssetsPerFrame; - int32 SplitNum = (InCluser.AssetDetails.Num() / OtherAssetNumPerFrame) + 1; - - const TArray> SplitedAssets= THotPatcherTemplateHelper::SplitArray(InCluser.AssetDetails,SplitNum); - for(const auto& AssetDetails:SplitedAssets) - { - FCookCluster NewCluster; - NewCluster.AssetDetails = std::move(AssetDetails); - NewCluster.Platforms = GetSettingObject()->CookTargetPlatforms; - NewCluster.bPreGeneratePlatformData = GetSettingObject()->bPreGeneratePlatformData; - NewCluster.CookActionCallback.OnAssetCooked = GetOnPackageSavedCallback(); - NewCluster.CookActionCallback.OnCookBegin = GetOnCookAssetBeginCallback(); - CookCluserQueue.Enqueue(NewCluster); - ++MakeClusterCount; - } - DumpCookerInfo.Append(FString::Printf(TEXT("\tOther Assets -- %d, make %d cluster.\n"),InCluser.AssetDetails.Num(),SplitedAssets.Num())); + FCookCluster NewCluster; + NewCluster.AssetDetails = std::move(AssetDetails); + NewCluster.Platforms = GetSettingObject()->CookTargetPlatforms; + NewCluster.bPreGeneratePlatformData = GetSettingObject()->bPreGeneratePlatformData; + NewCluster.CookActionCallback.OnAssetCooked = GetOnPackageSavedCallback(); + NewCluster.CookActionCallback.OnCookBegin = GetOnCookAssetBeginCallback(); + CookCluserQueue.Enqueue(NewCluster); + ++MakeClusterCount; } + DumpCookerInfo.Append(FString::Printf(TEXT("\tOther Assets -- %d, make %d cluster.\n"),InCluser.AssetDetails.Num(),SplitedAssets.Num())); } + DumpCookerInfo.Append(TEXT("---------------------------------------------------------------------\n")); UE_LOG(LogHotPatcher,Display,TEXT("%s"),*DumpCookerInfo); return MakeClusterCount; @@ -294,8 +291,8 @@ void USingleCookerProxy::ExecCookCluster(const FCookCluster& CookCluster) return; } + bool bContainShaderCluster = UFlibHotPatcherCoreHelper::AssetDetailsHasClasses(CookCluster.AssetDetails,UFlibHotPatcherCoreHelper::GetAllMaterialClassesNames()); DumpCluster(CookCluster,GCookLog); - TArray TargetPlatforms = UFlibHotPatcherCoreHelper::GetTargetPlatformsByNames(CookCluster.Platforms); FString CookBaseDir = GetSettingObject()->GetStorageCookedAbsDir(); CleanOldCooked(CookBaseDir,CookCluster.AsSoftObjectPaths(),CookCluster.Platforms); @@ -303,6 +300,8 @@ void USingleCookerProxy::ExecCookCluster(const FCookCluster& CookCluster) #if WITH_PACKAGE_CONTEXT TMap SavePackageContextsNameMapping = GetPlatformSavePackageContextsNameMapping(); #endif + + TSharedPtr ClassesPackageTracker = MakeShareable(new FClassesPackageTracker); TArray PreCachePackages = UFlibAssetManageHelper::LoadPackagesForCooking(CookCluster.AsSoftObjectPaths(),GetSettingObject()->bConcurrentSave); bool bCanConcurrentSave = GetSettingObject()->bConcurrentSave && CookCluster.bPreGeneratePlatformData; @@ -371,7 +370,6 @@ void USingleCookerProxy::ExecCookCluster(const FCookCluster& CookCluster) // clean cached ddd / release memory // CleanClusterCachedPlatformData(CookCluster); - UFlibShaderCodeLibraryHelper::WaitShaderCompilingComplete(); UFlibHotPatcherCoreHelper::WaitForAsyncFileWrites(); // for GC @@ -603,10 +601,14 @@ FCookCluster USingleCookerProxy::GetPackageTrackerAsCluster() if(PackageTracker && GetSettingObject()->bCookPackageTrackerAssets) { PackageTrackerCluster.AssetDetails.Empty(); - for(FName PackagePath:PackageTracker->GetPendingPackageSet()) + for(FName LongPackageName:PackageTracker->GetPendingPackageSet()) { // make asset data to asset registry - FSoftObjectPath ObjectPath(PackagePath.ToString()); + FSoftObjectPath ObjectPath( + UFlibAssetManageHelper::LongPackageNameToPackagePath(LongPackageName.ToString()) + ); + UFlibAssetManageHelper::UpdateAssetRegistryData(ObjectPath.GetLongPackageName()); + FAssetData AssetData; if(UAssetManager::Get().GetAssetDataForPath(ObjectPath,AssetData)) { @@ -618,7 +620,7 @@ FCookCluster USingleCookerProxy::GetPackageTrackerAsCluster() } else { - UE_LOG(LogHotPatcher,Warning,TEXT("[GetPackageTrackerAsCluster] Get %s AssetData Failed!"),*PackagePath.ToString()); + UE_LOG(LogHotPatcher,Warning,TEXT("[GetPackageTrackerAsCluster] Get %s AssetData Failed!"),*LongPackageName.ToString()); } } } @@ -695,6 +697,7 @@ bool USingleCookerProxy::DoExport() void USingleCookerProxy::CleanOldCooked(const FString& CookBaseDir,const TArray& ObjectPaths,const TArray& CookPlatforms) { + SCOPED_NAMED_EVENT_TEXT("CleanOldCooked",FColor::Red); TArray CookPlatfotms = UFlibHotPatcherCoreHelper::GetTargetPlatformsByNames(CookPlatforms); { SCOPED_NAMED_EVENT_TEXT("Delete Old Cooked Files",FColor::Red); @@ -724,7 +727,6 @@ void USingleCookerProxy::CleanOldCooked(const FString& CookBaseDir,const TArray< } } - bool USingleCookerProxy::HasError() { SCOPED_NAMED_EVENT_TEXT("USingleCookerProxy::HasError",FColor::Red); diff --git a/HotPatcher/Source/HotPatcherCore/Private/Cooker/PackageWriter/HotWorldPartitionCookPackageSplitter.cpp b/HotPatcher/Source/HotPatcherCore/Private/Cooker/PackageWriter/HotWorldPartitionCookPackageSplitter.cpp index caae8b03..d3ad5c78 100644 --- a/HotPatcher/Source/HotPatcherCore/Private/Cooker/PackageWriter/HotWorldPartitionCookPackageSplitter.cpp +++ b/HotPatcher/Source/HotPatcherCore/Private/Cooker/PackageWriter/HotWorldPartitionCookPackageSplitter.cpp @@ -15,7 +15,7 @@ #include "Editor.h" // Register FHotWorldPartitionCookPackageSplitter for UWorld class -REGISTER_COOKPACKAGE_SPLITTER(FHotWorldPartitionCookPackageSplitter, UWorld); +// REGISTER_COOKPACKAGE_SPLITTER(FHotWorldPartitionCookPackageSplitter, UWorld); bool FHotWorldPartitionCookPackageSplitter::ShouldSplit(UObject* SplitData) { diff --git a/HotPatcher/Source/HotPatcherCore/Private/CreatePatch/PatcherProxy.cpp b/HotPatcher/Source/HotPatcherCore/Private/CreatePatch/PatcherProxy.cpp index 8ecab750..4a51b5da 100644 --- a/HotPatcher/Source/HotPatcherCore/Private/CreatePatch/PatcherProxy.cpp +++ b/HotPatcher/Source/HotPatcherCore/Private/CreatePatch/PatcherProxy.cpp @@ -158,6 +158,7 @@ namespace PatchWorker void UPatcherProxy::Init(FPatcherEntitySettingBase* InSetting) { + SCOPED_NAMED_EVENT_TEXT("UPatcherProxy::Init",FColor::Red); Super::Init(InSetting); #if WITH_PACKAGE_CONTEXT PlatformSavePackageContexts = UFlibHotPatcherCoreHelper::CreatePlatformsPackageContexts( @@ -200,6 +201,7 @@ void UPatcherProxy::Shutdown() bool UPatcherProxy::DoExport() { + SCOPED_NAMED_EVENT_TEXT("UPatcherProxy::DoExport",FColor::Red); PatchContext = MakeShareable(new FHotPatcherPatchContext); PatchContext->PatchProxy = this; PatchContext->OnPaking.AddLambda([this](const FString& One,const FString& Msg){this->OnPaking.Broadcast(One,Msg);}); @@ -584,9 +586,17 @@ namespace PatchWorker EmptySetting.bConcurrentSave = false; // for current impl arch EmptySetting.bForceCookInOneFrame = true; + EmptySetting.NumberOfAssetsPerFrame = 200; EmptySetting.bDisplayConfig = false; EmptySetting.StorageCookedDir = Context.GetSettingObject()->GetStorageCookedDir();//FPaths::Combine(FPaths::ConvertRelativePathToFull(FPaths::ProjectSavedDir()),TEXT("Cooked")); - EmptySetting.StorageMetadataDir = FPaths::Combine(Context.GetSettingObject()->GetSaveAbsPath(),Context.CurrentVersion.VersionId,TEXT("Metadatas"),Chunk.ChunkName); + FReplacePakRegular PakSaveDirRegular{ + Context.CurrentVersion.VersionId, + Context.CurrentVersion.BaseVersionId, + Chunk.ChunkName, + PlatformName + }; + FString ReplacedPakSaveDirRegular = UFlibHotPatcherCoreHelper::ReplacePakRegular(PakSaveDirRegular,Context.GetSettingObject()->GetPakSaveDirRegular()); + EmptySetting.StorageMetadataDir = FPaths::Combine(Context.GetSettingObject()->GetSaveAbsPath(),ReplacedPakSaveDirRegular,TEXT("Metadatas")); #if WITH_PACKAGE_CONTEXT EmptySetting.bOverrideSavePackageContext = true; EmptySetting.PlatformSavePackageContexts = Context.PatchProxy->GetPlatformSavePackageContexts(); @@ -865,7 +875,14 @@ namespace PatchWorker Context.OnPaking.Broadcast(TEXT("ExportPatch"),*Dialog.ToString()); Context.UnrealPakSlowTask->EnterProgressFrame(1.0, Dialog); } - FString ChunkSaveBasePath = FPaths::Combine(Context.GetSettingObject()->GetSaveAbsPath(), Context.CurrentVersion.VersionId, PlatformName); + FReplacePakRegular PakSaveDirRegular{ + Context.CurrentVersion.VersionId, + Context.CurrentVersion.BaseVersionId, + Chunk.ChunkName, + PlatformName + }; + FString ReplacedPakSaveDirRegular = UFlibHotPatcherCoreHelper::ReplacePakRegular(PakSaveDirRegular,Context.GetSettingObject()->GetPakSaveDirRegular()); + FString ChunkSaveBasePath = FPaths::Combine(Context.GetSettingObject()->GetSaveAbsPath(),ReplacedPakSaveDirRegular); TArray ChunkPakListCommands; { @@ -961,8 +978,8 @@ namespace PatchWorker SinglePakForChunk.PakCommands = ChunkPakListCommands; // add extern file to pak(version file) SinglePakForChunk.PakCommands.Append(Context.AdditionalFileToPak); - - const FString ChunkPakName = UFlibHotPatcherCoreHelper::MakePakShortName(Context.CurrentVersion,Chunk,PlatformName,Context.GetSettingObject()->GetPakNameRegular()); + + const FString ChunkPakName = UFlibHotPatcherCoreHelper::ReplacePakRegular(PakSaveDirRegular,Context.GetSettingObject()->GetPakNameRegular()); SinglePakForChunk.ChunkStoreName = ChunkPakName; SinglePakForChunk.StorageDirectory = ChunkSaveBasePath; Chunk.GetPakFileProxys().Add(SinglePakForChunk); diff --git a/HotPatcher/Source/HotPatcherCore/Private/FlibHotPatcherCoreHelper.cpp b/HotPatcher/Source/HotPatcherCore/Private/FlibHotPatcherCoreHelper.cpp index 3a4cf9ca..da63966d 100644 --- a/HotPatcher/Source/HotPatcherCore/Private/FlibHotPatcherCoreHelper.cpp +++ b/HotPatcher/Source/HotPatcherCore/Private/FlibHotPatcherCoreHelper.cpp @@ -949,8 +949,7 @@ FString UFlibHotPatcherCoreHelper::PatchSummary(const FPatchVersionDiff& DiffInf return result; } - -FString UFlibHotPatcherCoreHelper::MakePakShortName(const FHotPatcherVersion& InCurrentVersion, const FChunkInfo& InChunkInfo, const FString& InPlatform,const FString& InRegular) +FString UFlibHotPatcherCoreHelper::ReplacePakRegular(const FReplacePakRegular& RegularConf, const FString& InRegular) { struct FResularOperator { @@ -961,15 +960,21 @@ FString UFlibHotPatcherCoreHelper::MakePakShortName(const FHotPatcherVersion& In }; TArray RegularOpList; - RegularOpList.Emplace(TEXT("{VERSION}"),[&InCurrentVersion]()->FString{return InCurrentVersion.VersionId;}); - RegularOpList.Emplace(TEXT("{BASEVERSION}"),[&InCurrentVersion]()->FString{return InCurrentVersion.BaseVersionId;}); - RegularOpList.Emplace(TEXT("{PLATFORM}"),[&InPlatform]()->FString{return InPlatform;}); - RegularOpList.Emplace(TEXT("{CHUNKNAME}"),[&InChunkInfo,&InCurrentVersion]()->FString + RegularOpList.Emplace(TEXT("{VERSION}"),[&RegularConf]()->FString{return RegularConf.VersionId;}); + RegularOpList.Emplace(TEXT("{BASEVERSION}"),[&RegularConf]()->FString{return RegularConf.BaseVersionId;}); + RegularOpList.Emplace(TEXT("{PLATFORM}"),[&RegularConf]()->FString{return RegularConf.PlatformName;}); + RegularOpList.Emplace(TEXT("{CHUNKNAME}"),[&RegularConf,InRegular]()->FString { - if(!InCurrentVersion.VersionId.Equals(InChunkInfo.ChunkName)) - return InChunkInfo.ChunkName; - else + if(InRegular.Contains(TEXT("{VERSION}")) && + InRegular.Contains(TEXT("{CHUNKNAME}")) && + RegularConf.VersionId.Equals(RegularConf.ChunkName)) + { return TEXT(""); + } + else + { + return RegularConf.ChunkName; + } }); auto CustomPakNameRegular = [](const TArray& Operators,const FString& Regular)->FString @@ -979,10 +984,15 @@ FString UFlibHotPatcherCoreHelper::MakePakShortName(const FHotPatcherVersion& In { Result = Result.Replace(*Operator.Name,*(Operator.Do())); } - while(Result.Contains(TEXT("__"))) + auto ReplaceDoubleLambda = [](FString& Src,const FString& From,const FString& To) { - Result = Result.Replace(TEXT("__"),TEXT("_")); - } + while(Src.Contains(From)) + { + Src = Src.Replace(*From,*To); + } + }; + ReplaceDoubleLambda(Result,TEXT("__"),TEXT("_")); + ReplaceDoubleLambda(Result,TEXT("--"),TEXT("-")); return Result; }; @@ -1155,6 +1165,7 @@ ITargetPlatform* UFlibHotPatcherCoreHelper::GetPlatformByName(const FString& Nam FPatchVersionDiff UFlibHotPatcherCoreHelper::DiffPatchVersionWithPatchSetting(const FExportPatchSettings& PatchSetting, const FHotPatcherVersion& Base, const FHotPatcherVersion& New) { + SCOPED_NAMED_EVENT_TEXT("DiffPatchVersionWithPatchSetting",FColor::Red); FPatchVersionDiff VersionDiffInfo; const FAssetDependenciesInfo& BaseVersionAssetDependInfo = Base.AssetInfo; const FAssetDependenciesInfo& CurrentVersionAssetDependInfo = New.AssetInfo; @@ -2554,8 +2565,10 @@ TempSandboxFile.Get() #endif return bresult; } + TArray UFlibHotPatcherCoreHelper::GetClassesByNames(const TArray& ClassesNames) { + SCOPED_NAMED_EVENT_TEXT("GetClassesByNames",FColor::Red); TArray result; for(const auto& ClassesName:ClassesNames) { @@ -2573,6 +2586,7 @@ TArray UFlibHotPatcherCoreHelper::GetClassesByNames(const TArray TArray UFlibHotPatcherCoreHelper::GetAllMaterialClasses() { + SCOPED_NAMED_EVENT_TEXT("GetAllMaterialClasses",FColor::Red); TArray Classes; TArray ParentClassesName = { // materials @@ -2589,8 +2603,34 @@ TArray UFlibHotPatcherCoreHelper::GetAllMaterialClasses() return Classes; } +bool UFlibHotPatcherCoreHelper::IsMaterialClasses(UClass* Class) +{ + return UFlibHotPatcherCoreHelper::IsMaterialClassName(Class->GetFName()); +}; + +bool UFlibHotPatcherCoreHelper::IsMaterialClassName(FName ClassName) +{ + return UFlibHotPatcherCoreHelper::GetAllMaterialClassesNames().Contains(ClassName); +} + +bool UFlibHotPatcherCoreHelper::AssetDetailsHasClasses(const TArray& AssetDetails, TSet ClasssName) +{ + SCOPED_NAMED_EVENT_TEXT("AssetDetailsHasClasses",FColor::Red); + bool bHas = false; + for(const auto& Detail:AssetDetails) + { + if(ClasssName.Contains(Detail.AssetType)) + { + bHas = true; + break; + } + } + return bHas; +} + TSet UFlibHotPatcherCoreHelper::GetAllMaterialClassesNames() { + SCOPED_NAMED_EVENT_TEXT("GetAllMaterialClassesNames",FColor::Red); TSet result; for(const auto& Class:GetAllMaterialClasses()) { @@ -2654,3 +2694,24 @@ TArray UFlibHotPatcherCoreHelper::GetPreCacheClasses() } return Results.Array(); } + +void UFlibHotPatcherCoreHelper::DumpActiveTargetPlatforms() +{ + FString ActiveTargetPlatforms; + ITargetPlatformManagerModule* TPM = GetTargetPlatformManager(); + if (TPM && (TPM->RestrictFormatsToRuntimeOnly() == false)) + { + TArray Platforms = TPM->GetActiveTargetPlatforms(); + for(auto& Platform:Platforms) + { + ActiveTargetPlatforms += FString::Printf(TEXT("%s,"),*Platform->PlatformName()); + } + ActiveTargetPlatforms.RemoveFromEnd(TEXT(",")); + } + UE_LOG(LogHotPatcherCoreHelper,Display,TEXT("[IMPORTTENT] ActiveTargetPlatforms: %s"),*ActiveTargetPlatforms); +} + +FString UFlibHotPatcherCoreHelper::GetPlatformsStr(TArray Platforms) +{ + return UFlibPatchParserHelper::GetPlatformsStr(Platforms); +} \ No newline at end of file diff --git a/HotPatcher/Source/HotPatcherCore/Private/ShaderPatch/FlibShaderCodeLibraryHelper.cpp b/HotPatcher/Source/HotPatcherCore/Private/ShaderPatch/FlibShaderCodeLibraryHelper.cpp index e09b3613..f1136024 100644 --- a/HotPatcher/Source/HotPatcherCore/Private/ShaderPatch/FlibShaderCodeLibraryHelper.cpp +++ b/HotPatcher/Source/HotPatcherCore/Private/ShaderPatch/FlibShaderCodeLibraryHelper.cpp @@ -120,7 +120,9 @@ bool UFlibShaderCodeLibraryHelper::SaveShaderLibrary(const ITargetPlatform* Targ #if ENGINE_MAJOR_VERSION > 4 || ENGINE_MINOR_VERSION > 25 #if ENGINE_MAJOR_VERSION > 4 || ENGINE_MINOR_VERSION > 26 FString ErrorString; - bool bOutHasData; +#if !UE_VERSION_OLDER_THAN(5,1,0) + bool bOutHasData = false; +#endif bSaved = SHADER_COOKER_CLASS::SaveShaderLibraryWithoutChunking(TargetPlatform, FApp::GetProjectName(), ShaderCodeDir, RootMetaDataPath, PlatformSCLCSVPaths, ErrorString #if !UE_VERSION_OLDER_THAN(5,1,0) ,bOutHasData @@ -244,7 +246,6 @@ void UFlibShaderCodeLibraryHelper::WaitShaderCompilingComplete() if (GShaderCompilingManager) { SCOPED_NAMED_EVENT_TEXT("WaitShaderCompileComplete",FColor::Red); - int32 LastRemainingJob = 0; while(GShaderCompilingManager->IsCompiling()) { GShaderCompilingManager->ProcessAsyncResults(false, false); @@ -254,12 +255,14 @@ void UFlibShaderCodeLibraryHelper::WaitShaderCompilingComplete() // UE_LOG(LogHotPatcher,Display,TEXT("Remaining Shader %d"),CurrentNumRemaingingJobs); // LastRemainingJob = CurrentNumRemaingingJobs; // } + // GShaderCompilingManager->FinishAllCompilation(); FPlatformProcess::Sleep(0.5f); GLog->Flush(); } // One last process to get the shaders that were compiled at the very end GShaderCompilingManager->ProcessAsyncResults(false, false); + GShaderCompilingManager->FinishAllCompilation(); } } @@ -270,3 +273,18 @@ void UFlibShaderCodeLibraryHelper::CleanShaderWorkerDir() UE_LOG(LogHotPatcher, Warning, TEXT("Could not delete the shader compiler working directory '%s'."), *FPaths::ShaderWorkingDir()); } } + +void UFlibShaderCodeLibraryHelper::CancelMaterialShaderCompile(UMaterialInterface* MaterialInterface) +{ + if(MaterialInterface) + { + UMaterial* Material = MaterialInterface->GetMaterial(); + for (int32 FeatureLevel = 0; FeatureLevel < ERHIFeatureLevel::Num; ++FeatureLevel) + { + if (FMaterialResource* Res = Material->GetMaterialResource((ERHIFeatureLevel::Type)FeatureLevel)) + { + Res->CancelCompilation(); + } + } + } +} diff --git a/HotPatcher/Source/HotPatcherCore/Public/CommandletBase/CommandletHelper.h b/HotPatcher/Source/HotPatcherCore/Public/CommandletBase/CommandletHelper.h index 54cec00b..64849b37 100644 --- a/HotPatcher/Source/HotPatcherCore/Public/CommandletBase/CommandletHelper.h +++ b/HotPatcher/Source/HotPatcherCore/Public/CommandletBase/CommandletHelper.h @@ -1,9 +1,7 @@ #pragma once +#include "CoreMinimal.h" #include "ETargetPlatform.h" -#include "FlibPatchParserHelper.h" -#include "ThreadUtils/FProcWorkerThread.hpp" -#include "HotPatcherLog.h" #define PATCHER_CONFIG_PARAM_NAME TEXT("-config=") #define ADD_PATCH_PLATFORMS TEXT("AddPatchPlatforms") diff --git a/HotPatcher/Source/HotPatcherCore/Public/FlibHotPatcherCoreHelper.h b/HotPatcher/Source/HotPatcherCore/Public/FlibHotPatcherCoreHelper.h index c57dcd28..dae45068 100644 --- a/HotPatcher/Source/HotPatcherCore/Public/FlibHotPatcherCoreHelper.h +++ b/HotPatcher/Source/HotPatcherCore/Public/FlibHotPatcherCoreHelper.h @@ -59,6 +59,17 @@ struct FProjectPackageAssetCollection TArray NeverCookPackages; }; + +struct HOTPATCHERCORE_API FReplacePakRegular +{ + FReplacePakRegular()=default; + FReplacePakRegular(const FString& InVersionId,const FString& InBaseVersionId,const FString& InChunkName,const FString& InPlatformName): + VersionId(InVersionId),BaseVersionId(InBaseVersionId),ChunkName(InChunkName),PlatformName(InPlatformName){} + FString VersionId; + FString BaseVersionId; + FString ChunkName; + FString PlatformName; +}; /** * */ @@ -161,7 +172,8 @@ class HOTPATCHERCORE_API UFlibHotPatcherCoreHelper : public UBlueprintFunctionLi static FString ReleaseSummary(const FHotPatcherVersion& NewVersion); static FString PatchSummary(const FPatchVersionDiff& DiffInfo); - static FString MakePakShortName(const FHotPatcherVersion& InCurrentVersion, const FChunkInfo& InChunkInfo, const FString& InPlatform,const FString& InRegular); + + static FString ReplacePakRegular(const FReplacePakRegular& RegularConf, const FString& InRegular); static bool CheckSelectedAssetsCookStatus(const FString& OverrideCookedDir,const TArray& PlatformNames, const FAssetDependenciesInfo& SelectedAssets, FString& OutMsg); static bool CheckPatchRequire(const FString& OverrideCookedDir,const FPatchVersionDiff& InDiff,const TArray& PlatformNames,FString& OutMsg); @@ -277,15 +289,17 @@ class HOTPATCHERCORE_API UFlibHotPatcherCoreHelper : public UBlueprintFunctionLi static void AdaptorOldVersionConfig(FAssetScanConfig& ScanConfig,const FString& JsonContent); static bool GetIniPlatformName(const FString& PlatformName,FString& OutIniPlatformName); - - // need add UNREALED_API to FAssetRegistryGenerator // all chunksinfo.csv / pakchunklist.txt / assetregistry.bin static bool SerializeChunksManifests(ITargetPlatform* TargetPlatform, const TSet&, const TSet&, bool bGenerateStreamingInstallManifest = true); static TArray GetClassesByNames(const TArray& ClassesNames); static TArray GetAllMaterialClasses(); + static bool IsMaterialClasses(UClass* Class); + static bool IsMaterialClassName(FName ClassName); + static bool AssetDetailsHasClasses(const TArray& AssetDetails,TSet ClasssName); static TSet GetAllMaterialClassesNames(); static TArray GetPreCacheClasses(); - + static void DumpActiveTargetPlatforms(); + static FString GetPlatformsStr(TArray Platforms); }; diff --git a/HotPatcher/Source/HotPatcherCore/Public/ShaderPatch/FlibShaderCodeLibraryHelper.h b/HotPatcher/Source/HotPatcherCore/Public/ShaderPatch/FlibShaderCodeLibraryHelper.h index fd2fd6ef..54345cc8 100644 --- a/HotPatcher/Source/HotPatcherCore/Public/ShaderPatch/FlibShaderCodeLibraryHelper.h +++ b/HotPatcher/Source/HotPatcherCore/Public/ShaderPatch/FlibShaderCodeLibraryHelper.h @@ -58,6 +58,7 @@ class HOTPATCHERCORE_API UFlibShaderCodeLibraryHelper : public UBlueprintFunctio static TArray FindCookedShaderLibByShaderFrmat(const FString& ShaderFormatName,const FString& Directory); static void WaitShaderCompilingComplete(); - + static void CancelMaterialShaderCompile(UMaterialInterface* MaterialInterface); + static void CleanShaderWorkerDir(); }; diff --git a/HotPatcher/Source/HotPatcherEditor/Private/CreatePatch/AssetActions/AssetTypeActions_PrimaryAssetLabel.cpp b/HotPatcher/Source/HotPatcherEditor/Private/CreatePatch/AssetActions/AssetTypeActions_PrimaryAssetLabel.cpp index 58758897..618441db 100644 --- a/HotPatcher/Source/HotPatcherEditor/Private/CreatePatch/AssetActions/AssetTypeActions_PrimaryAssetLabel.cpp +++ b/HotPatcher/Source/HotPatcherEditor/Private/CreatePatch/AssetActions/AssetTypeActions_PrimaryAssetLabel.cpp @@ -138,7 +138,7 @@ void FAssetTypeActions_PrimaryAssetLabel::MakeCookAndPakActionsSubMenu(UToolMenu FToolMenuSection& Section = Menu->AddSection("CookAndPakActionsSection"); UHotPatcherSettings* Settings = GetMutableDefault(); Settings->ReloadConfig(); - for (auto Platform : FHotPatcherEditorModule::Get().GetAllCookPlatforms()) + for (auto Platform : FHotPatcherEditorModule::Get().GetAllowCookPlatforms()) { if(Settings->bWhiteListCookInEditor && !Settings->PlatformWhitelists.Contains(Platform)) continue; diff --git a/HotPatcher/Source/HotPatcherEditor/Private/CreatePatch/SHotPatcherPatchWidget.cpp b/HotPatcher/Source/HotPatcherEditor/Private/CreatePatch/SHotPatcherPatchWidget.cpp index 899febf5..d6c56727 100644 --- a/HotPatcher/Source/HotPatcherEditor/Private/CreatePatch/SHotPatcherPatchWidget.cpp +++ b/HotPatcher/Source/HotPatcherEditor/Private/CreatePatch/SHotPatcherPatchWidget.cpp @@ -476,25 +476,10 @@ bool SHotPatcherPatchWidget::CanExportPatch()const FReply SHotPatcherPatchWidget::DoExportPatch() { - if(!GetConfigSettings()->IsStandaloneMode()) - { - UPatcherProxy* PatcherProxy = NewObject(); - PatcherProxy->AddToRoot(); - PatcherProxy->Init(ExportPatchSetting.Get()); - PatcherProxy->OnShowMsg.AddRaw(this,&SHotPatcherPatchWidget::ShowMsg); - PatcherProxy->DoExport(); - } - else - { - FString CurrentConfig; - THotPatcherTemplateHelper::TSerializeStructAsJsonString(*GetConfigSettings(),CurrentConfig); - FString SaveConfigTo = FPaths::ConvertRelativePathToFull(FPaths::Combine(FPaths::ProjectSavedDir(),TEXT("HotPatcher"),TEXT("PatchConfig.json"))); - FFileHelper::SaveStringToFile(CurrentConfig,*SaveConfigTo); - FString ProfilingCmd = GetConfigSettings()->IsEnableProfiling() ? TEXT("-trace=cpu,loadtimetrace") : TEXT(""); - FString MissionCommand = FString::Printf(TEXT("\"%s\" -run=HotPatcher -config=\"%s\" %s %s"),*UFlibPatchParserHelper::GetProjectFilePath(),*SaveConfigTo,*GetConfigSettings()->GetCombinedAdditionalCommandletArgs(),*ProfilingCmd); - UE_LOG(LogHotPatcher,Log,TEXT("HotPatcher %s Mission: %s %s"),*GetMissionName(),*UFlibHotPatcherCoreHelper::GetUECmdBinary(),*MissionCommand); - FHotPatcherEditorModule::Get().RunProcMission(UFlibHotPatcherCoreHelper::GetUECmdBinary(),MissionCommand,GetMissionName()); - } + TSharedPtr PatchSettings = MakeShareable(new FExportPatchSettings); + *PatchSettings = *GetConfigSettings(); + FHotPatcherEditorModule::Get().CookAndPakByPatchSettings(PatchSettings,PatchSettings->IsStandaloneMode()); + return FReply::Handled(); } diff --git a/HotPatcher/Source/HotPatcherEditor/Private/FlibHotPatcherEditorHelper.cpp b/HotPatcher/Source/HotPatcherEditor/Private/FlibHotPatcherEditorHelper.cpp index af12fa9c..7afc3d6f 100644 --- a/HotPatcher/Source/HotPatcherEditor/Private/FlibHotPatcherEditorHelper.cpp +++ b/HotPatcher/Source/HotPatcherEditor/Private/FlibHotPatcherEditorHelper.cpp @@ -1,4 +1,7 @@ #include "FlibHotPatcherEditorHelper.h" + +#include "DesktopPlatformModule.h" +#include "IDesktopPlatform.h" #include "Async/Async.h" void UFlibHotPatcherEditorHelper::CreateSaveFileNotify(const FText& InMsg, const FString& InSavedFile, @@ -25,3 +28,45 @@ void UFlibHotPatcherEditorHelper::CreateSaveFileNotify(const FText& InMsg, const FSlateNotificationManager::Get().AddNotification(Info)->SetCompletionState(NotifyType); }); } + +TArray UFlibHotPatcherEditorHelper::SaveFileDialog(const FString& DialogTitle, const FString& DefaultPath, + const FString& DefaultFile, const FString& FileTypes, uint32 Flags) +{ + IDesktopPlatform* DesktopPlatform = FDesktopPlatformModule::Get(); + + TArray SaveFilenames; + if (DesktopPlatform) + { + const bool bOpened = DesktopPlatform->SaveFileDialog( + nullptr, + DialogTitle, + DefaultPath, + DefaultFile, + FileTypes, + Flags, + SaveFilenames + ); + } + return SaveFilenames; +} + +TArray UFlibHotPatcherEditorHelper::OpenFileDialog(const FString& DialogTitle, const FString& DefaultPath, + const FString& DefaultFile, const FString& FileTypes, uint32 Flags) +{ + IDesktopPlatform* DesktopPlatform = FDesktopPlatformModule::Get(); + TArray SelectedFiles; + + if (DesktopPlatform) + { + const bool bOpened = DesktopPlatform->OpenFileDialog( + nullptr, + DialogTitle, + DefaultPath, + DefaultFile, + FileTypes, + Flags, + SelectedFiles + ); + } + return SelectedFiles; +} diff --git a/HotPatcher/Source/HotPatcherEditor/Private/HotPatcherEditor.cpp b/HotPatcher/Source/HotPatcherEditor/Private/HotPatcherEditor.cpp index 2024bc3b..15397470 100644 --- a/HotPatcher/Source/HotPatcherEditor/Private/HotPatcherEditor.cpp +++ b/HotPatcher/Source/HotPatcherEditor/Private/HotPatcherEditor.cpp @@ -330,13 +330,8 @@ void FHotPatcherEditorModule::ExtendContentBrowserPathSelectionMenu() void FHotPatcherEditorModule::MakeCookActionsSubMenu(UToolMenu* Menu) { FToolMenuSection& Section = Menu->AddSection("CookActionsSection"); - UHotPatcherSettings* Settings = GetMutableDefault(); - Settings->ReloadConfig(); - - for (auto Platform : GetAllCookPlatforms()) + for (auto Platform : GetAllowCookPlatforms()) { - if(Settings->bWhiteListCookInEditor && !Settings->PlatformWhitelists.Contains(Platform)) - continue; Section.AddMenuEntry( FName(*THotPatcherTemplateHelper::GetEnumNameByValue(Platform)), FText::Format(LOCTEXT("Platform", "{0}"), UKismetTextLibrary::Conv_StringToText(THotPatcherTemplateHelper::GetEnumNameByValue(Platform))), @@ -354,13 +349,9 @@ void FHotPatcherEditorModule::MakeCookAndPakActionsSubMenu(UToolMenu* Menu) FToolMenuSection& Section = Menu->AddSection("CookAndPakActionsSection"); UHotPatcherSettings* Settings = GetMutableDefault(); Settings->ReloadConfig(); - for (ETargetPlatform Platform : GetAllCookPlatforms()) + for (ETargetPlatform Platform : GetAllowCookPlatforms()) { FString PlatformName = THotPatcherTemplateHelper::GetEnumNameByValue(Platform); - if(PlatformName.StartsWith(TEXT("All"))) - continue; - if(Settings->bWhiteListCookInEditor && !Settings->PlatformWhitelists.Contains(Platform)) - continue; FToolMenuEntry& PlatformEntry = Section.AddSubMenu(FName(*PlatformName), FText::Format(LOCTEXT("Platform", "{0}"), UKismetTextLibrary::Conv_StringToText(THotPatcherTemplateHelper::GetEnumNameByValue(Platform))), FText(), @@ -463,29 +454,9 @@ void FHotPatcherEditorModule::OnAddToPatchSettings(const FToolMenuContext& MenuC void FHotPatcherEditorModule::OnPakPreset(FExportPatchSettings Config) { - UHotPatcherSettings* Settings = GetMutableDefault(); - Settings->ReloadConfig(); - - UPatcherProxy* PatcherProxy = NewObject(); - PatcherProxy->AddToRoot(); - Proxys.Add(PatcherProxy); - - PatcherProxy->Init(&Config); - - if(!Config.IsStandaloneMode()) - { - PatcherProxy->DoExport(); - } - else - { - FString CurrentConfig; - THotPatcherTemplateHelper::TSerializeStructAsJsonString(Config,CurrentConfig); - FString SaveConfigTo = FPaths::ConvertRelativePathToFull(FPaths::Combine(FPaths::ProjectSavedDir(),TEXT("HotPatcher"),TEXT("PatchConfig.json"))); - FFileHelper::SaveStringToFile(CurrentConfig,*SaveConfigTo); - FString MissionCommand = FString::Printf(TEXT("\"%s\" -run=HotPatcher -config=\"%s\" %s"),*UFlibPatchParserHelper::GetProjectFilePath(),*SaveConfigTo,*Config.GetCombinedAdditionalCommandletArgs()); - UE_LOG(LogHotPatcher,Log,TEXT("HotPatcher %s Mission: %s %s"),*Config.VersionId,*UFlibHotPatcherCoreHelper::GetUECmdBinary(),*MissionCommand); - RunProcMission(UFlibHotPatcherCoreHelper::GetUECmdBinary(),MissionCommand,FString::Printf(TEXT("Mission: %s"),*Config.VersionId)); - } + TSharedPtr TmpPatchSettings = MakeShareable(new FExportPatchSettings); + *TmpPatchSettings = Config; + CookAndPakByPatchSettings(TmpPatchSettings,TmpPatchSettings->IsStandaloneMode()); } #endif @@ -578,17 +549,20 @@ void FHotPatcherEditorModule::CookAndPakByAssetsAndFilters(TArray InPatchSettings,bool bForceStandalone) { if(bForceStandalone || InPatchSettings->IsStandaloneMode()) { FString CurrentConfig; THotPatcherTemplateHelper::TSerializeStructAsJsonString(*InPatchSettings.Get(),CurrentConfig); - FString SaveConfigTo = FPaths::ConvertRelativePathToFull(FPaths::Combine(FPaths::ProjectSavedDir(),TEXT("HotPatcher"),TEXT("PatchConfig.json"))); + FString SaveConfigTo = FPaths::ConvertRelativePathToFull(FPaths::Combine(FPaths::ProjectSavedDir(),TEXT("HotPatcher"),FString::Printf(TEXT("%s_PatchConfig.json"),*InPatchSettings->VersionId))); FFileHelper::SaveStringToFile(CurrentConfig,*SaveConfigTo); FString MissionCommand = FString::Printf(TEXT("\"%s\" -run=HotPatcher -config=\"%s\" %s"),*UFlibPatchParserHelper::GetProjectFilePath(),*SaveConfigTo,*InPatchSettings->GetCombinedAdditionalCommandletArgs()); UE_LOG(LogHotPatcher,Log,TEXT("HotPatcher %s Mission: %s %s"),*InPatchSettings->VersionId,*UFlibHotPatcherCoreHelper::GetUECmdBinary(),*MissionCommand); - RunProcMission(UFlibHotPatcherCoreHelper::GetUECmdBinary(),MissionCommand,FString::Printf(TEXT("Mission: %s"),*InPatchSettings->VersionId)); + + FText DisplayText = UKismetTextLibrary::Conv_StringToText(FString::Printf(TEXT("Packaging %s for %s..."),*InPatchSettings->VersionId,*UFlibHotPatcherCoreHelper::GetPlatformsStr(InPatchSettings->PakTargetPlatforms))); + RunProcMission(UFlibHotPatcherCoreHelper::GetUECmdBinary(),MissionCommand,InPatchSettings->VersionId,DisplayText); } else { @@ -695,7 +669,29 @@ TArray FHotPatcherEditorModule::GetAllCookPlatforms() const return TargetPlatforms; } -TSharedPtr FHotPatcherEditorModule::RunProcMission(const FString& Bin, const FString& Command, const FString& MissionName) +TArray FHotPatcherEditorModule::GetAllowCookPlatforms() const +{ + TArray results; + UHotPatcherSettings* Settings = GetMutableDefault(); + Settings->ReloadConfig(); + for (auto Platform : GetAllCookPlatforms()) + { + if(Settings->bWhiteListCookInEditor && !Settings->PlatformWhitelists.Contains(Platform)) + continue; + FString PlatformName = THotPatcherTemplateHelper::GetEnumNameByValue(Platform); + if(PlatformName.StartsWith(TEXT("All"))) + continue; + results.AddUnique(Platform); + } + return results; +} + +void FHotPatcherEditorModule::OnCookPlatformForExterner(ETargetPlatform Platform) +{ + FHotPatcherEditorModule::Get().OnCookPlatform(Platform); +} + +TSharedPtr FHotPatcherEditorModule::RunProcMission(const FString& Bin, const FString& Command, const FString& MissionName, const FText& NotifyTextOverride) { if (mProcWorkingThread.IsValid() && mProcWorkingThread->GetThreadStatus()==EThreadStatus::Busy) { @@ -709,8 +705,13 @@ TSharedPtr FHotPatcherEditorModule::RunProcMission(const FStr mProcWorkingThread->ProcSuccessedDelegate.AddUObject(MissionNotifyProay,&UMissionNotificationProxy::SpawnMissionSuccessedNotification); mProcWorkingThread->ProcFaildDelegate.AddUObject(MissionNotifyProay,&UMissionNotificationProxy::SpawnMissionFaildNotification); MissionNotifyProay->SetMissionName(*FString::Printf(TEXT("%s"),*MissionName)); + FText DisplayText = NotifyTextOverride; + if(DisplayText.IsEmpty()) + { + DisplayText = FText::FromString(FString::Printf(TEXT("%s in progress"),*MissionName)); + } MissionNotifyProay->SetMissionNotifyText( - FText::FromString(FString::Printf(TEXT("%s in progress"),*MissionName)), + DisplayText, LOCTEXT("RunningCookNotificationCancelButton", "Cancel"), FText::FromString(FString::Printf(TEXT("%s Mission Finished!"),*MissionName)), FText::FromString(FString::Printf(TEXT("%s Failed!"),*MissionName)) diff --git a/HotPatcher/Source/HotPatcherEditor/Private/SHotPatcherWidgetBase.cpp b/HotPatcher/Source/HotPatcherEditor/Private/SHotPatcherWidgetBase.cpp index 5468b90e..3e7c1764 100644 --- a/HotPatcher/Source/HotPatcherEditor/Private/SHotPatcherWidgetBase.cpp +++ b/HotPatcher/Source/HotPatcherEditor/Private/SHotPatcherWidgetBase.cpp @@ -10,6 +10,7 @@ // engine header #include "IDesktopPlatform.h" #include "DesktopPlatformModule.h" +#include "FlibHotPatcherEditorHelper.h" #include "Misc/FileHelper.h" #include "Widgets/Input/SHyperlink.h" @@ -44,42 +45,28 @@ FText SHotPatcherWidgetInterface::GetGenerateTooltipText() const TArray SHotPatcherWidgetInterface::OpenFileDialog()const { - IDesktopPlatform* DesktopPlatform = FDesktopPlatformModule::Get(); TArray SelectedFiles; - - if (DesktopPlatform) - { - const bool bOpened = DesktopPlatform->OpenFileDialog( - nullptr, + SelectedFiles = UFlibHotPatcherEditorHelper::OpenFileDialog( LOCTEXT("OpenHotPatchConfigDialog", "Open .json").ToString(), FString(TEXT("")), TEXT(""), TEXT("HotPatcher json (*.json)|*.json"), - EFileDialogFlags::None, - SelectedFiles + EFileDialogFlags::None ); - } return SelectedFiles; } TArray SHotPatcherWidgetInterface::SaveFileDialog()const { - IDesktopPlatform* DesktopPlatform = FDesktopPlatformModule::Get(); - TArray SaveFilenames; - if (DesktopPlatform) - { - const bool bOpened = DesktopPlatform->SaveFileDialog( - nullptr, + SaveFilenames = UFlibHotPatcherEditorHelper::SaveFileDialog( LOCTEXT("SvedHotPatcherConfig", "Save .json").ToString(), FString(TEXT("")), TEXT(""), TEXT("HotPatcher json (*.json)|*.json"), - EFileDialogFlags::None, - SaveFilenames + EFileDialogFlags::None ); - } return SaveFilenames; } diff --git a/HotPatcher/Source/HotPatcherEditor/Public/FlibHotPatcherEditorHelper.h b/HotPatcher/Source/HotPatcherEditor/Public/FlibHotPatcherEditorHelper.h index 2461b3d3..e9207228 100644 --- a/HotPatcher/Source/HotPatcherEditor/Public/FlibHotPatcherEditorHelper.h +++ b/HotPatcher/Source/HotPatcherEditor/Public/FlibHotPatcherEditorHelper.h @@ -3,6 +3,7 @@ // engine header #include "Templates/SharedPointer.h" #include "Dom/JsonObject.h" +#include "IDesktopPlatform.h" #include "CoreMinimal.h" #include "Framework/Notifications/NotificationManager.h" #include "Widgets/Notifications/SNotificationList.h" @@ -14,7 +15,8 @@ class HOTPATCHEREDITOR_API UFlibHotPatcherEditorHelper : public UBlueprintFuncti GENERATED_BODY() public: static void CreateSaveFileNotify(const FText& InMsg,const FString& InSavedFile,SNotificationItem::ECompletionState NotifyType = SNotificationItem::CS_Success); - + static TArray SaveFileDialog(const FString& DialogTitle, const FString& DefaultPath, const FString& DefaultFile, const FString& FileTypes, uint32 Flags); + static TArray OpenFileDialog(const FString& DialogTitle, const FString& DefaultPath, const FString& DefaultFile, const FString& FileTypes, uint32 Flags); }; diff --git a/HotPatcher/Source/HotPatcherEditor/Public/HotPatcherEditor.h b/HotPatcher/Source/HotPatcherEditor/Public/HotPatcherEditor.h index ae446b33..ffee617d 100644 --- a/HotPatcher/Source/HotPatcherEditor/Public/HotPatcherEditor.h +++ b/HotPatcher/Source/HotPatcherEditor/Public/HotPatcherEditor.h @@ -56,7 +56,7 @@ class HOTPATCHEREDITOR_API FHotPatcherEditorModule : public IModuleInterface TSharedPtr CreateProcMissionThread(const FString& Bin, const FString& Command, const FString& MissionName); - TSharedPtr RunProcMission(const FString& Bin, const FString& Command, const FString& MissionName); + TSharedPtr RunProcMission(const FString& Bin, const FString& Command, const FString& MissionName, const FText& NotifyTextOverride = FText{}); #if WITH_EDITOR_SECTION @@ -70,7 +70,8 @@ class HOTPATCHEREDITOR_API FHotPatcherEditorModule : public IModuleInterface void MakeHotPatcherPresetsActionsSubMenu(UToolMenu* Menu); void OnAddToPatchSettings(const FToolMenuContext& MenuContent); #endif - TArray GetAllCookPlatforms()const; + TArray GetAllowCookPlatforms() const; + static void OnCookPlatformForExterner(ETargetPlatform Platform);; void OnCookPlatform(ETargetPlatform Platform); void OnCookAndPakPlatform(ETargetPlatform Platform, bool bAnalysicDependencies); void CookAndPakByAssetsAndFilters(TArray IncludeAssets,TArray IncludePaths,TArray Platforms,bool bForceStandalone = false); @@ -89,6 +90,7 @@ class HOTPATCHEREDITOR_API FHotPatcherEditorModule : public IModuleInterface bool bCook ); private: + TArray GetAllCookPlatforms()const; TSharedRef OnSpawnPluginTab(const class FSpawnTabArgs& InSpawnTabArgs); void OnTabClosed(TSharedRef InTab); TArray GetSelectedAssetsInBrowserContent(); diff --git a/HotPatcher/Source/HotPatcherRuntime/Private/BaseTypes/FChunkInfo.cpp b/HotPatcher/Source/HotPatcherRuntime/Private/BaseTypes/FChunkInfo.cpp index 5a2646a4..ddb91085 100644 --- a/HotPatcher/Source/HotPatcherRuntime/Private/BaseTypes/FChunkInfo.cpp +++ b/HotPatcher/Source/HotPatcherRuntime/Private/BaseTypes/FChunkInfo.cpp @@ -31,6 +31,7 @@ FString FChunkInfo::GetShaderLibraryName() const TArray FChunkInfo::GetManagedAssets() const { + SCOPED_NAMED_EVENT_TEXT("FChunkInfo::GetManagedAssets",FColor::Red); TArray NewPaths; UAssetManager& Manager = UAssetManager::Get(); diff --git a/HotPatcher/Source/HotPatcherRuntime/Private/CreatePatch/FExportPatchSettings.cpp b/HotPatcher/Source/HotPatcherRuntime/Private/CreatePatch/FExportPatchSettings.cpp index c84169f7..203f9186 100644 --- a/HotPatcher/Source/HotPatcherRuntime/Private/CreatePatch/FExportPatchSettings.cpp +++ b/HotPatcher/Source/HotPatcherRuntime/Private/CreatePatch/FExportPatchSettings.cpp @@ -127,6 +127,15 @@ FString FExportPatchSettings::GetCurrentVersionSavePath() const return CurrentVersionSavePath; } +FString FExportPatchSettings::GetCombinedAdditionalCommandletArgs() const +{ + return UFlibPatchParserHelper::MergeOptionsAsCmdline(TArray{ + FHotPatcherSettingBase::GetCombinedAdditionalCommandletArgs(), + UFlibPatchParserHelper::GetTargetPlatformsCmdLine(GetPakTargetPlatforms()), + IsEnableProfiling() ? TEXT("-trace=cpu,loadtimetrace") : TEXT("") + }); +} + FString FExportPatchSettings::GetStorageCookedDir() const { return UFlibPatchParserHelper::ReplaceMark(StorageCookedDir); diff --git a/HotPatcher/Source/HotPatcherRuntime/Private/CreatePatch/HotPatcherSettingBase.cpp b/HotPatcher/Source/HotPatcherRuntime/Private/CreatePatch/HotPatcherSettingBase.cpp index 05f8479c..9ce4abee 100644 --- a/HotPatcher/Source/HotPatcherRuntime/Private/CreatePatch/HotPatcherSettingBase.cpp +++ b/HotPatcher/Source/HotPatcherRuntime/Private/CreatePatch/HotPatcherSettingBase.cpp @@ -3,6 +3,8 @@ #include "FPlatformExternFiles.h" #include "HotPatcherLog.h" +#include "Misc/EngineVersionComparison.h" + FHotPatcherSettingBase::FHotPatcherSettingBase() { GetAssetScanConfigRef().bAnalysisFilterDependencies = true; @@ -101,9 +103,18 @@ FString FHotPatcherSettingBase::GetSaveAbsPath()const return TEXT(""); } +FString FHotPatcherSettingBase::GetCombinedAdditionalCommandletArgs() const +{ + FString Result; + TArray Options = GetAdditionalCommandletArgs(); +#if UE_VERSION_OLDER_THAN(5,0,0) + Options.AddUnique(TEXT("-NoPostLoadCacheDDC")); +#endif + Result = UFlibPatchParserHelper::MergeOptionsAsCmdline(Options); + return Result; +} TArray FHotPatcherSettingBase::GetAllSkipContents() const - { TArray AllSkipContents;; AllSkipContents.Append(UFlibAssetManageHelper::DirectoriesToStrings(GetForceSkipContentRules())); diff --git a/HotPatcher/Source/HotPatcherRuntime/Private/DependenciesParser/FDefaultAssetDependenciesParser.cpp b/HotPatcher/Source/HotPatcherRuntime/Private/DependenciesParser/FDefaultAssetDependenciesParser.cpp index a924b9aa..cb898cce 100644 --- a/HotPatcher/Source/HotPatcherRuntime/Private/DependenciesParser/FDefaultAssetDependenciesParser.cpp +++ b/HotPatcher/Source/HotPatcherRuntime/Private/DependenciesParser/FDefaultAssetDependenciesParser.cpp @@ -65,7 +65,7 @@ void FAssetDependenciesParser::Parse(const FAssetDependencies& InParseConfig) continue; } FAssetData AssetData; - if(UFlibAssetManageHelper::GetAssetsDataByPackageName(LongPackageName,AssetData)) + if(!LongPackageName.IsEmpty() && UFlibAssetManageHelper::GetAssetsDataByPackageName(LongPackageName,AssetData)) { if(IsIgnoreAsset(AssetData)) { diff --git a/HotPatcher/Source/HotPatcherRuntime/Private/FlibAssetManageHelper.cpp b/HotPatcher/Source/HotPatcherRuntime/Private/FlibAssetManageHelper.cpp index b0fcc5f2..034c297c 100644 --- a/HotPatcher/Source/HotPatcherRuntime/Private/FlibAssetManageHelper.cpp +++ b/HotPatcher/Source/HotPatcherRuntime/Private/FlibAssetManageHelper.cpp @@ -82,6 +82,7 @@ bool UFlibAssetManageHelper::FilenameToPackagePath(const FString& InAbsPath, FSt void UFlibAssetManageHelper::UpdateAssetMangerDatabase(bool bForceRefresh) { + SCOPED_NAMED_EVENT_TEXT("UpdateAssetMangerDatabase",FColor::Red); #if WITH_EDITOR UAssetManager& AssetManager = UAssetManager::Get(); AssetManager.UpdateManagementDatabase(bForceRefresh); @@ -135,6 +136,18 @@ bool UFlibAssetManageHelper::GetAssetPackageGUID(const FString& InPackageName, F return bResult; } +FSoftObjectPath UFlibAssetManageHelper::CreateSoftObjectPathByPackage(UPackage* Package) +{ + FString AssetPathName = Package->GetPathName(); + FSoftObjectPath Path(UFlibAssetManageHelper::LongPackageNameToPackagePath(AssetPathName)); + return Path; +} + +FName UFlibAssetManageHelper::GetAssetTypeByPackage(UPackage* Package) +{ + return UFlibAssetManageHelper::GetAssetType(CreateSoftObjectPathByPackage(Package)); +} + FAssetDependenciesInfo UFlibAssetManageHelper::CombineAssetDependencies(const FAssetDependenciesInfo& A, const FAssetDependenciesInfo& B) { @@ -702,7 +715,8 @@ SCOPED_NAMED_EVENT_TEXT("UFlibAssetManageHelper::GetAllInValidAssetInProject",FC } } } - +#pragma warning(push) +#pragma warning(disable:4172) FAssetPackageData* UFlibAssetManageHelper::GetPackageDataByPackageName(const FString& InPackageName) { FAssetPackageData* AssetPackageData = nullptr; @@ -733,6 +747,7 @@ FAssetPackageData* UFlibAssetManageHelper::GetPackageDataByPackageName(const FSt return NULL; } +#pragma warning(pop) bool UFlibAssetManageHelper::ConvLongPackageNameToCookedPath( const FString& InProjectAbsDir, @@ -831,11 +846,12 @@ bool UFlibAssetManageHelper::MakePakCommandFromAssetDependencies( // TArray resault; TArray Keys; InAssetDependencies.AssetsDependenciesMap.GetKeys(Keys); - - for (const auto& Key : Keys) + FCriticalSection LocalSynchronizationObject; + ParallelFor(Keys.Num(),[&](int32 index) { + const FString& Key = Keys[index]; if (Key.Equals(TEXT("Script"))) - continue; + return; TArray AssetList; InAssetDependencies.AssetsDependenciesMap.Find(Key)->AssetDependencyDetails.GetKeys(AssetList); for (const auto& AssetLongPackageName : AssetList) @@ -843,10 +859,11 @@ bool UFlibAssetManageHelper::MakePakCommandFromAssetDependencies( TArray FinalCookedCommand; if (UFlibAssetManageHelper::MakePakCommandFromLongPackageName(InProjectDir, OverrideCookedDir,InPlatformName, AssetLongPackageName, /*InCookParams, */FinalCookedCommand,InReceivePakCommand,IsIoStoreAsset)) { + FScopeLock Lock(&LocalSynchronizationObject); OutCookCommand.Append(FinalCookedCommand); } } - } + },GForceSingleThread); return true; } @@ -1585,4 +1602,14 @@ FName UFlibAssetManageHelper::GetObjectPathByAssetData(const FAssetData& Data) return NAME_None; } +void UFlibAssetManageHelper::UpdateAssetRegistryData(const FString& PackageName) +{ + FAssetRegistryModule& AssetRegistryModule = FModuleManager::LoadModuleChecked("AssetRegistry"); + IAssetRegistry& AssetRegistry = AssetRegistryModule.Get(); + FString PackageFilename; + if (FPackageName::FindPackageFileWithoutExtension(FPackageName::LongPackageNameToFilename(PackageName), PackageFilename)) + { + AssetRegistry.ScanModifiedAssetFiles(TArray{PackageFilename}); + } +} // PRAGMA_ENABLE_DEPRECATION_WARNINGS diff --git a/HotPatcher/Source/HotPatcherRuntime/Private/FlibPatchParserHelper.cpp b/HotPatcher/Source/HotPatcherRuntime/Private/FlibPatchParserHelper.cpp index e1c24840..f1e3b598 100644 --- a/HotPatcher/Source/HotPatcherRuntime/Private/FlibPatchParserHelper.cpp +++ b/HotPatcher/Source/HotPatcherRuntime/Private/FlibPatchParserHelper.cpp @@ -927,113 +927,117 @@ FChunkAssetDescribe UFlibPatchParserHelper::CollectFChunkAssetsDescribeByChunk( const FChunkInfo& Chunk, TArray Platforms ) { + SCOPED_NAMED_EVENT_TEXT("CollectFChunkAssetsDescribeByChunk",FColor::Red); FChunkAssetDescribe ChunkAssetDescribe; // Collect Chunk Assets - { - - FAssetDependenciesInfo SpecifyDependAssets; - - FAssetDependenciesParser Parser; - FAssetDependencies Conf; - TArray AssetFilterPaths = UFlibAssetManageHelper::DirectoriesToStrings(Chunk.AssetIncludeFilters); - Conf.IncludeFilters = AssetFilterPaths; - Conf.IgnoreFilters = UFlibAssetManageHelper::DirectoriesToStrings(Chunk.AssetIgnoreFilters); - Conf.ForceSkipPackageNames = UFlibAssetManageHelper::SoftObjectPathsToStrings(Chunk.ForceSkipAssets); - Conf.InIncludeSpecifyAsset = Chunk.IncludeSpecifyAssets; - Conf.AssetRegistryDependencyTypes = Chunk.AssetRegistryDependencyTypes; - Conf.AnalysicFilterDependencies = Chunk.bAnalysisFilterDependencies; - Conf.ForceSkipContents = UFlibAssetManageHelper::DirectoriesToStrings(Chunk.ForceSkipContentRules); - Conf.ForceSkipContents.Append(UFlibAssetManageHelper::DirectoriesToStrings(PatcheSettings->GetAssetScanConfig().ForceSkipContentRules)); - - auto AddForceSkipAssets = [&Conf](const TArray& Assets) - { - for(const auto& forceSkipAsset:Assets) - { - Conf.ForceSkipContents.Add(forceSkipAsset.GetLongPackageName()); - } - }; - AddForceSkipAssets(Chunk.ForceSkipAssets); - AddForceSkipAssets(PatcheSettings->GetAssetScanConfig().ForceSkipAssets); - - auto AddSkipClassesLambda = [&Conf](const TArray& Classes) - { - for(const auto& ForceSkipClass:Classes) - { - Conf.IgnoreAseetTypes.Add(*ForceSkipClass->GetName()); - } - }; - AddSkipClassesLambda(Chunk.ForceSkipClasses); - AddSkipClassesLambda(PatcheSettings->GetAssetScanConfig().ForceSkipClasses); - - Parser.Parse(Conf); - TSet AssetLongPackageNames = Parser.GetrParseResults(); - - for(FName LongPackageName:AssetLongPackageNames) - { - if(LongPackageName.IsNone()) - { - continue; - } - AssetFilterPaths.AddUnique(LongPackageName.ToString()); - } - - const FAssetDependenciesInfo& AddAssetsRef = DiffInfo.AssetDiffInfo.AddAssetDependInfo; - const FAssetDependenciesInfo& ModifyAssetsRef = DiffInfo.AssetDiffInfo.ModifyAssetDependInfo; - - - auto CollectChunkAssets = [](const FAssetDependenciesInfo& SearchBase, const TArray& SearchFilters)->FAssetDependenciesInfo - { - FAssetDependenciesInfo ResultAssetDependInfos; - - for (const auto& SearchItem : SearchFilters) - { - if (SearchItem.IsEmpty()) - continue; - - FString SearchModuleName; - int32 findedPos = SearchItem.Find(TEXT("/"), ESearchCase::IgnoreCase, ESearchDir::FromStart, 1); - if (findedPos != INDEX_NONE) - { - SearchModuleName = UKismetStringLibrary::GetSubstring(SearchItem, 1, findedPos - 1); - } - else - { - SearchModuleName = UKismetStringLibrary::GetSubstring(SearchItem, 1, SearchItem.Len() - 1); - } - - if (!SearchModuleName.IsEmpty() && (SearchBase.AssetsDependenciesMap.Contains(SearchModuleName))) - { - if (!ResultAssetDependInfos.AssetsDependenciesMap.Contains(SearchModuleName)) - ResultAssetDependInfos.AssetsDependenciesMap.Add(SearchModuleName, FAssetDependenciesDetail(SearchModuleName, TMap{})); - - const FAssetDependenciesDetail& SearchBaseModule = *SearchBase.AssetsDependenciesMap.Find(SearchModuleName); - - TArray AllAssetKeys; - SearchBaseModule.AssetDependencyDetails.GetKeys(AllAssetKeys); - - for (const auto& KeyItem : AllAssetKeys) - { - if (KeyItem.StartsWith(SearchItem)) - { - FAssetDetail FindedAsset = *SearchBaseModule.AssetDependencyDetails.Find(KeyItem); - if (!ResultAssetDependInfos.AssetsDependenciesMap.Find(SearchModuleName)->AssetDependencyDetails.Contains(KeyItem)) - { - ResultAssetDependInfos.AssetsDependenciesMap.Find(SearchModuleName)->AssetDependencyDetails.Add(KeyItem, FindedAsset); - } - } - } - } - } - return ResultAssetDependInfos; - }; - - ChunkAssetDescribe.AddAssets = CollectChunkAssets(AddAssetsRef, AssetFilterPaths); - ChunkAssetDescribe.ModifyAssets = CollectChunkAssets(ModifyAssetsRef, AssetFilterPaths); + // { + // + // FAssetDependenciesInfo SpecifyDependAssets; + // + // FAssetDependenciesParser Parser; + // FAssetDependencies Conf; + // TArray AssetFilterPaths = UFlibAssetManageHelper::DirectoriesToStrings(Chunk.AssetIncludeFilters); + // Conf.IncludeFilters = AssetFilterPaths; + // Conf.IgnoreFilters = UFlibAssetManageHelper::DirectoriesToStrings(Chunk.AssetIgnoreFilters); + // Conf.ForceSkipPackageNames = UFlibAssetManageHelper::SoftObjectPathsToStrings(Chunk.ForceSkipAssets); + // Conf.InIncludeSpecifyAsset = Chunk.IncludeSpecifyAssets; + // Conf.AssetRegistryDependencyTypes = Chunk.AssetRegistryDependencyTypes; + // Conf.AnalysicFilterDependencies = Chunk.bAnalysisFilterDependencies; + // Conf.ForceSkipContents = UFlibAssetManageHelper::DirectoriesToStrings(Chunk.ForceSkipContentRules); + // Conf.ForceSkipContents.Append(UFlibAssetManageHelper::DirectoriesToStrings(PatcheSettings->GetAssetScanConfig().ForceSkipContentRules)); + // + // auto AddForceSkipAssets = [&Conf](const TArray& Assets) + // { + // for(const auto& forceSkipAsset:Assets) + // { + // Conf.ForceSkipContents.Add(forceSkipAsset.GetLongPackageName()); + // } + // }; + // AddForceSkipAssets(Chunk.ForceSkipAssets); + // AddForceSkipAssets(PatcheSettings->GetAssetScanConfig().ForceSkipAssets); + // + // auto AddSkipClassesLambda = [&Conf](const TArray& Classes) + // { + // for(const auto& ForceSkipClass:Classes) + // { + // Conf.IgnoreAseetTypes.Add(*ForceSkipClass->GetName()); + // } + // }; + // AddSkipClassesLambda(Chunk.ForceSkipClasses); + // AddSkipClassesLambda(PatcheSettings->GetAssetScanConfig().ForceSkipClasses); + // + // Parser.Parse(Conf); + // TSet AssetLongPackageNames = Parser.GetrParseResults(); + // + // for(FName LongPackageName:AssetLongPackageNames) + // { + // if(LongPackageName.IsNone()) + // { + // continue; + // } + // AssetFilterPaths.AddUnique(LongPackageName.ToString()); + // } + // + // const FAssetDependenciesInfo& AddAssetsRef = DiffInfo.AssetDiffInfo.AddAssetDependInfo; + // const FAssetDependenciesInfo& ModifyAssetsRef = DiffInfo.AssetDiffInfo.ModifyAssetDependInfo; + // + // + // auto CollectChunkAssets = [](const FAssetDependenciesInfo& SearchBase, const TArray& SearchFilters)->FAssetDependenciesInfo + // { + // SCOPED_NAMED_EVENT_TEXT("CollectChunkAssets",FColor::Red); + // FAssetDependenciesInfo ResultAssetDependInfos; + // + // for (const auto& SearchItem : SearchFilters) + // { + // if (SearchItem.IsEmpty()) + // continue; + // + // FString SearchModuleName; + // int32 findedPos = SearchItem.Find(TEXT("/"), ESearchCase::IgnoreCase, ESearchDir::FromStart, 1); + // if (findedPos != INDEX_NONE) + // { + // SearchModuleName = UKismetStringLibrary::GetSubstring(SearchItem, 1, findedPos - 1); + // } + // else + // { + // SearchModuleName = UKismetStringLibrary::GetSubstring(SearchItem, 1, SearchItem.Len() - 1); + // } + // + // if (!SearchModuleName.IsEmpty() && (SearchBase.AssetsDependenciesMap.Contains(SearchModuleName))) + // { + // if (!ResultAssetDependInfos.AssetsDependenciesMap.Contains(SearchModuleName)) + // ResultAssetDependInfos.AssetsDependenciesMap.Add(SearchModuleName, FAssetDependenciesDetail(SearchModuleName, TMap{})); + // + // const FAssetDependenciesDetail& SearchBaseModule = *SearchBase.AssetsDependenciesMap.Find(SearchModuleName); + // + // TArray AllAssetKeys; + // SearchBaseModule.AssetDependencyDetails.GetKeys(AllAssetKeys); + // + // for (const auto& KeyItem : AllAssetKeys) + // { + // if (KeyItem.StartsWith(SearchItem)) + // { + // FAssetDetail FindedAsset = *SearchBaseModule.AssetDependencyDetails.Find(KeyItem); + // if (!ResultAssetDependInfos.AssetsDependenciesMap.Find(SearchModuleName)->AssetDependencyDetails.Contains(KeyItem)) + // { + // ResultAssetDependInfos.AssetsDependenciesMap.Find(SearchModuleName)->AssetDependencyDetails.Add(KeyItem, FindedAsset); + // } + // } + // } + // } + // } + // return ResultAssetDependInfos; + // }; + //} + { + ChunkAssetDescribe.AddAssets = DiffInfo.AssetDiffInfo.AddAssetDependInfo; // CollectChunkAssets(AddAssetsRef, AssetFilterPaths); + ChunkAssetDescribe.ModifyAssets = DiffInfo.AssetDiffInfo.ModifyAssetDependInfo; //CollectChunkAssets(ModifyAssetsRef, AssetFilterPaths); ChunkAssetDescribe.Assets = UFlibAssetManageHelper::CombineAssetDependencies(ChunkAssetDescribe.AddAssets, ChunkAssetDescribe.ModifyAssets); } auto CoolectPlatformExFiles = [](const FPatchVersionDiff& DiffInfo,const auto& Chunk,ETargetPlatform Platform)->TArray { + SCOPED_NAMED_EVENT_TEXT("CoolectPlatformExFiles",FColor::Red); // Collect Extern Files TArray AllFiles; @@ -1125,16 +1129,15 @@ TArray UFlibPatchParserHelper::CollectPakCommandByChunk( const FExportPatchSettings* PatcheSettings ) { - auto CollectPakCommandsByChunkLambda = [PatcheSettings](const FPatchVersionDiff& DiffInfo, const FChunkInfo& Chunk, const FString& PlatformName/*, const TArray& PakOptions*/)->TArray + TArray PakCommands; + auto CollectPakCommandsByChunkLambda = [&PakCommands,PatcheSettings](const FPatchVersionDiff& DiffInfo, const FChunkInfo& Chunk, const FString& PlatformName/*, const TArray& PakOptions*/) { ETargetPlatform Platform; THotPatcherTemplateHelper::GetEnumValueByName(PlatformName,Platform); TArray CollectPlatforms = {ETargetPlatform::AllPlatforms}; CollectPlatforms.AddUnique(Platform); FChunkAssetDescribe ChunkAssetsDescrible = UFlibPatchParserHelper::CollectFChunkAssetsDescribeByChunk(PatcheSettings, DiffInfo ,Chunk, CollectPlatforms); - - TArray PakCommands; - + bool bIoStore =false; bool bAllowBulkDataInIoStore = false; #if ENGINE_MAJOR_VERSION > 4 ||ENGINE_MINOR_VERSION > 25 @@ -1250,10 +1253,9 @@ TArray UFlibPatchParserHelper::CollectPakCommandByChunk( // NotCompressOptions.Remove(TEXT("-compress")); auto ReceivePakCommandExFilesLambda = [&PakCommands](const FPakCommand& InCommand){ PakCommands.Emplace(InCommand); }; UFlibPatchParserHelper::GetPakCommandsFromInternalInfo(ChunkAssetsDescrible.InternalFiles, PlatformName,/* NotCompressOptions,*/ ReceivePakCommandExFilesLambda); - return PakCommands; }; - - return CollectPakCommandsByChunkLambda(DiffInfo, Chunk, PlatformName/*, PakOptions*/); + CollectPakCommandsByChunkLambda(DiffInfo, Chunk, PlatformName/*, PakOptions*/); + return PakCommands; } FHotPatcherVersion UFlibPatchParserHelper::ExportReleaseVersionInfoByChunk( @@ -1281,6 +1283,7 @@ FHotPatcherVersion UFlibPatchParserHelper::ExportReleaseVersionInfoByChunk( } FAssetScanConfig ScanConfig; + ScanConfig.bPackageTracker = false; ScanConfig.AssetIncludeFilters = InChunkInfo.AssetIncludeFilters; ScanConfig.AssetIgnoreFilters = InChunkInfo.AssetIgnoreFilters; ScanConfig.bAnalysisFilterDependencies = bInAnalysisFilterDependencies; @@ -2137,3 +2140,72 @@ bool UFlibPatchParserHelper::IsValidPatchSettings(const FExportPatchSettings* Pa } return bCanExport; } + +FString UFlibPatchParserHelper::GetTargetPlatformsCmdLine(const TArray& Platforms) +{ + FString Result; + if(Platforms.Num()) + { + FString PlatformArrayStr; + for(ETargetPlatform Platform:Platforms) + { + FString PlatformName = THotPatcherTemplateHelper::GetEnumNameByValue(Platform); + PlatformArrayStr += FString::Printf(TEXT("%s+"),*PlatformName); + } + PlatformArrayStr.RemoveFromEnd(TEXT("+")); + if(!PlatformArrayStr.IsEmpty()) + { + Result = FString::Printf(TEXT("-TargetPlatform=%s"),*PlatformArrayStr); + } + } + return Result; +} + +void UFlibPatchParserHelper::SetPropertyTransient(UStruct* Struct,const FString& PropertyName,bool bTransient) +{ + for(TFieldIterator PropertyIter(Struct);PropertyIter;++PropertyIter) + { + FProperty* PropertyIns = *PropertyIter; + if(PropertyName.Equals(*PropertyIns->GetName(),ESearchCase::IgnoreCase)) + { + if(bTransient) + { + PropertyIns->SetPropertyFlags(CPF_Transient); + } + else + { + PropertyIns->ClearPropertyFlags(CPF_Transient); + } + break; + } + } +} + +FString UFlibPatchParserHelper::MergeOptionsAsCmdline(const TArray& InOptions) +{ + FString Cmdline; + for(const auto& Option:InOptions) + { + FString InOption = Option; + while(InOption.RemoveFromStart(TEXT(" "))){} + while(InOption.RemoveFromEnd(TEXT(" "))){} + if(!InOption.IsEmpty()) + { + Cmdline += FString::Printf(TEXT("%s "),*InOption); + } + } + Cmdline.RemoveFromEnd(TEXT(" ")); + return Cmdline; +}; + +FString UFlibPatchParserHelper::GetPlatformsStr(TArray Platforms) +{ + FString result; + for(auto Platform:Platforms) + { + FString PlatformStr = THotPatcherTemplateHelper::GetEnumNameByValue(Platform,false); + result+=FString::Printf(TEXT("%s,"),*PlatformStr); + } + result.RemoveFromEnd(TEXT(",")); + return result; +} \ No newline at end of file diff --git a/HotPatcher/Source/HotPatcherRuntime/Public/BaseTypes/FHotPatcherVersion.h b/HotPatcher/Source/HotPatcherRuntime/Public/BaseTypes/FHotPatcherVersion.h index 7b6bf705..1b941165 100644 --- a/HotPatcher/Source/HotPatcherRuntime/Public/BaseTypes/FHotPatcherVersion.h +++ b/HotPatcher/Source/HotPatcherRuntime/Public/BaseTypes/FHotPatcherVersion.h @@ -21,7 +21,8 @@ struct FHotPatcherVersion GENERATED_USTRUCT_BODY() public: - + FHotPatcherVersion()=default; + UPROPERTY(EditAnywhere,BlueprintReadWrite) FString VersionId; UPROPERTY(EditAnywhere, BlueprintReadWrite) diff --git a/HotPatcher/Source/HotPatcherRuntime/Public/BaseTypes/FPackageTracker.h b/HotPatcher/Source/HotPatcherRuntime/Public/BaseTypes/FPackageTracker.h index c76f6142..195d94da 100644 --- a/HotPatcher/Source/HotPatcherRuntime/Public/BaseTypes/FPackageTracker.h +++ b/HotPatcher/Source/HotPatcherRuntime/Public/BaseTypes/FPackageTracker.h @@ -111,3 +111,36 @@ struct FPackageTracker : public FPackageTrackerBase TSet PackagesPendingSave; TSet& ExisitAssets; }; + +struct FClassesPackageTracker : public FPackageTrackerBase +{ + virtual void OnPackageCreated(UPackage* Package) override + { + FName ClassName = UFlibAssetManageHelper::GetAssetTypeByPackage(Package); + + if(!ClassMapping.Contains(ClassName)) + { + ClassMapping.Add(ClassName,TArray{}); + } + ClassMapping.Find(ClassName)->AddUnique(Package); + }; + virtual void OnPackageDeleted(UPackage* Package) override + { + FName ClassName = UFlibAssetManageHelper::GetAssetTypeByPackage(Package); + if(ClassMapping.Contains(ClassName)) + { + ClassMapping.Find(ClassName)->Remove(Package); + } + } + TArray GetPackagesByClassName(FName ClassName) + { + TArray result; + if(ClassMapping.Contains(ClassName)) + { + result = *ClassMapping.Find(ClassName); + } + return result; + } +protected: + TMap> ClassMapping; +}; \ No newline at end of file diff --git a/HotPatcher/Source/HotPatcherRuntime/Public/CreatePatch/FExportPatchSettings.h b/HotPatcher/Source/HotPatcherRuntime/Public/CreatePatch/FExportPatchSettings.h index 1909249f..7faded44 100644 --- a/HotPatcher/Source/HotPatcherRuntime/Public/CreatePatch/FExportPatchSettings.h +++ b/HotPatcher/Source/HotPatcherRuntime/Public/CreatePatch/FExportPatchSettings.h @@ -98,12 +98,12 @@ struct HOTPATCHERRUNTIME_API FExportPatchSettings:public FHotPatcherSettingBase FORCEINLINE bool IsCustomPakNameRegular()const {return bCustomPakNameRegular;} FORCEINLINE FString GetPakNameRegular()const { return PakNameRegular;} + FORCEINLINE bool IsCustomPakSaveDirRegular()const {return bCustomPakSaveDirRegular;} + FORCEINLINE FString GetPakSaveDirRegular()const { return PakSaveDirRegular;} FORCEINLINE bool IsCookPatchAssets()const {return bCookPatchAssets;} FORCEINLINE bool IsIgnoreDeletedAssetsInfo()const {return bIgnoreDeletedAssetsInfo;} FORCEINLINE bool IsSaveDeletedAssetsToNewReleaseJson()const {return bStorageDeletedAssetsToNewReleaseJson;} - - FORCEINLINE FIoStoreSettings GetIoStoreSettings()const { return IoStoreSettings; } FORCEINLINE FUnrealPakSettings GetUnrealPakSettings()const {return UnrealPakSettings;} FORCEINLINE TArray GetDefaultPakListOptions()const {return DefaultPakListOptions;} @@ -124,6 +124,8 @@ struct HOTPATCHERRUNTIME_API FExportPatchSettings:public FHotPatcherSettingBase FORCEINLINE FCookShaderOptions GetCookShaderOptions()const {return CookShaderOptions;} FORCEINLINE FAssetRegistryOptions GetSerializeAssetRegistryOptions()const{return SerializeAssetRegistryOptions;} FORCEINLINE bool IsImportProjectSettings()const{ return bImportProjectSettings; } + + virtual FString GetCombinedAdditionalCommandletArgs()const override; public: UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "BaseVersion") bool bByBaseVersion = false; @@ -223,8 +225,13 @@ struct HOTPATCHERRUNTIME_API FExportPatchSettings:public FHotPatcherSettingBase UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Pak Options") bool bCustomPakNameRegular = false; // Can use value: {VERSION} {BASEVERSION} {CHUNKNAME} {PLATFORM} - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Pak Options",meta=(EditCondition = "bCustomPakNameRegular")) + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Pak Options|Regular",meta=(EditCondition = "bCustomPakNameRegular")) FString PakNameRegular = TEXT("{VERSION}_{CHUNKNAME}_{PLATFORM}_001_P"); + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Pak Options|Regular") + bool bCustomPakSaveDirRegular = false; + // Can use value: {VERSION} {BASEVERSION} {CHUNKNAME} {PLATFORM} + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Pak Options",meta=(EditCondition = "bCustomPakSaveDirRegular")) + FString PakSaveDirRegular = TEXT("{CHUNKNAME}/{PLATFORM}"); UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "SaveTo") bool bStorageNewRelease = true; diff --git a/HotPatcher/Source/HotPatcherRuntime/Public/CreatePatch/HotPatcherSettingBase.h b/HotPatcher/Source/HotPatcherRuntime/Public/CreatePatch/HotPatcherSettingBase.h index 3ecd99de..18e2d824 100644 --- a/HotPatcher/Source/HotPatcherRuntime/Public/CreatePatch/HotPatcherSettingBase.h +++ b/HotPatcher/Source/HotPatcherRuntime/Public/CreatePatch/HotPatcherSettingBase.h @@ -37,15 +37,7 @@ struct HOTPATCHERRUNTIME_API FHotPatcherSettingBase:public FPatcherEntitySetting FORCEINLINE virtual bool IsStandaloneMode()const {return bStandaloneMode;} FORCEINLINE virtual bool IsSaveConfig()const {return bStorageConfig;} FORCEINLINE virtual TArray GetAdditionalCommandletArgs()const{return AdditionalCommandletArgs;} - FORCEINLINE virtual FString GetCombinedAdditionalCommandletArgs()const - { - FString Result; - for(const auto& Option:GetAdditionalCommandletArgs()) - { - Result+=FString::Printf(TEXT("%s "),*Option); - } - return Result; - } + virtual FString GetCombinedAdditionalCommandletArgs()const; FORCEINLINE virtual bool IsForceSkipContent()const{return GetAssetScanConfig().bForceSkipContent;} FORCEINLINE virtual TArray GetForceSkipContentRules()const {return GetAssetScanConfig().ForceSkipContentRules;} diff --git a/HotPatcher/Source/HotPatcherRuntime/Public/CreatePatch/TimeRecorder.h b/HotPatcher/Source/HotPatcherRuntime/Public/CreatePatch/TimeRecorder.h index 72efcf41..85d6273d 100644 --- a/HotPatcher/Source/HotPatcherRuntime/Public/CreatePatch/TimeRecorder.h +++ b/HotPatcher/Source/HotPatcherRuntime/Public/CreatePatch/TimeRecorder.h @@ -1,8 +1,9 @@ #pragma once #include "CoreMinimal.h" +#include "HotPatcherLog.h" -struct TimeRecorder +struct HOTPATCHERRUNTIME_API TimeRecorder { TimeRecorder(const FString& InDisplay=TEXT(""),bool InAuto = true):Display(InDisplay),bAuto(InAuto) { diff --git a/HotPatcher/Source/HotPatcherRuntime/Public/FlibAssetManageHelper.h b/HotPatcher/Source/HotPatcherRuntime/Public/FlibAssetManageHelper.h index 9a903030..fd1baba1 100644 --- a/HotPatcher/Source/HotPatcherRuntime/Public/FlibAssetManageHelper.h +++ b/HotPatcher/Source/HotPatcherRuntime/Public/FlibAssetManageHelper.h @@ -86,7 +86,9 @@ class HOTPATCHERRUNTIME_API UFlibAssetManageHelper : public UBlueprintFunctionLi static FAssetPackageData* GetPackageDataByPackageName(const FString& InPackageName); UFUNCTION(BlueprintCallable, Category = "GWorld|Flib|AssetManagerExEx") static bool GetAssetPackageGUID(const FString& InPackageName, FName& OutGUID); - + + static FSoftObjectPath CreateSoftObjectPathByPackage(UPackage* Package); + static FName GetAssetTypeByPackage(UPackage* Package); static FName GetAssetType(FSoftObjectPath InPackageName); // Combine AssetDependencies Filter repeat asset @@ -251,6 +253,8 @@ class HOTPATCHERRUNTIME_API UFlibAssetManageHelper : public UBlueprintFunctionLi static FName GetAssetDataClasses(const FAssetData& Data); static FName GetObjectPathByAssetData(const FAssetData& Data); static bool bIncludeOnlyOnDiskAssets; + + static void UpdateAssetRegistryData(const FString& PackageName); }; diff --git a/HotPatcher/Source/HotPatcherRuntime/Public/FlibPatchParserHelper.h b/HotPatcher/Source/HotPatcherRuntime/Public/FlibPatchParserHelper.h index b8a8e0c6..64ca6b3f 100644 --- a/HotPatcher/Source/HotPatcherRuntime/Public/FlibPatchParserHelper.h +++ b/HotPatcher/Source/HotPatcherRuntime/Public/FlibPatchParserHelper.h @@ -239,6 +239,9 @@ class HOTPATCHERRUNTIME_API UFlibPatchParserHelper : public UBlueprintFunctionLi } static bool IsValidPatchSettings(const FExportPatchSettings* PatchSettings,bool bExternalFilesCheck); - + static void SetPropertyTransient(UStruct* Struct,const FString& PropertyName,bool bTransient); + static FString GetTargetPlatformsCmdLine(const TArray& Platforms); + static FString MergeOptionsAsCmdline(const TArray& InOptions); + static FString GetPlatformsStr(TArray Platforms); }; diff --git a/Mods/HotChunker b/Mods/HotChunker index ce47eff0..c4cecb36 160000 --- a/Mods/HotChunker +++ b/Mods/HotChunker @@ -1 +1 @@ -Subproject commit ce47eff048e94ec7d6a690bc6153d7ca8918f332 +Subproject commit c4cecb3659e2d3461d82c44f9519c23eae45ca38 diff --git a/Mods/HotMultiCooker b/Mods/HotMultiCooker index dff1f4a9..0c7ac4d2 160000 --- a/Mods/HotMultiCooker +++ b/Mods/HotMultiCooker @@ -1 +1 @@ -Subproject commit dff1f4a9b606b4a74f0940f0bbabd8787bfa7fb9 +Subproject commit 0c7ac4d27fe84d24da01eb0dcaeae369a85097d6