Merge branch 'master' of https://github.com/space-wizards/RobustToolbox…

… into placement-changes-for-rcd

RELEASE-NOTES.md

@@ -47,7 +47,7 @@ END TEMPLATE-->
 
 ### Other
 
-*None yet*
+* The replay system now allows loading a replay with a mismatching serializer type hash. This means replays should be more robust against future version updates (engine security patches or .NET updates).
 
 ### Internal
 

Robust.Client/Console/Commands/Debug.cs

@@ -555,7 +555,7 @@ internal static List<MemberInfo> GetAllMembers(Control control)
                 if (type != typeof(Control))
                     cname = $"Control > {cname}";
 
-                returnVal.GetOrNew(cname).Add((member.Name, member.GetValue(control)?.ToString() ?? "null"));
+                returnVal.GetOrNew(cname).Add((member.Name, GetMemberValue(member, control, ", ")));
             }
 
             foreach (var (attachedProperty, value) in control.AllAttachedProperties)
@@ -570,6 +570,28 @@ internal static List<MemberInfo> GetAllMembers(Control control)
             }
             return returnVal;
         }
+
+        internal static string PropertyValuesString(Control control, string key)
+        {
+            var member = GetAllMembers(control).Find(m => m.Name == key);
+            return GetMemberValue(member, control, "\n", "\"{0}\"");
+        }
+
+        private static string GetMemberValue(MemberInfo? member, Control control, string separator, string
+                wrap = "{0}")
+        {
+            var value = member?.GetValue(control);
+            var o = value switch
+            {
+                ICollection<Control> controls => string.Join(separator,
+                    controls.Select(ctrl => $"{ctrl.Name}({ctrl.GetType()})")),
+                ICollection<string> list => string.Join(separator, list),
+                null => null,
+                _ => value.ToString()
+            };
+            // Convert to quote surrounded string or null with no quotes
+            return o is not null ? string.Format(wrap, o) : "null";
+        }
     }
 
     internal sealed class SetClipboardCommand : LocalizedCommands

Robust.Client/GameStates/ClientGameStateManager.cs

@@ -700,7 +700,7 @@ public IEnumerable<NetEntity> ApplyGameState(GameState curState, GameState? next
         {
             using var _ = _timing.StartStateApplicationArea();
 
-            // TODO repays optimize this.
+            // TODO replays optimize this.
             // This currently just saves game states as they are applied.
             // However this is inefficient and may have redundant data.
             // E.g., we may record states: [10 to 15] [11 to 16] *error* [0 to 18] [18 to 19] [18 to 20] ...

Robust.Client/Replays/Loading/ReplayLoadManager.Read.cs

@@ -129,39 +129,37 @@ public async Task<ReplayData> LoadReplayAsync(IReplayFileReader fileReader, Load
         return parsed.FirstOrDefault()?.Root as MappingDataNode;
     }
 
-    private (MappingDataNode YamlData, HashSet<string> CVars, TimeSpan Duration, TimeSpan StartTime, bool ClientSide)
+    private (MappingDataNode YamlData, HashSet<string> CVars, TimeSpan? Duration, TimeSpan StartTime, bool ClientSide)
         LoadMetadata(IReplayFileReader fileReader)
     {
         _sawmill.Info($"Reading replay metadata");
         var data = LoadYamlMetadata(fileReader);
         if (data == null)
             throw new Exception("Failed to load yaml metadata");
 
-        TimeSpan duration;
         var finalData = LoadYamlFinalMetadata(fileReader);
-        if (finalData == null)
-        {
-            var msg = "Failed to load final yaml metadata";
-            if (!_confMan.GetCVar(CVars.ReplayIgnoreErrors))
-                throw new Exception(msg);
+        TimeSpan? duration = finalData == null
+            ? null
+            : TimeSpan.Parse(((ValueDataNode) finalData[MetaFinalKeyDuration]).Value);
 
-            _sawmill.Error(msg);
-            duration = TimeSpan.FromDays(1);
-        }
-        else
-        {
-            duration = TimeSpan.Parse(((ValueDataNode) finalData[MetaFinalKeyDuration]).Value);
-        }
+        if (finalData == null)
+            _sawmill.Warning("Failed to load final yaml metadata. Partial/incomplete replay?");
 
-        var typeHash = Convert.FromHexString(((ValueDataNode) data[MetaKeyTypeHash]).Value);
+        var typeHashString = ((ValueDataNode) data[MetaKeyTypeHash]).Value;
+        var typeHash = Convert.FromHexString(typeHashString);
         var stringHash = Convert.FromHexString(((ValueDataNode) data[MetaKeyStringHash]).Value);
         var startTick = ((ValueDataNode) data[MetaKeyStartTick]).Value;
         var timeBaseTick = ((ValueDataNode) data[MetaKeyBaseTick]).Value;
         var timeBaseTimespan = ((ValueDataNode) data[MetaKeyBaseTime]).Value;
         var clientSide = bool.Parse(((ValueDataNode) data[MetaKeyIsClientRecording]).Value);
 
         if (!typeHash.SequenceEqual(_serializer.GetSerializableTypesHash()))
-            throw new Exception($"{nameof(IRobustSerializer)} hashes do not match. Loading replays using a bad replay-client version?");
+        {
+            if (!_confMan.GetCVar(CVars.ReplayIgnoreErrors))
+                throw new Exception($"RobustSerializer hash mismatch. do not match. Client hash: {_serializer.GetSerializableTypesHashString()}, replay hash: {typeHashString}.");
+
+            _sawmill.Warning($"RobustSerializer hash mismatch. Replay may fail to load!");
+        }
 
         using var stringFile = fileReader.Open(FileStrings);
         var stringData = new byte[stringFile.Length];

Robust.Client/Replays/UI/ReplayControlWidget.cs

@@ -90,6 +90,7 @@ protected override void FrameUpdate(FrameEventArgs args)
         var maxIndex = Math.Max(1, replay.States.Count - 1);
         var state = replay.States[index];
         var replayTime = TimeSpan.FromSeconds(TickSlider.Value);
+        var end = replay.Duration == null ? "N/A" : replay.Duration.Value.ToString(TimeFormat);
 
         IndexLabel.Text = Loc.GetString("replay-time-box-index-label",
             ("current", index), ("total", maxIndex), ("percentage", percentage));
@@ -98,10 +99,10 @@ protected override void FrameUpdate(FrameEventArgs args)
             ("current", state.ToSequence), ("total", replay.States[^1].ToSequence), ("percentage", percentage));
 
         TimeLabel.Text = Loc.GetString("replay-time-box-replay-time-label",
-            ("current", replayTime.ToString(TimeFormat)), ("end", replay.Duration.ToString(TimeFormat)), ("percentage", percentage));
+            ("current", replayTime.ToString(TimeFormat)), ("end", end), ("percentage", percentage));
 
         var serverTime = (replayTime + replay.StartTime).ToString(TimeFormat);
-        var duration = (replay.Duration + replay.StartTime).ToString(TimeFormat);
+        string duration = replay.Duration == null ? "N/A" : (replay.Duration + replay.StartTime).Value.ToString(TimeFormat);
         ServerTimeLabel.Text = Loc.GetString("replay-time-box-server-time-label",
             ("current", serverTime), ("end", duration), ("percentage", percentage));
 

Robust.Client/UserInterface/DevWindow/DevWindowTabUI.xaml

@@ -20,10 +20,14 @@
                         <BoxContainer Name="ControlTreeRoot" Orientation="Vertical" MouseFilter="Stop"  />
                     </PanelContainer>
                 </ScrollContainer>
-                <ScrollContainer HScrollEnabled="False">
+                <ScrollContainer>
                     <BoxContainer Name="ControlProperties" Orientation="Vertical" MouseFilter="Stop" />
                 </ScrollContainer>
             </SplitContainer>
         </BoxContainer>
     </PanelContainer>
+    <!-- TODO Remove and replace with a popup container on WindowRoot -->
+    <PopupContainer Name="PopupContainer" Access="Public">
+        <DevWindowTabUIPopup Name="UIPopup" />
+    </PopupContainer>
 </Control>

Robust.Client/UserInterface/DevWindow/DevWindowTabUI.xaml.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Numerics;
 using Robust.Client.AutoGenerated;
 using Robust.Client.Console.Commands;
 using Robust.Client.Graphics;
@@ -208,25 +209,40 @@ private void Refresh()
             });
             foreach (var (prop, value) in values)
             {
-                ControlProperties.AddChild(new BoxContainer
+                var button = new ContainerButton
                 {
-                    Orientation = BoxContainer.LayoutOrientation.Horizontal,
-                    SeparationOverride = 3,
-                    Margin = new Thickness(3, 1),
                     Children =
                     {
                         new BoxContainer
                         {
                             Orientation = BoxContainer.LayoutOrientation.Horizontal,
+                            SeparationOverride = 3,
+                            Margin = new Thickness(3, 1),
                             Children =
                             {
-                                new Label { Text = $"{prop}", FontColorOverride = Color.GreenYellow },
-                                new Label { Text = ":" }, // this is for the non colored ":", intentional
+                                new BoxContainer
+                                {
+                                    Orientation = BoxContainer.LayoutOrientation.Horizontal,
+                                    Children =
+                                    {
+                                        new Label { Text = $"{prop}", FontColorOverride = Color.GreenYellow },
+                                        new Label { Text = ":" }, // this is for the non colored ":", intentional
+                                    }
+                                },
+                                new Label { Text = $"{value}" },
                             }
-                        },
-                        new Label { Text = $"{value}" },
+                        }
                     }
-                });
+                };
+                button.OnPressed += _ =>
+                {
+                    // TODO replace with parenting to popup container on WindowRoot
+                    UIPopup.Text = GuiDumpCommand.PropertyValuesString(SelectedControl, prop);
+                    var box = UIBox2.FromDimensions(UserInterfaceManager.MousePositionScaled.Position
+                                                    - GlobalPosition, Vector2.One);
+                    UIPopup.Open(box);
+                };
+                ControlProperties.AddChild(button);
             }
         }
     }

Robust.Client/UserInterface/DevWindow/DevWindowTabUIPopup.xaml

@@ -0,0 +1,11 @@
+<DevWindowTabUIPopup xmlns="https://spacestation14.io"
+                xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+                x:Class="Robust.Client.UserInterface.DevWindowTabUIPopup"
+                xmlns:gfx="clr-namespace:Robust.Client.Graphics">
+    <PanelContainer>
+        <PanelContainer.PanelOverride>
+            <gfx:StyleBoxFlat BackgroundColor="#40404C" />
+        </PanelContainer.PanelOverride>
+    </PanelContainer>
+    <Label Name="TextLabel" />
+</DevWindowTabUIPopup>

Robust.Client/UserInterface/DevWindow/DevWindowTabUIPopup.xaml.cs

@@ -0,0 +1,20 @@
+using Robust.Client.AutoGenerated;
+using Robust.Client.UserInterface.Controls;
+using Robust.Client.UserInterface.XAML;
+
+namespace Robust.Client.UserInterface;
+
+[GenerateTypedNameReferences]
+internal sealed partial class DevWindowTabUIPopup : Popup
+{
+    public string? Text
+    {
+        get => TextLabel.Text;
+        set => TextLabel.Text = value;
+    }
+
+    public DevWindowTabUIPopup()
+    {
+        RobustXamlLoader.Load(this);
+    }
+}

Robust.Server/GameStates/PvsSystem.ToSendSet.cs

@@ -48,30 +48,17 @@ private void AddPvsChunk(PvsChunk chunk, float distance, PvsSession session)
         // We add chunk-size here so that its consistent with the normal PVS range setting.
         // I.e., distance here is the Chebyshev distance to the centre of each chunk, but the normal pvs range only
         // required that the chunk be touching the box, not the centre.
-        var limit = distance < (_lowLodDistance + ChunkSize) / 2
+        var count = distance < (_lowLodDistance + ChunkSize) / 2
             ? chunk.Contents.Count
             : chunk.LodCounts[0];
 
-        // If the PVS budget is exceeded, it should still be safe to send all of the chunk's direct children, though
-        // after that we have no guarantee that an entity's parent got sent.
-        var directChildren = Math.Min(limit, chunk.LodCounts[2]);
-
         // Send entities on the chunk.
-        var span = CollectionsMarshal.AsSpan(chunk.Contents);
-        for (var i = 0; i < limit; i++)
+        var span = CollectionsMarshal.AsSpan(chunk.Contents)[..count];
+        foreach (ref var ent in span)
         {
-            var ent = span[i];
             ref var meta = ref _metadataMemory.GetRef(ent.Ptr.Index);
-
-            if ((mask & meta.VisMask) != meta.VisMask)
-                continue;
-
-            // TODO PVS improve this somehow
-            // Having entities "leave" pvs view just because the pvs entry budget was exceeded sucks.
-            // This probably requires changing client game state manager to support receiving entities with unknown parents.
-            // Probably needs to do something similar to pending net entity states, but for entity spawning.
-            if (!AddEntity(session, ref ent, ref meta, fromTick))
-                limit = directChildren;
+            if ((mask & meta.VisMask) == meta.VisMask)
+                AddEntity(session, ref ent, ref meta, fromTick);
         }
     }
 
@@ -80,26 +67,26 @@ private void AddPvsChunk(PvsChunk chunk, float distance, PvsSession session)
     /// </summary>
     /// <returns>Returns false if the entity would exceed the client's PVS budget.</returns>
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
-    private bool AddEntity(PvsSession session, ref PvsChunk.ChunkEntity ent, ref PvsMetadata meta,
+    private void AddEntity(PvsSession session, ref PvsChunk.ChunkEntity ent, ref PvsMetadata meta,
         GameTick fromTick)
     {
         DebugTools.Assert(fromTick < _gameTiming.CurTick);
         ref var data = ref session.DataMemory.GetRef(ent.Ptr.Index);
 
         if (data.LastSeen == _gameTiming.CurTick)
-            return true;
+            return;
 
         if (meta.LifeStage >= EntityLifeStage.Terminating)
         {
             Log.Error($"Attempted to send deleted entity: {ToPrettyString(ent.Uid)}");
             EntityManager.QueueDeleteEntity(ent.Uid);
-            return true;
+            return;
         }
 
         var (entered,budgetExceeded) = IsEnteringPvsRange(ref data, fromTick, ref session.Budget);
 
         if (budgetExceeded)
-            return false;
+            return;
 
         data.LastSeen = _gameTiming.CurTick;
         session.ToSend!.Add(ent.Ptr);
@@ -108,25 +95,23 @@ private bool AddEntity(PvsSession session, ref PvsChunk.ChunkEntity ent, ref Pvs
         {
             var state = GetFullEntityState(session.Session, ent.Uid, ent.Meta);
             session.States.Add(state);
-            return true;
+            return;
         }
 
         if (entered)
         {
             var state = GetEntityState(session.Session, ent.Uid, data.EntityLastAcked, ent.Meta);
             session.States.Add(state);
-            return true;
+            return;
         }
 
         if (meta.LastModifiedTick <= fromTick)
-            return true;
+            return;
 
         var entState = GetEntityState(session.Session, ent.Uid, fromTick , ent.Meta);
 
         if (!entState.Empty)
             session.States.Add(entState);
-
-        return true;
     }
 
     /// <summary>

Robust.Server/GameStates/PvsSystem.cs

@@ -372,13 +372,6 @@ private void VerifySessionData(PvsSession pvsSession)
         {
             ref var data = ref pvsSession.DataMemory.GetRef(intPtr.Index);
             DebugTools.AssertEqual(data.LastSeen, _gameTiming.CurTick);
-
-            // if an entity is visible, its parents should always be visible.
-            if (_xformQuery.GetComponent(GetEntity(IndexToNetEntity(intPtr))).ParentUid is not {Valid: true} pUid)
-                continue;
-
-            DebugTools.Assert(toSendSet.Contains(GetNetEntity(pUid)),
-                $"Attempted to send an entity without sending it's parents. Entity: {ToPrettyString(pUid)}.");
         }
 
         pvsSession.PreviouslySent.TryGetValue(_gameTiming.CurTick - 1, out var lastSent);

Robust.Shared/CVars.cs

@@ -1611,7 +1611,8 @@ protected CVars()
         /// original exception rather than sending people on a wild-goose chase to find a non-existent bug.
         /// </remarks>
         public static readonly CVarDef<bool> ReplayIgnoreErrors =
-            CVarDef.Create("replay.ignore_errors", false, CVar.CLIENTONLY | CVar.ARCHIVE);
+            CVarDef.Create("replay.ignore_errors", false, CVar.CLIENTONLY);
+
         /*
          * CFG
          */

Robust.Shared/Console/Commands/DumpSerializerTypeMapCommand.cs

@@ -1,4 +1,4 @@
-using System.Linq;
+using System.IO;
 using Robust.Shared.IoC;
 using Robust.Shared.Serialization;
 
@@ -12,11 +12,16 @@ internal sealed class DumpSerializerTypeMapCommand : LocalizedCommands
 
     public override void Execute(IConsoleShell shell, string argStr, string[] args)
     {
-        foreach (var (type, index) in _robustSerializer.GetTypeMap().OrderBy(x => x.Value))
-        {
-            shell.WriteLine($"{index}: {type}");
-        }
+        var stream = new MemoryStream();
+        ((RobustSerializer)_robustSerializer).GetHashManifest(stream, true);
+        stream.Position = 0;
 
+        using var streamReader = new StreamReader(stream);
         shell.WriteLine($"Hash: {_robustSerializer.GetSerializableTypesHashString()}");
+        shell.WriteLine("Manifest:");
+        while (streamReader.ReadLine() is { } line)
+        {
+            shell.WriteLine(line);
+        }
     }
 }

Robust.Shared/GameObjects/Systems/SharedTransformSystem.Component.cs

@@ -722,10 +722,7 @@ internal void OnHandleState(EntityUid uid, TransformComponent xform, ref Compone
     {
         if (args.Current is TransformComponentState newState)
         {
-            var parent = GetEntity(newState.ParentID);
-            if (!parent.IsValid() && newState.ParentID.IsValid())
-                Log.Error($"Received transform component state with an unknown parent Id. Entity: {ToPrettyString(uid)}. Net parent: {newState.ParentID}");
-
+            var parent = EnsureEntity<TransformComponent>(newState.ParentID, uid);
             var oldAnchored = xform.Anchored;
 
             // update actual position data, if required