Merge pull request #66 from koculu/65-bug-count-method-fails-when-dis…

…k-segment-is-dropped

Attack to the disk segment during count to delay the drops.

src/ZoneTree.UnitTests/CountTests.cs

@@ -0,0 +1,124 @@
+using Tenray.ZoneTree.Comparers;
+using Tenray.ZoneTree.Options;
+using Tenray.ZoneTree.Serializers;
+
+namespace Tenray.ZoneTree.UnitTests;
+
+public sealed class CountTests
+{
+    [TestCase(DiskSegmentMode.MultiPartDiskSegment, 33, 77, false)]
+    [TestCase(DiskSegmentMode.MultiPartDiskSegment, 33, 77, true)]
+    public void CountRecordsDuringAMerge(
+        DiskSegmentMode diskSegmentMode,
+        int minimumRecordCount,
+        int maximumRecordCount,
+        bool useFullScan)
+    {
+        var dataPath = "data/CountRecordsDuringAMerge" + diskSegmentMode + minimumRecordCount + maximumRecordCount + useFullScan;
+        if (Directory.Exists(dataPath))
+            Directory.Delete(dataPath, true);
+
+        using var zoneTree = new ZoneTreeFactory<string, int>()
+            .SetIsValueDeletedDelegate((in int x) => x == -1)
+            .SetMarkValueDeletedDelegate((ref int x) => x = -1)
+            .SetMutableSegmentMaxItemCount(100)
+            .SetDataDirectory(dataPath)
+            .SetWriteAheadLogDirectory(dataPath)
+            .SetComparer(new StringCurrentCultureComparerAscending())
+            .SetKeySerializer(new Utf8StringSerializer())
+            .SetValueSerializer(new Int32Serializer())
+            .ConfigureDiskSegmentOptions(x =>
+            {
+                x.DiskSegmentMode = diskSegmentMode;
+                if (minimumRecordCount > 0)
+                    x.MinimumRecordCount = minimumRecordCount;
+                if (maximumRecordCount > 0)
+                    x.MaximumRecordCount = maximumRecordCount;
+            })
+            .OpenOrCreate();
+        var recordCount = 300;
+        PrepareData1(zoneTree, recordCount, "a");
+        zoneTree.Maintenance.MoveMutableSegmentForward();
+        zoneTree.Maintenance.StartMergeOperation()?.Join();
+        var stepSize = 37;
+        PrepareData1(zoneTree, recordCount);
+        var deletedCount = DeleteData1(zoneTree, recordCount, stepSize);
+
+        var expectedCount = recordCount * 8 - deletedCount;
+        var failedCount = 0;
+
+        var mergeThread = zoneTree.Maintenance.StartMergeOperation();
+
+        var isMergeFinished = false;
+        var task = Task.Factory.StartNew(() =>
+        {
+            while (!isMergeFinished)
+            {
+                var actualCount = useFullScan ? zoneTree.CountFullScan() : zoneTree.Count();
+                if (actualCount != expectedCount)
+                    ++failedCount;
+            }
+        });
+        mergeThread?.Join();
+        isMergeFinished = true;
+        task.Wait();
+        zoneTree.Maintenance.DestroyTree();
+
+        Assert.That(failedCount, Is.EqualTo(0));
+    }
+
+    static void PrepareData1(IZoneTree<string, int> zoneTree, int n, string postfix = "")
+    {
+        for (var i = 0; i < n; ++i)
+        {
+            var key = $"myprefix1|string{i}{postfix}";
+            zoneTree.Upsert(key, i);
+        }
+        for (var i = 0; i < n; ++i)
+        {
+            var key = $"myprefix2|string{i}{postfix}";
+            zoneTree.Upsert(key, i);
+        }
+        for (var i = 0; i < n; ++i)
+        {
+            var key = $"myprefix3|string{i}{postfix}";
+            zoneTree.Upsert(key, i);
+        }
+        for (var i = 0; i < n; ++i)
+        {
+            var key = $"myprefix4|string{i}{postfix}";
+            zoneTree.Upsert(key, i);
+        }
+    }
+
+    static int DeleteData1(IZoneTree<string, int> zoneTree, int n, int step)
+    {
+        var deletedCount = 0;
+        for (var i = step; i < n; i += step)
+        {
+            var key = $"myprefix1|string{i}";
+            zoneTree.ForceDelete(key);
+            ++deletedCount;
+
+        }
+        for (var i = step; i < n; i += step)
+        {
+            var key = $"myprefix2|string{i}";
+            zoneTree.ForceDelete(key);
+            ++deletedCount;
+        }
+        for (var i = step; i < n; i += step)
+        {
+            var key = $"myprefix3|string{i}";
+            zoneTree.ForceDelete(key);
+            ++deletedCount;
+        }
+        for (var i = step; i < n; i += step)
+        {
+            var key = $"myprefix4|string{i}";
+            zoneTree.ForceDelete(key);
+            ++deletedCount;
+        }
+        return deletedCount;
+    }
+}

src/ZoneTree.UnitTests/IteratorTests.cs

@@ -486,7 +486,7 @@ public void SeekIteratorsAfterMerge(
         int minimumRecordCount,
         int maximumRecordCount)
     {
-        var dataPath = "data/SeekIteratorsAfterMerge" + merge + diskSegmentMode + minimumRecordCount + maximumRecordCount; ;
+        var dataPath = "data/SeekIteratorsAfterMerge" + merge + diskSegmentMode + minimumRecordCount + maximumRecordCount;
         if (Directory.Exists(dataPath))
             Directory.Delete(dataPath, true);
 

src/ZoneTree/Core/ZoneTree.cs

@@ -283,37 +283,46 @@ public long Count()
         using var iterator = CreateInMemorySegmentsIterator(
             autoRefresh: false,
             includeDeletedRecords: true);
-
         IDiskSegment<TKey, TValue> diskSegment = null;
-        lock (ShortMergerLock)
-            lock (AtomicUpdateLock)
-            {
-                // 2 things to synchronize with
-                // MoveSegmentForward and disk merger segment swap.
-                diskSegment = DiskSegment;
-                iterator.Refresh();
-            }
-
-        if (!BottomSegmentQueue.IsEmpty)
-            return CountFullScan();
-
-        var count = diskSegment.Length;
-        while (iterator.Next())
+        try
         {
-            var hasKey = diskSegment.ContainsKey(iterator.CurrentKey);
-            var isValueDeleted = IsValueDeleted(iterator.CurrentValue);
-            if (hasKey)
-            {
-                if (isValueDeleted)
-                    --count;
-            }
-            else
+            lock (ShortMergerLock)
+                lock (AtomicUpdateLock)
+                {
+                    // 2 things to synchronize with
+                    // MoveSegmentForward and disk merger segment swap.
+                    diskSegment = DiskSegment;
+
+                    // ShortMergerLock ensures the diskSegment drop is not requested at the moment.
+                    // We can safely attach an iterator to the disk segment.
+                    diskSegment.AttachIterator();
+                    iterator.Refresh();
+                }
+
+            if (!BottomSegmentQueue.IsEmpty)
+                return CountFullScan();
+            var count = diskSegment.Length;
+            while (iterator.Next())
             {
-                if (!isValueDeleted)
-                    ++count;
+                var hasKey = diskSegment.ContainsKey(iterator.CurrentKey);
+                var isValueDeleted = IsValueDeleted(iterator.CurrentValue);
+                if (hasKey)
+                {
+                    if (isValueDeleted)
+                        --count;
+                }
+                else
+                {
+                    if (!isValueDeleted)
+                        ++count;
+                }
             }
+            return count;
+        }
+        finally
+        {
+            diskSegment.DetachIterator();
         }
-        return count;
     }
 
     public long CountFullScan()

src/ZoneTree/Directory.Build.props

@@ -5,8 +5,8 @@
     <Authors>Ahmed Yasin Koculu</Authors>
     <PackageId>ZoneTree</PackageId>
     <Title>ZoneTree</Title>
-    <ProductVersion>1.7.2.0</ProductVersion>
-    <Version>1.7.2.0</Version>
+    <ProductVersion>1.7.3.0</ProductVersion>
+    <Version>1.7.3.0</Version>
     <Authors>Ahmed Yasin Koculu</Authors>
     <AssemblyTitle>ZoneTree</AssemblyTitle>
     <Description>ZoneTree is a persistent, high-performance, transactional, ACID-compliant ordered key-value database for NET. It can operate in memory or on local/cloud storage.</Description>

src/ZoneTree/docs/ZoneTree/guide/quick-start.md

@@ -55,7 +55,8 @@ using var zoneTree = new ZoneTreeFactory<int, int>()
     .OpenOrCreate();
 
 using var maintainer = zoneTree.CreateMaintainer();
-for (var i = 0 ; i < 10_000_000; ++i){
+maintainer.EnableJobForCleaningInactiveCaches = true;
+for (var i = 0; i < 10_000_000; ++i){
    zoneTree.Insert(i, i+i);
 }
 
@@ -71,7 +72,7 @@ using var zoneTree = new ZoneTreeFactory<int, int>()
     .SetDataDirectory(dataPath)
     .OpenOrCreate();
 
-for (var i = 0 ; i < 10_000_000; ++i){
+for (var i = 0; i < 10_000_000; ++i){
    zoneTree.Insert(i, i+i);
 }