From 05c7a3c20d1d4555c893bdb99539221910a4c501 Mon Sep 17 00:00:00 2001 From: Ahmed Yasin Koculu Date: Fri, 9 Aug 2024 22:12:44 +0200 Subject: [PATCH 1/2] Attack to the disk segment during count to delay the drops. --- src/ZoneTree.UnitTests/CountTests.cs | 124 ++++++++++++++++++ src/ZoneTree.UnitTests/IteratorTests.cs | 2 +- src/ZoneTree/Core/ZoneTree.cs | 61 +++++---- .../docs/ZoneTree/guide/quick-start.md | 5 +- 4 files changed, 163 insertions(+), 29 deletions(-) create mode 100644 src/ZoneTree.UnitTests/CountTests.cs diff --git a/src/ZoneTree.UnitTests/CountTests.cs b/src/ZoneTree.UnitTests/CountTests.cs new file mode 100644 index 0000000..2f19480 --- /dev/null +++ b/src/ZoneTree.UnitTests/CountTests.cs @@ -0,0 +1,124 @@ +using Tenray.ZoneTree.Comparers; +using Tenray.ZoneTree.Options; +using Tenray.ZoneTree.Serializers; + +namespace Tenray.ZoneTree.UnitTests; + +public sealed class CountTests +{ + [TestCase(DiskSegmentMode.MultiPartDiskSegment, 33, 77, false)] + [TestCase(DiskSegmentMode.MultiPartDiskSegment, 33, 77, true)] + public void CountRecordsDuringAMerge( + DiskSegmentMode diskSegmentMode, + int minimumRecordCount, + int maximumRecordCount, + bool useFullScan) + { + var dataPath = "data/CountRecordsDuringAMerge" + diskSegmentMode + minimumRecordCount + maximumRecordCount + useFullScan; + if (Directory.Exists(dataPath)) + Directory.Delete(dataPath, true); + + using var zoneTree = new ZoneTreeFactory() + .SetIsValueDeletedDelegate((in int x) => x == -1) + .SetMarkValueDeletedDelegate((ref int x) => x = -1) + .SetMutableSegmentMaxItemCount(100) + .SetDataDirectory(dataPath) + .SetWriteAheadLogDirectory(dataPath) + .SetComparer(new StringCurrentCultureComparerAscending()) + .SetKeySerializer(new Utf8StringSerializer()) + .SetValueSerializer(new Int32Serializer()) + .ConfigureDiskSegmentOptions(x => + { + x.DiskSegmentMode = diskSegmentMode; + if (minimumRecordCount > 0) + x.MinimumRecordCount = minimumRecordCount; + if (maximumRecordCount > 0) + x.MaximumRecordCount = maximumRecordCount; + }) + .OpenOrCreate(); + var recordCount = 300; + PrepareData1(zoneTree, recordCount, "a"); + zoneTree.Maintenance.MoveMutableSegmentForward(); + zoneTree.Maintenance.StartMergeOperation()?.Join(); + var stepSize = 37; + PrepareData1(zoneTree, recordCount); + var deletedCount = DeleteData1(zoneTree, recordCount, stepSize); + + var expectedCount = recordCount * 8 - deletedCount; + var failedCount = 0; + + var mergeThread = zoneTree.Maintenance.StartMergeOperation(); + + var isMergeFinished = false; + var task = Task.Factory.StartNew(() => + { + while (!isMergeFinished) + { + var actualCount = useFullScan ? zoneTree.CountFullScan() : zoneTree.Count(); + if (actualCount != expectedCount) + ++failedCount; + } + }); + mergeThread?.Join(); + isMergeFinished = true; + task.Wait(); + zoneTree.Maintenance.DestroyTree(); + + Assert.That(failedCount, Is.EqualTo(0)); + } + + static void PrepareData1(IZoneTree zoneTree, int n, string postfix = "") + { + for (var i = 0; i < n; ++i) + { + var key = $"myprefix1|string{i}{postfix}"; + zoneTree.Upsert(key, i); + } + for (var i = 0; i < n; ++i) + { + var key = $"myprefix2|string{i}{postfix}"; + zoneTree.Upsert(key, i); + } + for (var i = 0; i < n; ++i) + { + var key = $"myprefix3|string{i}{postfix}"; + zoneTree.Upsert(key, i); + } + for (var i = 0; i < n; ++i) + { + var key = $"myprefix4|string{i}{postfix}"; + zoneTree.Upsert(key, i); + } + } + + static int DeleteData1(IZoneTree zoneTree, int n, int step) + { + var deletedCount = 0; + for (var i = step; i < n; i += step) + { + var key = $"myprefix1|string{i}"; + zoneTree.ForceDelete(key); + ++deletedCount; + + } + for (var i = step; i < n; i += step) + { + var key = $"myprefix2|string{i}"; + zoneTree.ForceDelete(key); + ++deletedCount; + } + for (var i = step; i < n; i += step) + { + var key = $"myprefix3|string{i}"; + zoneTree.ForceDelete(key); + ++deletedCount; + } + for (var i = step; i < n; i += step) + { + var key = $"myprefix4|string{i}"; + zoneTree.ForceDelete(key); + ++deletedCount; + } + return deletedCount; + } +} diff --git a/src/ZoneTree.UnitTests/IteratorTests.cs b/src/ZoneTree.UnitTests/IteratorTests.cs index 691911c..649636b 100644 --- a/src/ZoneTree.UnitTests/IteratorTests.cs +++ b/src/ZoneTree.UnitTests/IteratorTests.cs @@ -486,7 +486,7 @@ public void SeekIteratorsAfterMerge( int minimumRecordCount, int maximumRecordCount) { - var dataPath = "data/SeekIteratorsAfterMerge" + merge + diskSegmentMode + minimumRecordCount + maximumRecordCount; ; + var dataPath = "data/SeekIteratorsAfterMerge" + merge + diskSegmentMode + minimumRecordCount + maximumRecordCount; if (Directory.Exists(dataPath)) Directory.Delete(dataPath, true); diff --git a/src/ZoneTree/Core/ZoneTree.cs b/src/ZoneTree/Core/ZoneTree.cs index f5e4298..f04d788 100644 --- a/src/ZoneTree/Core/ZoneTree.cs +++ b/src/ZoneTree/Core/ZoneTree.cs @@ -283,37 +283,46 @@ public long Count() using var iterator = CreateInMemorySegmentsIterator( autoRefresh: false, includeDeletedRecords: true); - IDiskSegment diskSegment = null; - lock (ShortMergerLock) - lock (AtomicUpdateLock) - { - // 2 things to synchronize with - // MoveSegmentForward and disk merger segment swap. - diskSegment = DiskSegment; - iterator.Refresh(); - } - - if (!BottomSegmentQueue.IsEmpty) - return CountFullScan(); - - var count = diskSegment.Length; - while (iterator.Next()) + try { - var hasKey = diskSegment.ContainsKey(iterator.CurrentKey); - var isValueDeleted = IsValueDeleted(iterator.CurrentValue); - if (hasKey) - { - if (isValueDeleted) - --count; - } - else + lock (ShortMergerLock) + lock (AtomicUpdateLock) + { + // 2 things to synchronize with + // MoveSegmentForward and disk merger segment swap. + diskSegment = DiskSegment; + + // ShortMergerLock ensures the diskSegment drop is not requested at the moment. + // We can safely attach an iterator to the disk segment. + diskSegment.AttachIterator(); + iterator.Refresh(); + } + + if (!BottomSegmentQueue.IsEmpty) + return CountFullScan(); + var count = diskSegment.Length; + while (iterator.Next()) { - if (!isValueDeleted) - ++count; + var hasKey = diskSegment.ContainsKey(iterator.CurrentKey); + var isValueDeleted = IsValueDeleted(iterator.CurrentValue); + if (hasKey) + { + if (isValueDeleted) + --count; + } + else + { + if (!isValueDeleted) + ++count; + } } + return count; + } + finally + { + diskSegment.DetachIterator(); } - return count; } public long CountFullScan() diff --git a/src/ZoneTree/docs/ZoneTree/guide/quick-start.md b/src/ZoneTree/docs/ZoneTree/guide/quick-start.md index f127938..a9771c4 100644 --- a/src/ZoneTree/docs/ZoneTree/guide/quick-start.md +++ b/src/ZoneTree/docs/ZoneTree/guide/quick-start.md @@ -55,7 +55,8 @@ using var zoneTree = new ZoneTreeFactory() .OpenOrCreate(); using var maintainer = zoneTree.CreateMaintainer(); -for (var i = 0 ; i < 10_000_000; ++i){ +maintainer.EnableJobForCleaningInactiveCaches = true; +for (var i = 0; i < 10_000_000; ++i){ zoneTree.Insert(i, i+i); } @@ -71,7 +72,7 @@ using var zoneTree = new ZoneTreeFactory() .SetDataDirectory(dataPath) .OpenOrCreate(); -for (var i = 0 ; i < 10_000_000; ++i){ +for (var i = 0; i < 10_000_000; ++i){ zoneTree.Insert(i, i+i); } From 4a2e900c3f7a74bf3f3934c2970c52b6d16c0c3e Mon Sep 17 00:00:00 2001 From: Ahmed Yasin Koculu Date: Fri, 9 Aug 2024 22:44:00 +0200 Subject: [PATCH 2/2] Bump version. --- src/ZoneTree/Directory.Build.props | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ZoneTree/Directory.Build.props b/src/ZoneTree/Directory.Build.props index 4e3bfb6..aa26fa8 100644 --- a/src/ZoneTree/Directory.Build.props +++ b/src/ZoneTree/Directory.Build.props @@ -5,8 +5,8 @@ Ahmed Yasin Koculu ZoneTree ZoneTree - 1.7.2.0 - 1.7.2.0 + 1.7.3.0 + 1.7.3.0 Ahmed Yasin Koculu ZoneTree ZoneTree is a persistent, high-performance, transactional, ACID-compliant ordered key-value database for NET. It can operate in memory or on local/cloud storage.