From bcfcbd1db1af53f61b964ab3cb64d41c821f8f7f Mon Sep 17 00:00:00 2001 From: Maksim Dik Date: Tue, 9 Jul 2024 10:59:59 +0200 Subject: [PATCH 1/7] TECH-5558 Upgrade to MongoDb 5 --- ShardEqualizer/BsonSplitter.cs | 4 +-- .../ConfigRepositories/ChunkRepository.cs | 7 +++-- ShardEqualizer/ConfigServices/ChunkService.cs | 15 ++++++---- .../ShardedCollectionService.cs | 10 ++++++- ShardEqualizer/Models/Chunk.cs | 26 +++-------------- .../Models/ShardedCollectionInfo.cs | 8 ++--- ShardEqualizer/MongoClientBuilder.cs | 3 ++ ShardEqualizer/MongoCommands/CollStats.cs | 4 +-- .../MongoCommands/CollStatsResult.cs | 4 +-- .../Operations/BalancerStateOperation.cs | 6 +++- .../Operations/ConfigInitOperation.cs | 2 +- .../Operations/ConfigUpdateOperation.cs | 11 +------ .../Operations/EqualizeOperation.cs | 3 +- .../Operations/MergeChunksOperation.cs | 13 ++++++++- .../Operations/PresplitDataOperation.cs | 2 +- .../Operations/ScanJumboChunksOperation.cs | 28 +++++++++--------- ShardEqualizer/Properties/launchSettings.json | 29 +++++++++---------- paket.dependencies | 6 ++-- 18 files changed, 93 insertions(+), 88 deletions(-) diff --git a/ShardEqualizer/BsonSplitter.cs b/ShardEqualizer/BsonSplitter.cs index 1c351ab..10ca8ce 100644 --- a/ShardEqualizer/BsonSplitter.cs +++ b/ShardEqualizer/BsonSplitter.cs @@ -39,13 +39,13 @@ public static IList Split(BsonValue min, BsonValue max, int zonesCoun if (IsUuidLegacy(min) && IsUuidLegacy(max)) { return Split(min.AsByteArray, max.AsByteArray, zonesCount) - .Select(_ => (BsonValue)new BsonBinaryData(_, BsonBinarySubType.UuidLegacy, GuidRepresentation.CSharpLegacy)).ToList(); + .Select(_ => (BsonValue)new BsonBinaryData(_, BsonBinarySubType.UuidLegacy)).ToList(); } if (IsUuidStandard(min) && IsUuidStandard(max)) { return Split(min.AsByteArray, max.AsByteArray, zonesCount) - .Select(_ => (BsonValue)new BsonBinaryData(_, BsonBinarySubType.UuidStandard, GuidRepresentation.Standard)).ToList(); + .Select(_ => (BsonValue)new BsonBinaryData(_, BsonBinarySubType.UuidStandard)).ToList(); } if (min.IsObjectId && max.IsObjectId) diff --git a/ShardEqualizer/ConfigRepositories/ChunkRepository.cs b/ShardEqualizer/ConfigRepositories/ChunkRepository.cs index 0fec545..7f82114 100644 --- a/ShardEqualizer/ConfigRepositories/ChunkRepository.cs +++ b/ShardEqualizer/ConfigRepositories/ChunkRepository.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; @@ -20,9 +21,9 @@ public Task Find(string id) return _coll.Find(_ => _.Id == id).SingleOrDefaultAsync(); } - public Filtered ByNamespace(CollectionNamespace ns) + public Filtered ByUuid(Guid namespaceId) { - return new Filtered(_coll, Builders.Filter.Eq(_ => _.Namespace, ns)); + return new Filtered(_coll, Builders.Filter.Eq(_ => _.Uuid, namespaceId)); } public class Filtered @@ -41,7 +42,7 @@ public async Task> Find(CancellationToken token) return await _coll.FindAsync(_filter, new FindOptions() { Sort = Builders.Sort - .Ascending(_ => _.Namespace) + .Ascending(_ => _.Uuid) .Ascending(_ => _.Min) }, token); } diff --git a/ShardEqualizer/ConfigServices/ChunkService.cs b/ShardEqualizer/ConfigServices/ChunkService.cs index d32c36a..01bde7e 100644 --- a/ShardEqualizer/ConfigServices/ChunkService.cs +++ b/ShardEqualizer/ConfigServices/ChunkService.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.Linq; using System.Threading; @@ -17,20 +18,22 @@ public class ChunkService private readonly ChunkRepository _repo; private readonly ProgressRenderer _progressRenderer; private readonly INsLocalStore _store; + private readonly ShardedCollectionService _shardedCollectionService; public ChunkService( ChunkRepository repo, ProgressRenderer progressRenderer, - LocalStoreProvider storeProvider) + LocalStoreProvider storeProvider, ShardedCollectionService shardedCollectionService) { _repo = repo; _progressRenderer = progressRenderer; + _shardedCollectionService = shardedCollectionService; _store = storeProvider.Get("chunks", uploadChunks); } - public async Task> Get(IEnumerable nss, CancellationToken token) + public async Task> Get(IEnumerable ns, CancellationToken token) { - var nsList = nss.ToList(); + var nsList = ns.ToList(); await using var reporter = _progressRenderer.Start($"Load chunks", nsList.Count); { @@ -49,9 +52,11 @@ public async Task> Get(IEn private async Task uploadChunks(CollectionNamespace ns, CancellationToken t) { - var expectedCount = await _repo.ByNamespace(ns).Count(t); + var collection = await _shardedCollectionService.Get(ns, t); + + var expectedCount = await _repo.ByUuid(collection.Uuid).Count(t); var chunks = new List((int)expectedCount); - using var cursor = await _repo.ByNamespace(ns).Find(t); + using var cursor = await _repo.ByUuid(collection.Uuid).Find(t); while (await cursor.MoveNextAsync(t)) chunks.AddRange(cursor.Current.Select(_ => new ChunkInfo(_))); diff --git a/ShardEqualizer/ConfigServices/ShardedCollectionService.cs b/ShardEqualizer/ConfigServices/ShardedCollectionService.cs index 6debae3..beac5e6 100644 --- a/ShardEqualizer/ConfigServices/ShardedCollectionService.cs +++ b/ShardEqualizer/ConfigServices/ShardedCollectionService.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.Linq; using System.Threading; @@ -34,12 +35,19 @@ public async Task Get(CollectionNamespace ns, CancellationToken token) + { + var container = await _store.Get(token); + return container.ShardedCollection.GetValueOrDefault(ns) + ?? throw new InvalidOperationException($"Cannot find collection by namespace '{ns}'"); + } + private async Task uploadData(CancellationToken token) { await using var reporter = _progressRenderer.Start("Load sharded collections"); var result = await _repo.FindAll(false, token); - var message = $"found {result.Count(x => !x.Dropped)} collections"; + var message = $"found {result.Count} collections"; reporter.SetCompleteMessage(message); return new Container() diff --git a/ShardEqualizer/Models/Chunk.cs b/ShardEqualizer/Models/Chunk.cs index 3b4c703..99a63fd 100644 --- a/ShardEqualizer/Models/Chunk.cs +++ b/ShardEqualizer/Models/Chunk.cs @@ -1,23 +1,17 @@ -using System.Collections.Generic; +using System; using MongoDB.Bson; using MongoDB.Bson.Serialization.Attributes; -using MongoDB.Driver; namespace ShardEqualizer.Models { + [BsonIgnoreExtraElements] public class Chunk { [BsonId] public BsonValue Id { get; set; } - [BsonElement("lastmodEpoch"), BsonRequired] - public ObjectId LastmodEpoch { get; set; } - - [BsonElement("lastmod"), BsonRequired] - public BsonTimestamp Lastmod { get; set; } - - [BsonElement("ns"), BsonRequired] - public CollectionNamespace Namespace { get; set; } + [BsonElement("uuid"), BsonRequired, BsonGuidRepresentation(GuidRepresentation.Standard)] + public Guid Uuid { get; set; } [BsonElement("min"), BsonRequired] public BsonBound Min { get; set; } @@ -30,17 +24,5 @@ public class Chunk [BsonElement("jumbo"), BsonIgnoreIfDefault] public bool Jumbo { get; set; } - - [BsonElement("history"), BsonIgnoreIfNull] //BsonIgnoreIfNull - required for backward compatibility with MongoDB v3.6 - public IReadOnlyList History { get; set; } - - public class HistoryEntry - { - [BsonElement("shard"), BsonRequired] - public ShardIdentity Shard { get; set; } - - [BsonElement("validAfter"), BsonRequired] - public BsonTimestamp ValidAfter { get; set; } - } } } diff --git a/ShardEqualizer/Models/ShardedCollectionInfo.cs b/ShardEqualizer/Models/ShardedCollectionInfo.cs index 4e581c0..e488792 100644 --- a/ShardEqualizer/Models/ShardedCollectionInfo.cs +++ b/ShardEqualizer/Models/ShardedCollectionInfo.cs @@ -5,6 +5,7 @@ namespace ShardEqualizer.Models { + [BsonIgnoreExtraElements] public class ShardedCollectionInfo { [BsonId] @@ -16,9 +17,6 @@ public class ShardedCollectionInfo [BsonElement("lastmod"), BsonRequired] public DateTime Lastmod { get; private set; } - [BsonElement("dropped"), BsonRequired] - public bool Dropped { get; private set; } - [BsonElement("key"), BsonIgnoreIfNull] public BsonDocument Key { get; private set; } @@ -28,8 +26,8 @@ public class ShardedCollectionInfo [BsonElement("noBalance"), BsonIgnoreIfNull] public bool? NoBalance { get; private set; } - [BsonElement("uuid"), BsonIgnoreIfNull] - public Guid? UUID { get; private set; } + [BsonElement("uuid"), BsonRequired, BsonGuidRepresentation(GuidRepresentation.Standard)] + public Guid Uuid { get; private set; } [BsonElement("distributionMode"), BsonIgnoreIfNull] private string DistributionMode { get; set; } diff --git a/ShardEqualizer/MongoClientBuilder.cs b/ShardEqualizer/MongoClientBuilder.cs index a1956ea..97dca0f 100644 --- a/ShardEqualizer/MongoClientBuilder.cs +++ b/ShardEqualizer/MongoClientBuilder.cs @@ -1,4 +1,5 @@ using System.Linq; +using MongoDB.Bson; using MongoDB.Driver; using NLog; using ShardEqualizer.Config; @@ -20,6 +21,8 @@ public MongoClientBuilder(ConnectionConfig connectionConfig, ProgressRenderer pr public IMongoClient Build() { + BsonDefaults.GuidRepresentationMode = GuidRepresentationMode.V3; + _progressRenderer.WriteLine($"Connecting to {_connectionConfig.Servers}"); _log.Info("Connecting to {0}", _connectionConfig.Servers); diff --git a/ShardEqualizer/MongoCommands/CollStats.cs b/ShardEqualizer/MongoCommands/CollStats.cs index e37592b..57cbe49 100644 --- a/ShardEqualizer/MongoCommands/CollStats.cs +++ b/ShardEqualizer/MongoCommands/CollStats.cs @@ -74,7 +74,7 @@ public class CollStats: CommandResult [BsonElement("scaleFactor")] public double? ScaleFactor; - //[BsonExtraElements] - //public BsonDocument ExtraElements; + [BsonExtraElements] + public BsonDocument ExtraElements; } } diff --git a/ShardEqualizer/MongoCommands/CollStatsResult.cs b/ShardEqualizer/MongoCommands/CollStatsResult.cs index e132347..67bc987 100644 --- a/ShardEqualizer/MongoCommands/CollStatsResult.cs +++ b/ShardEqualizer/MongoCommands/CollStatsResult.cs @@ -78,7 +78,7 @@ public class CollStatsResult: CommandResult [BsonElement("totalSize")] public long? TotalSize; - //[BsonExtraElements] - //public BsonDocument ExtraElements; + [BsonExtraElements] + public BsonDocument ExtraElements; } } diff --git a/ShardEqualizer/Operations/BalancerStateOperation.cs b/ShardEqualizer/Operations/BalancerStateOperation.cs index 58d2a3a..f60b43e 100644 --- a/ShardEqualizer/Operations/BalancerStateOperation.cs +++ b/ShardEqualizer/Operations/BalancerStateOperation.cs @@ -15,6 +15,7 @@ public class BalancerStateOperation: IOperation { private readonly ShardListService _shardListService; private readonly TagRangeService _tagRangeService; + private readonly ShardedCollectionService _shardedCollectionService; private readonly ChunkRepository _chunkRepo; private readonly IReadOnlyList _intervals; private readonly ProgressRenderer _progressRenderer; @@ -22,12 +23,14 @@ public class BalancerStateOperation: IOperation public BalancerStateOperation( ShardListService shardListService, TagRangeService tagRangeService, + ShardedCollectionService shardedCollectionService, ChunkRepository chunkRepo, IReadOnlyList intervals, ProgressRenderer progressRenderer) { _shardListService = shardListService; _tagRangeService = tagRangeService; + _shardedCollectionService = shardedCollectionService; _chunkRepo = chunkRepo; _intervals = intervals; _progressRenderer = progressRenderer; @@ -51,7 +54,8 @@ private async Task> scanInterval(Interval interval, IReadOnl //TODO here supports multiple shards to scan all collections in the future - var unMovedChunks = await (await _chunkRepo.ByNamespace(interval.Namespace) + var collectionInfo = await _shardedCollectionService.Get(interval.Namespace, token); + var unMovedChunks = await (await _chunkRepo.ByUuid(collectionInfo.Uuid) .From(tagRange.Min).To(tagRange.Max).NoJumbo().ExcludeShards(validShards).Find(token)) .ToListAsync(token); diff --git a/ShardEqualizer/Operations/ConfigInitOperation.cs b/ShardEqualizer/Operations/ConfigInitOperation.cs index c67ae13..fefb4e8 100644 --- a/ShardEqualizer/Operations/ConfigInitOperation.cs +++ b/ShardEqualizer/Operations/ConfigInitOperation.cs @@ -80,7 +80,7 @@ public async Task Run(CancellationToken token) { Servers = _connectionConfig.Servers, DefaultZones = string.Join(",", zones.Select(_ => _.zoneName)), - ShardedCollections = shardedCollections.Values.Where(_ => !_.Dropped).Select(_ => _.Id.ToString()).ToList(), //TODO exclude hashed keys + ShardedCollections = shardedCollections.Values.Select(_ => _.Id.ToString()).ToList(), //TODO exclude hashed keys SecretFileName = secretFileName }; diff --git a/ShardEqualizer/Operations/ConfigUpdateOperation.cs b/ShardEqualizer/Operations/ConfigUpdateOperation.cs index f58862e..3edc245 100644 --- a/ShardEqualizer/Operations/ConfigUpdateOperation.cs +++ b/ShardEqualizer/Operations/ConfigUpdateOperation.cs @@ -40,11 +40,8 @@ private void analyseIntervals() { foreach (var ns in _intervals.Select(_ => _.Namespace)) { - if (_shardedCollections.TryGetValue(ns, out var shardedCollection)) + if (_shardedCollections.ContainsKey(ns)) { - if(shardedCollection.Dropped) - Console.WriteLine("\tcollection '{0}' dropped", ns); - _shardedCollections.Remove(ns); } else @@ -52,12 +49,6 @@ private void analyseIntervals() Console.WriteLine("\tcollection '{0}' not sharded", ns); } } - - foreach (var ns in _shardedCollections.Keys.ToList()) - { - if(_shardedCollections[ns].Dropped) - _shardedCollections.Remove(ns); - } } public async Task Run(CancellationToken token) diff --git a/ShardEqualizer/Operations/EqualizeOperation.cs b/ShardEqualizer/Operations/EqualizeOperation.cs index f8b5d6b..73c5bfa 100644 --- a/ShardEqualizer/Operations/EqualizeOperation.cs +++ b/ShardEqualizer/Operations/EqualizeOperation.cs @@ -277,7 +277,8 @@ public async Task Run(CancellationToken token) _tagRangesByNs = _adjustableIntervals.ToDictionary(_ => _.Namespace, _ => allTagRangesByNs[_.Namespace].InRange(_.Min, _.Max)); - var allChunksByNs = await _chunkService.Get(_adjustableIntervals.Select(_ => _.Namespace), token); + var allChunksByNs = await _chunkService.Get(_adjustableIntervals.Select(_ =>_.Namespace), token); + _chunksByCollection = _adjustableIntervals.ToDictionary(_ => _.Namespace, _ => (IReadOnlyList) allChunksByNs[_.Namespace].FromInterval(_.Min, _.Max)); diff --git a/ShardEqualizer/Operations/MergeChunksOperation.cs b/ShardEqualizer/Operations/MergeChunksOperation.cs index d4aecba..c9dae6e 100644 --- a/ShardEqualizer/Operations/MergeChunksOperation.cs +++ b/ShardEqualizer/Operations/MergeChunksOperation.cs @@ -18,12 +18,14 @@ public class MergeChunksOperation : IOperation private readonly ProgressRenderer _progressRenderer; private readonly ShardListService _shardListService; private readonly TagRangeService _tagRangeService; + private readonly ShardedCollectionService _shardedCollectionService; private readonly ChunkRepository _chunkRepo; private readonly CommandPlanWriter _commandPlanWriter; public MergeChunksOperation( ShardListService shardListService, TagRangeService tagRangeService, + ShardedCollectionService shardedCollectionService, ChunkRepository chunkRepo, //UNDONE use ChunkService IReadOnlyList intervals, ProgressRenderer progressRenderer, @@ -31,6 +33,7 @@ public MergeChunksOperation( { _shardListService = shardListService; _tagRangeService = tagRangeService; + _shardedCollectionService = shardedCollectionService; _chunkRepo = chunkRepo; _commandPlanWriter = commandPlanWriter; @@ -46,9 +49,17 @@ private async Task, int>> mergeInterval(IDictionary(); var mergedChunks = 0; + + if (!shardByTag.TryGetValue(zone.TagRange.Tag, out var shard)) + { + _progressRenderer.WriteLine($"Shard for tag '{zone.TagRange.Tag}' not found"); + return new Tuple, int>(new List(), 0); + } + var validShardId = shardByTag[zone.TagRange.Tag].Id; - var mergeCandidates = await (await _chunkRepo.ByNamespace(zone.Interval.Namespace) + var collectionInfo = await _shardedCollectionService.Get(zone.Interval.Namespace, token); + var mergeCandidates = await (await _chunkRepo.ByUuid(collectionInfo.Uuid) .From(zone.TagRange.Min).To(zone.TagRange.Max).NoJumbo().ByShards(new [] { validShardId }).Find(token)) .ToListAsync(token); diff --git a/ShardEqualizer/Operations/PresplitDataOperation.cs b/ShardEqualizer/Operations/PresplitDataOperation.cs index 0b5aab9..50b1b68 100644 --- a/ShardEqualizer/Operations/PresplitDataOperation.cs +++ b/ShardEqualizer/Operations/PresplitDataOperation.cs @@ -102,7 +102,7 @@ private void presplitInterval(Interval interval, TagRangeCommandBuffer buffer) { var collInfo = _shardedCollectionByNs[interval.Namespace]; - if (collInfo == null || collInfo.Dropped) + if (collInfo == null) throw new InvalidOperationException($"collection {interval.Namespace.FullName} not sharded"); if (!interval.Adjustable) diff --git a/ShardEqualizer/Operations/ScanJumboChunksOperation.cs b/ShardEqualizer/Operations/ScanJumboChunksOperation.cs index 75ab45c..a4b9e36 100644 --- a/ShardEqualizer/Operations/ScanJumboChunksOperation.cs +++ b/ShardEqualizer/Operations/ScanJumboChunksOperation.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Threading; @@ -34,36 +33,39 @@ public ScanJumboChunksOperation( _chunkRepo = chunkRepo; } - private async Task> findJumboChunks(IReadOnlyCollection namespaces, CancellationToken token) + private async Task>> findJumboChunks(IReadOnlyCollection namespaces, CancellationToken token) { await using var reporter = _progressRenderer.Start($"Find jumbo chunks", namespaces.Count); { - async Task> loadCollChunks(CollectionNamespace ns, CancellationToken t) + async Task<(CollectionNamespace ns, List chunks)> loadCollChunks(CollectionNamespace ns, CancellationToken t) { + var collection = await _shardedCollectionService.Get(ns, token); var allChunks = await (await _chunkRepo - .ByNamespace(ns) + .ByUuid(collection.Uuid) .OnlyJumbo() .Find(t)).ToListAsync(t); reporter.Increment(); - return allChunks; + return (ns, allChunks); } var results = await namespaces.ParallelsAsync(loadCollChunks, 16, token); - var jumboChunks = results.SelectMany(_ => _).ToList(); - reporter.SetCompleteMessage($"found {jumboChunks.Count} chunks."); + var jumboChunks = results.ToDictionary(x => x.ns, x => x.chunks); + reporter.SetCompleteMessage($"found {jumboChunks.SelectMany(x => x.Value).Count()} chunks."); return jumboChunks; } } - private async Task> scanJumboChunks(List jumboChunks, + private async Task> scanJumboChunks( + IReadOnlyDictionary> jumboChunksInfo, IReadOnlyDictionary collectionsInfo, CancellationToken token) { - await using var reporter = _progressRenderer.Start($"Scan jumbo chunks", jumboChunks.Count); + await using var reporter = _progressRenderer.Start($"Scan jumbo chunks", jumboChunksInfo.SelectMany(x => x.Value).Count()); { - async Task scanChunk(Chunk chunk, CancellationToken t) + async Task scanChunk((CollectionNamespace ns, Chunk chunk) chunkInfo, CancellationToken t) { - var ns = chunk.Namespace; + var (ns, chunk) = chunkInfo; + var db = _mongoClient.GetDatabase(ns.DatabaseNamespace.DatabaseName); var collInfo = collectionsInfo[ns]; @@ -79,7 +81,7 @@ async Task scanChunk(Chunk chunk, CancellationToken t) return null; } - var results = await jumboChunks.ParallelsAsync(scanChunk, 32, token); + var results = await jumboChunksInfo.SelectMany(x => x.Value.Select(y => (x.Key,y))).ToList().ParallelsAsync(scanChunk, 32, token); return results.Where(_ => _ != null).ToList(); } @@ -127,7 +129,7 @@ private void renderResults(ICollection chunkDataSizes) public async Task Run(CancellationToken token) { var collectionsInfo = await _shardedCollectionService.Get(token); - var shardedNamespaces = collectionsInfo.Values.Where(_ => !_.Dropped).Select(_ => _.Id).ToList(); + var shardedNamespaces = collectionsInfo.Values.Select(_ => _.Id).ToList(); var jumboChunks = await findJumboChunks(shardedNamespaces, token); var chunkDataSizes = await scanJumboChunks(jumboChunks, collectionsInfo, token); diff --git a/ShardEqualizer/Properties/launchSettings.json b/ShardEqualizer/Properties/launchSettings.json index c6f3119..e55cc63 100644 --- a/ShardEqualizer/Properties/launchSettings.json +++ b/ShardEqualizer/Properties/launchSettings.json @@ -10,59 +10,59 @@ }, "config-init": { "commandName": "Project", - "commandLineArgs": "config-init --config=newConfig.xml -hlocalhost:27017,localhost:27018" + "commandLineArgs": "config-init --config=newConfig.xml -h localhost:27017,localhost:27018" }, "config-update": { "commandName": "Project", - "commandLineArgs": "config-update -cdev.xml" + "commandLineArgs": "config-update -c dev.xml" }, "deviation": { "commandName": "Project", - "commandLineArgs": "deviation -cdev.xml -sG --format=md --layouts=\"default,balance\"" + "commandLineArgs": "deviation -c dev.xml -sG --format=md --layouts=\"default,balance\"" }, "scan-jumbo": { "commandName": "Project", - "commandLineArgs": "scan-jumbo -cdev.xml" + "commandLineArgs": "scan-jumbo -c dev.xml" }, "equalize planOnly": { "commandName": "Project", - "commandLineArgs": "equalize -cdev.xml --dry-run" + "commandLineArgs": "equalize -c dev.xml --dry-run" }, "equalize planOnly from cache": { "commandName": "Project", - "commandLineArgs": "equalize -cdev.xml --dry-run --store-mode=rw" + "commandLineArgs": "equalize -c dev.xml --dry-run --store-mode=rw" }, "equalize": { "commandName": "Project", - "commandLineArgs": "equalize -cdev.xml" + "commandLineArgs": "equalize -c dev.xml" }, "equalize correctionPercent": { "commandName": "Project", - "commandLineArgs": "equalize -cdev.xml --correction-percent=20" + "commandLineArgs": "equalize -c dev.xml --correction-percent=20" }, "equalize limited": { "commandName": "Project", - "commandLineArgs": "equalize -cdev.xml --move-limit=51200" + "commandLineArgs": "equalize -c dev.xml --move-limit=51200" }, "equalize offline": { "commandName": "Project", - "commandLineArgs": "equalize -cdev.xml --move-limit=200" + "commandLineArgs": "equalize -c dev.xml --move-limit=200" }, "merge": { "commandName": "Project", - "commandLineArgs": "merge -cdev.xml" + "commandLineArgs": "merge -c dev.xml" }, "presplit": { "commandName": "Project", - "commandLineArgs": "presplit -cdev.xml" + "commandLineArgs": "presplit -c dev.xml" }, "presplit renew": { "commandName": "Project", - "commandLineArgs": "presplit -cdev.xml --renew" + "commandLineArgs": "presplit -c dev.xml --renew" }, "balancer": { "commandName": "Project", - "commandLineArgs": "balancer -cdev.xml" + "commandLineArgs": "balancer -c dev.xml" }, "balancer help": { "commandName": "Project", @@ -83,7 +83,6 @@ "config-init help": { "commandName": "Project", "commandLineArgs": "config-init --help" - }, } } } diff --git a/paket.dependencies b/paket.dependencies index 851f19d..bc87e0d 100644 --- a/paket.dependencies +++ b/paket.dependencies @@ -5,9 +5,9 @@ framework: auto-detect nuget Accord.Math ~> 3.8 nuget CommandLineParser ~> 2.8 -nuget MongoDB.Bson ~> 2.12 -nuget MongoDB.Driver ~> 2.12 -nuget MongoDB.Driver.Core ~> 2.12 +nuget MongoDB.Bson ~> 2.25 +nuget MongoDB.Driver ~> 2.25 +nuget MongoDB.Driver.Core ~> 2.25 nuget NConfiguration ~> 3.0 nuget Newtonsoft.Json ~> 12.0 nuget Ninject ~> 3.3 From 2e8b5cdf47225205cc820174bd66e356ecf5be32 Mon Sep 17 00:00:00 2001 From: Maksim Dik Date: Thu, 11 Jul 2024 11:06:04 +0200 Subject: [PATCH 2/7] Cleanup --- ShardEqualizer/MongoCommands/CollStats.cs | 4 +--- ShardEqualizer/MongoCommands/CollStatsResult.cs | 4 +--- ShardEqualizer/Operations/EqualizeOperation.cs | 1 - 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/ShardEqualizer/MongoCommands/CollStats.cs b/ShardEqualizer/MongoCommands/CollStats.cs index 57cbe49..7e377f4 100644 --- a/ShardEqualizer/MongoCommands/CollStats.cs +++ b/ShardEqualizer/MongoCommands/CollStats.cs @@ -6,6 +6,7 @@ namespace ShardEqualizer.MongoCommands { + [BsonIgnoreExtraElements] public class CollStats: CommandResult { [BsonElement("ns"), BsonRequired] @@ -73,8 +74,5 @@ public class CollStats: CommandResult [BsonElement("scaleFactor")] public double? ScaleFactor; - - [BsonExtraElements] - public BsonDocument ExtraElements; } } diff --git a/ShardEqualizer/MongoCommands/CollStatsResult.cs b/ShardEqualizer/MongoCommands/CollStatsResult.cs index 67bc987..56e1681 100644 --- a/ShardEqualizer/MongoCommands/CollStatsResult.cs +++ b/ShardEqualizer/MongoCommands/CollStatsResult.cs @@ -7,6 +7,7 @@ namespace ShardEqualizer.MongoCommands { + [BsonIgnoreExtraElements] public class CollStatsResult: CommandResult { [BsonElement("primary"), BsonIgnoreIfNull] @@ -77,8 +78,5 @@ public class CollStatsResult: CommandResult [BsonElement("totalSize")] public long? TotalSize; - - [BsonExtraElements] - public BsonDocument ExtraElements; } } diff --git a/ShardEqualizer/Operations/EqualizeOperation.cs b/ShardEqualizer/Operations/EqualizeOperation.cs index 73c5bfa..78d4fd4 100644 --- a/ShardEqualizer/Operations/EqualizeOperation.cs +++ b/ShardEqualizer/Operations/EqualizeOperation.cs @@ -278,7 +278,6 @@ public async Task Run(CancellationToken token) _ => allTagRangesByNs[_.Namespace].InRange(_.Min, _.Max)); var allChunksByNs = await _chunkService.Get(_adjustableIntervals.Select(_ =>_.Namespace), token); - _chunksByCollection = _adjustableIntervals.ToDictionary(_ => _.Namespace, _ => (IReadOnlyList) allChunksByNs[_.Namespace].FromInterval(_.Min, _.Max)); From 8096267513973af3bd3dcbfb7d3d8faa78269a4c Mon Sep 17 00:00:00 2001 From: Maksim Dik Date: Thu, 11 Jul 2024 11:06:20 +0200 Subject: [PATCH 3/7] Publish as tool --- ShardEqualizer/ShardEqualizer.csproj | 13 +++++++------ build.ps1 | 6 ++++++ pack.ps1 | 5 +++++ push_nuget.ps1 | 21 +++++++++++++++++++++ 4 files changed, 39 insertions(+), 6 deletions(-) create mode 100644 build.ps1 create mode 100644 pack.ps1 create mode 100644 push_nuget.ps1 diff --git a/ShardEqualizer/ShardEqualizer.csproj b/ShardEqualizer/ShardEqualizer.csproj index fabd06c..e009dcc 100644 --- a/ShardEqualizer/ShardEqualizer.csproj +++ b/ShardEqualizer/ShardEqualizer.csproj @@ -8,16 +8,17 @@ ShardEqualizer true true - ShardEqualizer + SmartcatShardEqualizer Anatoliy Koperin <a.koperin@gmail.com> Tool for aligning the size of stored data on shards in a MongoDB cluster - https://github.com/ExM/ShardEqualizer - https://github.com/ExM/ShardEqualizer/blob/develop/LICENSE.txt - https://github.com/ExM/ShardEqualizer + http://gitlab.sc.local/commontools/forks/shardequalizer + Smartcat.Fork.ShardEqualizer + http://gitlab.sc.local/commontools/forks/shardequalizer/blob/develop/LICENSE.txt + http://gitlab.sc.local/commontools/forks/shardequalizer mongodb sharding 1.0.0 - 1.0.0-beta.7 - 1.0.0-beta.7 + 1.0.0-beta-8-TECH-5558 + 1.0.0-beta-8-TECH-5558 diff --git a/build.ps1 b/build.ps1 new file mode 100644 index 0000000..1ec4c31 --- /dev/null +++ b/build.ps1 @@ -0,0 +1,6 @@ +$ErrorActionPreference = "Stop" +$mainFolder = Resolve-Path (Split-Path -Path $MyInvocation.MyCommand.Definition -Parent) + +& dotnet tool restore +& dotnet paket restore +& dotnet build -c Release diff --git a/pack.ps1 b/pack.ps1 new file mode 100644 index 0000000..11b8572 --- /dev/null +++ b/pack.ps1 @@ -0,0 +1,5 @@ +$ErrorActionPreference = "Stop" +$mainFolder = Resolve-Path (Split-Path -Path $MyInvocation.MyCommand.Definition -Parent) + +& "$mainFolder/build.ps1" +& dotnet pack --output "$mainFolder/Published" diff --git a/push_nuget.ps1 b/push_nuget.ps1 new file mode 100644 index 0000000..c490606 --- /dev/null +++ b/push_nuget.ps1 @@ -0,0 +1,21 @@ +param ( + [string]$apikey = "" +) + +$ErrorActionPreference = "Stop" +$mainFolder = Resolve-Path (Split-Path -Path $MyInvocation.MyCommand.Definition -Parent) +if ($apikey -Eq "") +{ + Get-ChildItem -Path "$mainFolder\Published\*.nupkg" | foreach { & dotnet nuget push $_ -s http://nexus-repo.sc.local/repository/nuget/ } +} +else +{ + Get-ChildItem -Path "$mainFolder\Published\*.nupkg" | foreach { & dotnet nuget push $_ -s http://nexus-repo.sc.local/repository/nuget/ -k $apikey } +} + +if ($lastexitcode -ne 0) +{ + Write-Host "Error while pushing" -ForegroundColor Red + Exit 1 +} + From 7aae679c89b2521bd6d68a911899a9dc524165af Mon Sep 17 00:00:00 2001 From: Maksim Dik Date: Thu, 25 Jul 2024 11:54:10 +0200 Subject: [PATCH 4/7] Fix Guid presentation --- ShardEqualizer/BsonSplitter.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ShardEqualizer/BsonSplitter.cs b/ShardEqualizer/BsonSplitter.cs index 10ca8ce..1c351ab 100644 --- a/ShardEqualizer/BsonSplitter.cs +++ b/ShardEqualizer/BsonSplitter.cs @@ -39,13 +39,13 @@ public static IList Split(BsonValue min, BsonValue max, int zonesCoun if (IsUuidLegacy(min) && IsUuidLegacy(max)) { return Split(min.AsByteArray, max.AsByteArray, zonesCount) - .Select(_ => (BsonValue)new BsonBinaryData(_, BsonBinarySubType.UuidLegacy)).ToList(); + .Select(_ => (BsonValue)new BsonBinaryData(_, BsonBinarySubType.UuidLegacy, GuidRepresentation.CSharpLegacy)).ToList(); } if (IsUuidStandard(min) && IsUuidStandard(max)) { return Split(min.AsByteArray, max.AsByteArray, zonesCount) - .Select(_ => (BsonValue)new BsonBinaryData(_, BsonBinarySubType.UuidStandard)).ToList(); + .Select(_ => (BsonValue)new BsonBinaryData(_, BsonBinarySubType.UuidStandard, GuidRepresentation.Standard)).ToList(); } if (min.IsObjectId && max.IsObjectId) From 258cd5146b25a993ea13b376bcaeb9c0d95744b1 Mon Sep 17 00:00:00 2001 From: Maksim Dik Date: Thu, 25 Jul 2024 17:19:27 +0200 Subject: [PATCH 5/7] Fix concurrency writes to files --- ShardEqualizer/LocalStoring/BaseLocalStore.cs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/ShardEqualizer/LocalStoring/BaseLocalStore.cs b/ShardEqualizer/LocalStoring/BaseLocalStore.cs index fd0121b..06360fb 100644 --- a/ShardEqualizer/LocalStoring/BaseLocalStore.cs +++ b/ShardEqualizer/LocalStoring/BaseLocalStore.cs @@ -11,6 +11,7 @@ public abstract class BaseLocalStore { private readonly bool _read; private readonly bool _write; + private readonly object _lock = new object() ; protected BaseLocalStore(bool read, bool write) { @@ -30,11 +31,14 @@ protected async Task Get(string fileName, Func> up if (_write) { - await using var stream = File.Open(fileName, FileMode.Create); - using var bsonWriter = new BsonBinaryWriter(stream); - BsonSerializer.Serialize(bsonWriter, data); - bsonWriter.Flush(); - await stream.FlushAsync(token); + lock (_lock) + { + using var stream = File.Open(fileName, FileMode.Create); + using var bsonWriter = new BsonBinaryWriter(stream); + BsonSerializer.Serialize(bsonWriter, data); + bsonWriter.Flush(); + stream.Flush(true); + } } return data; From bcadccc02b6f94e55cf510b11417c079e82dcc78 Mon Sep 17 00:00:00 2001 From: Maksim Dik Date: Mon, 24 Mar 2025 09:16:05 +0100 Subject: [PATCH 6/7] Cleanup --- .../Operations/EqualizeOperation.cs | 2 +- ShardEqualizer/ShardEqualizer.csproj | 13 ++++++------ build.ps1 | 6 ------ pack.ps1 | 5 ----- push_nuget.ps1 | 21 ------------------- 5 files changed, 7 insertions(+), 40 deletions(-) delete mode 100644 build.ps1 delete mode 100644 pack.ps1 delete mode 100644 push_nuget.ps1 diff --git a/ShardEqualizer/Operations/EqualizeOperation.cs b/ShardEqualizer/Operations/EqualizeOperation.cs index 78d4fd4..f8b5d6b 100644 --- a/ShardEqualizer/Operations/EqualizeOperation.cs +++ b/ShardEqualizer/Operations/EqualizeOperation.cs @@ -277,7 +277,7 @@ public async Task Run(CancellationToken token) _tagRangesByNs = _adjustableIntervals.ToDictionary(_ => _.Namespace, _ => allTagRangesByNs[_.Namespace].InRange(_.Min, _.Max)); - var allChunksByNs = await _chunkService.Get(_adjustableIntervals.Select(_ =>_.Namespace), token); + var allChunksByNs = await _chunkService.Get(_adjustableIntervals.Select(_ => _.Namespace), token); _chunksByCollection = _adjustableIntervals.ToDictionary(_ => _.Namespace, _ => (IReadOnlyList) allChunksByNs[_.Namespace].FromInterval(_.Min, _.Max)); diff --git a/ShardEqualizer/ShardEqualizer.csproj b/ShardEqualizer/ShardEqualizer.csproj index e009dcc..1d50635 100644 --- a/ShardEqualizer/ShardEqualizer.csproj +++ b/ShardEqualizer/ShardEqualizer.csproj @@ -8,17 +8,16 @@ ShardEqualizer true true - SmartcatShardEqualizer + ShardEqualizer Anatoliy Koperin <a.koperin@gmail.com> Tool for aligning the size of stored data on shards in a MongoDB cluster - http://gitlab.sc.local/commontools/forks/shardequalizer - Smartcat.Fork.ShardEqualizer - http://gitlab.sc.local/commontools/forks/shardequalizer/blob/develop/LICENSE.txt - http://gitlab.sc.local/commontools/forks/shardequalizer + https://github.com/ExM/ShardEqualizer + https://github.com/ExM/ShardEqualizer/blob/develop/LICENSE.txt + https://github.com/ExM/ShardEqualizer mongodb sharding 1.0.0 - 1.0.0-beta-8-TECH-5558 - 1.0.0-beta-8-TECH-5558 + 1.0.0-beta.8 + 1.0.0-beta.8 diff --git a/build.ps1 b/build.ps1 deleted file mode 100644 index 1ec4c31..0000000 --- a/build.ps1 +++ /dev/null @@ -1,6 +0,0 @@ -$ErrorActionPreference = "Stop" -$mainFolder = Resolve-Path (Split-Path -Path $MyInvocation.MyCommand.Definition -Parent) - -& dotnet tool restore -& dotnet paket restore -& dotnet build -c Release diff --git a/pack.ps1 b/pack.ps1 deleted file mode 100644 index 11b8572..0000000 --- a/pack.ps1 +++ /dev/null @@ -1,5 +0,0 @@ -$ErrorActionPreference = "Stop" -$mainFolder = Resolve-Path (Split-Path -Path $MyInvocation.MyCommand.Definition -Parent) - -& "$mainFolder/build.ps1" -& dotnet pack --output "$mainFolder/Published" diff --git a/push_nuget.ps1 b/push_nuget.ps1 deleted file mode 100644 index c490606..0000000 --- a/push_nuget.ps1 +++ /dev/null @@ -1,21 +0,0 @@ -param ( - [string]$apikey = "" -) - -$ErrorActionPreference = "Stop" -$mainFolder = Resolve-Path (Split-Path -Path $MyInvocation.MyCommand.Definition -Parent) -if ($apikey -Eq "") -{ - Get-ChildItem -Path "$mainFolder\Published\*.nupkg" | foreach { & dotnet nuget push $_ -s http://nexus-repo.sc.local/repository/nuget/ } -} -else -{ - Get-ChildItem -Path "$mainFolder\Published\*.nupkg" | foreach { & dotnet nuget push $_ -s http://nexus-repo.sc.local/repository/nuget/ -k $apikey } -} - -if ($lastexitcode -ne 0) -{ - Write-Host "Error while pushing" -ForegroundColor Red - Exit 1 -} - From 5c8fd788fc191707373f06f83ef8ae9315654462 Mon Sep 17 00:00:00 2001 From: Maksim Dik Date: Thu, 27 Mar 2025 17:45:42 +0100 Subject: [PATCH 7/7] Use CSharpLegacy representation for the UuidLegacy subtype --- ShardEqualizer/BsonSplitter.cs | 4 ++-- ShardEqualizer/JsonSerialization/ShellJsonWriter.cs | 9 ++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/ShardEqualizer/BsonSplitter.cs b/ShardEqualizer/BsonSplitter.cs index 1c351ab..10ca8ce 100644 --- a/ShardEqualizer/BsonSplitter.cs +++ b/ShardEqualizer/BsonSplitter.cs @@ -39,13 +39,13 @@ public static IList Split(BsonValue min, BsonValue max, int zonesCoun if (IsUuidLegacy(min) && IsUuidLegacy(max)) { return Split(min.AsByteArray, max.AsByteArray, zonesCount) - .Select(_ => (BsonValue)new BsonBinaryData(_, BsonBinarySubType.UuidLegacy, GuidRepresentation.CSharpLegacy)).ToList(); + .Select(_ => (BsonValue)new BsonBinaryData(_, BsonBinarySubType.UuidLegacy)).ToList(); } if (IsUuidStandard(min) && IsUuidStandard(max)) { return Split(min.AsByteArray, max.AsByteArray, zonesCount) - .Select(_ => (BsonValue)new BsonBinaryData(_, BsonBinarySubType.UuidStandard, GuidRepresentation.Standard)).ToList(); + .Select(_ => (BsonValue)new BsonBinaryData(_, BsonBinarySubType.UuidStandard)).ToList(); } if (min.IsObjectId && max.IsObjectId) diff --git a/ShardEqualizer/JsonSerialization/ShellJsonWriter.cs b/ShardEqualizer/JsonSerialization/ShellJsonWriter.cs index cd1f369..e8acfdd 100644 --- a/ShardEqualizer/JsonSerialization/ShellJsonWriter.cs +++ b/ShardEqualizer/JsonSerialization/ShellJsonWriter.cs @@ -57,9 +57,12 @@ public override void WriteBinaryData(BsonBinaryData binaryData) } else { - guidRepresentation = subType == BsonBinarySubType.UuidStandard - ? GuidRepresentation.Standard - : GuidRepresentation.Unspecified; + guidRepresentation = subType switch + { + BsonBinarySubType.UuidLegacy => GuidRepresentation.CSharpLegacy, + BsonBinarySubType.UuidStandard => GuidRepresentation.Standard, + _ => GuidRepresentation.Unspecified + }; } #pragma warning restore 618