Deploy

apps/backend/Helpers/UserUploadCharacterProcessor.cs

@@ -1219,7 +1219,10 @@ private void HandleQuests()
             if (scanTime >= _character.AddonQuests.CompletedQuestsScannedAt)
             {
                 _character.AddonQuests.CompletedQuestsScannedAt = scanTime;
-                _character.AddonQuests.CompletedQuests = SquishUtilities.Unsquish(_characterData.CompletedQuestsSquish);
+
+                var completedQuestIds = SquishUtilities.Unsquish(_characterData.CompletedQuestsSquish);
+                _character.AddonQuests.CompressedCompletedIds = SerializationUtilities.SerializeToBitmap(completedQuestIds);
+                _character.AddonQuests.CompletedQuests = null;
             }
         }
 
@@ -1232,9 +1235,6 @@ private void HandleQuests()
             {
                 _character.AddonQuests.QuestsScannedAt = scanTime;
 
-                _character.AddonQuests.DailyQuests = _characterData.DailyQuests.EmptyIfNull();
-                _character.AddonQuests.OtherQuests = _characterData.OtherQuests.EmptyIfNull();
-
                 _character.AddonQuests.ProgressQuests = new();
                 foreach (string packedProgress in _characterData.ProgressQuests.EmptyIfNull())
                 {

apps/backend/Jobs/Character/CharacterQuestsCompletedJob.cs

@@ -1,8 +1,11 @@
 using System.Net.Http;
+using Equativ.RoaringBitmaps;
+using Wowthing.Backend.Models;
 using Wowthing.Backend.Models.API.Character;
 using Wowthing.Lib.Constants;
 using Wowthing.Lib.Models.Player;
 using Wowthing.Lib.Models.Query;
+using Wowthing.Lib.Utilities;
 
 namespace Wowthing.Backend.Jobs.Character;
 
@@ -21,18 +24,16 @@ public override void Setup(string[] data)
     public override async Task Run(string[] data)
     {
         // Fetch API data
-        ApiCharacterQuestsCompleted resultData;
+        JobHttpResult<ApiCharacterQuestsCompleted> result;
         var uri = GenerateUri(_query, ApiPath);
         try
         {
-            var result = await GetUriAsJsonAsync<ApiCharacterQuestsCompleted>(uri, useLastModified: false);
+            result = await GetUriAsJsonAsync<ApiCharacterQuestsCompleted>(uri, useLastModified: false);
             if (result.NotModified)
             {
                 LogNotModified();
                 return;
             }
-
-            resultData = result.Data;
         }
         catch (HttpRequestException e)
         {
@@ -51,15 +52,20 @@ public override async Task Run(string[] data)
             Context.PlayerCharacterQuests.Add(pcQuests);
         }
 
-        var completedIds = resultData.Quests
+        pcQuests.ScannedAt = result.LastModified;
+
+        var completedIds = result.Data.Quests
             .EmptyIfNull()
             .Select(quest => quest.Id)
-            .OrderBy(id => id)
+            .Order()
             .ToList();
 
-        if (pcQuests.CompletedIds == null || !completedIds.SequenceEqual(pcQuests.CompletedIds))
+        byte[] compressedCompletedIds = SerializationUtilities.SerializeToBitmap(completedIds);
+
+        if (pcQuests.CompressedCompletedIds == null || !compressedCompletedIds.SequenceEqual(pcQuests.CompressedCompletedIds))
         {
-            pcQuests.CompletedIds = completedIds;
+            pcQuests.CompletedIds = null;
+            pcQuests.CompressedCompletedIds = compressedCompletedIds;
         }
 
         int updated = await Context.SaveChangesAsync(CancellationToken);

apps/backend/Models/Uploads/UploadCharacter.cs

@@ -63,8 +63,6 @@ public class UploadCharacter
     public Dictionary<string, int> ScanTimes { get; set; }
     public string Transmog { get; set; }
 
-    public List<int> DailyQuests { get; set; }
-    public List<int> OtherQuests { get; set; }
     public List<string> ProgressQuests { get; set; }
     public Dictionary<short, Dictionary<int, string[]>> WorldQuests { get; set; }
 

apps/backend/Wowthing.Backend.csproj

@@ -8,6 +8,7 @@
   <ItemGroup>
     <PackageReference Include="AWSSDK.S3" Version="4.0.24.1" />
     <PackageReference Include="CsvHelper" Version="33.1.0" />
+    <PackageReference Include="Equativ.RoaringBitmaps" Version="0.1.5" />
     <PackageReference Include="Imageflow.AllPlatforms" Version="0.15.1" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.Analyzers" Version="10.0.7" />
     <PackageReference Include="Microsoft.Extensions.Hosting" Version="10.0.7" />

apps/benchmark/Program.cs

@@ -1,5 +1,6 @@
 // See https://aka.ms/new-console-template for more information
 
+using System.Diagnostics;
 using BenchmarkDotNet.Attributes;
 using BenchmarkDotNet.Running;
 using Wowthing.Lib.Utilities;
@@ -9,24 +10,24 @@ namespace Wowthing.Benchmark;
 [MemoryDiagnoser]
 public class JsonConverterBench
 {
-    private readonly string _data;
+    private readonly string _luaData;
 
     public JsonConverterBench()
     {
-        _data = File.ReadAllText("/home/freddie/projects/wowthing-again/temp/WoWthing_Collector.lua");
+        _luaData = File.ReadAllText("/home/freddie/projects/wowthing-again/temp/WoWthing_Collector.lua");
     }
 
     [Benchmark]
-    public string V1() => LuaToJsonConverter.Convert(_data);
+    public string V1() => LuaToJsonConverter.Convert(_luaData);
 
     [Benchmark]
-    public string V2() => LuaToJsonConverter2.Convert(_data);
+    public string V2() => LuaToJsonConverter2.Convert(_luaData);
 
     [Benchmark]
-    public string V3() => LuaToJsonConverter3.Convert(_data);
+    public string V3() => LuaToJsonConverter3.Convert(_luaData);
 
     [Benchmark]
-    public string V4() => LuaToJsonConverter4.Convert(_data);
+    public string V4() => LuaToJsonConverter4.Convert(_luaData);
 }
 
 public class Program

docker-compose.yml

@@ -24,6 +24,8 @@ services:
       - ./:/app/
       - /app/apps/backend/bin/container/
       - /app/apps/backend/obj/container/
+      - /app/packages/csharp-lib/bin/container/
+      - /app/packages/csharp-lib/obj/container/
 
   web:
     build:
@@ -49,6 +51,8 @@ services:
       - ./:/app/
       - /app/apps/web/bin/container/
       - /app/apps/web/obj/container/
+      - /app/packages/csharp-lib/bin/container/
+      - /app/packages/csharp-lib/obj/container/
 
   frontend:
     build: