diff --git a/DeveLanCacheUI_Backend/Program.cs b/DeveLanCacheUI_Backend/Program.cs index f318450..56b23ed 100644 --- a/DeveLanCacheUI_Backend/Program.cs +++ b/DeveLanCacheUI_Backend/Program.cs @@ -84,8 +84,8 @@ public static void Main(string[] args) } else { - builder.Services.AddHostedService(); builder.Services.AddHostedService(); + builder.Services.AddSingleton(); builder.Services.AddSingleton(); } diff --git a/DeveLanCacheUI_Backend/Services/OriginalDepotEnricher/SteamDepotDownloaderHostedService.cs b/DeveLanCacheUI_Backend/Services/OriginalDepotEnricher/SteamDepotDownloaderHostedService.cs index b4a7653..6bc209a 100644 --- a/DeveLanCacheUI_Backend/Services/OriginalDepotEnricher/SteamDepotDownloaderHostedService.cs +++ b/DeveLanCacheUI_Backend/Services/OriginalDepotEnricher/SteamDepotDownloaderHostedService.cs @@ -7,6 +7,7 @@ public class SteamDepotDownloaderHostedService : BackgroundService public IServiceProvider Services { get; } private readonly DeveLanCacheConfiguration _deveLanCacheConfiguration; + private readonly SteamDepotEnricherHostedService _steamDepotEnricherHostedService; private readonly ILogger _logger; private readonly HttpClient _httpClient; @@ -14,10 +15,12 @@ public class SteamDepotDownloaderHostedService : BackgroundService public SteamDepotDownloaderHostedService(IServiceProvider services, DeveLanCacheConfiguration deveLanCacheConfiguration, + SteamDepotEnricherHostedService steamDepotEnricherHostedService, ILogger logger) { Services = services; _deveLanCacheConfiguration = deveLanCacheConfiguration; + _steamDepotEnricherHostedService = steamDepotEnricherHostedService; _logger = logger; _httpClient = new HttpClient(); _httpClient.DefaultRequestHeaders.Add("User-Agent", "request"); @@ -40,31 +43,34 @@ private async Task GoRun(CancellationToken stoppingToken) var depotFileDirectory = Path.Combine(deveLanCacheUIDataDirectory, "depotdir"); - if (!Directory.Exists(depotFileDirectory)) + if (Directory.Exists(depotFileDirectory)) { - Directory.CreateDirectory(depotFileDirectory); + // This is purely cleanup since we don't use the depot directory anymore. We just directly download and process. + Directory.Delete(depotFileDirectory, true); } while (!stoppingToken.IsCancellationRequested) { var shouldDownload = await NewDepotFileAvailable(); + _logger.LogInformation("New Depot File Available: {NewVersionAvailable}, LatestVersion: {LatestVersion}, DownloadUrl: {DownloadUrl}", + shouldDownload.NewVersionAvailable, shouldDownload.LatestVersion, shouldDownload.DownloadUrl); + if (shouldDownload.NewVersionAvailable) { - var createdFileName = await GoDownload(depotFileDirectory, shouldDownload); + if (shouldDownload.LatestVersion == null || string.IsNullOrWhiteSpace(shouldDownload.DownloadUrl)) + { + throw new UnreachableException($"New version available, but LatestVersion or DownloadUrl is null. This should not happen. LatestVersion: {shouldDownload.LatestVersion}, DownloadUrl: {shouldDownload.DownloadUrl}"); + } + var downloadedBytes = await GoDownload(depotFileDirectory, shouldDownload); - //Remove all other csv files in DepotDir except this one - var depotFiles = Directory.GetFiles(depotFileDirectory).Where(t => Path.GetExtension(t).Equals(".csv", StringComparison.OrdinalIgnoreCase) && !t.EndsWith(createdFileName, StringComparison.OrdinalIgnoreCase)); - foreach (var depotFile in depotFiles) + if (downloadedBytes != null) + { + await _steamDepotEnricherHostedService.GoProcess(shouldDownload.LatestVersion, downloadedBytes, stoppingToken); + } + else { - try - { - File.Delete(depotFile); - } - catch (Exception ex) - { - _logger.LogWarning("Could not delete depot file: {DepotFile}, exception: {Exception}", depotFile, ex); - } + _logger.LogWarning("Downloaded depot file was null, not processing further."); } } @@ -72,7 +78,7 @@ private async Task GoRun(CancellationToken stoppingToken) } } - private async Task GoDownload(string depotFileDirectory, (bool NewVersionAvailable, Version? LatestVersion, string? DownloadUrl) shouldDownload) + private async Task GoDownload(string depotFileDirectory, (bool NewVersionAvailable, Version? LatestVersion, string? DownloadUrl) shouldDownload) { _logger.LogInformation("Detected that new version '{NewVersionAvailable}' of Depot File is available, so downloading: {DownloadUrl}...", shouldDownload.NewVersionAvailable, shouldDownload.DownloadUrl); @@ -84,31 +90,7 @@ private async Task GoRun(CancellationToken stoppingToken) } var bytes = await downloadedCsv.Content.ReadAsByteArrayAsync(); - - var fileName = $"depot_{shouldDownload.LatestVersion}.csv"; - var filePath = Path.Combine(depotFileDirectory, fileName); - File.WriteAllBytes(filePath, bytes); - - await using (var scope = Services.CreateAsyncScope()) - { - using var dbContext = scope.ServiceProvider.GetRequiredService(); - var foundSetting = await dbContext.Settings.FirstOrDefaultAsync(); - if (foundSetting == null || foundSetting.Value == null) - { - foundSetting = new DbSetting() - { - Key = DbSetting.SettingKey_DepotVersion, - Value = shouldDownload.LatestVersion?.ToString() - }; - dbContext.Settings.Add(foundSetting); - } - else - { - foundSetting.Value = shouldDownload.LatestVersion?.ToString(); - } - await dbContext.SaveChangesAsync(); - } - return fileName; + return bytes; } private async Task<(bool NewVersionAvailable, Version? LatestVersion, string? DownloadUrl)> NewDepotFileAvailable() @@ -148,7 +130,7 @@ private async Task GoRun(CancellationToken stoppingToken) await using (var scope = Services.CreateAsyncScope()) { using var dbContext = scope.ServiceProvider.GetRequiredService(); - var foundSetting = await dbContext.Settings.FirstOrDefaultAsync(); + var foundSetting = await dbContext.Settings.FirstOrDefaultAsync(t => t.Key == DbSetting.SettingKey_DepotVersion); if (foundSetting == null || foundSetting.Value == null) { _logger.LogInformation("Update of Depot File required because CurrentVersion could not be found"); diff --git a/DeveLanCacheUI_Backend/Services/OriginalDepotEnricher/SteamDepotEnricherHostedService.cs b/DeveLanCacheUI_Backend/Services/OriginalDepotEnricher/SteamDepotEnricherHostedService.cs index da97f3d..4e2d1c4 100644 --- a/DeveLanCacheUI_Backend/Services/OriginalDepotEnricher/SteamDepotEnricherHostedService.cs +++ b/DeveLanCacheUI_Backend/Services/OriginalDepotEnricher/SteamDepotEnricherHostedService.cs @@ -1,6 +1,6 @@ namespace DeveLanCacheUI_Backend.Services.OriginalDepotEnricher { - public class SteamDepotEnricherHostedService : BackgroundService + public class SteamDepotEnricherHostedService { public IServiceProvider Services { get; } @@ -16,157 +16,113 @@ public SteamDepotEnricherHostedService(IServiceProvider services, _logger = logger; } - protected override async Task ExecuteAsync(CancellationToken stoppingToken) + public async Task GoProcess(Version depotFileVersion, byte[] downloadedDepotFile, CancellationToken stoppingToken) { - await Task.Delay(TimeSpan.FromSeconds(1), stoppingToken); - - await GoRun(stoppingToken); - } + _logger.LogInformation("Processing depot file with version {DepotFileVersion}", depotFileVersion); + var desiredSteamAppToDepots = new List(); + //var depotToAppDict = new Dictionary(); - private async Task GoRun(CancellationToken stoppingToken) - { - var deveLanCacheUIDataDirectory = _deveLanCacheConfiguration.DeveLanCacheUIDataDirectory; - if (string.IsNullOrWhiteSpace(deveLanCacheUIDataDirectory)) + using (var reader = new StreamReader(new MemoryStream(downloadedDepotFile))) { - deveLanCacheUIDataDirectory = Directory.GetCurrentDirectory(); - } - - var depotFileDirectory = Path.Combine(deveLanCacheUIDataDirectory, "depotdir"); - - _logger.LogInformation("Watching directory: '{DepotFileDirectory}' for any .CSV files to update our Depot database...", depotFileDirectory); + string? line; - while (!stoppingToken.IsCancellationRequested) - { - if (Directory.Exists(depotFileDirectory)) + while ((line = reader.ReadLine()) != null) { - var firstFile = Directory.GetFiles(depotFileDirectory).Where(t => Path.GetExtension(t).Equals(".csv", StringComparison.OrdinalIgnoreCase)).FirstOrDefault(); + var values = line.Split(';'); - if (firstFile != null) + if (values.Length < 3) { - Console.WriteLine($"Found .CSV file to update our Depots Database: {firstFile}"); - - - var curFileSize = new FileInfo(firstFile).Length; - Console.WriteLine($"Waiting for file size to not increase anymore (as in, the copy is done). Current Size: {curFileSize}"); + //Console.WriteLine("Warning: Line does not contain sufficient data, skipping"); + continue; + } - var fileSizeTimer = Stopwatch.StartNew(); + bool appIdParsed = uint.TryParse(values[0], out uint appId); + var appName = values[1]; + bool depotIdParsed = uint.TryParse(values[2], out uint depotId); - while (fileSizeTimer.Elapsed.TotalSeconds < 5) - { - var newFileSize = new FileInfo(firstFile).Length; - if (curFileSize != newFileSize) - { - _logger.LogInformation("File size has increased, waiting 5 more seconds... from: {OldSize} to {NewSize}", curFileSize, newFileSize); - curFileSize = newFileSize; - fileSizeTimer.Restart(); - } - else - { - _logger.LogInformation("File size equal for {Elapsed}", fileSizeTimer.Elapsed); - } - await Task.Delay(1000); - } - - var desiredSteamAppToDepots = new List(); - //var depotToAppDict = new Dictionary(); + if (!appIdParsed || !depotIdParsed) + { + //Console.WriteLine("Warning: AppId or DepotId could not be parsed, skipping"); + continue; + } - try - { - using (var reader = new StreamReader(firstFile)) - { - string? line; - - while ((line = reader.ReadLine()) != null) - { - var values = line.Split(';'); - - if (values.Length < 3) - { - //Console.WriteLine("Warning: Line does not contain sufficient data, skipping"); - continue; - } - - bool appIdParsed = uint.TryParse(values[0], out uint appId); - var appName = values[1]; - bool depotIdParsed = uint.TryParse(values[2], out uint depotId); - - if (!appIdParsed || !depotIdParsed) - { - //Console.WriteLine("Warning: AppId or DepotId could not be parsed, skipping"); - continue; - } - - //create csv model - var csvModel = new SteamDepotEnricherCSVModel - { - SteamAppId = appId, - SteamAppName = appName, - SteamDepotId = depotId - }; - - desiredSteamAppToDepots.Add(csvModel); - } - } + //create csv model + var csvModel = new SteamDepotEnricherCSVModel + { + SteamAppId = appId, + SteamAppName = appName, + SteamDepotId = depotId + }; - Console.WriteLine($"Depot File {firstFile} read. Adding {desiredSteamAppToDepots.Count} entries to db..."); + desiredSteamAppToDepots.Add(csvModel); + } + } + _logger.LogInformation("Depot data read. Adding {DepotCount} entries to db...", desiredSteamAppToDepots.Count); - desiredSteamAppToDepots = desiredSteamAppToDepots.DistinctBy(t => new { t.SteamAppId, t.SteamDepotId }).ToList(); + desiredSteamAppToDepots = desiredSteamAppToDepots.DistinctBy(t => new { t.SteamAppId, t.SteamDepotId }).ToList(); - var retryPolicy = Policy - .Handle() - .WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)), - (exception, timeSpan, context) => - { - _logger.LogWarning("An error occurred while trying to save changes: {Message}", exception.Message); - }); + var retryPolicy = Policy + .Handle() + .WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)), + (exception, timeSpan, context) => + { + _logger.LogWarning("An error occurred while trying to save changes: {Message}", exception.Message); + }); - //Batch operations in groups of 1000 - for (int i = 0; i < desiredSteamAppToDepots.Count; i += 1000) + //Batch operations in groups of 1000 + for (int i = 0; i < desiredSteamAppToDepots.Count; i += 1000) + { + await retryPolicy.ExecuteAsync(async () => + { + await using (var scope = Services.CreateAsyncScope()) + { + using var dbContext = scope.ServiceProvider.GetRequiredService(); + var currentBatch = desiredSteamAppToDepots.Skip(i).Take(1000).ToList(); + int newDepots = 0; + foreach (var depot in currentBatch) + { + // Insert or update using Polly's retry policy + var dbDepot = await dbContext.SteamDepots.FirstOrDefaultAsync(d => d.SteamDepotId == depot.SteamDepotId && d.SteamAppId == depot.SteamAppId); + if (dbDepot == null) { - await retryPolicy.ExecuteAsync(async () => - { - await using (var scope = Services.CreateAsyncScope()) - { - using var dbContext = scope.ServiceProvider.GetRequiredService(); - var currentBatch = desiredSteamAppToDepots.Skip(i).Take(1000).ToList(); - int newDepots = 0; - foreach (var depot in currentBatch) - { - // Insert or update using Polly's retry policy - var dbDepot = await dbContext.SteamDepots.FirstOrDefaultAsync(d => d.SteamDepotId == depot.SteamDepotId && d.SteamAppId == depot.SteamAppId); - if (dbDepot == null) - { - //Depot does not exist, create it - dbDepot = new DbSteamDepot { SteamAppId = depot.SteamAppId, SteamDepotId = depot.SteamDepotId }; - dbContext.SteamDepots.Add(dbDepot); - newDepots++; - } - } - //Save changes - await dbContext.SaveChangesAsync(); - _logger.LogInformation($"Depots Processed: {i + currentBatch.Count}/{desiredSteamAppToDepots.Count}. Updated {currentBatch.Count - newDepots}, New {newDepots}"); - } - }); + //Depot does not exist, create it + dbDepot = new DbSteamDepot { SteamAppId = depot.SteamAppId, SteamDepotId = depot.SteamDepotId }; + dbContext.SteamDepots.Add(dbDepot); + newDepots++; } - - var processedDirectoryPath = Path.Combine(depotFileDirectory, "processed"); - Directory.CreateDirectory(processedDirectoryPath); - var newFileName = Path.GetFileNameWithoutExtension(firstFile) + "_" + DateTime.Now.ToString("yyyy-dd-M--HH-mm-ss") + Path.GetExtension(firstFile); - var newFilePath = Path.Combine(processedDirectoryPath, newFileName); - File.Move(firstFile, newFilePath); - Console.WriteLine($"File {firstFile} moved to {newFilePath}"); - } - catch (IOException ex) - { - _logger.LogWarning("IO Exception while reading/writing file. This could be because file is in use. Retrying..."); } + //Save changes + await dbContext.SaveChangesAsync(); + _logger.LogInformation($"Depots Processed: {i + currentBatch.Count}/{desiredSteamAppToDepots.Count}. Updated {currentBatch.Count - newDepots}, New {newDepots}"); } + }); + } + + _logger.LogInformation("Depot data processing completed. Total entries added: {TotalEntries}", desiredSteamAppToDepots.Count); + + await using (var scope = Services.CreateAsyncScope()) + { + using var dbContext = scope.ServiceProvider.GetRequiredService(); + var foundSetting = await dbContext.Settings.FirstOrDefaultAsync(t => t.Key == DbSetting.SettingKey_DepotVersion); + if (foundSetting == null || foundSetting.Value == null) + { + foundSetting = new DbSetting() + { + Key = DbSetting.SettingKey_DepotVersion, + Value = depotFileVersion.ToString() + }; + dbContext.Settings.Add(foundSetting); + } + else + { + foundSetting.Value = depotFileVersion.ToString(); } - await Task.Delay(1000); + await dbContext.SaveChangesAsync(); } + _logger.LogInformation("Depot file processing completed successfully. Version {DepotFileVersion} saved to database.", depotFileVersion); } } } diff --git a/DeveLanCacheUI_Backend/Services/SteamAppInfoService.cs b/DeveLanCacheUI_Backend/Services/SteamAppInfoService.cs index 6eefcdf..e082d9d 100644 --- a/DeveLanCacheUI_Backend/Services/SteamAppInfoService.cs +++ b/DeveLanCacheUI_Backend/Services/SteamAppInfoService.cs @@ -132,7 +132,11 @@ await retryPolicy.ExecuteAsync(async () => var item = dbContext.Settings.FirstOrDefault(t => t.Key == DbSetting.SettingKey_SteamChangeNumber); if (item == null) { - item = new DbSetting { Key = DbSetting.SettingKey_SteamChangeNumber, Value = _currentChangeNumber.ToString() }; + item = new DbSetting + { + Key = DbSetting.SettingKey_SteamChangeNumber, + Value = _currentChangeNumber.ToString() + }; dbContext.Settings.Add(item); } else