diff --git a/_magic/k8s-deployment-web.yml b/_magic/k8s-deployment-web.yml index dd08d6fd7..294b88885 100644 --- a/_magic/k8s-deployment-web.yml +++ b/_magic/k8s-deployment-web.yml @@ -6,7 +6,7 @@ metadata: labels: app: wowthing-again-web spec: - replicas: 2 + replicas: 4 selector: matchLabels: app: wowthing-again-web diff --git a/apps/backend/Jobs/JobBase.cs b/apps/backend/Jobs/JobBase.cs index 0a8ebd796..6846fa228 100644 --- a/apps/backend/Jobs/JobBase.cs +++ b/apps/backend/Jobs/JobBase.cs @@ -33,6 +33,7 @@ public abstract class JobBase : IJob, IDisposable internal JobRepository JobRepository; internal JsonSerializerOptions JsonSerializerOptions; internal MemoryCacheService MemoryCacheService; + internal S3Service S3Service; internal StateService StateService; internal long UserId; diff --git a/apps/backend/Jobs/JobFactory.cs b/apps/backend/Jobs/JobFactory.cs index 31689c710..98ef2402a 100644 --- a/apps/backend/Jobs/JobFactory.cs +++ b/apps/backend/Jobs/JobFactory.cs @@ -19,6 +19,7 @@ public class JobFactory private readonly JobRepository _jobRepository; private readonly JsonSerializerOptions _jsonSerializerOptions; private readonly MemoryCacheService _memoryCacheService; + private readonly S3Service _s3Service; private readonly StateService _stateService; private readonly ConnectionMultiplexer _redis; @@ -31,6 +32,7 @@ public JobFactory( JobRepository jobRepository, JsonSerializerOptions jsonSerializerOptions, MemoryCacheService memoryCacheService, + S3Service s3Service, StateService stateService, string redisConnectionString) { @@ -42,6 +44,7 @@ public JobFactory( _jsonSerializerOptions = jsonSerializerOptions; _logger = logger; _memoryCacheService = memoryCacheService; + _s3Service = s3Service; _stateService = stateService; _redis = RedisUtilities.GetConnection(redisConnectionString); @@ -56,6 +59,7 @@ public JobBase Create(Type type, IDbContextFactory contextFactory, obj.JsonSerializerOptions = _jsonSerializerOptions; obj.Logger = _logger; obj.Redis = _redis; + obj.S3Service = _s3Service; obj.StateService = _stateService; obj.CancellationToken = cancellationToken; diff --git a/apps/backend/Jobs/Misc/ImageJob.cs b/apps/backend/Jobs/Misc/ImageJob.cs index 8e848af2c..f714d5d42 100644 --- a/apps/backend/Jobs/Misc/ImageJob.cs +++ b/apps/backend/Jobs/Misc/ImageJob.cs @@ -23,6 +23,9 @@ public override async Task Run(string[] data) return; } + bool useS3 = S3Service.IsEnabled; + string sha256 = result.Data.Sha256(); + var image = await Context.Image.FindAsync(type, id, format); if (image == null) { @@ -34,6 +37,27 @@ public override async Task Run(string[] data) }; Context.Image.Add(image); } + else if (image.Sha256 == sha256 && ((useS3 && image.Data == null) || (!useS3 && image.Data != null))) + { + Logger.Debug("Hash matches"); + return; + } + else if (useS3) + { + // Hash is changing, delete the old file if a single row references it + int rowCount = await Context.Image + .Where(img => img.Type == image.Type && + img.Sha256 == image.Sha256 && + img.Format == image.Format) + .CountAsync(CancellationToken); + if (rowCount == 1) + { + await S3Service.DeleteImageAsync(image, CancellationToken); + timer.AddPoint("S3Delete"); + } + } + + image.Sha256 = sha256; if ( (format == ImageFormat.Jpeg && (url.EndsWith(".jpg") || url.EndsWith(".jpeg"))) || @@ -55,14 +79,12 @@ public override async Task Run(string[] data) .InProcessAsync(); var convertedBytes = converted.First.TryGetBytes(); - if (convertedBytes.HasValue) - { - image.Data = convertedBytes.Value.ToArray(); - } - else + if (!convertedBytes.HasValue) { throw new Exception($"Image conversion failed: type={type} id={id} format={format} url={url}"); } + + image.Data = convertedBytes.Value.ToArray(); } } else @@ -72,15 +94,21 @@ public override async Task Run(string[] data) timer.AddPoint("Convert"); - var sha256 = image.Data.Sha256(); - if (sha256 == image.Sha256) + // Upload the file instead if we're using S3 + if (useS3) { - Logger.Debug("Hash matches"); - return; + bool uploadOk = await S3Service.UploadImageAsync(image, CancellationToken); + if (uploadOk) + { + image.Data = null; + } + else + { + Logger.Warning("Upload failed: {path}", image.S3Path); + } + timer.AddPoint("S3Upload"); } - image.Sha256 = sha256; - await Context.SaveChangesAsync(CancellationToken); timer.AddPoint("Save", true); diff --git a/apps/backend/Models/BattleNetOptions.cs b/apps/backend/Models/BattleNetOptions.cs index 808fb9743..c76c43947 100644 --- a/apps/backend/Models/BattleNetOptions.cs +++ b/apps/backend/Models/BattleNetOptions.cs @@ -4,4 +4,4 @@ public class BattleNetOptions { public string ClientId { get; set; } public string ClientSecret { get; set; } -} \ No newline at end of file +} diff --git a/apps/backend/Models/WowthingBackendOptions.cs b/apps/backend/Models/WowthingBackendOptions.cs index 68a343b6e..908ffb758 100644 --- a/apps/backend/Models/WowthingBackendOptions.cs +++ b/apps/backend/Models/WowthingBackendOptions.cs @@ -2,10 +2,14 @@ public class WowthingBackendOptions { - public List AllAuctionRegions { get; set; } public int ApiRateLimit { get; set; } public int WorkerMaxAuction { get; set; } public int WorkerMaxBulk { get; set; } public int WorkerMaxHigh { get; set; } public int WorkerMaxLow { get; set; } + public string ImageAccessKeyId { get; set; } + public string ImageSecretAccessKey { get; set; } + public string ImageEndpoint { get; set; } + public string ImageBucket { get; set; } + public List AllAuctionRegions { get; set; } } diff --git a/apps/backend/Program.cs b/apps/backend/Program.cs index 1a7743dbc..d4aeb88df 100644 --- a/apps/backend/Program.cs +++ b/apps/backend/Program.cs @@ -146,6 +146,7 @@ private static void ConfigureServices(HostBuilderContext hostContext, IServiceCo // Services services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddHostedService(); diff --git a/apps/backend/Services/S3Service.cs b/apps/backend/Services/S3Service.cs new file mode 100644 index 000000000..48325d7e6 --- /dev/null +++ b/apps/backend/Services/S3Service.cs @@ -0,0 +1,76 @@ +using Amazon.Runtime; +using Amazon.S3; +using Amazon.S3.Model; +using Microsoft.Extensions.Options; +using Serilog; +using Wowthing.Backend.Models; +using Wowthing.Lib.Models; + +namespace Wowthing.Backend.Services; + +public class S3Service +{ + private readonly ILogger _logger; + private readonly WowthingBackendOptions _backendOptions; + + public S3Service(IOptions backendOptions) + { + _backendOptions = backendOptions.Value; + + _logger = Log.ForContext("Service", $"S3Service"); + + _logger.Information("Service starting"); + + _logger.Debug("ImageAccessKeyId {0}", _backendOptions.ImageAccessKeyId); + _logger.Debug("ImageSecretAccessKey {0}", _backendOptions.ImageSecretAccessKey); + _logger.Debug("ImageEndpoint {0}", _backendOptions.ImageEndpoint); + _logger.Debug("ImageBucket {0}", _backendOptions.ImageBucket); + } + + private AmazonS3Client S3Client + { + get + { + if (field == null) + { + var credentials = new BasicAWSCredentials(_backendOptions.ImageAccessKeyId, _backendOptions.ImageSecretAccessKey); + field = new AmazonS3Client(credentials, new AmazonS3Config + { + ServiceURL = _backendOptions.ImageEndpoint + }); + } + return field; + } + } + + public bool IsEnabled => !string.IsNullOrEmpty(_backendOptions.ImageBucket); + + public async Task UploadImageAsync(Image image, CancellationToken cancellationToken) + { + using var memoryStream = new MemoryStream(image.Data); + + var request = new PutObjectRequest + { + BucketName = _backendOptions.ImageBucket, + Key = image.S3Path, + InputStream = memoryStream, + ContentType = $"image/{image.Format.ToString().ToLower()}", + // Cloudflare R2 doesn't support SigV4 + DisableDefaultChecksumValidation = true, + DisablePayloadSigning = true, + }; + var response = await S3Client.PutObjectAsync(request, cancellationToken); + return response.HttpStatusCode == System.Net.HttpStatusCode.OK; + } + + public async Task DeleteImageAsync(Image image, CancellationToken cancellationToken) + { + var request = new DeleteObjectRequest + { + BucketName = _backendOptions.ImageBucket, + Key = image.S3Path, + }; + var response = await S3Client.DeleteObjectAsync(request, cancellationToken); + return response.HttpStatusCode == System.Net.HttpStatusCode.OK; + } +} diff --git a/apps/backend/Services/WorkerService.cs b/apps/backend/Services/WorkerService.cs index 5b854fa31..113a8f03b 100644 --- a/apps/backend/Services/WorkerService.cs +++ b/apps/backend/Services/WorkerService.cs @@ -45,6 +45,7 @@ public WorkerService( JobRepository jobRepository, JsonSerializerOptions jsonSerializerOptions, MemoryCacheService memoryCacheService, + S3Service s3Service, StateService stateService, JobMetrics jobMetrics ) @@ -69,6 +70,7 @@ JobMetrics jobMetrics jobRepository, jsonSerializerOptions, memoryCacheService, + s3Service, stateService, redisConnectionString ); diff --git a/apps/backend/Wowthing.Backend.csproj b/apps/backend/Wowthing.Backend.csproj index d31e185eb..c5d5a4000 100644 --- a/apps/backend/Wowthing.Backend.csproj +++ b/apps/backend/Wowthing.Backend.csproj @@ -6,6 +6,7 @@ + diff --git a/apps/frontend/components/characters/paperdoll/CharactersPaperdoll.svelte b/apps/frontend/components/characters/paperdoll/CharactersPaperdoll.svelte index f17907b42..177351b64 100644 --- a/apps/frontend/components/characters/paperdoll/CharactersPaperdoll.svelte +++ b/apps/frontend/components/characters/paperdoll/CharactersPaperdoll.svelte @@ -1,40 +1,38 @@ @@ -188,12 +192,16 @@ } } .crafted-modifier { - font-size: 90%; + --image-margin-top: -6px; + --scale: 1.2; + + border-radius: 50%; pointer-events: none; position: absolute; - left: 2px; - top: 2px; - width: 1.2rem; + left: 0px; + top: 1px; + height: 23px; + width: 23px; } .crafted-quality { pointer-events: none; @@ -231,7 +239,7 @@ {:else if (gear.equipped.count || 0) > 1} x{gear.equipped.count} {/if} - {:else} + {:else if showItemLevel} {gear.equipped.itemLevel} {/if} @@ -269,8 +277,8 @@ {:else if gear.equipped.craftedQuality > 0 || forceCrafted || item?.craftingQuality} {@const statModifier = getBonusIdCraftingStat(gear.equipped.bonusIds || [])} {#if statModifiers[statModifier]} -
- {statModifiers[statModifier]} +
+
{/if} diff --git a/apps/frontend/components/professions/equipment/TableRow.svelte b/apps/frontend/components/professions/equipment/TableRow.svelte index fca37effd..f1f1ef949 100644 --- a/apps/frontend/components/professions/equipment/TableRow.svelte +++ b/apps/frontend/components/professions/equipment/TableRow.svelte @@ -139,6 +139,7 @@ item.expansion !== Constants.expansion} forceCrafted={true} + showItemLevel={false} {gear} /> {:else if !userHas} diff --git a/apps/frontend/shared/icons/library.ts b/apps/frontend/shared/icons/library.ts index 87c6f7ce2..edd5ae6ef 100644 --- a/apps/frontend/shared/icons/library.ts +++ b/apps/frontend/shared/icons/library.ts @@ -147,9 +147,14 @@ export { default as mdiHammerWrench } from '~icons/mdi/hammer-wrench'; export { default as mdiHomeOutline } from '~icons/mdi/home-outline'; export { default as mdiImageBrokenVariant } from '~icons/mdi/image-broken-variant'; export { default as mdiLetterC } from '~icons/mdi/alpha-c-circle-outline'; +export { default as mdiLetterD } from '~icons/mdi/alpha-d-circle-outline'; +export { default as mdiLetterF } from '~icons/mdi/alpha-f-circle-outline'; +export { default as mdiLetterI } from '~icons/mdi/alpha-i-circle-outline'; export { default as mdiLetterL } from '~icons/mdi/alpha-l-circle-outline'; export { default as mdiLetterM } from '~icons/mdi/alpha-m-circle-outline'; export { default as mdiLetterP } from '~icons/mdi/alpha-p-circle-outline'; +export { default as mdiLetterR } from '~icons/mdi/alpha-r-circle-outline'; +export { default as mdiLetterS } from '~icons/mdi/alpha-s-circle-outline'; export { default as mdiLightningBoltOutline } from '~icons/mdi/lightning-bolt-outline'; export { default as mdiListStatus } from '~icons/mdi/list-status'; export { default as mdiLockOutline } from '~icons/mdi/lock-outline'; diff --git a/apps/web/Models/WowthingWebOptions.cs b/apps/web/Models/WowthingWebOptions.cs index a51e8591d..f16b58800 100644 --- a/apps/web/Models/WowthingWebOptions.cs +++ b/apps/web/Models/WowthingWebOptions.cs @@ -3,4 +3,5 @@ public class WowthingWebOptions { public string Hostname { get; set; } -} \ No newline at end of file + public string ImageUrl { get; set; } +} diff --git a/apps/web/Program.cs b/apps/web/Program.cs index 9421fe276..572553e65 100644 --- a/apps/web/Program.cs +++ b/apps/web/Program.cs @@ -10,7 +10,7 @@ public static void Main(string[] args) CreateHostBuilder(args).Build().Run(); } - public static IHostBuilder CreateHostBuilder(string[] args) => + private static IHostBuilder CreateHostBuilder(string[] args) => Host.CreateDefaultBuilder(args) .ConfigureWebHostDefaults(webBuilder => { diff --git a/apps/web/Services/UriService.cs b/apps/web/Services/UriService.cs index 538d52083..1679f0a29 100644 --- a/apps/web/Services/UriService.cs +++ b/apps/web/Services/UriService.cs @@ -31,13 +31,11 @@ public UriService( _webOptions = webOptions.Value; } - private Uri _baseUri; - public Uri BaseUri { get { - if (_baseUri == null) + if (field == null) { var host = _httpContextAccessor.HttpContext.Request.Host.Host; var builder = new UriBuilder(); @@ -60,14 +58,16 @@ public Uri BaseUri builder.Scheme = _httpContextAccessor.HttpContext.Request.Scheme; - _baseUri = builder.Uri; - _logger.LogDebug("Base URI is {Uri}", _baseUri); + field = builder.Uri; + _logger.LogDebug("Base URI is {Uri}", field); } - return _baseUri; + return field; } } + public string ImageUrl => _webOptions.ImageUrl; + public string GetBaseAction(string controller, string action, object values = null) { return _linkGenerator.GetUriByAction( diff --git a/apps/web/Views/User/Index.cshtml b/apps/web/Views/User/Index.cshtml index 35880d52f..855c02945 100644 --- a/apps/web/Views/User/Index.cshtml +++ b/apps/web/Views/User/Index.cshtml @@ -46,7 +46,9 @@ }
diff --git a/docker-compose.yml b/docker-compose.yml index 4fc083179..d4951542f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,10 +10,14 @@ services: BattleNet__ClientID: $BattleNet__ClientID BattleNet__ClientSecret: $BattleNet__ClientSecret SENTRY_DSN: ${SENTRY_DSN-} -# logging: -# options: -# max-size: "1m" -# max-file: "5" + WowthingBackend__ImageAccessKeyId: ${WowthingBackend__ImageAccessKeyId-} + WowthingBackend__ImageSecretAccessKey: ${WowthingBackend__ImageSecretAccessKey-} + WowthingBackend__ImageEndpoint: ${WowthingBackend__ImageEndpoint-} + WowthingBackend__ImageBucket: ${WowthingBackend__ImageBucket-} + # logging: + # options: + # max-size: "1m" + # max-file: "5" networks: - wowthing volumes: @@ -31,10 +35,11 @@ services: environment: BattleNet__ClientID: $BattleNet__ClientID BattleNet__ClientSecret: $BattleNet__ClientSecret -# logging: -# options: -# max-size: "1m" -# max-file: "5" + WowthingWeb__ImageUrl: ${WowthingWeb__ImageUrl-} + # logging: + # options: + # max-size: "1m" + # max-file: "5" networks: - wowthing ports: @@ -49,10 +54,10 @@ services: build: context: . dockerfile: apps/frontend/development.Dockerfile -# logging: -# options: -# max-size: "1m" -# max-file: "5" + # logging: + # options: + # max-size: "1m" + # max-file: "5" networks: - wowthing ports: diff --git a/packages/csharp-lib/Models/Image.cs b/packages/csharp-lib/Models/Image.cs index 41bd1e7ad..d3364f241 100644 --- a/packages/csharp-lib/Models/Image.cs +++ b/packages/csharp-lib/Models/Image.cs @@ -15,5 +15,7 @@ public class Image public byte[] Data { get; set; } - public string Url => $"/image/{(int)Type}/{Sha256}.{Format.ToString().ToLower()}"; + public string Url => Data == null ? $"/{S3Path}" : $"/image/{(int)Type}/{Sha256}.{Format.ToString().ToLower()}"; + + public string S3Path => $"{(int)Type}/{Sha256[0]}/{Sha256[1]}/{Sha256}.{Format.ToString().ToLower()}"; }