From 9285776659a7633fd627f8a99d6a448a5b961b11 Mon Sep 17 00:00:00 2001 From: Astemir Boziev Date: Sat, 23 May 2026 17:23:23 +0400 Subject: [PATCH 1/3] Add chart-position watchdog (backend) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Daily Queues job that polls Apple's iTunes RSS top-free chart for each watched app's primary genre across the storefronts where the app is available, and emits ChartEvent rows on every transition (entered / moved / exited). The watchdog stays silent for never-charted apps — Azri isn't in any chart's top-100 today but will be eventually, and the dashboard wants to know the moment that changes. - Three new tables: app_storefront_availability, chart_position_snapshot, chart_event. Snapshot rows hold only the latest position per (app, country, chart, genre) tuple so storage stays bounded. - WatchedApp gains primary_genre_id (backfilled by AvailabilityProber on next probe or lazily by ChartTrackerService on first refresh). - AvailabilityProber: batched iTunes lookup against all 175 storefronts; triggered from AppsController.create and POST /apps/:id/availability/refresh. - ChartTrackerService: per-(app, country) Fluent transaction wraps each diff so an interrupted job can't desync the snapshot from the event log. Diff logic extracted into a pure `decideChartTransition` for unit testing. - ITunesChartsClient: thin wrapper around the legacy //rss/topfreeapplications/limit=200/genre=/json endpoint (the v2 applemarketingtools API doesn't support a genre filter). Static `parseEntries` exposed for tests. - RefreshChartsScheduler runs daily at 04:00 UTC (an hour after the keyword refresh and a few hours past Apple's PT-midnight chart refresh window so the RSS feeds have settled). POST /charts/refresh spawns the same work via a detached Task. - Four new routes: GET /chart-positions, GET /chart-events?since=&limit=, POST /charts/refresh, POST /apps/:id/availability/refresh. - Tests cover all 7 transition cases in the diff matrix plus the RSS envelope decoder. --- Sources/App/Clients/ITunesChartsClient.swift | 113 ++++++++ Sources/App/Composition/Container.swift | 29 ++ Sources/App/Controllers/AppsController.swift | 17 +- .../App/Controllers/ChartsController.swift | 144 ++++++++++ Sources/App/Domain/DomainTypes.swift | 4 + Sources/App/Jobs/RefreshChartsScheduler.swift | 23 ++ .../Models/AppStorefrontAvailability.swift | 50 ++++ Sources/App/Models/ChartEvent.swift | 81 ++++++ .../App/Models/ChartPositionSnapshot.swift | 57 ++++ Sources/App/Models/WatchedApp.swift | 33 ++- Sources/App/Services/AppService.swift | 3 +- Sources/App/Services/AvailabilityProber.swift | 170 +++++++++++ .../App/Services/ChartTrackerService.swift | 271 ++++++++++++++++++ Sources/App/configure.swift | 10 + Sources/App/routes.swift | 1 + Tests/AppTests/AppServiceTests.swift | 5 +- Tests/AppTests/ChartTransitionTests.swift | 69 +++++ Tests/AppTests/ITunesChartsClientTests.swift | 75 +++++ 18 files changed, 1150 insertions(+), 5 deletions(-) create mode 100644 Sources/App/Clients/ITunesChartsClient.swift create mode 100644 Sources/App/Controllers/ChartsController.swift create mode 100644 Sources/App/Jobs/RefreshChartsScheduler.swift create mode 100644 Sources/App/Models/AppStorefrontAvailability.swift create mode 100644 Sources/App/Models/ChartEvent.swift create mode 100644 Sources/App/Models/ChartPositionSnapshot.swift create mode 100644 Sources/App/Services/AvailabilityProber.swift create mode 100644 Sources/App/Services/ChartTrackerService.swift create mode 100644 Tests/AppTests/ChartTransitionTests.swift create mode 100644 Tests/AppTests/ITunesChartsClientTests.swift diff --git a/Sources/App/Clients/ITunesChartsClient.swift b/Sources/App/Clients/ITunesChartsClient.swift new file mode 100644 index 0000000..0707da1 --- /dev/null +++ b/Sources/App/Clients/ITunesChartsClient.swift @@ -0,0 +1,113 @@ +import Vapor + +// Vapor's existing iTunes clients live under Services/ITunesSearchClient.swift +// etc., but the chart-RSS endpoint serves a different envelope shape and a +// different host path style ("//rss/topfreeapplications/...") so it +// gets its own thin client. Mirrors the timeout pattern of ITunesSearchClient. + +protocol ITunesChartsClientProtocol: Sendable { + func topFree(country: String, genreId: Int, limit: Int) async throws -> [ChartEntry] +} + +struct ChartEntry: Sendable, Equatable { + let appStoreId: Int64 + let position: Int // 1-indexed + let name: String +} + +struct ITunesChartsClient: ITunesChartsClientProtocol { + let client: any Client + let logger: Logger + static let requestTimeoutSeconds: UInt64 = 30 + + func topFree(country: String, genreId: Int, limit: Int) async throws -> [ChartEntry] { + let cc = country.lowercased() + // RSS feeds for category-scoped charts. The legacy + // //rss//limit=/genre=/json path is the only + // one that supports the genre filter; the newer applemarketingtools + // v2 API drops it. + let url = URI(string: + "https://itunes.apple.com/\(cc)/rss/topfreeapplications/limit=\(limit)/genre=\(genreId)/json" + ) + + let response: ClientResponse = try await withThrowingTaskGroup(of: ClientResponse.self) { group in + let theClient = client + let theURL = url + let timeoutSeconds = Self.requestTimeoutSeconds + group.addTask { try await theClient.get(theURL) } + group.addTask { + try await Task.sleep(nanoseconds: timeoutSeconds * 1_000_000_000) + throw Abort(.gatewayTimeout, reason: "iTunes charts timed out after \(timeoutSeconds)s") + } + guard let first = try await group.next() else { + throw Abort(.internalServerError, reason: "iTunes charts produced no result") + } + group.cancelAll() + return first + } + + guard response.status == .ok else { + logger.error("iTunes charts returned \(response.status) for country=\(cc) genre=\(genreId)") + throw Abort(.badGateway, reason: "iTunes charts failed with \(response.status)") + } + + guard let buffer = response.body else { return [] } + return try ITunesChartsClient.parseEntries(from: Data(buffer: buffer)) + } + + // Pure JSON-to-`[ChartEntry]` parser, exposed for unit testing without a + // live Vapor Client. Marked `static` because it doesn't touch `client` + // or `logger`. + static func parseEntries(from data: Data) throws -> [ChartEntry] { + let envelope = try JSONDecoder().decode(RSSEnvelope.self, from: data) + return envelope.feed.entry?.enumerated().compactMap { (idx, entry) -> ChartEntry? in + guard let idStr = entry.id.attributes.imId, let appStoreId = Int64(idStr) else { + return nil + } + return ChartEntry( + appStoreId: appStoreId, + position: idx + 1, + name: entry.imName.label + ) + } ?? [] + } +} + +// MARK: - RSS envelope (Apple's "iTunes RSS feed in JSON form" shape). +// +// The endpoint returns either an array of entries (when the chart has +// results) or omits the `entry` key entirely (for empty / unknown genres). +// `entry` is therefore optional. +private struct RSSEnvelope: Decodable { + let feed: RSSFeed +} + +private struct RSSFeed: Decodable { + let entry: [RSSEntry]? +} + +private struct RSSEntry: Decodable { + let imName: ImName + let id: RSSEntryId + + enum CodingKeys: String, CodingKey { + case imName = "im:name" + case id + } + + struct ImName: Decodable { + let label: String + } + + struct RSSEntryId: Decodable { + let attributes: Attributes + struct Attributes: Decodable { + // Apple uses dotted/colon keys ("im:id") which Swift cannot + // express as a property name; decode via a custom CodingKey. + let imId: String? + enum CodingKeys: String, CodingKey { + case imId = "im:id" + } + } + } +} diff --git a/Sources/App/Composition/Container.swift b/Sources/App/Composition/Container.swift index 2864006..8347d7f 100644 --- a/Sources/App/Composition/Container.swift +++ b/Sources/App/Composition/Container.swift @@ -74,6 +74,23 @@ extension Request { QueueStatusService(db: db) } + func chartTrackerService() -> any ChartTrackerServiceProtocol { + ChartTrackerService( + db: db, + chartsClient: ITunesChartsClient(client: client, logger: logger), + lookupClient: ITunesLookupClient(client: client), + logger: logger + ) + } + + func availabilityProber() -> any AvailabilityProberProtocol { + AvailabilityProber( + db: db, + lookupClient: ITunesLookupClient(client: client), + logger: logger + ) + } + func versionService() -> any VersionServiceProtocol { VersionService( client: client, @@ -126,4 +143,16 @@ extension Application { ) } } + + var chartTrackerServiceFactory: @Sendable (QueueContext) -> any ChartTrackerServiceProtocol { + { context in + let app = context.application + return ChartTrackerService( + db: app.db, + chartsClient: ITunesChartsClient(client: app.client, logger: context.logger), + lookupClient: ITunesLookupClient(client: app.client), + logger: context.logger + ) + } + } } diff --git a/Sources/App/Controllers/AppsController.swift b/Sources/App/Controllers/AppsController.swift index 59182f2..c63edbd 100644 --- a/Sources/App/Controllers/AppsController.swift +++ b/Sources/App/Controllers/AppsController.swift @@ -19,10 +19,25 @@ struct AppsController: RouteCollection { @Sendable func create(req: Request) async throws -> WatchedApp { let payload = try req.content.decode(CreatePayload.self) - return try await req.appService().create( + let app = try await req.appService().create( appStoreId: payload.appStoreId, lookupCountry: payload.lookupCountry ?? "us" ) + // Kick off the 175-storefront availability probe in the background so + // the chart-watchdog has a narrowed sweep target on its next pass. + // The HTTP response shouldn't wait on this — it can take ~60s. + if let appID = app.id { + let prober = req.availabilityProber() + let logger = req.logger + Task.detached { + do { + _ = try await prober.probe(watchedAppID: appID) + } catch { + logger.error("Initial availability probe failed for app=\(appID): \(error)") + } + } + } + return app } @Sendable func delete(req: Request) async throws -> HTTPStatus { diff --git a/Sources/App/Controllers/ChartsController.swift b/Sources/App/Controllers/ChartsController.swift new file mode 100644 index 0000000..b3350c1 --- /dev/null +++ b/Sources/App/Controllers/ChartsController.swift @@ -0,0 +1,144 @@ +import Fluent +import Foundation +import Vapor + +// HTTP surface for the chart-position watchdog. +// GET /chart-positions → currently-charted snapshot rows +// GET /chart-events?since=&limit= → activity feed for the SPA +// POST /charts/refresh → kick off ChartTrackerService now +// POST /apps/:id/availability/refresh → re-probe a single app's storefronts +struct ChartsController: RouteCollection { + func boot(routes: any RoutesBuilder) throws { + routes.get("chart-positions", use: chartPositions) + routes.get("chart-events", use: chartEvents) + routes.post("charts", "refresh", use: refreshNow) + routes.post("apps", ":id", "availability", "refresh", use: refreshAvailability) + } + + // MARK: - DTOs (kept here rather than DomainTypes.swift because they're + // the wire format for one feature; nothing else consumes them.) + + struct ChartPositionDTO: Content { + let appId: UUID + let appName: String + let country: String + let chartType: String + let genreId: Int + let position: Int + let observedAt: Date + } + + struct ChartEventDTO: Content { + let id: UUID + let appId: UUID + let appName: String + let country: String + let chartType: String + let genreId: Int + let kind: String + let position: Int? + let prevPosition: Int? + let createdAt: Date + } + + struct RefreshAcceptedDTO: Content { let queued: Bool } + + // MARK: - Handlers + + @Sendable func chartPositions(req: Request) async throws -> [ChartPositionDTO] { + // Snapshot rows with a non-null position == "currently charted". + let snaps = try await ChartPositionSnapshot.query(on: req.db) + .filter(\.$position != nil) + .with(\.$watchedApp) + .all() + + return snaps.compactMap { snap -> ChartPositionDTO? in + guard + let appID = snap.watchedApp.id, + let position = snap.position + else { return nil } + return ChartPositionDTO( + appId: appID, + appName: snap.watchedApp.name, + country: snap.country, + chartType: snap.chartType, + genreId: snap.genreId, + position: position, + observedAt: snap.observedAt + ) + } + } + + @Sendable func chartEvents(req: Request) async throws -> [ChartEventDTO] { + let sinceStr = try? req.query.get(String.self, at: "since") + let limit = (try? req.query.get(Int.self, at: "limit")).map { min(max($0, 1), 200) } ?? 50 + + var q = ChartEvent.query(on: req.db).with(\.$watchedApp) + if let s = sinceStr { + let formatter = ISO8601DateFormatter() + formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + let since = formatter.date(from: s) + ?? ISO8601DateFormatter().date(from: s) + if let since { + q = q.filter(\.$createdAt > since) + } + } + + let events = try await q + .sort(\.$createdAt, .descending) + .range(.. ChartEventDTO? in + guard let evID = ev.id, let appID = ev.watchedApp.id else { return nil } + return ChartEventDTO( + id: evID, + appId: appID, + appName: ev.watchedApp.name, + country: ev.country, + chartType: ev.chartType, + genreId: ev.genreId, + kind: ev.kind, + position: ev.position, + prevPosition: ev.prevPosition, + createdAt: ev.createdAt + ) + } + } + + @Sendable func refreshNow(req: Request) async throws -> Response { + // Detached task so the HTTP response returns immediately. The job + // can take tens of seconds for a few apps × 30 countries; tying the + // request lifetime to it would block the SPA on the "Check now" click. + let service = req.chartTrackerService() + let logger = req.logger + Task.detached { + do { + _ = try await service.refreshAll(now: Date()) + } catch { + logger.error("Manual chart refresh failed: \(error)") + } + } + let response = Response(status: .accepted) + try response.content.encode(RefreshAcceptedDTO(queued: true)) + return response + } + + @Sendable func refreshAvailability(req: Request) async throws -> Response { + guard let id = req.parameters.get("id", as: UUID.self) else { + throw Abort(.badRequest, reason: "Invalid app id") + } + let prober = req.availabilityProber() + let logger = req.logger + Task.detached { + do { + _ = try await prober.probe(watchedAppID: id) + } catch { + logger.error("Availability probe failed for app=\(id): \(error)") + } + } + let response = Response(status: .accepted) + try response.content.encode(RefreshAcceptedDTO(queued: true)) + return response + } +} diff --git a/Sources/App/Domain/DomainTypes.swift b/Sources/App/Domain/DomainTypes.swift index bff624c..c6284e1 100644 --- a/Sources/App/Domain/DomainTypes.swift +++ b/Sources/App/Domain/DomainTypes.swift @@ -15,6 +15,10 @@ struct LookupResultApp: Codable, Sendable, Equatable { let bundleId: String let trackName: String let artworkUrl100: String? + // Apple's "Primary Category" id (e.g. 6017 = Education). Used by the + // chart-tracking watchdog to pick the right RSS feed per app. Optional + // because not every iTunes lookup variant returns it consistently. + let primaryGenreId: Int? } struct DashboardRow: Codable, Sendable, Equatable { diff --git a/Sources/App/Jobs/RefreshChartsScheduler.swift b/Sources/App/Jobs/RefreshChartsScheduler.swift new file mode 100644 index 0000000..f6230b6 --- /dev/null +++ b/Sources/App/Jobs/RefreshChartsScheduler.swift @@ -0,0 +1,23 @@ +import Foundation +import Queues +import Vapor + +// Daily chart-watchdog pass. Mirrors DailyRefreshScheduler's shape: the +// scheduler runs the work inline rather than enqueueing per-app jobs because +// (a) ChartTrackerService already does its own bounded concurrency across +// countries and (b) we only have a handful of watched apps in practice. +// +// Cadence is configured in configure.swift to 04:00 UTC — a few hours after +// Apple's PT-midnight chart refresh window so the RSS feeds have settled. +struct RefreshChartsScheduler: AsyncScheduledJob { + func run(context: QueueContext) async throws { + let service = context.application.chartTrackerServiceFactory(context) + let summary = try await service.refreshAll(now: Date()) + context.logger.info( + """ + RefreshChartsScheduler done: \ + apps=\(summary.appsProcessed) charts=\(summary.chartsFetched) events=\(summary.eventsEmitted) + """ + ) + } +} diff --git a/Sources/App/Models/AppStorefrontAvailability.swift b/Sources/App/Models/AppStorefrontAvailability.swift new file mode 100644 index 0000000..19d8acf --- /dev/null +++ b/Sources/App/Models/AppStorefrontAvailability.swift @@ -0,0 +1,50 @@ +import Fluent +import Vapor + +// Records which App Store storefronts a watched app is actually published +// in. Populated by AvailabilityProber after an app is added (and on demand +// via /apps/:id/availability/refresh). ChartTrackerService consults this to +// avoid 175 wasted RSS fetches per cycle when the app only ships in a dozen +// countries. +final class AppStorefrontAvailability: Model, Content, @unchecked Sendable { + static let schema = "app_storefront_availability" + + @ID(custom: .id, generatedBy: .user) var id: UUID? + @Parent(key: "app_id") var watchedApp: WatchedApp + @Field(key: "country") var country: String + @Field(key: "available") var available: Bool + @Field(key: "checked_at") var checkedAt: Date + + init() {} + + init( + id: UUID? = nil, + watchedAppID: UUID, + country: String, + available: Bool, + checkedAt: Date + ) { + self.id = id ?? UUID() + self.$watchedApp.id = watchedAppID + self.country = country.lowercased() + self.available = available + self.checkedAt = checkedAt + } +} + +struct CreateAppStorefrontAvailability: AsyncMigration { + func prepare(on database: Database) async throws { + try await database.schema(AppStorefrontAvailability.schema) + .id() + .field("app_id", .uuid, .required, .references(WatchedApp.schema, "id", onDelete: .cascade)) + .field("country", .string, .required) + .field("available", .bool, .required) + .field("checked_at", .datetime, .required) + .unique(on: "app_id", "country") + .create() + } + + func revert(on database: Database) async throws { + try await database.schema(AppStorefrontAvailability.schema).delete() + } +} diff --git a/Sources/App/Models/ChartEvent.swift b/Sources/App/Models/ChartEvent.swift new file mode 100644 index 0000000..3f7ddeb --- /dev/null +++ b/Sources/App/Models/ChartEvent.swift @@ -0,0 +1,81 @@ +import Fluent +import SQLKit +import Vapor + +// Append-only audit log of chart transitions. Drives both the activity feed +// rendered on /charts and the polling loop that fires browser notifications. +// One row per detected change — entered (not charted → #N), moved (#M → #N), +// exited (#M → not charted). Stable transitions and "still not charted" both +// produce no row. +final class ChartEvent: Model, Content, @unchecked Sendable { + static let schema = "chart_event" + + @ID(custom: .id, generatedBy: .user) var id: UUID? + @Parent(key: "app_id") var watchedApp: WatchedApp + @Field(key: "country") var country: String + @Field(key: "chart_type") var chartType: String + @Field(key: "genre_id") var genreId: Int + @Field(key: "kind") var kind: String // entered | moved | exited + @OptionalField(key: "position") var position: Int? + @OptionalField(key: "prev_position") var prevPosition: Int? + @Field(key: "created_at") var createdAt: Date + + init() {} + + init( + id: UUID? = nil, + watchedAppID: UUID, + country: String, + chartType: String, + genreId: Int, + kind: Kind, + position: Int?, + prevPosition: Int?, + createdAt: Date + ) { + self.id = id ?? UUID() + self.$watchedApp.id = watchedAppID + self.country = country.lowercased() + self.chartType = chartType + self.genreId = genreId + self.kind = kind.rawValue + self.position = position + self.prevPosition = prevPosition + self.createdAt = createdAt + } + + enum Kind: String, Sendable { + case entered, moved, exited + } +} + +struct CreateChartEvent: AsyncMigration { + func prepare(on database: Database) async throws { + try await database.schema(ChartEvent.schema) + .id() + .field("app_id", .uuid, .required, .references(WatchedApp.schema, "id", onDelete: .cascade)) + .field("country", .string, .required) + .field("chart_type", .string, .required) + .field("genre_id", .int, .required) + .field("kind", .string, .required) + .field("position", .int) + .field("prev_position", .int) + .field("created_at", .datetime, .required) + .create() + + // Activity feed and the SPA polling loop both want newest-first by + // created_at. The composite (app_id, created_at DESC) covers both + // "events globally, newest first" (the polling query) and + // "events for this app, newest first" (the per-app history view). + if let sql = database as? SQLDatabase { + try await sql.raw(""" + CREATE INDEX IF NOT EXISTS chart_event_app_time + ON chart_event (app_id, created_at DESC) + """).run() + } + } + + func revert(on database: Database) async throws { + try await database.schema(ChartEvent.schema).delete() + } +} diff --git a/Sources/App/Models/ChartPositionSnapshot.swift b/Sources/App/Models/ChartPositionSnapshot.swift new file mode 100644 index 0000000..8652b55 --- /dev/null +++ b/Sources/App/Models/ChartPositionSnapshot.swift @@ -0,0 +1,57 @@ +import Fluent +import Vapor + +// Last-known position for a watched app on a single (country, chart, genre) +// chart. NULL position means "polled this cycle but the app wasn't in the +// top-100" — kept as a tombstone so the diff algorithm can distinguish +// "still not charted" (no event) from "just exited" (emit `exited`). +final class ChartPositionSnapshot: Model, Content, @unchecked Sendable { + static let schema = "chart_position_snapshot" + + @ID(custom: .id, generatedBy: .user) var id: UUID? + @Parent(key: "app_id") var watchedApp: WatchedApp + @Field(key: "country") var country: String + @Field(key: "chart_type") var chartType: String + @Field(key: "genre_id") var genreId: Int + @OptionalField(key: "position") var position: Int? + @Field(key: "observed_at") var observedAt: Date + + init() {} + + init( + id: UUID? = nil, + watchedAppID: UUID, + country: String, + chartType: String, + genreId: Int, + position: Int?, + observedAt: Date + ) { + self.id = id ?? UUID() + self.$watchedApp.id = watchedAppID + self.country = country.lowercased() + self.chartType = chartType + self.genreId = genreId + self.position = position + self.observedAt = observedAt + } +} + +struct CreateChartPositionSnapshot: AsyncMigration { + func prepare(on database: Database) async throws { + try await database.schema(ChartPositionSnapshot.schema) + .id() + .field("app_id", .uuid, .required, .references(WatchedApp.schema, "id", onDelete: .cascade)) + .field("country", .string, .required) + .field("chart_type", .string, .required) + .field("genre_id", .int, .required) + .field("position", .int) + .field("observed_at", .datetime, .required) + .unique(on: "app_id", "country", "chart_type", "genre_id") + .create() + } + + func revert(on database: Database) async throws { + try await database.schema(ChartPositionSnapshot.schema).delete() + } +} diff --git a/Sources/App/Models/WatchedApp.swift b/Sources/App/Models/WatchedApp.swift index 87e38bd..77705ee 100644 --- a/Sources/App/Models/WatchedApp.swift +++ b/Sources/App/Models/WatchedApp.swift @@ -9,16 +9,29 @@ final class WatchedApp: Model, Content, @unchecked Sendable { @Field(key: "bundle_id") var bundleId: String @Field(key: "name") var name: String @OptionalField(key: "icon_url") var iconURL: String? + // iTunes "Primary Category" (e.g. 6017 = Education, 6014 = Games). + // Optional only because the column was added later; AppService fills it + // on every newly-created row, and ChartTrackerService lazily backfills + // existing rows by re-running the iTunes lookup on first chart-refresh. + @OptionalField(key: "primary_genre_id") var primaryGenreId: Int? @Timestamp(key: "added_at", on: .create) var addedAt: Date? init() {} - init(id: UUID? = nil, appStoreId: Int64, bundleId: String, name: String, iconURL: String?) { + init( + id: UUID? = nil, + appStoreId: Int64, + bundleId: String, + name: String, + iconURL: String?, + primaryGenreId: Int? = nil + ) { self.id = id self.appStoreId = appStoreId self.bundleId = bundleId self.name = name self.iconURL = iconURL + self.primaryGenreId = primaryGenreId } } @@ -39,3 +52,21 @@ struct CreateWatchedApp: AsyncMigration { try await database.schema(WatchedApp.schema).delete() } } + +// Added to support chart-tracking: the primary genre drives which top-free +// chart we poll per app. Nullable for migration compatibility — existing +// rows get backfilled on the next iTunes lookup (either /availability/refresh +// or the first chart-refresh cycle). +struct AddPrimaryGenreIdToWatchedApp: AsyncMigration { + func prepare(on database: Database) async throws { + try await database.schema(WatchedApp.schema) + .field("primary_genre_id", .int) + .update() + } + + func revert(on database: Database) async throws { + try await database.schema(WatchedApp.schema) + .deleteField("primary_genre_id") + .update() + } +} diff --git a/Sources/App/Services/AppService.swift b/Sources/App/Services/AppService.swift index 0dafe4e..ec1f26f 100644 --- a/Sources/App/Services/AppService.swift +++ b/Sources/App/Services/AppService.swift @@ -24,7 +24,8 @@ struct AppService: AppServiceProtocol { appStoreId: info.trackId, bundleId: info.bundleId, name: info.trackName, - iconURL: info.artworkUrl100 + iconURL: info.artworkUrl100, + primaryGenreId: info.primaryGenreId ) try await repository.save(app) return app diff --git a/Sources/App/Services/AvailabilityProber.swift b/Sources/App/Services/AvailabilityProber.swift new file mode 100644 index 0000000..0b978ea --- /dev/null +++ b/Sources/App/Services/AvailabilityProber.swift @@ -0,0 +1,170 @@ +import Fluent +import Foundation +import Vapor + +// Batched probe that asks iTunes lookup "is this app published in country X?" +// for every App Store storefront. Used after an app is added and via +// /apps/:id/availability/refresh. Results land in app_storefront_availability +// and ChartTrackerService consults that table to avoid 175 RSS fetches per +// cycle when the app only ships in ~30 countries. +// +// Side effect: if WatchedApp.primaryGenreId is nil, this opportunistically +// backfills it from the first successful lookup. Existing rows added before +// the chart-tracking feature shipped get healed automatically the next time +// they're probed. + +protocol AvailabilityProberProtocol: Sendable { + func probe(watchedAppID: UUID) async throws -> Int +} + +struct AvailabilityProber: AvailabilityProberProtocol { + let db: any Database + let lookupClient: any ITunesLookupClientProtocol + let logger: Logger + // Run at most N lookups in flight against iTunes at once. iTunes' edge + // tolerates much higher fan-out but 8 is plenty to finish the 175-probe + // sweep in under a minute without being a thundering herd. + var concurrency: Int = 8 + + @discardableResult + func probe(watchedAppID: UUID) async throws -> Int { + guard let app = try await WatchedApp.find(watchedAppID, on: db) else { + throw Abort(.notFound, reason: "WatchedApp \(watchedAppID) not found") + } + let appStoreId = app.appStoreId + let now = Date() + + // Fan out lookups in batches of `concurrency` to keep iTunes happy. + // For each country we map to one of three states: + // .ok(primaryGenreId?) — app exists in this storefront + // .notAvailable — confirmed 404 from iTunes + // .skipped — transient error; leave existing row alone + let results = try await withThrowingTaskGroup(of: (String, ProbeOutcome).self) { group in + var iterator = AppStoreCountries.all.makeIterator() + var inFlight = 0 + var collected: [(String, ProbeOutcome)] = [] + + // Prime the pipeline. + while inFlight < concurrency, let country = iterator.next() { + group.addTask { + let outcome = await self.probeOne(appStoreId: appStoreId, country: country) + return (country, outcome) + } + inFlight += 1 + } + while let next = try await group.next() { + collected.append(next) + inFlight -= 1 + if let country = iterator.next() { + group.addTask { + let outcome = await self.probeOne(appStoreId: appStoreId, country: country) + return (country, outcome) + } + inFlight += 1 + } + } + return collected + } + + var written = 0 + var backfilledGenre: Int? = nil + for (country, outcome) in results { + switch outcome { + case .ok(let genreId): + if backfilledGenre == nil, let genreId { backfilledGenre = genreId } + try await upsertAvailability(appID: watchedAppID, country: country, available: true, at: now) + written += 1 + case .notAvailable: + try await upsertAvailability(appID: watchedAppID, country: country, available: false, at: now) + written += 1 + case .skipped: + continue + } + } + + // Backfill primary_genre_id if absent. Existing rows from before the + // chart-tracking feature shipped don't have this and we need it to + // know which top-free chart to poll. + if app.primaryGenreId == nil, let g = backfilledGenre { + app.primaryGenreId = g + try await app.save(on: db) + } + + logger.info("AvailabilityProber app=\(appStoreId) wrote=\(written) genre=\(app.primaryGenreId ?? -1)") + return written + } + + private enum ProbeOutcome { + case ok(primaryGenreId: Int?) + case notAvailable + case skipped + } + + private func probeOne(appStoreId: Int64, country: String) async -> ProbeOutcome { + do { + let result = try await lookupClient.lookup(appStoreId: appStoreId, country: country) + return .ok(primaryGenreId: result.primaryGenreId) + } catch let abort as AbortError where abort.status == .notFound { + return .notAvailable + } catch { + // Transient (timeout, 5xx, DNS). Don't write a stale row. + logger.warning("AvailabilityProber transient error app=\(appStoreId) country=\(country): \(error)") + return .skipped + } + } + + private func upsertAvailability( + appID: UUID, + country: String, + available: Bool, + at: Date + ) async throws { + let cc = country.lowercased() + if let existing = try await AppStorefrontAvailability.query(on: db) + .filter(\.$watchedApp.$id == appID) + .filter(\.$country == cc) + .first() + { + existing.available = available + existing.checkedAt = at + try await existing.save(on: db) + } else { + let row = AppStorefrontAvailability( + watchedAppID: appID, + country: cc, + available: available, + checkedAt: at + ) + try await row.save(on: db) + } + } +} + +// The 175 App Store storefronts, verified against +// developer.apple.com/help/app-store-connect/reference/pricing-and-availability/... +// Kept in sync with web/src/lib/countries.ts; if Apple adds territories, +// update both lists. +enum AppStoreCountries { + static let all: [String] = [ + "af", "al", "dz", "ao", "ai", "ag", "ar", "am", "au", "at", "az", + "bs", "bh", "bb", "by", "be", "bz", "bj", "bm", "bt", "bo", "ba", + "bw", "br", "vg", "bn", "bg", "bf", + "kh", "cm", "ca", "cv", "ky", "td", "cl", "cn", "co", "cd", "cg", + "cr", "ci", "hr", "cy", "cz", + "dk", "dm", "do", "ec", "eg", "sv", "ee", "sz", "fj", "fi", "fr", + "ga", "gm", "ge", "de", "gh", "gr", "gd", "gt", "gw", "gy", + "hn", "hk", "hu", "is", "in", "id", "iq", "ie", "il", "it", + "jm", "jp", "jo", "kz", "ke", "xk", "kw", "kg", + "la", "lv", "lb", "lr", "ly", "lt", "lu", + "mo", "mg", "mw", "my", "mv", "ml", "mt", "mr", "mu", "mx", + "fm", "md", "mn", "me", "ms", "ma", "mz", "mm", + "na", "nr", "np", "nl", "nz", "ni", "ng", "mk", "no", + "om", "pk", "pw", "pa", "pg", "py", "pe", "ph", "pl", "pt", + "qa", "kr", "ro", "ru", "rw", + "st", "sa", "sn", "rs", "sc", "sl", "sg", "sk", "si", "sb", "za", + "es", "lk", "kn", "lc", "vc", "sr", "se", "ch", + "tw", "tj", "tz", "th", "to", "tt", "tn", "tr", "tm", "tc", + "ug", "ua", "ae", "gb", "us", "uy", "uz", + "vu", "ve", "vn", "ye", "zm", "zw", + ] +} diff --git a/Sources/App/Services/ChartTrackerService.swift b/Sources/App/Services/ChartTrackerService.swift new file mode 100644 index 0000000..fd20942 --- /dev/null +++ b/Sources/App/Services/ChartTrackerService.swift @@ -0,0 +1,271 @@ +import Fluent +import Foundation +import Vapor + +// One refresh pass of the chart-position watchdog. For each watched app, +// for each country where the app is available, fetch the top-free chart in +// the app's primary genre and diff the result against the stored snapshot. +// Emits ChartEvent rows for entered / moved / exited transitions; no event +// for stable-charted or still-not-charted. +// +// Called from RefreshChartsJob on a daily Queues schedule and from +// POST /api/v1/charts/refresh. + +protocol ChartTrackerServiceProtocol: Sendable { + func refreshAll(now: Date) async throws -> ChartRefreshSummary +} + +struct ChartRefreshSummary: Sendable, Equatable { + let appsProcessed: Int + let chartsFetched: Int + let eventsEmitted: Int +} + +// What the diff algorithm decided for a single (app, country) result. +// Extracted from ChartTrackerService so it can be unit-tested without a DB. +enum ChartTransition: Sendable, Equatable { + case entered(position: Int) + case moved(from: Int, to: Int) + case exited(from: Int) + case stableCharted(at: Int) // unchanged position; bump observed_at only + case stableTombstone // had a tombstone row, still not charted; bump observed_at + case noop // never had a snapshot row and still not charted; write nothing + + var shouldWriteSnapshot: Bool { + if case .noop = self { return false } + return true + } + + var eventKind: ChartEvent.Kind? { + switch self { + case .entered: return .entered + case .moved: return .moved + case .exited: return .exited + case .stableCharted, .stableTombstone, .noop: return nil + } + } +} + +// Pure-function form of the watchdog's diff logic. Given the previous +// snapshot state and the position observed in the just-fetched chart, +// decide what transition happened. No DB, no I/O — easy to unit test. +func decideChartTransition(prev: Int?, new: Int?, hasPriorRow: Bool) -> ChartTransition { + switch (prev, new) { + case (nil, nil): + return hasPriorRow ? .stableTombstone : .noop + case (nil, .some(let n)): + return .entered(position: n) + case (.some(let p), nil): + return .exited(from: p) + case let (.some(p), .some(n)) where p != n: + return .moved(from: p, to: n) + case (.some(let p), .some): + return .stableCharted(at: p) + } +} + +struct ChartTrackerService: ChartTrackerServiceProtocol { + let db: any Database + let chartsClient: any ITunesChartsClientProtocol + let lookupClient: any ITunesLookupClientProtocol + let logger: Logger + // Max concurrent country-level fetches per app. Plays nice with iTunes + // while still finishing a ~30-country sweep in seconds. + var countryConcurrency: Int = 4 + // Only the top-free chart is watched in MVP. Hard-coded here rather than + // a column on the snapshot so adding top-paid/top-grossing later is just + // a loop expansion. + var chartType: String = "top-free" + + @discardableResult + func refreshAll(now: Date) async throws -> ChartRefreshSummary { + let apps = try await WatchedApp.query(on: db).all() + var totalCharts = 0 + var totalEvents = 0 + + for app in apps { + guard let appID = app.id else { continue } + // Backfill primary_genre_id for legacy rows by re-running the + // iTunes lookup. One-off cost; subsequent passes use the cached + // value. + let genreId: Int + if let cached = app.primaryGenreId { + genreId = cached + } else { + do { + let info = try await lookupClient.lookup(appStoreId: app.appStoreId, country: "us") + guard let g = info.primaryGenreId else { + logger.warning("ChartTracker skip app=\(app.appStoreId) — no primaryGenreId from iTunes") + continue + } + app.primaryGenreId = g + try await app.save(on: db) + genreId = g + } catch { + logger.warning("ChartTracker backfill failed app=\(app.appStoreId): \(error)") + continue + } + } + + // Available countries for this app. If the prober hasn't run + // (e.g. the app was added before this feature shipped), fall + // back to every storefront — the prober will populate this + // table over time. + let availabilityRows = try await AppStorefrontAvailability.query(on: db) + .filter(\.$watchedApp.$id == appID) + .filter(\.$available == true) + .all() + let countries: [String] + if availabilityRows.isEmpty { + countries = AppStoreCountries.all + } else { + countries = availabilityRows.map { $0.country } + } + + let (charts, events) = await refreshOne( + appID: appID, + appStoreId: app.appStoreId, + genreId: genreId, + countries: countries, + now: now + ) + totalCharts += charts + totalEvents += events + } + + logger.info("ChartTracker pass: apps=\(apps.count) charts=\(totalCharts) events=\(totalEvents)") + return .init(appsProcessed: apps.count, chartsFetched: totalCharts, eventsEmitted: totalEvents) + } + + private func refreshOne( + appID: UUID, + appStoreId: Int64, + genreId: Int, + countries: [String], + now: Date + ) async -> (charts: Int, events: Int) { + // Bounded-concurrency country sweep. Each task returns the count of + // events it emitted (0 or 1). + return await withTaskGroup(of: Int.self) { group in + var iterator = countries.makeIterator() + var inFlight = 0 + var charts = 0 + var events = 0 + + while inFlight < countryConcurrency, let country = iterator.next() { + group.addTask { + await self.refreshOneCountry( + appID: appID, appStoreId: appStoreId, + genreId: genreId, country: country, now: now + ) + } + inFlight += 1 + } + while let emitted = await group.next() { + charts += 1 + events += emitted + inFlight -= 1 + if let country = iterator.next() { + group.addTask { + await self.refreshOneCountry( + appID: appID, appStoreId: appStoreId, + genreId: genreId, country: country, now: now + ) + } + inFlight += 1 + } + } + return (charts, events) + } + } + + private func refreshOneCountry( + appID: UUID, + appStoreId: Int64, + genreId: Int, + country: String, + now: Date + ) async -> Int { + do { + let entries = try await chartsClient.topFree(country: country, genreId: genreId, limit: 200) + let newPosition = entries.first(where: { $0.appStoreId == appStoreId })?.position + return try await applyDiff( + appID: appID, country: country, genreId: genreId, + newPosition: newPosition, now: now + ) + } catch { + // Don't emit "exited" off a failed fetch — the apparent absence + // is a network failure, not a real chart drop. Next cycle fixes it. + logger.warning("ChartTracker fetch failed app=\(appStoreId) country=\(country): \(error)") + return 0 + } + } + + // Heart of the watchdog. Returns the number of events emitted (0 or 1). + // Atomic per (app, country) so an interrupted job can't desync the + // snapshot from the event log. + private func applyDiff( + appID: UUID, + country: String, + genreId: Int, + newPosition: Int?, + now: Date + ) async throws -> Int { + return try await db.transaction { tx in + let cc = country.lowercased() + let existing = try await ChartPositionSnapshot.query(on: tx) + .filter(\.$watchedApp.$id == appID) + .filter(\.$country == cc) + .filter(\.$chartType == chartType) + .filter(\.$genreId == genreId) + .first() + + // Flatten the layered optional: + // no snapshot row → prev = nil + // tombstone (position) → prev = nil + // charted at #M → prev = M + let prevPosition: Int? = existing?.position ?? nil + let hasPriorRow = existing != nil + + let transition = decideChartTransition( + prev: prevPosition, new: newPosition, hasPriorRow: hasPriorRow + ) + if !transition.shouldWriteSnapshot { return 0 } + + // Upsert snapshot. + if let existing { + existing.position = newPosition + existing.observedAt = now + try await existing.save(on: tx) + } else { + let snap = ChartPositionSnapshot( + watchedAppID: appID, + country: cc, + chartType: chartType, + genreId: genreId, + position: newPosition, + observedAt: now + ) + try await snap.save(on: tx) + } + + // Emit event if needed. + if let kind = transition.eventKind { + let ev = ChartEvent( + watchedAppID: appID, + country: cc, + chartType: chartType, + genreId: genreId, + kind: kind, + position: newPosition, + prevPosition: prevPosition, + createdAt: now + ) + try await ev.save(on: tx) + return 1 + } + + return 0 + } + } +} diff --git a/Sources/App/configure.swift b/Sources/App/configure.swift index 8f3761b..14701ed 100644 --- a/Sources/App/configure.swift +++ b/Sources/App/configure.swift @@ -91,6 +91,10 @@ public func configure(_ app: Application) async throws { app.migrations.add(CreateSetting()) app.migrations.add(JobModelMigrate()) app.migrations.add(AddFirstSeenAtToRankCheck()) + app.migrations.add(AddPrimaryGenreIdToWatchedApp()) + app.migrations.add(CreateAppStorefrontAvailability()) + app.migrations.add(CreateChartPositionSnapshot()) + app.migrations.add(CreateChartEvent()) try await app.autoMigrate() @@ -112,6 +116,12 @@ public func configure(_ app: Application) async throws { app.queues.schedule(DailyRefreshScheduler()) .daily() .at("3:00am") + // Chart-position watchdog. Lands one hour after the keyword refresh so + // it doesn't pile on top of iTunes simultaneously, and 4 hours after + // Apple's midnight-PT chart refresh window so the RSS feeds are settled. + app.queues.schedule(RefreshChartsScheduler()) + .daily() + .at("4:00am") // Serial worker. The original plan called for "~1 req/sec to iTunes" // to stay below Apple's edge throttling; running multiple workers in diff --git a/Sources/App/routes.swift b/Sources/App/routes.swift index 77f472c..4d8bb06 100644 --- a/Sources/App/routes.swift +++ b/Sources/App/routes.swift @@ -16,4 +16,5 @@ func routes(_ app: Application) throws { try api.register(collection: DashboardController()) try api.register(collection: SettingsController()) try api.register(collection: VersionController()) + try api.register(collection: ChartsController()) } diff --git a/Tests/AppTests/AppServiceTests.swift b/Tests/AppTests/AppServiceTests.swift index 3efa444..cb027af 100644 --- a/Tests/AppTests/AppServiceTests.swift +++ b/Tests/AppTests/AppServiceTests.swift @@ -6,7 +6,7 @@ import Testing struct AppServiceTests { @Test("create enriches via lookup client and persists") func create_enrichesAndPersists() async throws { - let lookup = LookupResultApp(trackId: 42, bundleId: "com.azri", trackName: "Azri", artworkUrl100: "https://icon") + let lookup = LookupResultApp(trackId: 42, bundleId: "com.azri", trackName: "Azri", artworkUrl100: "https://icon", primaryGenreId: 6017) let repo = InMemoryWatchedAppRepository() let service = AppService(repository: repo, lookupClient: StubLookupClient(canned: lookup)) @@ -16,6 +16,7 @@ struct AppServiceTests { #expect(result.name == "Azri") #expect(result.bundleId == "com.azri") #expect(result.iconURL == "https://icon") + #expect(result.primaryGenreId == 6017) let all = try await repo.all() #expect(all.count == 1) @@ -25,7 +26,7 @@ struct AppServiceTests { func list_returnsRepository() async throws { let app = WatchedApp(id: UUID(), appStoreId: 1, bundleId: "a", name: "A", iconURL: nil) let repo = InMemoryWatchedAppRepository([app]) - let lookup = LookupResultApp(trackId: 1, bundleId: "a", trackName: "A", artworkUrl100: nil) + let lookup = LookupResultApp(trackId: 1, bundleId: "a", trackName: "A", artworkUrl100: nil, primaryGenreId: nil) let service = AppService(repository: repo, lookupClient: StubLookupClient(canned: lookup)) let result = try await service.list() diff --git a/Tests/AppTests/ChartTransitionTests.swift b/Tests/AppTests/ChartTransitionTests.swift new file mode 100644 index 0000000..d4095bc --- /dev/null +++ b/Tests/AppTests/ChartTransitionTests.swift @@ -0,0 +1,69 @@ +@testable import App +import Foundation +import Testing + +// Pure-function coverage of the watchdog's diff logic. The full +// ChartTrackerService wraps each transition in a Fluent transaction; that +// path is verified by the manual smoke test in the plan. These tests own +// the decision matrix that determines what event (if any) we emit. +@Suite("decideChartTransition") +struct ChartTransitionTests { + @Test("never charted, still not charted — no-op (no snapshot row written)") + func noPriorRow_stillNotCharted_isNoop() { + let t = decideChartTransition(prev: nil, new: nil, hasPriorRow: false) + #expect(t == .noop) + #expect(t.eventKind == nil) + #expect(t.shouldWriteSnapshot == false) + } + + @Test("tombstone, still not charted — bump observed_at only") + func tombstone_stillNotCharted_isStableTombstone() { + let t = decideChartTransition(prev: nil, new: nil, hasPriorRow: true) + #expect(t == .stableTombstone) + #expect(t.eventKind == nil) + #expect(t.shouldWriteSnapshot) + } + + @Test("never charted → #57 emits entered") + func enteredFromNothing() { + let t = decideChartTransition(prev: nil, new: 57, hasPriorRow: false) + #expect(t == .entered(position: 57)) + #expect(t.eventKind == .entered) + } + + @Test("tombstone → #87 emits entered (resumes from exit)") + func enteredFromTombstone() { + let t = decideChartTransition(prev: nil, new: 87, hasPriorRow: true) + #expect(t == .entered(position: 87)) + #expect(t.eventKind == .entered) + } + + @Test("#45 → #30 emits moved") + func movedUp() { + let t = decideChartTransition(prev: 45, new: 30, hasPriorRow: true) + #expect(t == .moved(from: 45, to: 30)) + #expect(t.eventKind == .moved) + } + + @Test("#89 → #87 emits moved (down inside chart counts too)") + func movedDown() { + let t = decideChartTransition(prev: 89, new: 87, hasPriorRow: true) + #expect(t == .moved(from: 89, to: 87)) + #expect(t.eventKind == .moved) + } + + @Test("#94 → not found emits exited") + func exited() { + let t = decideChartTransition(prev: 94, new: nil, hasPriorRow: true) + #expect(t == .exited(from: 94)) + #expect(t.eventKind == .exited) + } + + @Test("#45 → #45 emits no event but still bumps observed_at") + func stableCharted() { + let t = decideChartTransition(prev: 45, new: 45, hasPriorRow: true) + #expect(t == .stableCharted(at: 45)) + #expect(t.eventKind == nil) + #expect(t.shouldWriteSnapshot) + } +} diff --git a/Tests/AppTests/ITunesChartsClientTests.swift b/Tests/AppTests/ITunesChartsClientTests.swift new file mode 100644 index 0000000..db29a35 --- /dev/null +++ b/Tests/AppTests/ITunesChartsClientTests.swift @@ -0,0 +1,75 @@ +@testable import App +import Foundation +import Testing + +// Verifies the RSS-feed JSON decoder. Apple's iTunes RSS endpoint returns a +// nested `feed.entry[]` shape with dotted keys ("im:id", "im:name") that +// Swift can't express directly; we rely on custom CodingKeys to map them. +@Suite("ITunesChartsClient.parseEntries") +struct ITunesChartsClientTests { + @Test("decodes positions and app ids in order") + func decodesEntriesInOrder() throws { + let json = """ + { + "feed": { + "entry": [ + { + "im:name": { "label": "Duolingo: Language Lessons" }, + "id": { "attributes": { "im:id": "570060128" } } + }, + { + "im:name": { "label": "Azri: AI Flashcards & FSRS" }, + "id": { "attributes": { "im:id": "1625870857" } } + }, + { + "im:name": { "label": "Toca Boca World" }, + "id": { "attributes": { "im:id": "863571574" } } + } + ] + } + } + """.data(using: .utf8)! + + let entries = try ITunesChartsClient.parseEntries(from: json) + #expect(entries.count == 3) + #expect(entries[0].position == 1) + #expect(entries[0].appStoreId == 570060128) + #expect(entries[0].name == "Duolingo: Language Lessons") + #expect(entries[1].position == 2) + #expect(entries[1].appStoreId == 1625870857) + #expect(entries[2].position == 3) + } + + @Test("handles an absent entry array (empty chart) without crashing") + func decodesEmptyEntries() throws { + let json = """ + { "feed": { } } + """.data(using: .utf8)! + let entries = try ITunesChartsClient.parseEntries(from: json) + #expect(entries.isEmpty) + } + + @Test("skips entries with malformed im:id rather than throwing") + func skipsBadEntries() throws { + let json = """ + { + "feed": { + "entry": [ + { + "im:name": { "label": "Good" }, + "id": { "attributes": { "im:id": "42" } } + }, + { + "im:name": { "label": "Bad — non-numeric id" }, + "id": { "attributes": { "im:id": "not-a-number" } } + } + ] + } + } + """.data(using: .utf8)! + let entries = try ITunesChartsClient.parseEntries(from: json) + #expect(entries.count == 1) + #expect(entries[0].name == "Good") + #expect(entries[0].appStoreId == 42) + } +} From 6ddc361bdfc56941c964b6cf857240d359578d48 Mon Sep 17 00:00:00 2001 From: Astemir Boziev Date: Sat, 23 May 2026 17:23:40 +0400 Subject: [PATCH 2/3] Add Charts page + browser-notification polling (frontend) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Surfaces the chart-watchdog from the backend. A new "Charts" button in the dashboard toolbar (with an unread-count badge fed by a 30s polling loop) opens a full-screen ChartsPage with three states: - Empty + permission CTA: first visit asks for Notification permission; the Later button is remembered via localStorage so we don't re-prompt. - Currently charted: cards showing rank + flag + country + genre + "seen X ago" for each non-null snapshot row. - Recent activity: feed of entered/moved/exited events with relative timestamps and a glyph per transition kind. A "Check now" button POSTs to /api/v1/charts/refresh and reloads after the backend job settles; "Re-probe availability" loops over watched apps and kicks /apps/:id/availability/refresh for each. chartEvents.ts owns the singleton 30s poll, deduping by event id and persisting lastSeenIso so reloads never re-fire stale notifications. notifications.ts wraps the Notification API with a graceful fallback when the user denies or the browser doesn't support it — the activity feed remains the durable record. Genre IDs decoded inline (small static map) since the backend stores the numeric id; only common App Store categories are hardcoded, the rest fall back to "Category #N". --- web/src/components/ChartActivityRow.svelte | 34 ++++ web/src/components/ChartPositionCard.svelte | 56 +++++ web/src/components/ChartsPage.svelte | 215 ++++++++++++++++++++ web/src/components/Dashboard.svelte | 36 ++++ web/src/lib/api.ts | 21 ++ web/src/lib/chartEvents.ts | 106 ++++++++++ web/src/lib/notifications.ts | 83 ++++++++ web/src/lib/types.ts | 27 +++ 8 files changed, 578 insertions(+) create mode 100644 web/src/components/ChartActivityRow.svelte create mode 100644 web/src/components/ChartPositionCard.svelte create mode 100644 web/src/components/ChartsPage.svelte create mode 100644 web/src/lib/chartEvents.ts create mode 100644 web/src/lib/notifications.ts diff --git a/web/src/components/ChartActivityRow.svelte b/web/src/components/ChartActivityRow.svelte new file mode 100644 index 0000000..57a23da --- /dev/null +++ b/web/src/components/ChartActivityRow.svelte @@ -0,0 +1,34 @@ + + +
+ {ICON[event.kind]} + + {event.appName} + {#if event.kind === 'entered'} + entered {isoCountryToFlag(event.country)} {event.country.toUpperCase()} at #{event.position} + {:else if event.kind === 'moved'} + moved in {isoCountryToFlag(event.country)} {event.country.toUpperCase()}: #{event.prevPosition} → #{event.position} + {:else} + exited {isoCountryToFlag(event.country)} {event.country.toUpperCase()} (was #{event.prevPosition}) + {/if} + + {timeAgo(event.createdAt)} +
diff --git a/web/src/components/ChartPositionCard.svelte b/web/src/components/ChartPositionCard.svelte new file mode 100644 index 0000000..d493ea2 --- /dev/null +++ b/web/src/components/ChartPositionCard.svelte @@ -0,0 +1,56 @@ + + +
+
+ {pos.appName} + + {isoCountryToFlag(pos.country)} {pos.country.toUpperCase()} + +
+
#{pos.position}
+
{genre} · {pos.chartType}
+
Seen {timeAgo(pos.observedAt)}
+
diff --git a/web/src/components/ChartsPage.svelte b/web/src/components/ChartsPage.svelte new file mode 100644 index 0000000..97adaa9 --- /dev/null +++ b/web/src/components/ChartsPage.svelte @@ -0,0 +1,215 @@ + + +
+ +
+

Keywordista

+ + Charts + +
+ + + +
+
+ +
+ {#if showPermissionCTA} +
+ 🔔 + + Get notified when your apps chart.
+ We'll fire a browser notification when one of your watched apps enters, moves in, or exits a top-100 chart in any storefront where it's available. + +
+ + +
+
+ {:else if permission === 'denied'} +
+ Browser notifications are blocked. Re-enable them in your browser's site settings, then reload this page. +
+ {/if} + + {#if loading} +

Loading…

+ {:else if error} +

{error}

+ {:else} + +
+

+ Currently charted + {#if positions.length > 0} + + {positions.length} {positions.length === 1 ? 'position' : 'positions'} + + {/if} +

+ {#if positions.length === 0} +
+
📈
+
Nothing charted right now
+
+ Watching {$apps.length} {$apps.length === 1 ? 'app' : 'apps'} across every storefront they ship in. + Events show up here the moment something crosses into the top-100. +
+
+ {:else} +
+ {#each positions as pos (pos.appId + pos.country + pos.chartType + pos.genreId)} + + {/each} +
+ {/if} +
+ + +
+

Recent activity

+ {#if $chartEvents.length === 0} +

No chart events yet. They'll appear here as soon as something moves.

+ {:else} +
+ {#each $chartEvents as ev (ev.id)} + + {/each} +
+ {/if} +
+ {/if} +
+
diff --git a/web/src/components/Dashboard.svelte b/web/src/components/Dashboard.svelte index 420e1dc..67bc9a7 100644 --- a/web/src/components/Dashboard.svelte +++ b/web/src/components/Dashboard.svelte @@ -26,6 +26,8 @@ import GroupHeader from './GroupHeader.svelte'; import AppSwitcher from './AppSwitcher.svelte'; import SettingsPanel from './SettingsPanel.svelte'; + import ChartsPage from './ChartsPage.svelte'; + import { chartEvents, lastVisited, startChartEventPoll } from '../lib/chartEvents'; import type { DashboardRow as Row } from '../lib/types'; let loading = $state(true); @@ -33,8 +35,19 @@ let showAddKeyword = $state(false); let showAddApp = $state(false); let showSettings = $state(false); + let showCharts = $state(false); let historyTarget = $state(null); + // Unread chart events for the toolbar badge: count of events created after + // the last time the user opened the Charts page. Visible badge nudges them + // to look at the activity without competing with the keyword dashboard. + const chartsUnreadCount = $derived.by(() => { + const last = lastVisited(); + if (!last) return $chartEvents.length; + const lastDate = new Date(last).getTime(); + return $chartEvents.filter((e) => new Date(e.createdAt).getTime() > lastDate).length; + }); + // Refresh-all progress is derived directly from the row-level state: // total = number of keyword IDs in the active batch // done = total minus how many of those IDs are still in `refreshing` @@ -91,6 +104,10 @@ onMount(async () => { await load(); + // Start the singleton chart-event poll. It fires browser notifications + // for new transitions and keeps the unread badge live. Safe to call from + // anywhere; the loop dedupes itself. + startChartEventPoll(); // Pull the latest ASC keyword list in the background — localStorage gives // us instant-paint badges from the last session; this refresh keeps them // current if the user shipped a new version since the page was last open. @@ -191,6 +208,22 @@ Refresh all {/if} +