diff --git a/README.md b/README.md index abd5dcac..8e3c6217 100644 --- a/README.md +++ b/README.md @@ -797,6 +797,8 @@ source = "both" # use "discord" for bot-only sync or "wiretap" for desktop-cache concurrency = 16 repair_every = "6h" full_history = true +exclude_channel_ids = [] +exclude_channel_kinds = [] attachment_text = true attachment_media = false max_attachment_bytes = 104857600 diff --git a/docs/commands/sync.md b/docs/commands/sync.md index 4e330d8f..4a8ea0b2 100644 --- a/docs/commands/sync.md +++ b/docs/commands/sync.md @@ -27,6 +27,7 @@ discrawl sync --source discord # bot API only; aliases: key, bot, api discrawl sync --source wiretap # desktop cache only; aliases: desktop, cache discrawl sync --guild 123456789012345678 --all-channels discrawl sync --channels 111,222 --since 2026-03-01T00:00:00Z +discrawl sync --exclude-channels 333,444 --exclude-channel-kinds announcement discrawl sync --with-embeddings discrawl sync --with-media ``` @@ -59,8 +60,11 @@ discrawl sync --with-media - `--all` - ignore `default_guild_id` and fan out across every discovered guild - `--guild ` / `--guilds ` - target specific guilds - `--channels ` - target specific channels (forum ids expand to threads) +- `--exclude-channels ` - omit channel messages even if a target or repair would otherwise select them +- `--exclude-channel-kinds ` - omit message channels by stored Discord kind, such as `announcement` - `--since ` - limit initial history and `--full` backfill to messages at or after this timestamp - `--concurrency ` - override worker count (default auto-sized: floor 8, cap 32) +- `--channel-timeout ` - bound each channel crawl; the shorter parent deadline still wins - `--skip-members` - refresh guild/channel/message data without crawling members - `--with-members` - refresh guild members even during the default latest-only sync; fail if the member crawl cannot complete - `--with-embeddings` - also enqueue changed messages into `embedding_jobs` @@ -69,6 +73,8 @@ discrawl sync --with-media ## Notes - `--latest-only` is the default for untargeted `sync`. Use `--all-channels` to opt out without doing a full historical crawl. +- Collection exclusions apply to Discord API sync and the wiretap portion of `source=both`; exclusions win over explicit targets. +- With `--update=auto`, the same exclusions filter message-scoped rows from the safe upstream merge while retaining channel metadata and existing local rows. - `--with-media` records expired or removed Discord CDN URLs as failed fetches with the HTTP status, commonly `404`. - `--with-media` updates the local cache only; run `publish --push` afterward to include cached non-DM media in the Git backup as gzip-compressed files. - `--since` does not mark older history as complete, so a later `sync --full` without `--since` can continue the backfill. diff --git a/docs/commands/tail.md b/docs/commands/tail.md index d094aa11..96fdc9e4 100644 --- a/docs/commands/tail.md +++ b/docs/commands/tail.md @@ -9,6 +9,7 @@ discrawl tail discrawl tail --guild 123456789012345678 discrawl tail --repair-every 30m discrawl tail --replay-failures-only +discrawl tail --repair-every 6h --repair-offset 15m ``` ## What it does @@ -18,6 +19,8 @@ discrawl tail --replay-failures-only - periodically runs a repair pass to catch anything the live stream missed - can replay a bounded set of unresolved exact-message failures without starting the Gateway tail +- ignores messages, edits, and deletes from channels excluded by + `[sync].exclude_channel_ids` or `[sync].exclude_channel_kinds` ## Flags @@ -25,6 +28,7 @@ discrawl tail --replay-failures-only - `--repair-every ` - frequency of the repair sweep - `--replay-failures-only` - replay unresolved exact-message tail failures and exit - `--replay-limit ` - maximum failures to inspect in replay-only mode (default and maximum: `25`) +- `--repair-offset ` - for positive values, align repairs to that wall-clock offset within each repair period ## Notes @@ -33,6 +37,9 @@ discrawl tail --replay-failures-only - terminates cleanly on SIGINT / SIGTERM and treats cancellation as normal exit - replay-only mode uses the normal exclusive writer lock and does not create Gateway message events or update `tail:last_event` +- retains excluded channel metadata while omitting their message traffic +- positive-offset repairs skip missed aligned slots after a long repair rather than running catch-up sweeps +- separation from another scheduled job is only stable when that job also uses a fixed calendar schedule; a drifting `StartInterval` schedule cannot guarantee the offset ## See also diff --git a/docs/commands/update.md b/docs/commands/update.md index 954a91af..c411fe9c 100644 --- a/docs/commands/update.md +++ b/docs/commands/update.md @@ -37,6 +37,12 @@ discrawl update --force --ref backup-2026-06-19 Normal updates preserve rows learned from live Discord or the desktop cache, even when those rows are absent from the Git snapshot. Generated event history and local sync cursors are not replayed during routine merges. If a safe merge is impossible, Discrawl keeps the current database, marks the snapshot as needing attention in `status --json`, and asks you to rerun with `--force`. +Routine safe merges also honor `[sync].exclude_channel_ids` and +`[sync].exclude_channel_kinds`. Excluded channel metadata and existing local +rows remain in place, while new message-scoped snapshot rows for those channels +are skipped. `--force` is still an explicit exact-reconciliation path and does +not provide this preservation guarantee. + ## How `sync` interacts `discrawl sync` does **not** auto-import the share unless `--update=auto` (safe merge when stale) or `--update=force` (exact replacement before live deltas). Routine live refreshes stay fast; explicit imports happen via `update`. diff --git a/docs/configuration.md b/docs/configuration.md index 0212aa04..b37d7362 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -60,7 +60,10 @@ token_keyring_account = "discord_bot_token" source = "both" # "discord" for bot-only sync, "wiretap" for desktop-cache-only import concurrency = 16 repair_every = "6h" +repair_offset = "0s" full_history = true +exclude_channel_ids = [] +exclude_channel_kinds = [] # for example, ["announcement"] attachment_text = true attachment_media = false max_attachment_bytes = 104857600 @@ -104,6 +107,13 @@ token_env = "DISCRAWL_REMOTE_TOKEN" stale_after = "" ``` +`sync.exclude_channel_ids` and `sync.exclude_channel_kinds` apply to Discord API +sync, live tail events, periodic tail repair, Discord Desktop cache import, and +routine safe Git snapshot merges. Exclusions win over an explicit +`sync --channels` target. Channel metadata and existing local rows are retained, +but new message-scoped rows from excluded channels are omitted. Forced exact +snapshot replacement remains a separate destructive operation. + `concurrency` is auto-sized at `init` to `min(32, max(8, GOMAXPROCS*2))`. ## Token resolution diff --git a/docs/guides/git-snapshots.md b/docs/guides/git-snapshots.md index 10c44776..b7014520 100644 --- a/docs/guides/git-snapshots.md +++ b/docs/guides/git-snapshots.md @@ -65,6 +65,11 @@ discrawl subscribe --no-auto-update https://github.com/example/discord-archive.g `discrawl update` runs the same safe pull/merge step manually. `discrawl update --force --ref ` reads historical Git objects directly and leaves the share checkout unchanged. Snapshot imports are delta-planned from crawlkit shard fingerprints. Older manifests without those fields fall back to Git blob identity, so the common publish shape only imports changed canonical shards. Routine merges preserve destination-only rows and do not replay generated event history or remote sync cursors. +In hybrid mode, routine safe merges honor the collection exclusions under +`[sync]`. They keep excluded channel metadata and previously stored local rows, +but skip new message-scoped rows from excluded channels. Forced replacement is +an explicit destructive exception. + Discrawl does not silently fall back to a full import. Removed shards and incompatible table changes leave the current database intact and require `discrawl update --force`. Forced updates replace public snapshot tables and rebuild search indexes; local DM rows remain untouched. `discrawl sync` does **not** auto-import the share unless `--update=auto` or `--update=force` is provided. Auto mode uses the safe merge; force mode performs exact replacement before live deltas. diff --git a/docs/guides/sync-sources.md b/docs/guides/sync-sources.md index f451bbec..ea827012 100644 --- a/docs/guides/sync-sources.md +++ b/docs/guides/sync-sources.md @@ -35,8 +35,16 @@ Run one explicit `--full` pass when you want a complete historical guild archive - `--guilds 123,456` runs an explicit set - `--all` ignores `default_guild_id` and fans out across every discovered guild - `--channels 111,222` targets specific channels (forum ids expand to their threads) +- `--exclude-channels 333,444` omits specific channel messages +- `--exclude-channel-kinds announcement` omits message channels by Discord kind - `--since ` limits initial history and `--full` backfill to messages at or after the timestamp; older history is not marked complete, so a later `sync --full` without `--since` can continue the backfill +The same exclusions apply to routine sync, historical backfill, live tail, +periodic tail repair, Discord Desktop cache import, and safe upstream snapshot +merges. Exclusions win over explicit channel targets. Safe merges retain channel +metadata and existing local rows while omitting new message-scoped rows for +excluded channels. + ## Performance and resilience - Long runs emit periodic progress logs to stderr. diff --git a/go.mod b/go.mod index bf8fd1c4..9029db1e 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/zalando/go-keyring v0.2.8 golang.org/x/sys v0.47.0 golang.org/x/text v0.40.0 + modernc.org/sqlite v1.54.0 ) require ( @@ -17,7 +18,6 @@ require ( github.com/clipperhouse/displaywidth v0.11.0 // indirect github.com/clipperhouse/uax29/v2 v2.7.0 // indirect github.com/pelletier/go-toml/v2 v2.4.3 // indirect - modernc.org/sqlite v1.54.0 // indirect ) require ( diff --git a/internal/cli/admin_commands.go b/internal/cli/admin_commands.go index dd413718..5b0c8b2e 100644 --- a/internal/cli/admin_commands.go +++ b/internal/cli/admin_commands.go @@ -107,6 +107,10 @@ func (r *runtime) runInit(args []string) error { } func (r *runtime) runSync(args []string) error { + return r.runSyncWithShareUpdate(args, shareUpdateNever) +} + +func (r *runtime) runSyncWithShareUpdate(args []string, shareUpdate shareUpdateMode) error { fs := flag.NewFlagSet("sync", flag.ContinueOnError) fs.SetOutput(io.Discard) full := fs.Bool("full", false, "") @@ -114,6 +118,8 @@ func (r *runtime) runSync(args []string) error { allChannels := fs.Bool("all-channels", false, "") since := fs.String("since", "", "") channels := fs.String("channels", "", "") + excludeChannels := fs.String("exclude-channels", strings.Join(r.cfg.Sync.ExcludeChannelIDs, ","), "") + excludeChannelKinds := fs.String("exclude-channel-kinds", strings.Join(r.cfg.Sync.ExcludeChannelKinds, ","), "") concurrency := fs.Int("concurrency", r.cfg.Sync.Concurrency, "") source := fs.String("source", r.cfg.Sync.Source, "") withEmbeddings := fs.Bool("with-embeddings", false, "") @@ -121,6 +127,7 @@ func (r *runtime) runSync(args []string) error { skipMembers := fs.Bool("skip-members", false, "") withMembers := fs.Bool("with-members", false, "") latestOnly := fs.Bool("latest-only", false, "") + channelTimeout := fs.String("channel-timeout", "", "") guildsFlag := fs.String("guilds", "", "") guildFlag := fs.String("guild", "", "") updateMode := fs.String("update", "", "") @@ -151,23 +158,42 @@ func (r *runtime) runSync(args []string) error { } sinceTime = parsed } + var messageChannelTimeout time.Duration + if strings.TrimSpace(*channelTimeout) != "" { + parsed, err := time.ParseDuration(strings.TrimSpace(*channelTimeout)) + if err != nil || parsed <= 0 { + return usageErr(fmt.Errorf("invalid --channel-timeout %q", *channelTimeout)) + } + messageChannelTimeout = parsed + } guildIDs, err := r.resolveSyncGuildsAll(*guildFlag, *guildsFlag, *all) if err != nil { return usageErr(err) } defaultLatest := defaultLatestSyncMode(*full, *allChannels, *since, *channels) opts := syncer.SyncOptions{ - Full: *full, - GuildIDs: guildIDs, - ChannelIDs: csvList(*channels), - Concurrency: *concurrency, - Since: sinceTime, - Embeddings: *withEmbeddings, - SkipMembers: syncSkipsMembers(*skipMembers, *withMembers, defaultLatest), - RequireMembers: *withMembers, - LatestOnly: syncLatestOnly(*latestOnly, defaultLatest), + Full: *full, + GuildIDs: guildIDs, + ChannelIDs: csvList(*channels), + Concurrency: *concurrency, + Since: sinceTime, + Embeddings: *withEmbeddings, + SkipMembers: syncSkipsMembers(*skipMembers, *withMembers, defaultLatest), + RequireMembers: *withMembers, + LatestOnly: syncLatestOnly(*latestOnly, defaultLatest), + MessageChannelTimeout: messageChannelTimeout, + ExcludeChannelIDs: csvList(*excludeChannels), + ExcludeChannelKinds: csvList(*excludeChannelKinds), } return r.withSyncLock(func() error { + if shareUpdate != shareUpdateNever && os.Getenv("DISCRAWL_NO_AUTO_UPDATE") != "1" { + if err := r.autoUpdateShareWithExclusions(shareUpdate, &shareMergeExclusions{ + channelIDs: opts.ExcludeChannelIDs, + channelKinds: opts.ExcludeChannelKinds, + }); err != nil { + return err + } + } return r.runSyncLocked(sources, opts, *withMedia) }) } @@ -193,10 +219,12 @@ func (r *runtime) runSyncLocked(sources syncSources, opts syncer.SyncOptions, wi if sources.wiretap { r.setSyncLockPhase("wiretap import") stats, err := discorddesktop.Import(r.ctx, r.store, discorddesktop.Options{ - Path: r.cfg.Desktop.Path, - MaxFileBytes: r.cfg.Desktop.MaxFileBytes, - FullCache: r.cfg.Desktop.FullCache, - Now: r.now, + Path: r.cfg.Desktop.Path, + MaxFileBytes: r.cfg.Desktop.MaxFileBytes, + FullCache: r.cfg.Desktop.FullCache, + ExcludeChannelIDs: opts.ExcludeChannelIDs, + ExcludeChannelKinds: opts.ExcludeChannelKinds, + Now: r.now, }) if err != nil { return err @@ -332,6 +360,15 @@ func (r *runtime) runTail(args []string) error { repairEvery := fs.Duration("repair-every", mustDuration(r.cfg.Sync.RepairEvery), "") replayFailuresOnly := fs.Bool("replay-failures-only", false, "") replayLimit := fs.Int("replay-limit", syncer.TailMessageReplayLimit, "") + repairOffsetDefault := time.Duration(0) + if raw := strings.TrimSpace(r.cfg.Sync.RepairOffset); raw != "" { + parsed, err := time.ParseDuration(raw) + if err != nil { + return configErr(fmt.Errorf("parse sync.repair_offset: %w", err)) + } + repairOffsetDefault = parsed + } + repairOffset := fs.Duration("repair-offset", repairOffsetDefault, "") guildsFlag := fs.String("guilds", "", "") guildFlag := fs.String("guild", "", "") if err := fs.Parse(args); err != nil { @@ -368,6 +405,9 @@ func (r *runtime) runTail(args []string) error { } ctx, stop := signal.NotifyContext(r.ctx, os.Interrupt, syscall.SIGTERM) defer stop() + if configurable, ok := r.syncer.(repairOffsetConfigurer); ok { + configurable.SetRepairOffset(*repairOffset) + } if configurable, ok := r.syncer.(tailReadyConfigurer); ok { configurable.SetTailReadyCallback(func(context.Context) error { return r.activateTailSyncLock() @@ -402,11 +442,13 @@ func (r *runtime) runWiretap(args []string) error { var previousCoverage *store.CoverageReport runOnce := func(ctx context.Context) error { stats, err := discorddesktop.Import(ctx, r.store, discorddesktop.Options{ - Path: *path, - MaxFileBytes: *maxFileBytes, - FullCache: *fullCache, - DryRun: *dryRun, - Now: r.now, + Path: *path, + MaxFileBytes: *maxFileBytes, + FullCache: *fullCache, + DryRun: *dryRun, + ExcludeChannelIDs: r.cfg.Sync.ExcludeChannelIDs, + ExcludeChannelKinds: r.cfg.Sync.ExcludeChannelKinds, + Now: r.now, }) if err != nil { return err diff --git a/internal/cli/cli.go b/internal/cli/cli.go index ade6d93c..c4f6964a 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -8,6 +8,7 @@ import ( "log/slog" "os" "strings" + "sync" "time" "github.com/alecthomas/kong" @@ -264,6 +265,7 @@ type runtime struct { lockOperation string lockToken string lockTokenFree func() error + postLockGuard func() error openStore func(context.Context, string) (*store.Store, error) newDiscord func(config.Config) (discordClient, error) newRemote func(config.Config) (remoteArchiveClient, error) @@ -290,6 +292,48 @@ type discordClient interface { Guilds(context.Context) ([]*discordgo.UserGuild, error) } +// coordinatedDiscordClient lets the tail shutdown path own the first Close +// while making outer service cleanup idempotent and non-blocking if that close +// is still in progress. +type coordinatedDiscordClient struct { + discordClient + closeMu sync.Mutex + closeStarted bool + closeDone chan struct{} + closeErr error +} + +func (c *coordinatedDiscordClient) Close() error { + if c == nil || c.discordClient == nil { + return nil + } + c.closeMu.Lock() + if c.closeStarted { + done := c.closeDone + c.closeMu.Unlock() + select { + case <-done: + c.closeMu.Lock() + err := c.closeErr + c.closeMu.Unlock() + return err + default: + return nil + } + } + c.closeStarted = true + c.closeDone = make(chan struct{}) + done := c.closeDone + c.closeMu.Unlock() + + err := c.discordClient.Close() + c.closeMu.Lock() + c.closeErr = err + close(done) + c.closeMu.Unlock() + return err +} + type syncService interface { DiscoverGuilds(context.Context) ([]*discordgo.UserGuild, error) Sync(context.Context, syncer.SyncOptions) (syncer.SyncStats, error) @@ -304,10 +348,18 @@ type tailMessageFailureReplayer interface { ReplayTailMessageFailures(context.Context, []string, int) (syncer.TailMessageReplayStats, error) } +type repairOffsetConfigurer interface { + SetRepairOffset(time.Duration) +} + type attachmentTextConfigurer interface { SetAttachmentTextEnabled(bool) } +type channelExclusionConfigurer interface { + SetChannelExclusions([]string, []string) +} + func (r *runtime) dispatch(rest []string) error { switch rest[0] { case "metadata": @@ -321,7 +373,9 @@ func (r *runtime) dispatch(rest []string) error { if err != nil { return usageErr(err) } - return r.withLocalStoreUpdateLocked(updateMode, true, func() error { return r.runSync(rest[1:]) }) + return r.withLocalStoreUpdateLocked(shareUpdateNever, true, func() error { + return r.runSyncWithShareUpdate(rest[1:], updateMode) + }) case "tail": operation := "tail-starting" if tailReplayOnlyEnabled(rest[1:]) { @@ -755,7 +809,7 @@ func (r *runtime) ensureDiscordServices() error { if err != nil { return authErr(err) } - r.client = client + r.client = &coordinatedDiscordClient{discordClient: client} syncerFactory := r.newSyncer if syncerFactory == nil { syncerFactory = func(client syncer.Client, s *store.Store, logger *slog.Logger) syncService { @@ -766,10 +820,22 @@ func (r *runtime) ensureDiscordServices() error { if configurable, ok := r.syncer.(attachmentTextConfigurer); ok { configurable.SetAttachmentTextEnabled(r.cfg.AttachmentTextEnabled()) } + if configurable, ok := r.syncer.(channelExclusionConfigurer); ok { + configurable.SetChannelExclusions(r.cfg.Sync.ExcludeChannelIDs, r.cfg.Sync.ExcludeChannelKinds) + } return nil } +type shareMergeExclusions struct { + channelIDs []string + channelKinds []string +} + func (r *runtime) autoUpdateShare(mode shareUpdateMode) error { + return r.autoUpdateShareWithExclusions(mode, nil) +} + +func (r *runtime) autoUpdateShareWithExclusions(mode shareUpdateMode, exclusions *shareMergeExclusions) error { if !r.cfg.ShareEnabled() || (mode == shareUpdateConfigured && !r.cfg.Share.AutoUpdate) { return nil } @@ -784,6 +850,10 @@ func (r *runtime) autoUpdateShare(mode shareUpdateMode) error { if err != nil { return err } + if exclusions != nil { + opts.MergeExcludeChannelIDs = append([]string(nil), exclusions.channelIDs...) + opts.MergeExcludeChannelKinds = append([]string(nil), exclusions.channelKinds...) + } r.setSyncLockPhase("share pull") r.logger.Info("share update pulling", "repo_path", opts.RepoPath, "remote", opts.Remote) if err := share.Pull(r.ctx, opts); err != nil { @@ -815,11 +885,13 @@ func (r *runtime) shareOptions() (share.Options, error) { return share.Options{}, configErr(err) } return share.Options{ - RepoPath: repoPath, - CacheDir: cacheDir, - Remote: r.cfg.Share.Remote, - Branch: r.cfg.Share.Branch, - IncludeMedia: r.cfg.ShareMediaEnabled(), - Progress: r.shareProgress, + RepoPath: repoPath, + CacheDir: cacheDir, + Remote: r.cfg.Share.Remote, + Branch: r.cfg.Share.Branch, + MergeExcludeChannelIDs: append([]string(nil), r.cfg.Sync.ExcludeChannelIDs...), + MergeExcludeChannelKinds: append([]string(nil), r.cfg.Sync.ExcludeChannelKinds...), + IncludeMedia: r.cfg.ShareMediaEnabled(), + Progress: r.shareProgress, }, nil } diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index 16fb86e1..0de5115e 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -18,6 +18,7 @@ import ( "path/filepath" goruntime "runtime" "strings" + "sync" "testing" "time" @@ -2418,6 +2419,84 @@ func TestSyncSkipsGitShareByDefaultAndCanImportBeforeLiveDiscord(t *testing.T) { require.Contains(t, contents, "live discord filled the delta") } +func TestSyncUpdateUsesCommandExclusionsForPrecedingShareMerge(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + remoteRepo := filepath.Join(dir, "remote.git") + runGit(t, dir, "init", "--bare", remoteRepo) + + publisher := seedCLIStore(t, filepath.Join(dir, "publisher.db")) + defer func() { _ = publisher.Close() }() + now := time.Now().UTC().Format(time.RFC3339Nano) + for _, channel := range []store.ChannelRecord{ + {ID: "feed", GuildID: "g1", Kind: "text", Name: "feed", RawJSON: `{}`}, + {ID: "announcements", GuildID: "g1", Kind: "announcement", Name: "announcements", RawJSON: `{}`}, + } { + require.NoError(t, publisher.UpsertChannel(ctx, channel)) + } + for _, message := range []store.MessageRecord{ + { + ID: "m-feed", GuildID: "g1", ChannelID: "feed", ChannelName: "feed", + AuthorID: "bot", AuthorName: "Bot", CreatedAt: now, + Content: "command excluded feed", NormalizedContent: "command excluded feed", RawJSON: `{}`, + }, + { + ID: "m-announcement", GuildID: "g1", ChannelID: "announcements", ChannelName: "announcements", + AuthorID: "bot", AuthorName: "Bot", CreatedAt: now, + Content: "command excluded announcement", NormalizedContent: "command excluded announcement", RawJSON: `{}`, + }, + } { + require.NoError(t, publisher.UpsertMessage(ctx, message)) + } + publisherOpts := share.Options{ + RepoPath: filepath.Join(dir, "publisher-share"), + Remote: remoteRepo, + Branch: "main", + } + publishSnapshot(t, ctx, publisher, publisherOpts, "test: command exclusions") + + cfgPath := filepath.Join(dir, "reader.toml") + cfg := config.Default() + cfg.DBPath = filepath.Join(dir, "reader.db") + cfg.Share.Remote = remoteRepo + cfg.Share.RepoPath = filepath.Join(dir, "reader-share") + cfg.Share.StaleAfter = "15m" + require.NoError(t, config.Write(cfgPath, cfg)) + + fakeSync := &fakeSyncService{} + rt := &runtime{ + ctx: ctx, + configPath: cfgPath, + stdout: &bytes.Buffer{}, + stderr: &bytes.Buffer{}, + logger: discardLogger(), + openStore: store.Open, + newDiscord: func(config.Config) (discordClient, error) { + return &fakeDiscordClient{guilds: []*discordgo.UserGuild{{ID: "g1"}}, self: &discordgo.User{ID: "bot"}}, nil + }, + newSyncer: func(syncer.Client, *store.Store, *slog.Logger) syncService { + return fakeSync + }, + } + + require.NoError(t, rt.dispatch([]string{ + "sync", + "--all", + "--update=auto", + "--exclude-channels", "feed", + "--exclude-channel-kinds", "announcement", + })) + require.Equal(t, []string{"feed"}, fakeSync.lastSync.ExcludeChannelIDs) + require.Equal(t, []string{"announcement"}, fakeSync.lastSync.ExcludeChannelKinds) + + reader, err := store.Open(ctx, cfg.DBPath) + require.NoError(t, err) + defer func() { _ = reader.Close() }() + _, rows, err := reader.ReadOnlyQuery(ctx, `select id from messages order by id`) + require.NoError(t, err) + require.Equal(t, [][]string{{"m100"}}, rows) +} + func TestSyncLockSerializesConcurrentRuns(t *testing.T) { if goruntime.GOOS == "windows" { t.Skip("sync lock is currently a no-op on Windows") @@ -2451,6 +2530,64 @@ func TestSyncLockSerializesConcurrentRuns(t *testing.T) { require.ErrorIs(t, err, context.DeadlineExceeded) } +func TestPostLockGuardRunsWhileArchiveLockIsHeld(t *testing.T) { + if goruntime.GOOS == "windows" { + t.Skip("sync lock timing is flaky on Windows") + } + ctx := context.Background() + dir := t.TempDir() + cfg := config.Default() + cfg.DBPath = filepath.Join(dir, "discrawl.db") + lockPath := filepath.Join(dir, ".discrawl-sync.lock") + guardErr := errors.New("guard stopped writer") + callbackCalled := false + rt := &runtime{ + ctx: ctx, + cfg: cfg, + postLockGuard: func() error { + held, known, err := syncLockState(lockPath) + require.NoError(t, err) + require.True(t, known) + require.True(t, held) + return guardErr + }, + } + + err := rt.withSyncLock(func() error { + callbackCalled = true + return nil + }) + + require.ErrorIs(t, err, guardErr) + require.False(t, callbackCalled) + held, known, err := syncLockState(lockPath) + require.NoError(t, err) + require.True(t, known) + require.False(t, held) +} + +func TestLockedResourceGuardBlocksBeforeWriterCallback(t *testing.T) { + if goruntime.GOOS == "windows" { + t.Skip("locked free-space guard is unsupported on Windows") + } + ctx := context.Background() + dir := t.TempDir() + cfg := config.Default() + cfg.DBPath = filepath.Join(dir, "discrawl.db") + t.Setenv(lockedMinFreeKiBEnv, "18446744073709551615") + t.Setenv(lockedMaxWALBytesEnv, "4294967296") + callbackCalled := false + rt := &runtime{ctx: ctx, cfg: cfg} + + err := rt.withSyncLock(func() error { + callbackCalled = true + return nil + }) + + require.ErrorContains(t, err, "free space at or below threshold") + require.False(t, callbackCalled) +} + func TestReadCommandsDoNotWaitForSyncLock(t *testing.T) { if goruntime.GOOS == "windows" { t.Skip("sync lock timing is flaky on Windows") @@ -2869,6 +3006,77 @@ func TestTailReadyPromotesOnceAndCleansUpOnCancellation(t *testing.T) { require.Empty(t, syncLockOwnerFiles(t, lockPath)) } +func TestTailShutdownTimeoutDoesNotRecloseClientOrHoldWriterLock(t *testing.T) { + if goruntime.GOOS == "windows" { + t.Skip("sync lock is currently a no-op on Windows") + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + dir := t.TempDir() + cfg, cfgPath := writeTestConfig(t, dir) + s := seedCLIStore(t, cfg.DBPath) + require.NoError(t, s.Close()) + t.Setenv(config.DefaultTokenEnv, "env-token") + + client := &blockingCloseDiscordClient{ + closeStarted: make(chan struct{}), + closeRelease: make(chan struct{}), + closeFinished: make(chan struct{}), + } + tailService := &boundaryTimeoutTailService{started: make(chan struct{})} + rt := &runtime{ + ctx: ctx, + configPath: cfgPath, + stdout: &bytes.Buffer{}, + stderr: &bytes.Buffer{}, + logger: discardLogger(), + openStore: store.Open, + newDiscord: func(config.Config) (discordClient, error) { + return client, nil + }, + newSyncer: func(syncClient syncer.Client, _ *store.Store, _ *slog.Logger) syncService { + tailService.client = syncClient.(interface{ Close() error }) + tailService.closeStarted = client.closeStarted + return tailService + }, + } + done := make(chan error, 1) + go func() { + done <- rt.dispatch([]string{"tail"}) + }() + + select { + case <-tailService.started: + case <-time.After(time.Second): + t.Fatal("tail did not start") + } + cancel() + select { + case <-client.closeStarted: + case <-time.After(time.Second): + t.Fatal("tail close did not start") + } + select { + case err := <-done: + require.ErrorContains(t, err, "tail shutdown timed out") + case <-time.After(time.Second): + t.Fatal("CLI cleanup repeated the blocked client close") + } + + lockPath := filepath.Join(dir, ".discrawl-sync.lock") + release, err := acquireSyncLock(context.Background(), lockPath) + require.NoError(t, err) + require.NoError(t, release()) + require.Empty(t, syncLockOwnerFiles(t, lockPath)) + + close(client.closeRelease) + select { + case <-client.closeFinished: + case <-time.After(time.Second): + t.Fatal("blocked client close did not finish") + } +} + func TestSyncLockHelperEdges(t *testing.T) { ctx := context.Background() dir := t.TempDir() @@ -3620,6 +3828,10 @@ func (f *fakeDiscordClient) GuildChannels(context.Context, string) ([]*discordgo return nil, nil } +func (f *fakeDiscordClient) Channel(context.Context, string) (*discordgo.Channel, error) { + return nil, nil +} + func (f *fakeDiscordClient) ThreadsActive(context.Context, string) ([]*discordgo.Channel, error) { return nil, nil } @@ -3648,6 +3860,23 @@ func (f *fakeDiscordClient) Tail(context.Context, discordclient.EventHandler) er return nil } +type blockingCloseDiscordClient struct { + fakeDiscordClient + closeStarted chan struct{} + closeRelease chan struct{} + closeFinished chan struct{} + closeOnce sync.Once +} + +func (c *blockingCloseDiscordClient) Close() error { + c.closeOnce.Do(func() { + close(c.closeStarted) + <-c.closeRelease + close(c.closeFinished) + }) + return nil +} + type fakeSyncService struct { discovered []*discordgo.UserGuild lastSync syncer.SyncOptions @@ -3661,7 +3890,10 @@ type fakeSyncService struct { replayHook func() replayStats syncer.TailMessageReplayStats replayErr error + repairOffset time.Duration attachmentTextEnabled bool + excludedChannelIDs []string + excludedChannelKinds []string callTailReady bool tailReadyCalls int tailReady func(context.Context) error @@ -3708,10 +3940,46 @@ func (f *fakeSyncService) SetTailReadyCallback(fn func(context.Context) error) { f.tailReady = fn } +func (f *fakeSyncService) SetRepairOffset(offset time.Duration) { + f.repairOffset = offset +} + func (f *fakeSyncService) SetAttachmentTextEnabled(enabled bool) { f.attachmentTextEnabled = enabled } +func (f *fakeSyncService) SetChannelExclusions(channelIDs, channelKinds []string) { + f.excludedChannelIDs = append([]string(nil), channelIDs...) + f.excludedChannelKinds = append([]string(nil), channelKinds...) +} + +type boundaryTimeoutTailService struct { + fakeSyncService + client interface{ Close() error } + started chan struct{} + closeStarted <-chan struct{} +} + +func (f *boundaryTimeoutTailService) RunTail(ctx context.Context, guildIDs []string, repairEvery time.Duration) error { + f.lastTail = guildIDs + f.lastRepair = repairEvery + if f.tailReady != nil { + f.tailReadyCalls++ + if err := f.tailReady(ctx); err != nil { + return err + } + } + close(f.started) + <-ctx.Done() + go func() { _ = f.client.Close() }() + select { + case <-f.closeStarted: + case <-time.After(time.Second): + return errors.New("tail client close did not start") + } + return errors.New("tail shutdown timed out after 25ms") +} + type hybridSyncService struct { store *store.Store sawGitMessage bool @@ -3797,6 +4065,9 @@ func TestRuntimeInitSyncTailAndDoctor(t *testing.T) { require.Equal(t, "g2", cfg.DefaultGuildID) require.True(t, cfg.Search.Embeddings.Enabled) cfg.Desktop.Path = filepath.Join(dir, "empty-discord") + cfg.Sync.RepairOffset = "15m" + cfg.Sync.ExcludeChannelIDs = []string{"feed"} + cfg.Sync.ExcludeChannelKinds = []string{"announcement"} require.NoError(t, os.MkdirAll(cfg.Desktop.Path, 0o755)) require.NoError(t, config.Write(cfgPath, cfg)) @@ -3806,6 +4077,10 @@ func TestRuntimeInitSyncTailAndDoctor(t *testing.T) { require.True(t, fakeSync.lastSync.LatestOnly) require.True(t, fakeSync.lastSync.SkipMembers) require.True(t, fakeSync.attachmentTextEnabled) + require.Equal(t, []string{"feed"}, fakeSync.lastSync.ExcludeChannelIDs) + require.Equal(t, []string{"announcement"}, fakeSync.lastSync.ExcludeChannelKinds) + require.Equal(t, []string{"feed"}, fakeSync.excludedChannelIDs) + require.Equal(t, []string{"announcement"}, fakeSync.excludedChannelKinds) rt = newRuntime() require.NoError(t, rt.withServices(true, func() error { return rt.runSync([]string{"--guilds", "g2", "--with-members"}) })) @@ -3826,10 +4101,25 @@ func TestRuntimeInitSyncTailAndDoctor(t *testing.T) { require.False(t, fakeSync.lastSync.LatestOnly) require.False(t, fakeSync.lastSync.SkipMembers) + rt = newRuntime() + require.NoError(t, rt.withServices(true, func() error { + return rt.runSync([]string{"--guilds", "g2", "--channels", "c1", "--channel-timeout", "20m"}) + })) + require.Equal(t, []string{"g2"}, fakeSync.lastSync.GuildIDs) + require.Equal(t, []string{"c1"}, fakeSync.lastSync.ChannelIDs) + require.Equal(t, 20*time.Minute, fakeSync.lastSync.MessageChannelTimeout) + rt = newRuntime() require.NoError(t, rt.withServices(true, func() error { return rt.runTail([]string{"--repair-every", "30s"}) })) require.Equal(t, []string{"g2"}, fakeSync.lastTail) require.Equal(t, 30*time.Second, fakeSync.lastRepair) + require.Equal(t, 15*time.Minute, fakeSync.repairOffset) + + rt = newRuntime() + require.NoError(t, rt.withServices(true, func() error { + return rt.runTail([]string{"--repair-every", "30s", "--repair-offset", "20m"}) + })) + require.Equal(t, 20*time.Minute, fakeSync.repairOffset) rt = newRuntime() var out bytes.Buffer @@ -4255,6 +4545,12 @@ func TestHelpers(t *testing.T) { require.NoError(t, err) require.Equal(t, "git@example.com:org/archive.git", opts.Remote) require.Equal(t, "main", opts.Branch) + cfg := config.Default() + cfg.Sync.ExcludeChannelIDs = []string{"feed"} + cfg.Sync.ExcludeChannelKinds = []string{"announcement"} + applyCollectionExclusionsToShareMerge(&opts, cfg) + require.Equal(t, []string{"feed"}, opts.MergeExcludeChannelIDs) + require.Equal(t, []string{"announcement"}, opts.MergeExcludeChannelKinds) var out bytes.Buffer require.NoError(t, printHuman(&out, syncer.SyncStats{Guilds: 1})) require.Contains(t, out.String(), "guilds=1") @@ -4737,6 +5033,8 @@ func TestCommandUsageErrors(t *testing.T) { require.Equal(t, 2, ExitCode(rt.runSync([]string{"--all", "--guild", "g1"}))) require.Equal(t, 2, ExitCode(rt.runSync([]string{"--update", "bogus"}))) require.Equal(t, 2, ExitCode(rt.runSync([]string{"--update=force", "--no-update"}))) + require.Equal(t, 2, ExitCode(rt.runSync([]string{"--channel-timeout", "0s"}))) + require.Equal(t, 2, ExitCode(rt.runTail([]string{"--repair-offset", "not-a-duration"}))) require.Equal(t, 2, ExitCode(rt.runChannels(nil))) require.Equal(t, 2, ExitCode(rt.runStatus([]string{"extra"}))) require.NoError(t, (&runtime{stdout: &bytes.Buffer{}}).runDoctor(nil)) diff --git a/internal/cli/locked_resource_guard.go b/internal/cli/locked_resource_guard.go new file mode 100644 index 00000000..5b500238 --- /dev/null +++ b/internal/cli/locked_resource_guard.go @@ -0,0 +1,76 @@ +package cli + +import ( + "errors" + "fmt" + "os" + "strconv" + "strings" + + "github.com/openclaw/discrawl/internal/config" +) + +const ( + lockedMinFreeKiBEnv = "DISCRAWL_LOCKED_MIN_FREE_KIB" + lockedMaxWALBytesEnv = "DISCRAWL_LOCKED_MAX_WAL_BYTES" +) + +func lockedResourceGuard(rawDBPath string) error { + rawMinFree := strings.TrimSpace(os.Getenv(lockedMinFreeKiBEnv)) + rawMaxWAL := strings.TrimSpace(os.Getenv(lockedMaxWALBytesEnv)) + if rawMinFree == "" && rawMaxWAL == "" { + return nil + } + if rawMinFree == "" || rawMaxWAL == "" { + return errors.New("both locked resource guard thresholds are required") + } + minFreeKiB, err := strconv.ParseUint(rawMinFree, 10, 64) + if err != nil || minFreeKiB == 0 { + return fmt.Errorf("invalid %s value %q", lockedMinFreeKiBEnv, rawMinFree) + } + maxWALBytes, err := strconv.ParseUint(rawMaxWAL, 10, 64) + if err != nil || maxWALBytes == 0 { + return fmt.Errorf("invalid %s value %q", lockedMaxWALBytesEnv, rawMaxWAL) + } + dbPath, err := config.ExpandPath(rawDBPath) + if err != nil { + return fmt.Errorf("expand database path: %w", err) + } + freeKiB, err := lockedFreeKiB(dbPath) + if err != nil { + return fmt.Errorf("probe free space: %w", err) + } + walBytes, err := lockedWALBytes(dbPath + "-wal") + if err != nil { + return fmt.Errorf("probe WAL: %w", err) + } + if freeKiB <= minFreeKiB { + return fmt.Errorf( + "free space at or below threshold: free_kib=%d threshold_kib=%d", + freeKiB, + minFreeKiB, + ) + } + if walBytes >= maxWALBytes { + return fmt.Errorf( + "WAL at or above threshold: wal_bytes=%d threshold_bytes=%d", + walBytes, + maxWALBytes, + ) + } + return nil +} + +func lockedWALBytes(path string) (uint64, error) { + info, err := os.Lstat(path) + if errors.Is(err, os.ErrNotExist) { + return 0, nil + } + if err != nil { + return 0, err + } + if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() { + return 0, fmt.Errorf("WAL path is not a regular file: %s", path) + } + return uint64(info.Size()), nil +} diff --git a/internal/cli/locked_resource_guard_other.go b/internal/cli/locked_resource_guard_other.go new file mode 100644 index 00000000..ded851e0 --- /dev/null +++ b/internal/cli/locked_resource_guard_other.go @@ -0,0 +1,9 @@ +//go:build !unix + +package cli + +import "errors" + +func lockedFreeKiB(string) (uint64, error) { + return 0, errors.New("locked free-space guard is unsupported on this platform") +} diff --git a/internal/cli/locked_resource_guard_test.go b/internal/cli/locked_resource_guard_test.go new file mode 100644 index 00000000..aa48a904 --- /dev/null +++ b/internal/cli/locked_resource_guard_test.go @@ -0,0 +1,44 @@ +package cli + +import ( + "os" + "path/filepath" + goruntime "runtime" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestLockedResourceGuardRejectsWALAtThreshold(t *testing.T) { + if goruntime.GOOS == "windows" { + t.Skip("locked free-space guard is unsupported on Windows") + } + dir := t.TempDir() + dbPath := filepath.Join(dir, "discrawl.db") + require.NoError(t, os.WriteFile(dbPath, nil, 0o600)) + require.NoError(t, os.WriteFile(dbPath+"-wal", []byte{1}, 0o600)) + t.Setenv(lockedMinFreeKiBEnv, "1") + t.Setenv(lockedMaxWALBytesEnv, "1") + + err := lockedResourceGuard(dbPath) + + require.ErrorContains(t, err, "WAL at or above threshold") +} + +func TestLockedResourceGuardRejectsSymlinkedWAL(t *testing.T) { + if goruntime.GOOS == "windows" { + t.Skip("locked free-space guard is unsupported on Windows") + } + dir := t.TempDir() + dbPath := filepath.Join(dir, "discrawl.db") + target := filepath.Join(dir, "target") + require.NoError(t, os.WriteFile(dbPath, nil, 0o600)) + require.NoError(t, os.WriteFile(target, nil, 0o600)) + require.NoError(t, os.Symlink(target, dbPath+"-wal")) + t.Setenv(lockedMinFreeKiBEnv, "1") + t.Setenv(lockedMaxWALBytesEnv, "4294967296") + + err := lockedResourceGuard(dbPath) + + require.ErrorContains(t, err, "WAL path is not a regular file") +} diff --git a/internal/cli/locked_resource_guard_unix.go b/internal/cli/locked_resource_guard_unix.go new file mode 100644 index 00000000..8b6ef3dc --- /dev/null +++ b/internal/cli/locked_resource_guard_unix.go @@ -0,0 +1,17 @@ +//go:build unix + +package cli + +import ( + "path/filepath" + + "golang.org/x/sys/unix" +) + +func lockedFreeKiB(dbPath string) (uint64, error) { + var details unix.Statfs_t + if err := unix.Statfs(filepath.Dir(dbPath), &details); err != nil { + return 0, err + } + return details.Bavail * uint64(details.Bsize) / 1024, nil +} diff --git a/internal/cli/share_commands.go b/internal/cli/share_commands.go index a0b558b8..671f0e39 100644 --- a/internal/cli/share_commands.go +++ b/internal/cli/share_commands.go @@ -222,6 +222,7 @@ func (r *runtime) runSubscribe(args []string) error { return configErr(err) } opts := share.Options{RepoPath: expandedRepo, Remote: cfg.Share.Remote, Branch: cfg.Share.Branch, Progress: r.shareProgress} + applyCollectionExclusionsToShareMerge(&opts, cfg) if err := applyMediaShareOptions(&opts, cfg, cfg.ShareMediaEnabled() && !*noMedia); err != nil { return err } @@ -283,6 +284,7 @@ func (r *runtime) runUpdate(args []string) error { if err != nil { return err } + applyCollectionExclusionsToShareMerge(&opts, r.cfg) opts.Progress = r.shareProgress if *withEmbeddings { applyEmbeddingShareOptions(&opts, r.cfg) @@ -344,6 +346,14 @@ func shareOptionsFromFlags(repoPath, remote, branch string) (share.Options, erro return share.Options{RepoPath: expandedRepo, Remote: remote, Branch: branch}, nil } +func applyCollectionExclusionsToShareMerge(opts *share.Options, cfg config.Config) { + if opts == nil { + return + } + opts.MergeExcludeChannelIDs = append([]string(nil), cfg.Sync.ExcludeChannelIDs...) + opts.MergeExcludeChannelKinds = append([]string(nil), cfg.Sync.ExcludeChannelKinds...) +} + func applyEmbeddingShareOptions(opts *share.Options, cfg config.Config) { opts.IncludeEmbeddings = true opts.EmbeddingProvider = cfg.Search.Embeddings.Provider diff --git a/internal/cli/sync_lock.go b/internal/cli/sync_lock.go index 88955d0c..45f7aa97 100644 --- a/internal/cli/sync_lock.go +++ b/internal/cli/sync_lock.go @@ -124,9 +124,22 @@ func (r *runtime) runWithHeldSyncLock(lockPath string, release func() error, ope } _ = release() }() + if err := r.runPostLockGuard(); err != nil { + return err + } return fn() } +func (r *runtime) runPostLockGuard() error { + if r.postLockGuard != nil { + return r.postLockGuard() + } + if err := lockedResourceGuard(r.cfg.DBPath); err != nil { + return dbErr(fmt.Errorf("post-lock resource guard: %w", err)) + } + return nil +} + func (r *runtime) activateTailSyncLock() error { if !r.dbLockHeld { return nil diff --git a/internal/config/config.go b/internal/config/config.go index 2977692d..5ea432f8 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -51,13 +51,16 @@ type DesktopConfig struct { } type SyncConfig struct { - Source string `toml:"source"` - Concurrency int `toml:"concurrency"` - RepairEvery string `toml:"repair_every"` - FullHistory bool `toml:"full_history"` - AttachmentText *bool `toml:"attachment_text"` - AttachmentMedia *bool `toml:"attachment_media"` - MaxAttachmentBytes int64 `toml:"max_attachment_bytes"` + Source string `toml:"source"` + Concurrency int `toml:"concurrency"` + RepairEvery string `toml:"repair_every"` + RepairOffset string `toml:"repair_offset"` + FullHistory bool `toml:"full_history"` + ExcludeChannelIDs []string `toml:"exclude_channel_ids,omitempty"` + ExcludeChannelKinds []string `toml:"exclude_channel_kinds,omitempty"` + AttachmentText *bool `toml:"attachment_text"` + AttachmentMedia *bool `toml:"attachment_media"` + MaxAttachmentBytes int64 `toml:"max_attachment_bytes"` } type SearchConfig struct { @@ -138,6 +141,7 @@ func Default() Config { Source: "both", Concurrency: defaultSyncConcurrency(), RepairEvery: "6h", + RepairOffset: "0s", FullHistory: true, AttachmentText: new(true), AttachmentMedia: new(false), @@ -268,6 +272,15 @@ func (c *Config) Normalize() error { if c.Sync.RepairEvery == "" { c.Sync.RepairEvery = "6h" } + c.Sync.RepairOffset = strings.TrimSpace(c.Sync.RepairOffset) + if c.Sync.RepairOffset == "" { + c.Sync.RepairOffset = "0s" + } + if _, err := time.ParseDuration(c.Sync.RepairOffset); err != nil { + return fmt.Errorf("parse sync.repair_offset: %w", err) + } + c.Sync.ExcludeChannelIDs = uniqueStrings(c.Sync.ExcludeChannelIDs) + c.Sync.ExcludeChannelKinds = uniqueLowerStrings(c.Sync.ExcludeChannelKinds) if c.Sync.AttachmentText == nil { c.Sync.AttachmentText = new(true) } @@ -439,3 +452,11 @@ func uniqueStrings(in []string) []string { } return out } + +func uniqueLowerStrings(in []string) []string { + out := uniqueStrings(in) + for i := range out { + out[i] = strings.ToLower(out[i]) + } + return uniqueStrings(out) +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 5910032c..a979fb44 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -24,6 +24,10 @@ func TestNormalizeFillsDefaults(t *testing.T) { require.Equal(t, defaultSyncConcurrency(), cfg.Sync.Concurrency) require.GreaterOrEqual(t, cfg.Sync.Concurrency, 8) require.LessOrEqual(t, cfg.Sync.Concurrency, 32) + require.Equal(t, "6h", cfg.Sync.RepairEvery) + require.Equal(t, "0s", cfg.Sync.RepairOffset) + require.Empty(t, cfg.Sync.ExcludeChannelIDs) + require.Empty(t, cfg.Sync.ExcludeChannelKinds) require.NotNil(t, cfg.Sync.AttachmentText) require.True(t, *cfg.Sync.AttachmentText) require.Equal(t, "fts", cfg.Search.DefaultMode) @@ -325,6 +329,7 @@ func TestWriteAndLoadRoundTrip(t *testing.T) { cfg := Default() cfg.DefaultGuildID = "g1" cfg.GuildIDs = []string{"g1", "g2"} + cfg.Sync.RepairOffset = "15m" require.NoError(t, Write(path, cfg)) loaded, err := Load(path) @@ -332,10 +337,24 @@ func TestWriteAndLoadRoundTrip(t *testing.T) { require.Equal(t, "g1", loaded.EffectiveDefaultGuildID()) require.Equal(t, []string{"g1", "g2"}, loaded.GuildIDs) require.Equal(t, DefaultRemoteTokenEnv, loaded.Remote.TokenEnv) + require.Equal(t, "15m", loaded.Sync.RepairOffset) require.NotNil(t, loaded.Sync.AttachmentText) require.True(t, *loaded.Sync.AttachmentText) } +func TestNormalizeRejectsInvalidRepairOffset(t *testing.T) { + t.Parallel() + + cfg := Default() + cfg.Sync.RepairOffset = "not-a-duration" + require.ErrorContains(t, cfg.Normalize(), "parse sync.repair_offset") + + cfg = Default() + cfg.Sync.RepairOffset = "-15m" + require.NoError(t, cfg.Normalize()) + require.Equal(t, "-15m", cfg.Sync.RepairOffset) +} + func TestWriteRejectsNonPositiveEmbeddingTimeout(t *testing.T) { t.Parallel() @@ -489,9 +508,13 @@ func TestEffectiveDefaultGuildAndDirs(t *testing.T) { cfg := Default() cfg.GuildIDs = []string{"g1"} + cfg.Sync.ExcludeChannelIDs = []string{" c1 ", "", "c2", "c1"} + cfg.Sync.ExcludeChannelKinds = []string{" Announcement ", "announcement", "TEXT"} cfg.Share.Filter.IncludeChannelIDs = []string{" c1 ", "", "c2", "c1"} cfg.Share.Filter.ExcludeChannelIDs = []string{" c3 ", "c3"} require.NoError(t, cfg.Normalize()) + require.Equal(t, []string{"c1", "c2"}, cfg.Sync.ExcludeChannelIDs) + require.Equal(t, []string{"announcement", "text"}, cfg.Sync.ExcludeChannelKinds) require.Equal(t, []string{"c1", "c2"}, cfg.Share.Filter.IncludeChannelIDs) require.Equal(t, []string{"c3"}, cfg.Share.Filter.ExcludeChannelIDs) require.Equal(t, "g1", cfg.EffectiveDefaultGuildID()) diff --git a/internal/discord/client.go b/internal/discord/client.go index 91e4d817..95a1e397 100644 --- a/internal/discord/client.go +++ b/internal/discord/client.go @@ -26,6 +26,10 @@ type tailGuildFilter interface { TailAllowsGuild(string) bool } +type messageUpdateRefetchFailureHandler interface { + OnMessageUpdateRefetchFailure(context.Context, *discordgo.Message, error) (bool, error) +} + type TailReadyHandler interface { OnTailReady(context.Context) error } @@ -294,6 +298,12 @@ func (c *Client) GuildChannels(ctx context.Context, guildID string) ([]*discordg return c.session.GuildChannels(guildID, discordgo.WithContext(reqCtx)) } +func (c *Client) Channel(ctx context.Context, channelID string) (*discordgo.Channel, error) { + reqCtx, cancel := c.requestContext(ctx) + defer cancel() + return c.session.Channel(channelID, discordgo.WithContext(reqCtx)) +} + func (c *Client) ThreadsActive(ctx context.Context, channelID string) ([]*discordgo.Channel, error) { reqCtx, cancel := c.requestContext(ctx) defer cancel() @@ -549,10 +559,29 @@ func (c *Client) Tail(ctx context.Context, handler EventHandler) error { if msg == nil { return refetchErr } - // A failed refetch does not suppress the partial update, but it remains - // an event failure even when the handler accepts that recovery input. + // A failed refetch does not suppress the partial update. It remains an + // event failure unless an explicitly capable handler reconciles it. UpdateTailFailureStage(taskCtx, TailFailureStageHandler) - handlerErr := handler.OnMessageUpdate(taskCtx, msg) + var ( + refetchConsumed bool + handlerErr error + ) + if refetchErr != nil { + if refetchHandler, ok := handler.(messageUpdateRefetchFailureHandler); ok { + refetchConsumed, handlerErr = refetchHandler.OnMessageUpdateRefetchFailure( + taskCtx, + msg, + refetchErr, + ) + } else { + handlerErr = handler.OnMessageUpdate(taskCtx, msg) + } + } else { + handlerErr = handler.OnMessageUpdate(taskCtx, msg) + } + if refetchConsumed && handlerErr == nil { + refetchErr = nil + } if refetchErr != nil { UpdateTailFailureStage(taskCtx, TailFailureStageMessageUpdateRefetch) } @@ -603,6 +632,34 @@ func (c *Client) Tail(ctx context.Context, handler EventHandler) error { before, )) }) + addHandler(func(_ *discordgo.Session, evt *discordgo.ThreadCreate) { + var channel *discordgo.Channel + if evt != nil { + channel = evt.Channel + } + c.enqueueTailTask(tailCtx, orderedWorkCh, fatal, newChannelTailTask( + "THREAD_CREATE", + func(taskCtx context.Context) error { + return handler.OnChannelUpsert(taskCtx, channel) + }, + channel, + )) + }) + addHandler(func(_ *discordgo.Session, evt *discordgo.ThreadUpdate) { + var channel, before *discordgo.Channel + if evt != nil { + channel = evt.Channel + before = evt.BeforeUpdate + } + c.enqueueTailTask(tailCtx, orderedWorkCh, fatal, newChannelTailTask( + "THREAD_UPDATE", + func(taskCtx context.Context) error { + return handler.OnChannelUpsert(taskCtx, channel) + }, + channel, + before, + )) + }) addHandler(func(_ *discordgo.Session, evt *discordgo.GuildMemberAdd) { var member *discordgo.Member if evt != nil { diff --git a/internal/discord/client_test.go b/internal/discord/client_test.go index 66331fb8..5149c4a7 100644 --- a/internal/discord/client_test.go +++ b/internal/discord/client_test.go @@ -34,6 +34,9 @@ func TestClientRESTWrappers(t *testing.T) { mux.HandleFunc("/api/v10/guilds/g1/channels", writeJSON([]map[string]any{ {"id": "c1", "guild_id": "g1", "name": "general", "type": 0}, })) + mux.HandleFunc("/api/v10/channels/c1", writeJSON(map[string]any{ + "id": "c1", "guild_id": "g1", "name": "general", "type": 0, + })) mux.HandleFunc("/api/v10/guilds/g1/threads/active", writeJSON(map[string]any{ "threads": []map[string]any{ {"id": "tg1", "guild_id": "g1", "parent_id": "c1", "name": "guild-thread", "type": 11}, @@ -140,6 +143,10 @@ func TestClientRESTWrappers(t *testing.T) { require.NoError(t, err) require.Len(t, channels, 1) + channel, err := client.Channel(ctx, "c1") + require.NoError(t, err) + require.Equal(t, "c1", channel.ID) + members, err := client.GuildMembers(ctx, "g1") require.NoError(t, err) require.Len(t, members, 1) @@ -1099,8 +1106,10 @@ func TestTailReceivesGatewayEvents(t *testing.T) { {"op": 0, "t": "MESSAGE_UPDATE", "s": 3, "d": map[string]any{"id": "m1", "guild_id": "g1", "channel_id": "c1", "content": "hello 2", "timestamp": now, "author": map[string]any{"id": "u1", "username": "user"}}}, {"op": 0, "t": "MESSAGE_DELETE", "s": 4, "d": map[string]any{"id": "m1", "guild_id": "g1", "channel_id": "c1"}}, {"op": 0, "t": "CHANNEL_CREATE", "s": 5, "d": map[string]any{"id": "c1", "guild_id": "g1", "name": "general", "type": 0}}, - {"op": 0, "t": "GUILD_MEMBER_ADD", "s": 6, "d": map[string]any{"guild_id": "g1", "user": map[string]any{"id": "u1", "username": "user"}, "roles": []string{}}}, - {"op": 0, "t": "GUILD_MEMBER_REMOVE", "s": 7, "d": map[string]any{"guild_id": "g1", "user": map[string]any{"id": "u1", "username": "user"}}}, + {"op": 0, "t": "THREAD_CREATE", "s": 6, "d": map[string]any{"id": "t1", "guild_id": "g1", "parent_id": "c1", "name": "thread", "type": 11, "newly_created": true}}, + {"op": 0, "t": "THREAD_UPDATE", "s": 7, "d": map[string]any{"id": "t1", "guild_id": "g1", "parent_id": "c1", "name": "thread updated", "type": 11}}, + {"op": 0, "t": "GUILD_MEMBER_ADD", "s": 8, "d": map[string]any{"guild_id": "g1", "user": map[string]any{"id": "u1", "username": "user"}, "roles": []string{}}}, + {"op": 0, "t": "GUILD_MEMBER_REMOVE", "s": 9, "d": map[string]any{"guild_id": "g1", "user": map[string]any{"id": "u1", "username": "user"}}}, } for _, event := range events { if err := conn.WriteJSON(event); err != nil { @@ -1120,6 +1129,7 @@ func TestTailReceivesGatewayEvents(t *testing.T) { require.NoError(t, err) defer func() { _ = client.Close() }() client.session.ShouldReconnectOnError = false + client.tailWorkerCount = 1 require.Zero(t, discordSessionHandlerCount(client.session)) ctx, cancel := context.WithCancel(context.Background()) @@ -1133,7 +1143,7 @@ func TestTailReceivesGatewayEvents(t *testing.T) { require.Equal(t, 1, handler.creates) require.Equal(t, 1, handler.updates) require.Equal(t, 1, handler.deletes) - require.Equal(t, 1, handler.channels) + require.Equal(t, 3, handler.channels) require.Equal(t, 1, handler.memberUpserts) require.Equal(t, 1, handler.memberDeletes) require.Equal(t, 1, handler.ready) @@ -2665,7 +2675,6 @@ func TestTailMessageUpdateRefetchTimeoutReportsRefetchStage(t *testing.T) { }, ) defer server.Close() - restore := patchDiscordEndpoints(server.URL + "/api/v10/") defer restore() @@ -2720,6 +2729,8 @@ func TestTailMessageUpdateImmediateHTTPRefetchFailureStillInvokesHandlerAndRecor recorded: make(chan TailFailure, 1), updates: make(chan *discordgo.Message, 1), } + _, consumesRefetchFailures := any(handler).(messageUpdateRefetchFailureHandler) + require.False(t, consumesRefetchFailures) var refetchCalls atomic.Int32 now := time.Now().UTC().Format(time.RFC3339) server := newTailTestGatewayWithRoutes( @@ -3018,6 +3029,137 @@ func TestTailReadyHandlerFailureRemainsTerminal(t *testing.T) { require.ErrorIs(t, err, readyErr) } +func TestTailSerializesThreadEventsInGatewayOrder(t *testing.T) { + mux := http.NewServeMux() + upgrader := websocket.Upgrader{} + gatewayURL := "" + serverRelease := make(chan struct{}) + mux.HandleFunc("/api/v10/gateway", func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"url": gatewayURL}) + }) + server := httptest.NewServer(mux) + defer server.Close() + defer close(serverRelease) + + gatewayURL = "ws" + server.URL[len("http"):] + "/gateway" + gatewayHandler := func(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + t.Errorf("upgrade gateway: %v", err) + return + } + defer func() { _ = conn.Close() }() + + if err := conn.WriteJSON(map[string]any{ + "op": 10, + "d": map[string]any{"heartbeat_interval": 1000}, + }); err != nil { + t.Errorf("write hello: %v", err) + return + } + if _, _, err := conn.ReadMessage(); err != nil { + t.Errorf("read identify: %v", err) + return + } + events := []map[string]any{ + { + "op": 0, + "t": "READY", + "s": 1, + "d": map[string]any{ + "session_id": "session", + "user": map[string]any{"id": "bot", "username": "bot"}, + }, + }, + { + "op": 0, + "t": "THREAD_CREATE", + "s": 2, + "d": map[string]any{ + "id": "t1", "guild_id": "g1", "parent_id": "c1", + "name": "created", "type": 11, "newly_created": true, + }, + }, + { + "op": 0, + "t": "THREAD_UPDATE", + "s": 3, + "d": map[string]any{ + "id": "t1", "guild_id": "g1", "parent_id": "c1", + "name": "updated", "type": 11, + }, + }, + { + "op": 0, + "t": "MESSAGE_CREATE", + "s": 4, + "d": map[string]any{ + "id": "m1", "guild_id": "g1", "channel_id": "t1", + "content": "after update", "timestamp": time.Now().UTC().Format(time.RFC3339), + "author": map[string]any{"id": "u1", "username": "user"}, + }, + }, + } + for _, event := range events { + if err := conn.WriteJSON(event); err != nil { + t.Errorf("write event %v: %v", event["t"], err) + return + } + } + select { + case <-serverRelease: + case <-r.Context().Done(): + } + } + mux.HandleFunc("/gateway", gatewayHandler) + mux.HandleFunc("/gateway/", gatewayHandler) + + restore := patchDiscordEndpoints(server.URL + "/api/v10/") + defer restore() + + client, err := New("token") + require.NoError(t, err) + defer func() { _ = client.Close() }() + client.session.ShouldReconnectOnError = false + client.tailWorkerCount = 4 + client.tailQueueSize = 8 + + handler := newOrderedChannelHandler() + ctx, cancel := context.WithCancel(context.Background()) + tailDone := make(chan error, 1) + go func() { + tailDone <- client.Tail(ctx, handler) + }() + + select { + case <-handler.firstStarted: + case <-time.After(time.Second): + t.Fatal("first thread event did not start") + } + require.True(t, client.session.SyncEvents) + select { + case <-handler.secondStarted: + t.Fatal("second thread event overtook the first") + case <-time.After(50 * time.Millisecond): + } + select { + case <-handler.messageStarted: + t.Fatal("message event overtook preceding thread events") + case <-time.After(50 * time.Millisecond): + } + close(handler.releaseFirst) + select { + case <-handler.done: + case <-time.After(time.Second): + t.Fatal("ordered thread events did not complete") + } + cancel() + require.NoError(t, <-tailDone) + require.Equal(t, []string{"created", "updated", "message"}, handler.names()) + require.True(t, client.session.SyncEvents) +} + func TestTailFailsFastWhenWorkerQueueFills(t *testing.T) { mux := http.NewServeMux() upgrader := websocket.Upgrader{} @@ -3997,3 +4139,81 @@ func (s *slowHandler) OnMemberUpsert(context.Context, string, *discordgo.Member) func (s *slowHandler) OnMemberDelete(context.Context, string, string) error { return nil } + +type orderedChannelHandler struct { + mu sync.Mutex + firstOnce sync.Once + secondOnce sync.Once + doneOnce sync.Once + firstStarted chan struct{} + secondStarted chan struct{} + messageStarted chan struct{} + releaseFirst chan struct{} + done chan struct{} + channelNames []string +} + +func newOrderedChannelHandler() *orderedChannelHandler { + return &orderedChannelHandler{ + firstStarted: make(chan struct{}), + secondStarted: make(chan struct{}), + messageStarted: make(chan struct{}), + releaseFirst: make(chan struct{}), + done: make(chan struct{}), + } +} + +func (h *orderedChannelHandler) OnMessageCreate(context.Context, *discordgo.Message) error { + close(h.messageStarted) + h.record("message") + return nil +} + +func (h *orderedChannelHandler) OnMessageUpdate(context.Context, *discordgo.Message) error { + return nil +} + +func (h *orderedChannelHandler) OnMessageDelete(context.Context, *discordgo.MessageDelete) error { + return nil +} + +func (h *orderedChannelHandler) OnChannelUpsert(ctx context.Context, channel *discordgo.Channel) error { + switch channel.Name { + case "created": + h.firstOnce.Do(func() { close(h.firstStarted) }) + select { + case <-ctx.Done(): + return ctx.Err() + case <-h.releaseFirst: + } + case "updated": + h.secondOnce.Do(func() { close(h.secondStarted) }) + } + + h.record(channel.Name) + return nil +} + +func (h *orderedChannelHandler) OnMemberUpsert(context.Context, string, *discordgo.Member) error { + return nil +} + +func (h *orderedChannelHandler) OnMemberDelete(context.Context, string, string) error { + return nil +} + +func (h *orderedChannelHandler) names() []string { + h.mu.Lock() + defer h.mu.Unlock() + return append([]string(nil), h.channelNames...) +} + +func (h *orderedChannelHandler) record(name string) { + h.mu.Lock() + h.channelNames = append(h.channelNames, name) + count := len(h.channelNames) + h.mu.Unlock() + if count == 3 { + h.doneOnce.Do(func() { close(h.done) }) + } +} diff --git a/internal/discorddesktop/import.go b/internal/discorddesktop/import.go index b304a36d..4fbb974e 100644 --- a/internal/discorddesktop/import.go +++ b/internal/discorddesktop/import.go @@ -38,11 +38,13 @@ var ( ) type Options struct { - Path string - MaxFileBytes int64 - DryRun bool - FullCache bool - Now func() time.Time + Path string + MaxFileBytes int64 + DryRun bool + FullCache bool + ExcludeChannelIDs []string + ExcludeChannelKinds []string + Now func() time.Time } type Stats struct { @@ -62,6 +64,8 @@ type Stats struct { GuildMessages int `json:"guild_messages"` SkippedMessages int `json:"skipped_messages"` SkippedChannels int `json:"skipped_channels"` + ExcludedMessages int `json:"excluded_messages"` + ExcludedChannels int `json:"excluded_channels"` Checkpoints int `json:"checkpoints"` DryRun bool `json:"dry_run,omitempty"` FullCache bool `json:"full_cache,omitempty"` @@ -106,14 +110,16 @@ type fileCandidate struct { } type scanTotals struct { - guilds map[string]struct{} - channels map[string]struct{} - messages map[string]struct{} - dmMessages map[string]struct{} - guildMessages map[string]struct{} - dmChannels map[string]struct{} - skippedMessages map[string]struct{} - skippedChannels map[string]struct{} + guilds map[string]struct{} + channels map[string]struct{} + messages map[string]struct{} + dmMessages map[string]struct{} + guildMessages map[string]struct{} + dmChannels map[string]struct{} + skippedMessages map[string]struct{} + skippedChannels map[string]struct{} + excludedMessages map[string]struct{} + excludedChannels map[string]struct{} } type unresolvedMessages map[string]string @@ -203,16 +209,18 @@ func loadScanState(ctx context.Context, st *store.Store, opts Options) (scanStat current: map[string]fileFingerprint{}, channels: map[string]store.ChannelRecord{}, } - if st == nil || opts.DryRun { + if st == nil { return state, nil } - raw, err := st.GetSyncState(ctx, fileIndexScope(opts)) - if err != nil { - return state, err - } - if strings.TrimSpace(raw) != "" { - if err := json.Unmarshal([]byte(raw), &state.previous); err != nil { - state.previous = map[string]fileFingerprint{} + if !opts.DryRun { + raw, err := st.GetSyncState(ctx, fileIndexScope(opts)) + if err != nil { + return state, err + } + if strings.TrimSpace(raw) != "" { + if err := json.Unmarshal([]byte(raw), &state.previous); err != nil { + state.previous = map[string]fileFingerprint{} + } } } channels, err := st.Channels(ctx, "") @@ -221,10 +229,13 @@ func loadScanState(ctx context.Context, st *store.Store, opts Options) (scanStat } for _, channel := range channels { state.channels[channel.ID] = store.ChannelRecord{ - ID: channel.ID, - GuildID: channel.GuildID, - Kind: channel.Kind, - Name: channel.Name, + ID: channel.ID, + GuildID: channel.GuildID, + ParentID: channel.ParentID, + Kind: channel.Kind, + Name: channel.Name, + IsPrivateThread: channel.IsPrivateThread, + ThreadParentID: channel.ThreadParentID, } } return state, nil @@ -421,6 +432,7 @@ func scanFullCache(ctx context.Context, opts Options, state scanState) (Stats, s return stats, snap, err } totals := newScanTotals() + filterExcludedMessages(snap, state.channels, opts, totals, &stats) finalizeSnapshot(snap, state.channels, totals, &stats, true) stats.FinishedAt = now().UTC() return stats, snap, nil @@ -571,15 +583,113 @@ func collectCacheRouteHints(ctx context.Context, rootFS *os.Root, candidates []f func newScanTotals() scanTotals { return scanTotals{ - guilds: map[string]struct{}{}, - channels: map[string]struct{}{}, - messages: map[string]struct{}{}, - dmMessages: map[string]struct{}{}, - guildMessages: map[string]struct{}{}, - dmChannels: map[string]struct{}{}, - skippedMessages: map[string]struct{}{}, - skippedChannels: map[string]struct{}{}, + guilds: map[string]struct{}{}, + channels: map[string]struct{}{}, + messages: map[string]struct{}{}, + dmMessages: map[string]struct{}{}, + guildMessages: map[string]struct{}{}, + dmChannels: map[string]struct{}{}, + skippedMessages: map[string]struct{}{}, + skippedChannels: map[string]struct{}{}, + excludedMessages: map[string]struct{}{}, + excludedChannels: map[string]struct{}{}, + } +} + +func filterExcludedMessages(snap snapshot, channelLookup map[string]store.ChannelRecord, opts Options, totals scanTotals, stats *Stats) { + excludedIDs := normalizedCollectionSet(opts.ExcludeChannelIDs, false) + excludedKinds := normalizedCollectionSet(opts.ExcludeChannelKinds, true) + if len(excludedIDs) == 0 && len(excludedKinds) == 0 { + return + } + channelByID := func(channelID string) (store.ChannelRecord, bool) { + if channel, ok := channelLookup[channelID]; ok && strings.TrimSpace(channel.Kind) != "" { + return channel, true + } + channel, ok := snap.channels[channelID] + return channel, ok + } + excludesKind := func(kind string) bool { + kind = strings.ToLower(strings.TrimSpace(kind)) + if _, ok := excludedKinds[kind]; ok { + return true + } + switch kind { + case "news", "thread_news", "thread_announcement": + _, ok := excludedKinds["announcement"] + return ok + default: + return false + } + } + decisions := map[string]bool{} + var excludesChannel func(string, map[string]struct{}) bool + excludesChannel = func(channelID string, visiting map[string]struct{}) bool { + if decision, ok := decisions[channelID]; ok { + return decision + } + if _, ok := excludedIDs[channelID]; ok { + decisions[channelID] = true + return true + } + channel, ok := channelByID(channelID) + if !ok { + decisions[channelID] = false + return false + } + if excludesKind(channel.Kind) { + decisions[channelID] = true + return true + } + parentID := inheritedChannelParentID(channel) + if parentID == "" { + decisions[channelID] = false + return false + } + if visiting == nil { + visiting = map[string]struct{}{} + } + if _, ok := visiting[channelID]; ok { + return false + } + visiting[channelID] = struct{}{} + excluded := excludesChannel(parentID, visiting) + delete(visiting, channelID) + decisions[channelID] = excluded + return excluded } + for messageID, message := range snap.messages { + channelID := message.Record.ChannelID + if !excludesChannel(channelID, nil) { + continue + } + totals.excludedMessages[messageID] = struct{}{} + totals.excludedChannels[channelID] = struct{}{} + delete(snap.messages, messageID) + } + stats.ExcludedMessages = len(totals.excludedMessages) + stats.ExcludedChannels = len(totals.excludedChannels) +} + +func inheritedChannelParentID(channel store.ChannelRecord) string { + if channel.ThreadParentID != "" { + return channel.ThreadParentID + } + return channel.ParentID +} + +func normalizedCollectionSet(values []string, lower bool) map[string]struct{} { + out := make(map[string]struct{}, len(values)) + for _, value := range values { + value = strings.TrimSpace(value) + if lower { + value = strings.ToLower(value) + } + if value != "" { + out[value] = struct{}{} + } + } + return out } func finalizeSnapshot(snap snapshot, channelLookup map[string]store.ChannelRecord, totals scanTotals, stats *Stats, recordSkipped bool) unresolvedMessages { @@ -900,6 +1010,9 @@ func collectValue(snap snapshot, channelLookup map[string]store.ChannelRecord, v collectUserLabel(snap, typed) collectSelectedDirectMessageRoutes(snap, typed) if channel, ok := parseChannel(typed); ok { + if existing, exists := channelLookup[channel.ID]; exists { + channel = mergeCachedChannelParentMetadata(existing, channel, typed) + } snap.channels[channel.ID] = channel channelLookup[channel.ID] = channel if channel.GuildID == DirectMessageGuildID { @@ -1003,14 +1116,42 @@ func parseChannel(raw map[string]any) (store.ChannelRecord, bool) { name = "channel-" + shortID(id) } } - rawJSON := channelRawJSON(raw, id, guildID, name, kindForChannelType(typeValue, isDM)) - return store.ChannelRecord{ - ID: id, - GuildID: guildID, - Kind: kindForChannelType(typeValue, isDM), - Name: name, - RawJSON: rawJSON, - }, true + kind := kindForChannelType(typeValue, isDM) + parentID := stringField(raw, "parent_id") + channel := store.ChannelRecord{ + ID: id, + GuildID: guildID, + ParentID: parentID, + Kind: kind, + Name: name, + RawJSON: channelRawJSON(raw, id, guildID, name, kind), + } + if isThreadChannelKind(kind) { + channel.ThreadParentID = parentID + channel.IsPrivateThread = kind == "thread_private" + } + return channel, true +} + +func mergeCachedChannelParentMetadata(existing, cached store.ChannelRecord, raw map[string]any) store.ChannelRecord { + if _, hasType := raw["type"]; !hasType && isThreadChannelKind(existing.Kind) { + cached.Kind = existing.Kind + cached.IsPrivateThread = existing.IsPrivateThread + } + if _, hasParentID := raw["parent_id"]; !hasParentID { + cached.ParentID = existing.ParentID + } + if cached.ThreadParentID == "" && (isThreadChannelKind(cached.Kind) || isThreadChannelKind(existing.Kind)) { + cached.ThreadParentID = firstNonEmpty(cached.ParentID, existing.ThreadParentID, existing.ParentID) + } + if cached.ParentID == "" && cached.ThreadParentID != "" { + cached.ParentID = cached.ThreadParentID + } + return cached +} + +func isThreadChannelKind(kind string) bool { + return strings.HasPrefix(strings.ToLower(strings.TrimSpace(kind)), "thread_") } func parseMessage(raw map[string]any, fallbackTime time.Time, channels map[string]store.ChannelRecord) (store.MessageMutation, bool) { @@ -1435,12 +1576,13 @@ func kindForChannelType(typeValue int, dm bool) string { func channelRawJSON(raw map[string]any, id, guildID, name, kind string) string { return marshalJSONString(map[string]any{ - "id": id, - "guild_id": guildID, - "name": name, - "kind": kind, - "source": "discord_desktop", - "type": raw["type"], + "id": id, + "guild_id": guildID, + "parent_id": raw["parent_id"], + "name": name, + "kind": kind, + "source": "discord_desktop", + "type": raw["type"], }, "{}") } diff --git a/internal/discorddesktop/import_helpers_test.go b/internal/discorddesktop/import_helpers_test.go index 911705dc..97480ad2 100644 --- a/internal/discorddesktop/import_helpers_test.go +++ b/internal/discorddesktop/import_helpers_test.go @@ -69,6 +69,52 @@ func TestSnapshotWithoutMessageEvents(t *testing.T) { require.True(t, snap.messages["333333333333333346"].Options.AppendEvent) } +func TestFilterExcludedMessages(t *testing.T) { + snap := newSnapshot() + snap.messages["m-news"] = store.MessageMutation{Record: store.MessageRecord{ID: "m-news", ChannelID: "news"}} + snap.messages["m-logs"] = store.MessageMutation{Record: store.MessageRecord{ID: "m-logs", ChannelID: "logs"}} + snap.messages["m-general"] = store.MessageMutation{Record: store.MessageRecord{ID: "m-general", ChannelID: "general"}} + lookup := map[string]store.ChannelRecord{ + "news": {ID: "news", Kind: "announcement"}, + "logs": {ID: "logs", Kind: "text"}, + "general": {ID: "general", Kind: "text"}, + } + totals := newScanTotals() + stats := &Stats{} + + filterExcludedMessages(snap, lookup, Options{ + ExcludeChannelIDs: []string{"logs"}, + ExcludeChannelKinds: []string{"announcement"}, + }, totals, stats) + + require.Len(t, snap.messages, 1) + require.Contains(t, snap.messages, "m-general") + require.Equal(t, 2, stats.ExcludedMessages) + require.Equal(t, 2, stats.ExcludedChannels) +} + +func TestFilterExcludedMessagesInheritsLegacyAnnouncementExclusions(t *testing.T) { + snap := newSnapshot() + snap.messages["m-news"] = store.MessageMutation{Record: store.MessageRecord{ID: "m-news", ChannelID: "news"}} + snap.messages["m-thread-news"] = store.MessageMutation{Record: store.MessageRecord{ID: "m-thread-news", ChannelID: "thread-news"}} + snap.messages["m-child"] = store.MessageMutation{Record: store.MessageRecord{ID: "m-child", ChannelID: "child"}} + lookup := map[string]store.ChannelRecord{ + "news": {ID: "news", Kind: "news"}, + "thread-news": {ID: "thread-news", Kind: "thread_news"}, + "child": {ID: "child", Kind: "thread_public", ThreadParentID: "news"}, + } + totals := newScanTotals() + stats := &Stats{} + + filterExcludedMessages(snap, lookup, Options{ + ExcludeChannelKinds: []string{"announcement"}, + }, totals, stats) + + require.Empty(t, snap.messages) + require.Equal(t, 3, stats.ExcludedMessages) + require.Equal(t, 3, stats.ExcludedChannels) +} + func TestRouteFilteredCacheHelpers(t *testing.T) { require.Equal(t, fileSourceCacheData, sourceForPath("/tmp/discord", "/tmp/discord/Cache/Cache_Data/entry", "Cache/Cache_Data/entry")) require.Equal(t, fileSourceCacheData, sourceForPath("/tmp/discord", "/tmp/discord/Service Worker/CacheStorage/cache/entry", "Service Worker/CacheStorage/cache/entry")) @@ -127,11 +173,21 @@ func TestImportAndStateEdgeBranches(t *testing.T) { require.True(t, stats.FullCache) require.NoError(t, s.SetSyncState(ctx, fileIndexScope(Options{}), "{not-json")) - require.NoError(t, s.UpsertChannel(ctx, store.ChannelRecord{ID: "c1", GuildID: "g1", Kind: "text", Name: "general", RawJSON: `{}`})) + require.NoError(t, s.UpsertChannel(ctx, store.ChannelRecord{ + ID: "c1", + GuildID: "g1", + ParentID: "parent", + Kind: "thread_public", + Name: "general", + ThreadParentID: "parent", + RawJSON: `{}`, + })) state, err := loadScanState(ctx, s, Options{}) require.NoError(t, err) require.Empty(t, state.previous) require.Equal(t, "general", state.channels["c1"].Name) + require.Equal(t, "parent", state.channels["c1"].ParentID) + require.Equal(t, "parent", state.channels["c1"].ThreadParentID) } func TestSnapshotFinalizeAndCommitBranches(t *testing.T) { diff --git a/internal/discorddesktop/import_pipeline_test.go b/internal/discorddesktop/import_pipeline_test.go index d466f8d4..f3b6f6af 100644 --- a/internal/discorddesktop/import_pipeline_test.go +++ b/internal/discorddesktop/import_pipeline_test.go @@ -363,6 +363,171 @@ func TestImportFastCachePreservesKnownChannelMetadataAcrossBatches(t *testing.T) require.Contains(t, rows[0][0], `"type":11`) } +func TestImportExcludesCachedThreadRowsThroughParent(t *testing.T) { + ctx := context.Background() + guildID := "999999999999999996" + parentID := "111111111111111120" + threadID := "111111111111111121" + + tests := []struct { + name string + parentType int + options Options + }{ + { + name: "parent id", + parentType: 0, + options: Options{ExcludeChannelIDs: []string{parentID}}, + }, + { + name: "parent kind", + parentType: 5, + options: Options{ExcludeChannelKinds: []string{"announcement"}}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + cachePath := filepath.Join(dir, "Cache", "Cache_Data") + require.NoError(t, os.MkdirAll(cachePath, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(cachePath, "entry_0"), bytesf(`https://discord.com/api/v9/channels/%s/messages?limit=50 +{"id":"%s","guild_id":"%s","type":%d,"name":"excluded-parent"} +{"id":"%s","guild_id":"%s","parent_id":"%s","type":11,"name":"cached-thread"} +{"id":"333333333333333346","channel_id":"%s","content":"excluded cached thread payload","timestamp":"2026-04-23T18:20:43Z","author":{"id":"222222222222222232","username":"alice"},"attachments":[{"id":"444444444444444445","filename":"trace.txt"}],"mentions":[{"id":"555555555555555556","username":"bob"}]} +`, threadID, parentID, guildID, tt.parentType, threadID, guildID, parentID, threadID), 0o600)) + + st, err := store.Open(ctx, filepath.Join(dir, "discrawl.db")) + require.NoError(t, err) + defer func() { _ = st.Close() }() + + opts := tt.options + opts.Path = dir + stats, err := Import(ctx, st, opts) + require.NoError(t, err) + require.Zero(t, stats.Messages) + require.Equal(t, 1, stats.ExcludedMessages) + require.Equal(t, 1, stats.ExcludedChannels) + for _, table := range []string{"messages", "message_events", "message_attachments", "mention_events"} { + requireMessageCount(t, ctx, st, table, 0) + } + + channels, err := st.Channels(ctx, guildID) + require.NoError(t, err) + channelsByID := make(map[string]store.ChannelRow, len(channels)) + for _, channel := range channels { + channelsByID[channel.ID] = channel + } + require.Equal(t, parentID, channelsByID[threadID].ParentID) + require.Equal(t, parentID, channelsByID[threadID].ThreadParentID) + }) + } +} + +func TestImportDoesNotEraseStoredThreadParentFromPoorerCacheChannel(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + cachePath := filepath.Join(dir, "Cache", "Cache_Data") + require.NoError(t, os.MkdirAll(cachePath, 0o755)) + + guildID := "999999999999999996" + parentID := "111111111111111120" + threadID := "111111111111111121" + require.NoError(t, os.WriteFile(filepath.Join(cachePath, "entry_0"), bytesf(`https://discord.com/api/v9/channels/%s/messages?limit=50 +{"id":"%s","guild_id":"%s","type":11,"name":"poorer-cached-thread"} +{"id":"333333333333333346","channel_id":"%s","content":"stored parent must still exclude","timestamp":"2026-04-23T18:20:43Z","author":{"id":"222222222222222232","username":"alice"},"attachments":[{"id":"444444444444444445","filename":"trace.txt"}],"mentions":[{"id":"555555555555555556","username":"bob"}]} +`, threadID, threadID, guildID, threadID), 0o600)) + + st, err := store.Open(ctx, filepath.Join(dir, "discrawl.db")) + require.NoError(t, err) + defer func() { _ = st.Close() }() + require.NoError(t, st.UpsertGuild(ctx, store.GuildRecord{ID: guildID, Name: "Guild", RawJSON: `{}`})) + require.NoError(t, st.UpsertChannel(ctx, store.ChannelRecord{ + ID: parentID, + GuildID: guildID, + Kind: "news", + Name: "legacy-announcement", + RawJSON: `{}`, + })) + require.NoError(t, st.UpsertChannel(ctx, store.ChannelRecord{ + ID: threadID, + GuildID: guildID, + ParentID: parentID, + Kind: "thread_public", + Name: "stored-thread", + ThreadParentID: parentID, + RawJSON: `{}`, + })) + + stats, err := Import(ctx, st, Options{ + Path: dir, + ExcludeChannelKinds: []string{"announcement"}, + }) + require.NoError(t, err) + require.Zero(t, stats.Messages) + require.Equal(t, 1, stats.ExcludedMessages) + require.Equal(t, 1, stats.ExcludedChannels) + for _, table := range []string{"messages", "message_events", "message_attachments", "mention_events"} { + requireMessageCount(t, ctx, st, table, 0) + } + + channels, err := st.Channels(ctx, guildID) + require.NoError(t, err) + channelsByID := make(map[string]store.ChannelRow, len(channels)) + for _, channel := range channels { + channelsByID[channel.ID] = channel + } + require.Equal(t, parentID, channelsByID[threadID].ParentID) + require.Equal(t, parentID, channelsByID[threadID].ThreadParentID) +} + +func TestImportDryRunUsesStoredThreadParentForExclusions(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + cachePath := filepath.Join(dir, "Cache", "Cache_Data") + require.NoError(t, os.MkdirAll(cachePath, 0o755)) + + guildID := "999999999999999996" + parentID := "111111111111111120" + threadID := "111111111111111121" + require.NoError(t, os.WriteFile(filepath.Join(cachePath, "entry_0"), bytesf(`https://discord.com/api/v9/channels/%s/messages?limit=50 +{"id":"%s","guild_id":"%s","type":11,"name":"poorer-cached-thread"} +{"id":"333333333333333346","channel_id":"%s","content":"dry run must inherit stored parent","timestamp":"2026-04-23T18:20:43Z","author":{"id":"222222222222222232","username":"alice"}} +`, threadID, threadID, guildID, threadID), 0o600)) + + st, err := store.Open(ctx, filepath.Join(dir, "discrawl.db")) + require.NoError(t, err) + defer func() { _ = st.Close() }() + require.NoError(t, st.UpsertGuild(ctx, store.GuildRecord{ID: guildID, Name: "Guild", RawJSON: `{}`})) + require.NoError(t, st.UpsertChannel(ctx, store.ChannelRecord{ + ID: parentID, + GuildID: guildID, + Kind: "news", + Name: "legacy-announcement", + RawJSON: `{}`, + })) + require.NoError(t, st.UpsertChannel(ctx, store.ChannelRecord{ + ID: threadID, + GuildID: guildID, + ParentID: parentID, + Kind: "thread_public", + Name: "stored-thread", + ThreadParentID: parentID, + RawJSON: `{}`, + })) + + stats, err := Import(ctx, st, Options{ + Path: dir, + DryRun: true, + ExcludeChannelKinds: []string{"announcement"}, + }) + require.NoError(t, err) + require.True(t, stats.DryRun) + require.Zero(t, stats.Messages) + require.Equal(t, 1, stats.ExcludedMessages) + require.Equal(t, 1, stats.ExcludedChannels) + requireMessageCount(t, ctx, st, "messages", 0) +} + func TestImportFastCacheRouteFiltersServiceWorkerCacheStorage(t *testing.T) { ctx := context.Background() dir := t.TempDir() diff --git a/internal/discorddesktop/import_run.go b/internal/discorddesktop/import_run.go index f06fb3cd..01201a8f 100644 --- a/internal/discorddesktop/import_run.go +++ b/internal/discorddesktop/import_run.go @@ -64,6 +64,7 @@ func (r *importRun) scanCacheBatches(candidates []fileCandidate) error { } func (r *importRun) finalizeAndCommit(candidates []fileCandidate, snap snapshot, recordSkipped bool) error { + filterExcludedMessages(snap, r.channelLookup, r.opts, r.totals, r.stats) unresolved := finalizeSnapshot(snap, r.channelLookup, r.totals, r.stats, recordSkipped) checkpoint := len(unresolved) == 0 if !checkpoint { @@ -97,6 +98,7 @@ func (r *importRun) retryPending() error { if err := scanCandidates(r.ctx, r.rootFS, r.opts, r.pending, retry, r.channelLookup, r.stats); err != nil { return err } + filterExcludedMessages(retry, r.channelLookup, r.opts, r.totals, r.stats) unresolved := finalizeSnapshot(retry, r.channelLookup, r.totals, r.stats, true) checkpoint := len(unresolved) == 0 if err := commitSnapshot(r.ctx, r.st, r.opts, r.state, r.pending, retry, checkpoint, r.stats); err != nil { diff --git a/internal/share/import_filter.go b/internal/share/import_filter.go new file mode 100644 index 00000000..6264889d --- /dev/null +++ b/internal/share/import_filter.go @@ -0,0 +1,180 @@ +package share + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" +) + +type mergeImportChannel struct { + parentID string + kind string +} + +type mergeImportFilter struct { + excludedIDs map[string]struct{} + excludedKinds map[string]struct{} + channels map[string]mergeImportChannel + decisions map[string]bool +} + +type mergeImportQuerier interface { + QueryContext(context.Context, string, ...any) (*sql.Rows, error) +} + +func newMergeImportFilter( + ctx context.Context, + db mergeImportQuerier, + opts Options, +) (*mergeImportFilter, error) { + filter := &mergeImportFilter{ + excludedIDs: normalizedImportSet(opts.MergeExcludeChannelIDs, false), + excludedKinds: normalizedImportSet(opts.MergeExcludeChannelKinds, true), + channels: map[string]mergeImportChannel{}, + decisions: map[string]bool{}, + } + if len(filter.excludedIDs) > 0 || len(filter.excludedKinds) > 0 { + if err := filter.loadChannels(ctx, db); err != nil { + return nil, err + } + } + return filter, nil +} + +func (f *mergeImportFilter) loadChannels(ctx context.Context, db mergeImportQuerier) error { + rows, err := db.QueryContext(ctx, ` + select id, coalesce(parent_id, ''), coalesce(thread_parent_id, ''), kind + from channels + `) + if err != nil { + return fmt.Errorf("load channels for snapshot merge exclusions: %w", err) + } + defer func() { _ = rows.Close() }() + for rows.Next() { + var id, parentID, threadParentID, kind string + if err := rows.Scan(&id, &parentID, &threadParentID, &kind); err != nil { + return fmt.Errorf("scan channel for snapshot merge exclusions: %w", err) + } + if threadParentID != "" { + parentID = threadParentID + } + f.channels[id] = mergeImportChannel{parentID: parentID, kind: kind} + } + if err := rows.Err(); err != nil { + return fmt.Errorf("iterate channels for snapshot merge exclusions: %w", err) + } + return nil +} + +func (f *mergeImportFilter) allowMessageID(ctx context.Context, lookup *sql.Stmt, messageID string) (bool, error) { + var guildID, channelID string + err := lookup.QueryRowContext(ctx, messageID).Scan(&guildID, &channelID) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("load embedding message %s: %w", messageID, err) + } + return f.allow("messages", map[string]any{ + "guild_id": guildID, + "channel_id": channelID, + }) +} + +func (f *mergeImportFilter) allow(table string, row map[string]any) (bool, error) { + if isDirectMessageSnapshotRow(table, row) { + return false, nil + } + switch table { + case "channels": + f.recordChannel(row) + return true, nil + case "messages", "message_events", "message_attachments", "mention_events": + return !f.excludesChannel(stringValue(row["channel_id"]), nil), nil + default: + return true, nil + } +} + +func (f *mergeImportFilter) recordChannel(row map[string]any) { + id := stringValue(row["id"]) + if id == "" { + return + } + parentID := stringValue(row["thread_parent_id"]) + if parentID == "" { + parentID = stringValue(row["parent_id"]) + } + f.channels[id] = mergeImportChannel{ + parentID: parentID, + kind: stringValue(row["kind"]), + } + clear(f.decisions) +} + +func (f *mergeImportFilter) excludesChannel(channelID string, visiting map[string]struct{}) bool { + if channelID == "" { + return false + } + if decision, ok := f.decisions[channelID]; ok { + return decision + } + if _, ok := f.excludedIDs[channelID]; ok { + f.decisions[channelID] = true + return true + } + channel, ok := f.channels[channelID] + if !ok { + f.decisions[channelID] = false + return false + } + if f.excludesKind(channel.kind) { + f.decisions[channelID] = true + return true + } + if channel.parentID == "" { + f.decisions[channelID] = false + return false + } + if visiting == nil { + visiting = map[string]struct{}{} + } + if _, ok := visiting[channelID]; ok { + return false + } + visiting[channelID] = struct{}{} + excluded := f.excludesChannel(channel.parentID, visiting) + delete(visiting, channelID) + f.decisions[channelID] = excluded + return excluded +} + +func (f *mergeImportFilter) excludesKind(kind string) bool { + kind = strings.ToLower(strings.TrimSpace(kind)) + if _, ok := f.excludedKinds[kind]; ok { + return true + } + switch kind { + case "news", "thread_news", "thread_announcement": + _, ok := f.excludedKinds["announcement"] + return ok + default: + return false + } +} + +func normalizedImportSet(values []string, lower bool) map[string]struct{} { + out := make(map[string]struct{}, len(values)) + for _, value := range values { + value = strings.TrimSpace(value) + if lower { + value = strings.ToLower(value) + } + if value != "" { + out[value] = struct{}{} + } + } + return out +} diff --git a/internal/share/merge.go b/internal/share/merge.go index 12347d90..89f32193 100644 --- a/internal/share/merge.go +++ b/internal/share/merge.go @@ -15,11 +15,12 @@ import ( ) const ( - LastCheckSyncScope = "share:last_check_at" - LastMergeSyncScope = "share:last_merge_at" - LastMergeManifestSyncScope = "share:last_merge_manifest_generated_at" - LastMergeManifestJSONScope = "share:last_merge_manifest_json" - PendingReplacementSyncScope = "share:pending_replacement" + LastCheckSyncScope = "share:last_check_at" + LastMergeSyncScope = "share:last_merge_at" + LastMergeManifestSyncScope = "share:last_merge_manifest_generated_at" + LastMergeManifestJSONScope = "share:last_merge_manifest_json" + LastMergeEmbeddingsSyncScope = "share:last_merge_embeddings_state_v1" + PendingReplacementSyncScope = "share:pending_replacement" ) var ErrReplacementRequired = errors.New("share snapshot replacement required") @@ -192,6 +193,73 @@ func PreviousMergedManifest(ctx context.Context, s *store.Store, opts Options) ( return PreviousImportedManifest(ctx, s, opts) } +type mergeEmbeddingsState struct { + Version int `json:"version"` + GeneratedAt string `json:"generated_at"` + Provider string `json:"provider"` + Model string `json:"model"` + InputVersion string `json:"input_version"` + ExcludeChannelIDs []string `json:"exclude_channel_ids"` + ExcludeChannelKinds []string `json:"exclude_channel_kinds"` + Manifests []EmbeddingManifest `json:"manifests"` +} + +func mergedEmbeddingsStateValue(manifest Manifest, opts Options) (string, error) { + inputVersion := strings.TrimSpace(opts.EmbeddingInputVersion) + if inputVersion == "" { + inputVersion = store.EmbeddingInputVersion + } + state := mergeEmbeddingsState{ + Version: 1, + GeneratedAt: manifest.GeneratedAt.Format(time.RFC3339Nano), + Provider: strings.ToLower(strings.TrimSpace(opts.EmbeddingProvider)), + Model: strings.TrimSpace(opts.EmbeddingModel), + InputVersion: inputVersion, + ExcludeChannelIDs: sortedNormalizedImportValues(opts.MergeExcludeChannelIDs, false), + ExcludeChannelKinds: sortedNormalizedImportValues(opts.MergeExcludeChannelKinds, true), + Manifests: []EmbeddingManifest{}, + } + for _, embeddingManifest := range manifest.Embeddings { + if embeddingManifestMatches(opts, embeddingManifest) { + state.Manifests = append(state.Manifests, embeddingManifest) + } + } + body, err := json.Marshal(state) + if err != nil { + return "", fmt.Errorf("marshal merged embedding state: %w", err) + } + return string(body), nil +} + +func sortedNormalizedImportValues(values []string, lower bool) []string { + set := normalizedImportSet(values, lower) + out := make([]string, 0, len(set)) + for value := range set { + out = append(out, value) + } + sort.Strings(out) + return out +} + +func mergedEmbeddingsNeedImport(ctx context.Context, s *store.Store, manifest Manifest, opts Options) (string, bool, error) { + state, err := mergedEmbeddingsStateValue(manifest, opts) + if err != nil { + return "", false, err + } + previous, err := s.GetSyncState(ctx, LastMergeEmbeddingsSyncScope) + if err != nil { + return "", false, fmt.Errorf("read merged embedding state: %w", err) + } + return state, strings.TrimSpace(previous) != state, nil +} + +func markMergedEmbeddings(ctx context.Context, s *store.Store, state string) error { + if err := s.SetSyncState(ctx, LastMergeEmbeddingsSyncScope, state); err != nil { + return fmt.Errorf("write merged embedding state: %w", err) + } + return nil +} + func MarkChecked(ctx context.Context, s *store.Store) error { return s.SetSyncState(ctx, LastCheckSyncScope, time.Now().UTC().Format(time.RFC3339Nano)) } diff --git a/internal/share/merge_test.go b/internal/share/merge_test.go index 46593ca7..1c08fc24 100644 --- a/internal/share/merge_test.go +++ b/internal/share/merge_test.go @@ -132,6 +132,108 @@ func TestMergeIfChangedPreservesLocalRowsUntilForcedReplacement(t *testing.T) { require.Equal(t, "0", rows[0][0], "force must reconcile even when the manifest is unchanged") } +func TestMergeIfChangedExcludesConfiguredFeedChannelRows(t *testing.T) { + ctx := context.Background() + src, err := store.Open(ctx, filepath.Join(t.TempDir(), "src.db")) + require.NoError(t, err) + defer func() { _ = src.Close() }() + + require.NoError(t, src.UpsertGuild(ctx, store.GuildRecord{ID: "g1", Name: "Guild", RawJSON: `{}`})) + for _, channel := range []store.ChannelRecord{ + {ID: "general", GuildID: "g1", Kind: "text", Name: "general", RawJSON: `{}`}, + {ID: "github", GuildID: "g1", Kind: "announcement", Name: "github", RawJSON: `{}`}, + {ID: "github-thread", GuildID: "g1", ParentID: "github", Kind: "thread_public", Name: "thread", RawJSON: `{}`}, + {ID: "logs", GuildID: "g1", Kind: "text", Name: "logs", RawJSON: `{}`}, + } { + require.NoError(t, src.UpsertChannel(ctx, channel)) + } + now := time.Now().UTC().Format(time.RFC3339Nano) + message := func(id, channelID, content string) store.MessageRecord { + return store.MessageRecord{ + ID: id, + GuildID: "g1", + ChannelID: channelID, + ChannelName: channelID, + AuthorID: "u1", + AuthorName: "Bot", + CreatedAt: now, + Content: content, + NormalizedContent: content, + RawJSON: `{}`, + } + } + require.NoError(t, src.UpsertMessages(ctx, []store.MessageMutation{ + {Record: message("m-general", "general", "human discussion")}, + { + Record: message("m-github", "github", "automated feed"), + Attachments: []store.AttachmentRecord{{ + AttachmentID: "a-github", + MessageID: "m-github", + GuildID: "g1", + ChannelID: "github", + Filename: "payload.json", + }}, + Mentions: []store.MentionEventRecord{{ + MessageID: "m-github", + GuildID: "g1", + ChannelID: "github", + TargetType: "user", + TargetID: "u2", + EventAt: now, + }}, + }, + {Record: message("m-thread", "github-thread", "feed thread")}, + {Record: message("m-logs", "logs", "automated logs")}, + })) + + repo := filepath.Join(t.TempDir(), "share") + _, err = Export(ctx, src, Options{RepoPath: repo, Branch: "main"}) + require.NoError(t, err) + + dst, err := store.Open(ctx, filepath.Join(t.TempDir(), "dst.db")) + require.NoError(t, err) + defer func() { _ = dst.Close() }() + require.NoError(t, dst.UpsertGuild(ctx, store.GuildRecord{ID: "g1", Name: "Guild", RawJSON: `{}`})) + require.NoError(t, dst.UpsertChannel(ctx, store.ChannelRecord{ + ID: "github", GuildID: "g1", Kind: "announcement", Name: "github", RawJSON: `{}`, + })) + require.NoError(t, dst.UpsertMessage(ctx, message("local-feed", "github", "preserved existing feed row"))) + + opts := Options{ + RepoPath: repo, + Branch: "main", + MergeExcludeChannelIDs: []string{"logs"}, + MergeExcludeChannelKinds: []string{"announcement"}, + } + _, changed, err := MergeIfChanged(ctx, dst, opts) + require.NoError(t, err) + require.True(t, changed) + + _, rows, err := dst.ReadOnlyQuery(ctx, `select id from messages order by id`) + require.NoError(t, err) + require.Equal(t, [][]string{{"local-feed"}, {"m-general"}}, rows) + _, rows, err = dst.ReadOnlyQuery(ctx, `select id from channels order by id`) + require.NoError(t, err) + require.Equal(t, [][]string{{"general"}, {"github"}, {"github-thread"}, {"logs"}}, rows) + _, rows, err = dst.ReadOnlyQuery(ctx, `select count(*) from message_attachments`) + require.NoError(t, err) + require.Equal(t, "0", rows[0][0]) + _, rows, err = dst.ReadOnlyQuery(ctx, `select count(*) from mention_events`) + require.NoError(t, err) + require.Equal(t, "0", rows[0][0]) + + require.NoError(t, src.UpsertMessage(ctx, message("m-general-2", "general", "new human discussion"))) + require.NoError(t, src.UpsertMessage(ctx, message("m-github-2", "github", "new automated feed"))) + _, err = Export(ctx, src, Options{RepoPath: repo, Branch: "main"}) + require.NoError(t, err) + _, changed, err = MergeIfChanged(ctx, dst, opts) + require.NoError(t, err) + require.True(t, changed) + _, rows, err = dst.ReadOnlyQuery(ctx, `select id from messages order by id`) + require.NoError(t, err) + require.Equal(t, [][]string{{"local-feed"}, {"m-general"}, {"m-general-2"}}, rows) +} + func TestMergeIfChangedMarksReplacementPendingWithoutChangingRows(t *testing.T) { ctx := context.Background() src := seedStore(t, filepath.Join(t.TempDir(), "src.db")) diff --git a/internal/share/share.go b/internal/share/share.go index 03d57c58..8af999a4 100644 --- a/internal/share/share.go +++ b/internal/share/share.go @@ -65,18 +65,20 @@ var SnapshotTables = []string{ } type Options struct { - RepoPath string - CacheDir string - Remote string - Branch string - Tag string - Filter FilterOptions - IncludeMedia bool - IncludeEmbeddings bool - EmbeddingProvider string - EmbeddingModel string - EmbeddingInputVersion string - Progress func(ImportProgress) + RepoPath string + CacheDir string + Remote string + Branch string + Tag string + Filter FilterOptions + MergeExcludeChannelIDs []string + MergeExcludeChannelKinds []string + IncludeMedia bool + IncludeEmbeddings bool + EmbeddingProvider string + EmbeddingModel string + EmbeddingInputVersion string + Progress func(ImportProgress) } type FilterOptions struct { @@ -536,7 +538,11 @@ func Import(ctx context.Context, s *store.Store, opts Options) (Manifest, error) return err } if opts.IncludeEmbeddings { - return importEmbeddings(ctx, tx, opts, manifest.Embeddings) + embeddingOpts := opts + embeddingOpts.MergeExcludeChannelIDs = nil + embeddingOpts.MergeExcludeChannelKinds = nil + _, err := importEmbeddings(ctx, tx, embeddingOpts, manifest.Embeddings) + return err } return nil }, @@ -552,6 +558,18 @@ func Import(ctx context.Context, s *store.Store, opts Options) (Manifest, error) return Manifest{}, err } } + if opts.IncludeEmbeddings { + embeddingOpts := opts + embeddingOpts.MergeExcludeChannelIDs = nil + embeddingOpts.MergeExcludeChannelKinds = nil + state, err := mergedEmbeddingsStateValue(manifest, embeddingOpts) + if err != nil { + return Manifest{}, err + } + if err := markMergedEmbeddings(ctx, s, state); err != nil { + return Manifest{}, err + } + } if err := MarkImported(ctx, s, manifest); err != nil { return Manifest{}, err } @@ -610,10 +628,21 @@ func importMergePlan( plan snapshot.ImportPlan, ) (Manifest, bool, error) { if !plan.Changed() { + embeddingRows := 0 if opts.IncludeEmbeddings { - if err := ImportEmbeddings(ctx, s, opts, manifest); err != nil { + state, needed, err := mergedEmbeddingsNeedImport(ctx, s, manifest, opts) + if err != nil { return Manifest{}, false, err } + if needed { + embeddingRows, err = importEmbeddingsIntoStore(ctx, s, opts, manifest) + if err != nil { + return Manifest{}, false, err + } + if err := markMergedEmbeddings(ctx, s, state); err != nil { + return Manifest{}, false, err + } + } } copied := 0 if opts.IncludeMedia { @@ -626,7 +655,11 @@ func importMergePlan( if err := MarkMerged(ctx, s, manifest); err != nil { return Manifest{}, false, err } - return manifest, copied > 0, nil + return manifest, embeddingRows > 0 || copied > 0, nil + } + rowFilter, err := newMergeImportFilter(ctx, s.DB(), opts) + if err != nil { + return Manifest{}, false, err } opts.reportProgress(ImportProgress{Phase: "start", TotalRows: importPlanRowCount(plan)}) restorePragmas, err := applyImportPragmas(ctx, s.DB()) @@ -657,9 +690,7 @@ func importMergePlan( TotalRows: progress.TotalRows, }) }, - Filter: func(table string, row map[string]any) (bool, error) { - return !isDirectMessageSnapshotRow(table, row), nil - }, + Filter: rowFilter.allow, BeforeImport: func(ctx context.Context, tx *sql.Tx) error { var err error existingMedia, err = attachmentMediaByID(ctx, tx) @@ -681,7 +712,8 @@ func importMergePlan( return err } if opts.IncludeEmbeddings { - return importEmbeddings(ctx, tx, opts, manifest.Embeddings) + _, err := importEmbeddingsWithFilter(ctx, tx, opts, manifest.Embeddings, rowFilter) + return err } return nil }, @@ -706,6 +738,15 @@ func importMergePlan( return Manifest{}, false, err } } + if opts.IncludeEmbeddings { + state, err := mergedEmbeddingsStateValue(manifest, opts) + if err != nil { + return Manifest{}, false, err + } + if err := markMergedEmbeddings(ctx, s, state); err != nil { + return Manifest{}, false, err + } + } if err := MarkMerged(ctx, s, manifest); err != nil { return Manifest{}, false, err } @@ -756,24 +797,37 @@ func importPlanRowCount(plan snapshot.ImportPlan) int { } func ImportEmbeddings(ctx context.Context, s *store.Store, opts Options, manifest Manifest) error { - tx, err := s.DB().BeginTx(ctx, nil) + _, err := importEmbeddingsIntoStore(ctx, s, opts, manifest) if err != nil { return err } + state, err := mergedEmbeddingsStateValue(manifest, opts) + if err != nil { + return err + } + return markMergedEmbeddings(ctx, s, state) +} + +func importEmbeddingsIntoStore(ctx context.Context, s *store.Store, opts Options, manifest Manifest) (int, error) { + tx, err := s.DB().BeginTx(ctx, nil) + if err != nil { + return 0, err + } committed := false defer func() { if !committed { _ = tx.Rollback() } }() - if err := importEmbeddings(ctx, tx, opts, manifest.Embeddings); err != nil { - return err + rows, err := importEmbeddings(ctx, tx, opts, manifest.Embeddings) + if err != nil { + return 0, err } if err := tx.Commit(); err != nil { - return err + return 0, err } committed = true - return nil + return rows, nil } func ManifestAlreadyImported(ctx context.Context, s *store.Store, manifest Manifest) bool { @@ -2197,9 +2251,28 @@ func isLocalOnlyGuildID(guildID string) bool { return guildID == directMessageGuildID } -func importEmbeddings(ctx context.Context, tx *sql.Tx, opts Options, manifests []EmbeddingManifest) error { +func importEmbeddings( + ctx context.Context, + tx *sql.Tx, + opts Options, + manifests []EmbeddingManifest, +) (int, error) { + filter, err := newMergeImportFilter(ctx, tx, opts) + if err != nil { + return 0, err + } + return importEmbeddingsWithFilter(ctx, tx, opts, manifests, filter) +} + +func importEmbeddingsWithFilter( + ctx context.Context, + tx *sql.Tx, + opts Options, + manifests []EmbeddingManifest, + filter *mergeImportFilter, +) (int, error) { if len(manifests) == 0 { - return nil + return 0, nil } stmt, err := tx.PrepareContext(ctx, ` insert into message_embeddings( @@ -2209,50 +2282,72 @@ func importEmbeddings(ctx context.Context, tx *sql.Tx, opts Options, manifests [ dimensions = excluded.dimensions, embedding_blob = excluded.embedding_blob, embedded_at = excluded.embedded_at + where message_embeddings.dimensions is not excluded.dimensions + or message_embeddings.embedding_blob is not excluded.embedding_blob + or message_embeddings.embedded_at is not excluded.embedded_at `) if err != nil { - return fmt.Errorf("prepare import message_embeddings: %w", err) + return 0, fmt.Errorf("prepare import message_embeddings: %w", err) } defer func() { _ = stmt.Close() }() + messageLookup, err := tx.PrepareContext(ctx, ` + select guild_id, channel_id + from messages + where id = ? + `) + if err != nil { + return 0, fmt.Errorf("prepare embedding message lookup: %w", err) + } + defer func() { _ = messageLookup.Close() }() + changed := 0 for _, manifest := range manifests { if err := ctx.Err(); err != nil { - return err + return 0, err } if !embeddingManifestMatches(opts, manifest) { continue } files := manifest.Files if len(files) == 0 { - return fmt.Errorf("embedding manifest %s/%s/%s has no files", manifest.Provider, manifest.Model, manifest.InputVersion) + return 0, fmt.Errorf("embedding manifest %s/%s/%s has no files", manifest.Provider, manifest.Model, manifest.InputVersion) } for _, rel := range files { if err := ctx.Err(); err != nil { - return err + return 0, err } - if err := importEmbeddingFile(ctx, stmt, opts.RepoPath, rel); err != nil { - return err + rows, err := importEmbeddingFile(ctx, stmt, messageLookup, filter, opts.RepoPath, rel) + if err != nil { + return 0, err } + changed += rows } } - return nil + return changed, nil } -func importEmbeddingFile(ctx context.Context, stmt *sql.Stmt, repoPath, rel string) error { +func importEmbeddingFile( + ctx context.Context, + stmt *sql.Stmt, + messageLookup *sql.Stmt, + filter *mergeImportFilter, + repoPath, + rel string, +) (int, error) { path, err := embeddingRepoPath(repoPath, rel) if err != nil { - return err + return 0, err } if _, err := regularFileInRoot(filepath.Join(repoPath, "embeddings"), path, rel, "embedding"); err != nil { - return err + return 0, err } file, err := os.Open(path) // #nosec G304 -- path is confined by embeddingRepoPath and regularFileInRoot. if err != nil { - return fmt.Errorf("open %s: %w", rel, err) + return 0, fmt.Errorf("open %s: %w", rel, err) } defer func() { _ = file.Close() }() gz, err := gzip.NewReader(file) if err != nil { - return fmt.Errorf("read gzip %s: %w", rel, err) + return 0, fmt.Errorf("read gzip %s: %w", rel, err) } defer func() { _ = gz.Close() }() dec := json.NewDecoder(&limitedReader{ @@ -2261,9 +2356,10 @@ func importEmbeddingFile(ctx context.Context, stmt *sql.Stmt, repoPath, rel stri label: "embedding", }) dec.UseNumber() + changed := 0 for { if err := ctx.Err(); err != nil { - return err + return 0, err } var row struct { MessageID string `json:"message_id"` @@ -2279,21 +2375,34 @@ func importEmbeddingFile(ctx context.Context, stmt *sql.Stmt, repoPath, rel stri break } if err != nil { - return fmt.Errorf("decode %s: %w", rel, err) + return 0, fmt.Errorf("decode %s: %w", rel, err) } dimensions, err := strconv.Atoi(row.Dimensions.String()) if err != nil { - return fmt.Errorf("decode dimensions in %s: %w", rel, err) + return 0, fmt.Errorf("decode dimensions in %s: %w", rel, err) } blob, err := base64.StdEncoding.DecodeString(row.EmbeddingBlob) if err != nil { - return fmt.Errorf("decode embedding blob in %s: %w", rel, err) + return 0, fmt.Errorf("decode embedding blob in %s: %w", rel, err) } - if _, err := stmt.ExecContext(ctx, row.MessageID, row.Provider, row.Model, row.InputVersion, dimensions, blob, row.EmbeddedAt); err != nil { - return fmt.Errorf("insert message_embeddings: %w", err) + allowed, err := filter.allowMessageID(ctx, messageLookup, row.MessageID) + if err != nil { + return 0, err + } + if !allowed { + continue + } + result, err := stmt.ExecContext(ctx, row.MessageID, row.Provider, row.Model, row.InputVersion, dimensions, blob, row.EmbeddedAt) + if err != nil { + return 0, fmt.Errorf("insert message_embeddings: %w", err) + } + rows, err := result.RowsAffected() + if err != nil { + return 0, fmt.Errorf("count imported message_embeddings: %w", err) } + changed += int(rows) } - return nil + return changed, nil } func embeddingRepoPath(repoPath, rel string) (string, error) { diff --git a/internal/share/share_test.go b/internal/share/share_test.go index b4921aa9..a0988af2 100644 --- a/internal/share/share_test.go +++ b/internal/share/share_test.go @@ -1535,6 +1535,213 @@ func TestImportEmbeddingsFiltersByConfiguredIdentity(t *testing.T) { require.Equal(t, "0", rows[0][0]) } +func TestMergeIfChangedFiltersExcludedFeedEmbeddings(t *testing.T) { + ctx := context.Background() + src := seedStore(t, filepath.Join(t.TempDir(), "src.db")) + defer func() { _ = src.Close() }() + require.NoError(t, src.UpsertChannel(ctx, store.ChannelRecord{ + ID: "feed", GuildID: "g1", Kind: "announcement", Name: "feed", RawJSON: `{}`, + })) + now := time.Now().UTC().Format(time.RFC3339Nano) + require.NoError(t, src.UpsertMessage(ctx, store.MessageRecord{ + ID: "m-feed", GuildID: "g1", ChannelID: "feed", ChannelName: "feed", + AuthorID: "bot", AuthorName: "Bot", CreatedAt: now, + Content: "automated feed", NormalizedContent: "automated feed", RawJSON: `{}`, + })) + insertShareTestEmbedding(t, ctx, src, "m1", []float32{1, 0}) + insertShareTestEmbedding(t, ctx, src, "m-feed", []float32{0, 1}) + + opts := Options{ + RepoPath: filepath.Join(t.TempDir(), "share"), + Branch: "main", + IncludeEmbeddings: true, + EmbeddingProvider: "openai", + EmbeddingModel: "text-embedding-3-small", + EmbeddingInputVersion: store.EmbeddingInputVersion, + } + _, err := Export(ctx, src, opts) + require.NoError(t, err) + + dst, err := store.Open(ctx, filepath.Join(t.TempDir(), "dst.db")) + require.NoError(t, err) + defer func() { _ = dst.Close() }() + opts.MergeExcludeChannelKinds = []string{"announcement"} + _, changed, err := MergeIfChanged(ctx, dst, opts) + require.NoError(t, err) + require.True(t, changed) + + _, rows, err := dst.ReadOnlyQuery(ctx, `select message_id from message_embeddings order by message_id`) + require.NoError(t, err) + require.Equal(t, [][]string{{"m1"}}, rows) + _, rows, err = dst.ReadOnlyQuery(ctx, `select id from messages order by id`) + require.NoError(t, err) + require.Equal(t, [][]string{{"m1"}}, rows) +} + +func TestExactImportDoesNotApplySafeMergeEmbeddingExclusions(t *testing.T) { + ctx := context.Background() + src := seedStore(t, filepath.Join(t.TempDir(), "src.db")) + defer func() { _ = src.Close() }() + require.NoError(t, src.UpsertChannel(ctx, store.ChannelRecord{ + ID: "feed", GuildID: "g1", Kind: "announcement", Name: "feed", RawJSON: `{}`, + })) + now := time.Now().UTC().Format(time.RFC3339Nano) + require.NoError(t, src.UpsertMessage(ctx, store.MessageRecord{ + ID: "m-feed", GuildID: "g1", ChannelID: "feed", ChannelName: "feed", + AuthorID: "bot", AuthorName: "Bot", CreatedAt: now, + Content: "exact feed row", NormalizedContent: "exact feed row", RawJSON: `{}`, + })) + insertShareTestEmbedding(t, ctx, src, "m-feed", []float32{0, 1}) + + opts := Options{ + RepoPath: filepath.Join(t.TempDir(), "share"), + Branch: "main", + IncludeEmbeddings: true, + EmbeddingProvider: "openai", + EmbeddingModel: "text-embedding-3-small", + EmbeddingInputVersion: store.EmbeddingInputVersion, + MergeExcludeChannelKinds: []string{"announcement"}, + } + _, err := Export(ctx, src, opts) + require.NoError(t, err) + + dst, err := store.Open(ctx, filepath.Join(t.TempDir(), "dst.db")) + require.NoError(t, err) + defer func() { _ = dst.Close() }() + _, err = Import(ctx, dst, opts) + require.NoError(t, err) + + _, rows, err := dst.ReadOnlyQuery(ctx, `select id from messages where id = 'm-feed'`) + require.NoError(t, err) + require.Equal(t, [][]string{{"m-feed"}}, rows) + _, rows, err = dst.ReadOnlyQuery(ctx, `select message_id from message_embeddings where message_id = 'm-feed'`) + require.NoError(t, err) + require.Equal(t, [][]string{{"m-feed"}}, rows) +} + +func TestImportEmbeddingsSkipsOrphans(t *testing.T) { + ctx := context.Background() + src := seedStore(t, filepath.Join(t.TempDir(), "src.db")) + defer func() { _ = src.Close() }() + insertShareTestEmbedding(t, ctx, src, "m1", []float32{1, 0}) + + opts := Options{ + RepoPath: filepath.Join(t.TempDir(), "share"), + Branch: "main", + IncludeEmbeddings: true, + EmbeddingProvider: "openai", + EmbeddingModel: "text-embedding-3-small", + EmbeddingInputVersion: store.EmbeddingInputVersion, + } + manifest, err := Export(ctx, src, opts) + require.NoError(t, err) + validLine := strings.TrimSpace(snapshotFilesText(t, opts.RepoPath, manifest.Embeddings[0].Files)) + var orphanRow map[string]any + require.NoError(t, json.Unmarshal([]byte(validLine), &orphanRow)) + orphanRow["message_id"] = "orphan" + orphanLine, err := json.Marshal(orphanRow) + require.NoError(t, err) + writeGzipJSONLines( + t, + filepath.Join(opts.RepoPath, filepath.FromSlash(manifest.Embeddings[0].Files[0])), + []string{validLine, string(orphanLine)}, + ) + manifest.Embeddings[0].Rows = 2 + require.Equal(t, 2, manifest.Embeddings[0].Rows) + + dst := seedStore(t, filepath.Join(t.TempDir(), "dst.db")) + defer func() { _ = dst.Close() }() + require.NoError(t, ImportEmbeddings(ctx, dst, opts, manifest)) + _, rows, err := dst.ReadOnlyQuery(ctx, `select message_id from message_embeddings order by message_id`) + require.NoError(t, err) + require.Equal(t, [][]string{{"m1"}}, rows) +} + +func TestMergeIfChangedImportsEmbeddingsWhenCanonicalTablesUnchanged(t *testing.T) { + ctx := context.Background() + src := seedStore(t, filepath.Join(t.TempDir(), "src.db")) + defer func() { _ = src.Close() }() + insertShareTestEmbedding(t, ctx, src, "m1", []float32{1, 0}) + + opts := Options{ + RepoPath: filepath.Join(t.TempDir(), "share"), + Branch: "main", + IncludeEmbeddings: true, + EmbeddingProvider: "openai", + EmbeddingModel: "text-embedding-3-small", + EmbeddingInputVersion: store.EmbeddingInputVersion, + } + manifest, err := Export(ctx, src, opts) + require.NoError(t, err) + + dst, err := store.Open(ctx, filepath.Join(t.TempDir(), "dst.db")) + require.NoError(t, err) + defer func() { _ = dst.Close() }() + canonicalOnly := opts + canonicalOnly.IncludeEmbeddings = false + _, changed, err := MergeIfChanged(ctx, dst, canonicalOnly) + require.NoError(t, err) + require.True(t, changed) + _, rows, err := dst.ReadOnlyQuery(ctx, `select count(*) from message_embeddings`) + require.NoError(t, err) + require.Equal(t, "0", rows[0][0]) + + _, changed, err = MergeIfChanged(ctx, dst, opts) + require.NoError(t, err) + require.True(t, changed) + _, rows, err = dst.ReadOnlyQuery(ctx, `select message_id from message_embeddings`) + require.NoError(t, err) + require.Equal(t, [][]string{{"m1"}}, rows) + + require.NoError(t, dst.SetSyncState(ctx, LastMergeEmbeddingsSyncScope, "")) + _, changed, err = MergeIfChanged(ctx, dst, opts) + require.NoError(t, err) + require.False(t, changed) + + require.NoError(t, os.Remove(filepath.Join(opts.RepoPath, filepath.FromSlash(manifest.Embeddings[0].Files[0])))) + _, changed, err = MergeIfChanged(ctx, dst, opts) + require.NoError(t, err) + require.False(t, changed) +} + +func TestImportEmbeddingsPreservesLocalDirectMessageRows(t *testing.T) { + ctx := context.Background() + src := seedStore(t, filepath.Join(t.TempDir(), "src.db")) + defer func() { _ = src.Close() }() + require.NoError(t, src.UpsertMessage(ctx, store.MessageRecord{ + ID: "dm1", GuildID: "g1", ChannelID: "c1", ChannelName: "general", + AuthorID: "u1", AuthorName: "Peter", CreatedAt: time.Now().UTC().Format(time.RFC3339Nano), + Content: "public collision", NormalizedContent: "public collision", RawJSON: `{}`, + })) + insertShareTestEmbedding(t, ctx, src, "dm1", []float32{1, 0}) + opts := Options{ + RepoPath: filepath.Join(t.TempDir(), "share"), + Branch: "main", + IncludeEmbeddings: true, + EmbeddingProvider: "openai", + EmbeddingModel: "text-embedding-3-small", + EmbeddingInputVersion: store.EmbeddingInputVersion, + } + manifest, err := Export(ctx, src, opts) + require.NoError(t, err) + + dst := seedStore(t, filepath.Join(t.TempDir(), "dst.db")) + defer func() { _ = dst.Close() }() + seedDirectMessageData(t, ctx, dst) + insertShareTestEmbedding(t, ctx, dst, "dm1", []float32{9, 9}) + require.NoError(t, ImportEmbeddings(ctx, dst, opts, manifest)) + + var blob []byte + require.NoError(t, dst.DB().QueryRowContext(ctx, ` + select embedding_blob + from message_embeddings + where message_id = 'dm1' + `).Scan(&blob)) + vector, err := store.DecodeEmbeddingVector(blob) + require.NoError(t, err) + require.Equal(t, []float32{9, 9}, vector) +} + func TestMergeIfChangedSkipsSameManifest(t *testing.T) { ctx := context.Background() src := seedStore(t, filepath.Join(t.TempDir(), "src.db")) @@ -2243,7 +2450,7 @@ func TestLegacyManifestFileImportAndEmbeddingDecodeErrors(t *testing.T) { }) tx, err = s.DB().BeginTx(ctx, nil) require.NoError(t, err) - require.ErrorContains(t, importEmbeddings(ctx, tx, Options{ + _, err = importEmbeddings(ctx, tx, Options{ RepoPath: repo, EmbeddingProvider: "openai", EmbeddingModel: "model", @@ -2253,7 +2460,8 @@ func TestLegacyManifestFileImportAndEmbeddingDecodeErrors(t *testing.T) { Model: "model", InputVersion: store.EmbeddingInputVersion, Files: []string{embeddingRel}, - }}), "decode dimensions") + }}) + require.ErrorContains(t, err, "decode dimensions") require.NoError(t, tx.Rollback()) writeGzipJSONLines(t, filepath.Join(repo, filepath.FromSlash(embeddingRel)), []string{ @@ -2261,7 +2469,7 @@ func TestLegacyManifestFileImportAndEmbeddingDecodeErrors(t *testing.T) { }) tx, err = s.DB().BeginTx(ctx, nil) require.NoError(t, err) - require.ErrorContains(t, importEmbeddings(ctx, tx, Options{ + _, err = importEmbeddings(ctx, tx, Options{ RepoPath: repo, EmbeddingProvider: "openai", EmbeddingModel: "model", @@ -2271,7 +2479,8 @@ func TestLegacyManifestFileImportAndEmbeddingDecodeErrors(t *testing.T) { Model: "model", InputVersion: store.EmbeddingInputVersion, Files: []string{embeddingRel}, - }}), "decode embedding blob") + }}) + require.ErrorContains(t, err, "decode embedding blob") require.NoError(t, tx.Rollback()) } @@ -2330,7 +2539,7 @@ func TestImportEmbeddingsRejectsUnsafeManifestFiles(t *testing.T) { } { tx, err := s.DB().BeginTx(ctx, nil) require.NoError(t, err) - require.ErrorContains(t, importEmbeddings(ctx, tx, Options{ + _, err = importEmbeddings(ctx, tx, Options{ RepoPath: repo, EmbeddingProvider: "openai", EmbeddingModel: "model", @@ -2340,7 +2549,8 @@ func TestImportEmbeddingsRejectsUnsafeManifestFiles(t *testing.T) { Model: "model", InputVersion: store.EmbeddingInputVersion, Files: []string{rel}, - }}), "invalid embedding manifest path") + }}) + require.ErrorContains(t, err, "invalid embedding manifest path") require.NoError(t, tx.Rollback()) } } @@ -2364,7 +2574,7 @@ func TestImportEmbeddingsBoundsDecompressedInput(t *testing.T) { tx, err := s.DB().BeginTx(ctx, nil) require.NoError(t, err) - require.ErrorContains(t, importEmbeddings(ctx, tx, Options{ + _, err = importEmbeddings(ctx, tx, Options{ RepoPath: repo, EmbeddingProvider: "openai", EmbeddingModel: "model", @@ -2374,7 +2584,8 @@ func TestImportEmbeddingsBoundsDecompressedInput(t *testing.T) { Model: "model", InputVersion: store.EmbeddingInputVersion, Files: []string{embeddingRel}, - }}), "embedding decompressed size exceeds") + }}) + require.ErrorContains(t, err, "embedding decompressed size exceeds") require.NoError(t, tx.Rollback()) } @@ -2493,6 +2704,18 @@ func seedStore(t *testing.T, path string) *store.Store { return s } +func insertShareTestEmbedding(t *testing.T, ctx context.Context, s *store.Store, messageID string, vector []float32) { + t.Helper() + blob, err := store.EncodeEmbeddingVector(vector) + require.NoError(t, err) + _, err = s.DB().ExecContext(ctx, ` + insert into message_embeddings( + message_id, provider, model, input_version, dimensions, embedding_blob, embedded_at + ) values (?, 'openai', 'text-embedding-3-small', ?, ?, ?, ?) + `, messageID, store.EmbeddingInputVersion, len(vector), blob, time.Now().UTC().Format(time.RFC3339Nano)) + require.NoError(t, err) +} + func addCachedAttachment(ctx context.Context, s *store.Store, mediaPath, hash string, size int64) error { now := time.Now().UTC().Format(time.RFC3339Nano) if err := s.UpsertMessages(ctx, []store.MessageMutation{{ diff --git a/internal/store/coverage.go b/internal/store/coverage.go index e77bc8d7..aafaa82e 100644 --- a/internal/store/coverage.go +++ b/internal/store/coverage.go @@ -11,7 +11,50 @@ import ( const wiretapStatsScope = "wiretap:last_stats:v1" -const coverageQueryTimeout = 2 * time.Minute +const ( + coverageQueryTimeout = 2 * time.Minute + globalCoverageQueryTimeout = 4 * time.Minute +) + +const filteredCoverageChannelQuery = ` + select + c.id, c.guild_id, c.name, c.kind, + case when exists ( + select 1 from sync_state s + where s.scope = 'channel:' || c.id || ':history_complete' + ) then 1 else 0 end, + count(m.id), coalesce(min(m.created_at), ''), coalesce(max(m.created_at), '') + from channels c + left join messages m on m.channel_id = c.id and m.deleted_at is null + where c.guild_id = ? + group by c.id, c.guild_id, c.name, c.kind + order by c.guild_id, count(m.id) desc, lower(c.name), c.id +` + +const globalCoverageChannelQuery = ` + with message_coverage as materialized ( + select + channel_id, + count(*) as message_count, + coalesce(min(created_at), '') as earliest_message_at, + coalesce(max(created_at), '') as latest_message_at + from messages not indexed + where deleted_at is null + group by channel_id + ) + select + c.id, c.guild_id, c.name, c.kind, + case when exists ( + select 1 from sync_state s + where s.scope = 'channel:' || c.id || ':history_complete' + ) then 1 else 0 end, + coalesce(m.message_count, 0), + coalesce(m.earliest_message_at, ''), + coalesce(m.latest_message_at, '') + from channels c + left join message_coverage m on m.channel_id = c.id + order by c.guild_id, coalesce(m.message_count, 0) desc, lower(c.name), c.id +` var messageChannelKinds = map[string]struct{}{ "text": {}, "news": {}, "announcement": {}, "dm": {}, "group_dm": {}, @@ -102,7 +145,7 @@ func (s *Store) SetWiretapImportStats(ctx context.Context, stats WiretapImportSt func (s *Store) Coverage(ctx context.Context, guildID string, generatedAt time.Time) (CoverageReport, error) { report := CoverageReport{GeneratedAt: generatedAt.UTC(), Guilds: []CoverageGuild{}} - queryCtx, cancel := context.WithTimeout(ctx, coverageQueryTimeout) + queryCtx, cancel := withCoverageQueryTimeout(ctx, guildID) defer cancel() rows, err := s.db.QueryContext(queryCtx, ` @@ -137,20 +180,13 @@ func (s *Store) Coverage(ctx context.Context, guildID string, generatedAt time.T for i := range report.Guilds { guilds[report.Guilds[i].ID] = &report.Guilds[i] } - channelRows, err := s.db.QueryContext(queryCtx, ` - select - c.id, c.guild_id, c.name, c.kind, - case when exists ( - select 1 from sync_state s - where s.scope = 'channel:' || c.id || ':history_complete' - ) then 1 else 0 end, - count(m.id), coalesce(min(m.created_at), ''), coalesce(max(m.created_at), '') - from channels c - left join messages m on m.channel_id = c.id and m.deleted_at is null - where ? = '' or c.guild_id = ? - group by c.id, c.guild_id, c.name, c.kind - order by c.guild_id, count(m.id) desc, lower(c.name), c.id - `, guildID, guildID) + channelQuery := filteredCoverageChannelQuery + channelArgs := []any{guildID} + if guildID == "" { + channelQuery = globalCoverageChannelQuery + channelArgs = nil + } + channelRows, err := s.db.QueryContext(queryCtx, channelQuery, channelArgs...) if err != nil { return CoverageReport{}, fmt.Errorf("query channel coverage: %w", err) } @@ -226,6 +262,14 @@ func (s *Store) Coverage(ctx context.Context, guildID string, generatedAt time.T return report, nil } +func withCoverageQueryTimeout(ctx context.Context, guildID string) (context.Context, context.CancelFunc) { + timeout := coverageQueryTimeout + if guildID == "" { + timeout = globalCoverageQueryTimeout + } + return context.WithTimeout(ctx, timeout) +} + func (s *Store) loadKnownFailureCoverage(ctx context.Context, guildID string, report *CoverageReport) error { rows, err := s.db.QueryContext(ctx, ` select guild_id, channel_id, count(*) diff --git a/internal/store/coverage_test.go b/internal/store/coverage_test.go index 8af199c7..c8ceb792 100644 --- a/internal/store/coverage_test.go +++ b/internal/store/coverage_test.go @@ -2,14 +2,64 @@ package store import ( "context" + "database/sql" + "database/sql/driver" "errors" + "fmt" "path/filepath" + "strings" + "sync/atomic" "testing" "time" + "github.com/openclaw/discrawl/internal/store/storedb" "github.com/stretchr/testify/require" + moderncsqlite "modernc.org/sqlite" ) +var coverageObserverDriverID atomic.Uint64 + +type coverageQueryObservation struct { + contextErrAtStart error + contextErrAtReturn error + queryErr error +} + +type coverageObserverDriver struct { + driver.Driver + observations chan<- coverageQueryObservation +} + +func (d *coverageObserverDriver) Open(name string) (driver.Conn, error) { + conn, err := d.Driver.Open(name) + if err != nil { + return nil, err + } + return &coverageObserverConn{Conn: conn, observations: d.observations}, nil +} + +type coverageObserverConn struct { + driver.Conn + observations chan<- coverageQueryObservation +} + +func (c *coverageObserverConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + queryer, ok := c.Conn.(driver.QueryerContext) + if !ok { + return nil, driver.ErrSkip + } + if query != globalCoverageChannelQuery { + return queryer.QueryContext(ctx, query, args) + } + + observation := coverageQueryObservation{contextErrAtStart: ctx.Err()} + rows, err := queryer.QueryContext(ctx, query, args) + observation.contextErrAtReturn = ctx.Err() + observation.queryErr = err + c.observations <- observation + return rows, err +} + func TestCoverageReportsGuildChannelAndWiretapState(t *testing.T) { t.Parallel() ctx := context.Background() @@ -21,14 +71,19 @@ func TestCoverageReportsGuildChannelAndWiretapState(t *testing.T) { require.NoError(t, s.UpsertGuild(ctx, GuildRecord{ID: "g2", Name: "Guild Two", RawJSON: `{}`})) require.NoError(t, s.UpsertChannel(ctx, ChannelRecord{ID: "c1", GuildID: "g1", Kind: "text", Name: "general", RawJSON: `{}`})) require.NoError(t, s.UpsertChannel(ctx, ChannelRecord{ID: "c2", GuildID: "g1", Kind: "text", Name: "channel-c2", RawJSON: `{"source":"discord_desktop"}`})) + require.NoError(t, s.UpsertChannel(ctx, ChannelRecord{ID: "c3", GuildID: "g2", Kind: "text", Name: "empty", RawJSON: `{}`})) require.NoError(t, s.UpsertChannel(ctx, ChannelRecord{ID: "v1", GuildID: "g1", Kind: "voice", Name: "Voice", RawJSON: `{}`})) for _, message := range []MessageRecord{ + {ID: "deleted-early", GuildID: "g1", ChannelID: "c1", CreatedAt: "2026-05-01T10:00:00Z", Content: "deleted early", NormalizedContent: "deleted early", RawJSON: `{}`}, {ID: "m1", GuildID: "g1", ChannelID: "c1", CreatedAt: "2026-06-01T10:00:00Z", Content: "one", NormalizedContent: "one", RawJSON: `{}`}, {ID: "m2", GuildID: "g1", ChannelID: "c1", CreatedAt: "2026-06-02T10:00:00Z", Content: "two", NormalizedContent: "two", RawJSON: `{}`}, {ID: "m3", GuildID: "g1", ChannelID: "c2", CreatedAt: "2026-06-03T10:00:00Z", Content: "three", NormalizedContent: "three", RawJSON: `{}`}, + {ID: "deleted-late", GuildID: "g1", ChannelID: "c1", CreatedAt: "2026-07-01T10:00:00Z", Content: "deleted late", NormalizedContent: "deleted late", RawJSON: `{}`}, } { require.NoError(t, s.UpsertMessage(ctx, message)) } + require.NoError(t, s.MarkMessageDeleted(ctx, "g1", "c1", "deleted-early", nil)) + require.NoError(t, s.MarkMessageDeleted(ctx, "g1", "c1", "deleted-late", nil)) require.NoError(t, s.SetSyncState(ctx, "channel:c1:history_complete", "1")) require.NoError(t, s.SetSyncState(ctx, "sync:last_success", "2026-06-04T10:00:00Z")) require.NoError(t, s.SetSyncState(ctx, "wiretap:last_import", "2026-06-04T11:00:00Z")) @@ -44,8 +99,8 @@ func TestCoverageReportsGuildChannelAndWiretapState(t *testing.T) { require.NoError(t, err) require.Equal(t, generatedAt, report.GeneratedAt) require.Equal(t, CoverageTotals{ - GuildCount: 2, MessageCount: 3, ChannelCount: 3, MessageChannelCount: 2, - NamedChannelCount: 2, SyntheticChannelCount: 1, HistoryCompleteChannelCount: 1, + GuildCount: 2, MessageCount: 3, ChannelCount: 4, MessageChannelCount: 3, + NamedChannelCount: 3, SyntheticChannelCount: 1, HistoryCompleteChannelCount: 1, KnownFailureCount: 2, UnscopedKnownFailureCount: 1, }, report.Totals) require.Equal(t, time.Date(2026, 6, 4, 10, 0, 0, 0, time.UTC), report.LastBotSyncAt) @@ -63,15 +118,125 @@ func TestCoverageReportsGuildChannelAndWiretapState(t *testing.T) { require.Equal(t, 1, report.Guilds[0].KnownFailureCount) require.Equal(t, 1, report.Guilds[0].Channels[0].KnownFailureCount) require.True(t, report.Guilds[0].Channels[1].Synthetic) + require.Equal(t, 0, report.Guilds[0].Channels[2].MessageCount) + require.Equal(t, "v1", report.Guilds[0].Channels[2].ID) + + filtered, err := s.Coverage(ctx, "g1", generatedAt) + require.NoError(t, err) + require.Equal(t, 3, filtered.Totals.MessageCount) + require.Equal(t, 3, filtered.Totals.ChannelCount) + require.Equal(t, "g1", filtered.Guilds[0].ID) + require.Equal(t, report.Guilds[0].Channels, filtered.Guilds[0].Channels) - filtered, err := s.Coverage(ctx, "g2", generatedAt) + empty, err := s.Coverage(ctx, "g2", generatedAt) require.NoError(t, err) - require.Equal(t, CoverageTotals{GuildCount: 1}, filtered.Totals) - require.Equal(t, "g2", filtered.Guilds[0].ID) + require.Equal(t, CoverageTotals{ + GuildCount: 1, ChannelCount: 1, MessageChannelCount: 1, NamedChannelCount: 1, + }, empty.Totals) + require.Equal(t, "c3", empty.Guilds[0].Channels[0].ID) + require.Equal(t, 0, empty.Guilds[0].Channels[0].MessageCount) _, err = s.Coverage(ctx, "missing", generatedAt) require.ErrorContains(t, err, `guild "missing" not found`) } +func TestCoverageQueryTimeoutsAndCancellation(t *testing.T) { + t.Parallel() + + started := time.Now() + globalCtx, globalCancel := withCoverageQueryTimeout(context.Background(), "") + globalDeadline, ok := globalCtx.Deadline() + require.True(t, ok) + require.WithinDuration(t, started.Add(4*time.Minute), globalDeadline, time.Second) + globalCancel() + + started = time.Now() + filteredCtx, filteredCancel := withCoverageQueryTimeout(context.Background(), "g1") + filteredDeadline, ok := filteredCtx.Deadline() + require.True(t, ok) + require.WithinDuration(t, started.Add(2*time.Minute), filteredDeadline, time.Second) + filteredCancel() + + parentDeadline := time.Now().Add(time.Minute) + parentCtx, parentCancel := context.WithDeadline(context.Background(), parentDeadline) + defer parentCancel() + childCtx, childCancel := withCoverageQueryTimeout(parentCtx, "") + defer childCancel() + childDeadline, ok := childCtx.Deadline() + require.True(t, ok) + require.Equal(t, parentDeadline, childDeadline) + + ctx := context.Background() + s, err := Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + for _, guildID := range []string{"", "g1"} { + canceledCtx, cancel := context.WithCancel(ctx) + cancel() + _, err := s.Coverage(canceledCtx, guildID, time.Now()) + require.ErrorIs(t, err, context.Canceled) + } +} + +func TestCoverageGlobalQueryPlanMaterializesSequentialMessageScan(t *testing.T) { + ctx := context.Background() + s, err := Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { require.NoError(t, s.Close()) }() + + rows, err := s.DB().QueryContext(ctx, "explain query plan "+globalCoverageChannelQuery) + require.NoError(t, err) + defer func() { require.NoError(t, rows.Close()) }() + + var details []string + for rows.Next() { + var selectID, parentID, unused int + var detail string + require.NoError(t, rows.Scan(&selectID, &parentID, &unused, &detail)) + details = append(details, detail) + } + require.NoError(t, rows.Err()) + + plan := strings.ToLower(strings.Join(details, "\n")) + require.Contains(t, strings.ToLower(globalCoverageChannelQuery), "with message_coverage as materialized") + require.Contains(t, strings.ToLower(globalCoverageChannelQuery), "from messages not indexed") + require.Contains(t, plan, "materialize message_coverage") + require.Contains(t, plan, "scan messages") + require.NotContains(t, plan, "idx_messages_channel_id") + require.NotContains(t, plan, "idx_messages_channel_created_id") +} + +func TestCoverageGlobalQueryHonorsParentDeadlineInFlight(t *testing.T) { + const ( + messageCount = 300_000 + parentTimeout = 50 * time.Millisecond + ) + + s, observations := newCoverageDeadlineTestStore(t, messageCount) + parentCtx, cancel := context.WithTimeout(context.Background(), parentTimeout) + defer cancel() + parentDeadline, ok := parentCtx.Deadline() + require.True(t, ok) + + started := time.Now() + _, err := s.Coverage(parentCtx, "", time.Now()) + elapsed := time.Since(started) + require.ErrorIs(t, err, context.DeadlineExceeded) + require.ErrorContains(t, err, "query channel coverage") + require.NotContains(t, err.Error(), "list coverage guilds") + require.False(t, time.Now().Before(parentDeadline)) + + select { + case observation := <-observations: + require.NoError(t, observation.contextErrAtStart) + require.ErrorIs(t, observation.contextErrAtReturn, context.DeadlineExceeded) + require.ErrorIs(t, observation.queryErr, context.DeadlineExceeded) + default: + t.Fatalf("global channel query was not observed; Coverage returned %v", err) + } + t.Logf("canceled global channel query over %d generated messages after %s", messageCount, elapsed) +} + func TestCoverageDeltaSince(t *testing.T) { previous := CoverageReport{Totals: CoverageTotals{MessageCount: 4, ChannelCount: 3, NamedChannelCount: 2, SyntheticChannelCount: 1}} current := CoverageReport{Totals: CoverageTotals{MessageCount: 7, ChannelCount: 4, NamedChannelCount: 4, SyntheticChannelCount: 0}} @@ -84,3 +249,83 @@ func TestSyntheticChannelClassificationUsesGeneratedPlaceholder(t *testing.T) { require.False(t, isSyntheticChannel("123456123456", "general")) require.False(t, isSyntheticChannel("123456123456", "channel-other")) } + +func newCoverageDeadlineTestStore(t *testing.T, messageCount int) (*Store, <-chan coverageQueryObservation) { + t.Helper() + ctx := context.Background() + dbPath := filepath.Join(t.TempDir(), "coverage-deadline.db") + seedDB, err := sql.Open("sqlite", dbPath) + require.NoError(t, err) + _, err = seedDB.ExecContext(ctx, ` + create table guilds ( + id text primary key, + name text not null + ); + create table channels ( + id text primary key, + guild_id text not null, + name text not null, + kind text not null + ); + create table messages ( + id text primary key, + channel_id text not null, + created_at text not null, + deleted_at text + ); + create table sync_state ( + scope text primary key, + cursor text, + updated_at text + ); + create table failure_ledger ( + guild_id text not null default '', + channel_id text not null default '', + resolved_at text + ); + insert into guilds (id, name) values ('g1', 'Guild One'); + insert into channels (id, guild_id, name, kind) values ('c1', 'g1', 'general', 'text'); + `) + require.NoError(t, err) + _, err = seedDB.ExecContext(ctx, ` + with + digit(n) as ( + values (0), (1), (2), (3), (4), (5), (6), (7), (8), (9) + ), + generated(n) as ( + select 1 + a.n + 10*b.n + 100*c.n + 1000*d.n + 10000*e.n + 100000*f.n + from digit a + cross join digit b + cross join digit c + cross join digit d + cross join digit e + cross join digit f + limit ? + ) + insert into messages (id, channel_id, created_at) + select + printf('message-%06d', n), + printf('generated-channel-%06d', n), + '2026-01-01T00:00:00Z' + from generated + `, messageCount) + require.NoError(t, err) + require.NoError(t, seedDB.Close()) + + observations := make(chan coverageQueryObservation, 1) + driverName := fmt.Sprintf("coverage-observer-%d", coverageObserverDriverID.Add(1)) + sql.Register(driverName, &coverageObserverDriver{ + Driver: &moderncsqlite.Driver{}, + observations: observations, + }) + db, err := sql.Open(driverName, dbPath) + require.NoError(t, err) + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + require.NoError(t, db.PingContext(ctx)) + t.Cleanup(func() { + require.NoError(t, db.Close()) + }) + + return &Store{db: db, q: storedb.New(db), path: dbPath}, observations +} diff --git a/internal/store/store_test.go b/internal/store/store_test.go index deba64dc..f7e140d9 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -227,6 +227,7 @@ func TestClosedStoreOperationsReturnErrors(t *testing.T) { require.Error(t, s.UpsertGuild(ctx, GuildRecord{ID: "g1"})) require.Error(t, s.UpsertChannel(ctx, ChannelRecord{ID: "c1"})) require.Error(t, s.ReplaceMembers(ctx, "g1", nil)) + require.Error(t, s.MergeMembers(ctx, nil)) require.Error(t, s.UpsertMember(ctx, MemberRecord{GuildID: "g1", UserID: "u1"})) require.Error(t, s.DeleteMember(ctx, "g1", "u1")) require.Error(t, s.UpsertMessageWithOptions(ctx, MessageRecord{ID: "m1"}, WriteOptions{})) @@ -1649,6 +1650,38 @@ func TestUpsertAndDeleteMember(t *testing.T) { require.Equal(t, "Other bio", rows[0].Bio) } +func TestMergeMembersPreservesExistingRows(t *testing.T) { + t.Parallel() + + ctx := context.Background() + dbPath := filepath.Join(t.TempDir(), "discrawl.db") + s, err := Open(ctx, dbPath) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + require.NoError(t, s.ReplaceMembers(ctx, "g1", []MemberRecord{{ + GuildID: "g1", + UserID: "stale", + Username: "stale-user", + RoleIDsJSON: `[]`, + RawJSON: `{"source":"preserved"}`, + }})) + require.NoError(t, s.MergeMembers(ctx, []MemberRecord{{ + GuildID: "g1", + UserID: "live", + Username: "live-user", + RoleIDsJSON: `[]`, + RawJSON: `{"source":"live"}`, + }})) + + rows, err := s.Members(ctx, "g1", "", 10) + require.NoError(t, err) + require.Len(t, rows, 2) + ids := []string{rows[0].UserID, rows[1].UserID} + require.Contains(t, ids, "stale") + require.Contains(t, ids, "live") +} + func TestOpenTightensDBFilePerms(t *testing.T) { t.Parallel() diff --git a/internal/store/write.go b/internal/store/write.go index 1021f58e..d6360c15 100644 --- a/internal/store/write.go +++ b/internal/store/write.go @@ -156,6 +156,25 @@ func (s *Store) ReplaceMembers(ctx context.Context, guildID string, members []Me return tx.Commit() } +func (s *Store) MergeMembers(ctx context.Context, members []MemberRecord) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer rollback(tx) + qtx := s.q.WithTx(tx) + now := time.Now().UTC().Format(timeLayout) + for _, member := range members { + if err := qtx.UpsertMember(ctx, upsertMemberParams(member, now)); err != nil { + return err + } + if err := upsertMemberFTSTx(ctx, tx, member); err != nil { + return err + } + } + return tx.Commit() +} + func (s *Store) UpsertMember(ctx context.Context, member MemberRecord) error { tx, err := s.db.BeginTx(ctx, nil) if err != nil { @@ -443,6 +462,23 @@ func (s *Store) markMessageDeleted( return tx.Commit() } +func (s *Store) MessageScope(ctx context.Context, messageID string) (string, string, bool, error) { + var guildID string + var channelID string + err := s.db.QueryRowContext( + ctx, + `select guild_id, channel_id from messages where id = ?`, + strings.TrimSpace(messageID), + ).Scan(&guildID, &channelID) + if errors.Is(err, sql.ErrNoRows) { + return "", "", false, nil + } + if err != nil { + return "", "", false, err + } + return guildID, channelID, true, nil +} + func (s *Store) AppendMessageEvent(ctx context.Context, guildID, channelID, messageID, eventType string, payload any) error { body, err := json.Marshal(payload) if err != nil { diff --git a/internal/syncer/channel_catalog.go b/internal/syncer/channel_catalog.go index 64ace597..ee114b1d 100644 --- a/internal/syncer/channel_catalog.go +++ b/internal/syncer/channel_catalog.go @@ -18,7 +18,13 @@ const ( channelCatalogIncremental ) -func (s *Syncer) channelList(ctx context.Context, guildID string, requested []string, mode channelCatalogMode) ([]*discordgo.Channel, bool, error) { +func (s *Syncer) channelList( + ctx context.Context, + guildID string, + requested []string, + mode channelCatalogMode, + exclusions channelExclusions, +) ([]*discordgo.Channel, bool, error) { if len(requested) == 0 { channels, err := s.liveChannelList(ctx, guildID, mode) if err != nil { @@ -34,6 +40,10 @@ func (s *Syncer) channelList(ctx context.Context, guildID string, requested []st if err != nil { return nil, false, err } + requestedSet = filterExcludedStoredTargets(rows, requestedSet, exclusions) + if len(requestedSet) == 0 { + return nil, true, nil + } storedByID = selectStoredChannels(rows, requestedSet) if canUseStoredTargets(storedByID, requestedSet) { selected := selectRequestedChannels(nil, storedByID, requestedSet) @@ -65,9 +75,54 @@ func (s *Syncer) channelList(ctx context.Context, guildID string, requested []st } selected = selectRequestedChannels(allChannels, storedByID, requestedSet) } + selected = filterExcludedDiscordTargets(selected, allChannels, exclusions) return selected, true, nil } +func filterExcludedStoredTargets( + rows []store.ChannelRow, + requested map[string]struct{}, + exclusions channelExclusions, +) map[string]struct{} { + if len(requested) == 0 || (len(exclusions.ids) == 0 && len(exclusions.kinds) == 0) { + return requested + } + channelByID := make(map[string]store.ChannelRow, len(rows)) + for _, row := range rows { + channelByID[row.ID] = row + } + out := make(map[string]struct{}, len(requested)) + for channelID := range requested { + channel, ok := channelByID[channelID] + if ok && exclusions.excludesStoredChannel(channel, channelByID) { + continue + } + if !ok && exclusions.excludesID(channelID) { + continue + } + out[channelID] = struct{}{} + } + return out +} + +func filterExcludedDiscordTargets( + channels []*discordgo.Channel, + channelByID map[string]*discordgo.Channel, + exclusions channelExclusions, +) []*discordgo.Channel { + if len(channels) == 0 || (len(exclusions.ids) == 0 && len(exclusions.kinds) == 0) { + return channels + } + out := make([]*discordgo.Channel, 0, len(channels)) + for _, channel := range channels { + if exclusions.excludesDiscordChannel(channel, channelByID) { + continue + } + out = append(out, channel) + } + return out +} + func (s *Syncer) liveChannelList(ctx context.Context, guildID string, mode channelCatalogMode) ([]*discordgo.Channel, error) { channels, err := s.client.GuildChannels(ctx, guildID) if err != nil { @@ -244,7 +299,7 @@ func channelFromRow(row store.ChannelRow) *discordgo.Channel { return &discordgo.Channel{ ID: row.ID, GuildID: row.GuildID, - ParentID: row.ParentID, + ParentID: storedChannelParentID(row), Name: row.Name, Topic: row.Topic, Position: row.Position, diff --git a/internal/syncer/channel_catalog_test.go b/internal/syncer/channel_catalog_test.go index 1272251d..7111ea32 100644 --- a/internal/syncer/channel_catalog_test.go +++ b/internal/syncer/channel_catalog_test.go @@ -396,6 +396,61 @@ func TestSyncChannelSubsetMergesStoredAndLiveTargets(t *testing.T) { require.Equal(t, "20", latest) } +func TestSyncTargetedStoredThreadInheritsExcludedParentKind(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + require.NoError(t, s.UpsertGuild(ctx, store.GuildRecord{ID: "g1", Name: "Guild", RawJSON: `{}`})) + require.NoError(t, s.UpsertChannel(ctx, store.ChannelRecord{ + ID: "news", + GuildID: "g1", + Kind: "announcement", + Name: "feed", + RawJSON: `{}`, + })) + require.NoError(t, s.UpsertChannel(ctx, store.ChannelRecord{ + ID: "thread", + GuildID: "g1", + Kind: "thread_public", + Name: "feed-thread", + ThreadParentID: "news", + RawJSON: `{}`, + })) + + client := &fakeClient{ + guilds: []*discordgo.UserGuild{{ID: "g1", Name: "Guild"}}, + guildByID: map[string]*discordgo.Guild{ + "g1": {ID: "g1", Name: "Guild"}, + }, + messages: map[string][]*discordgo.Message{ + "thread": {{ + ID: "10", + GuildID: "g1", + ChannelID: "thread", + Content: "must remain excluded", + Timestamp: time.Now().UTC(), + Author: &discordgo.User{ID: "u1", Username: "user"}, + }}, + }, + } + + svc := New(client, s, nil) + stats, err := svc.Sync(ctx, SyncOptions{ + Full: true, + GuildIDs: []string{"g1"}, + ChannelIDs: []string{"thread"}, + ExcludeChannelKinds: []string{"announcement"}, + }) + require.NoError(t, err) + require.Zero(t, stats.Messages) + require.Zero(t, client.guildChanCalls) + require.Zero(t, client.messageCalls["thread"]) +} + func TestSyncSkipsUnchangedThreadsWhenHistoryComplete(t *testing.T) { t.Parallel() diff --git a/internal/syncer/channel_exclusions.go b/internal/syncer/channel_exclusions.go new file mode 100644 index 00000000..bb0d3b99 --- /dev/null +++ b/internal/syncer/channel_exclusions.go @@ -0,0 +1,180 @@ +package syncer + +import ( + "context" + "strings" + + "github.com/bwmarrin/discordgo" + + "github.com/openclaw/discrawl/internal/store" +) + +type channelExclusions struct { + ids map[string]struct{} + kinds map[string]struct{} +} + +func newChannelExclusions(ids, kinds []string) channelExclusions { + return channelExclusions{ + ids: normalizedStringSet(ids, false), + kinds: normalizedStringSet(kinds, true), + } +} + +func normalizedStringSet(values []string, lower bool) map[string]struct{} { + out := make(map[string]struct{}, len(values)) + for _, value := range values { + value = strings.TrimSpace(value) + if lower { + value = strings.ToLower(value) + } + if value != "" { + out[value] = struct{}{} + } + } + return out +} + +func (e channelExclusions) merged(other channelExclusions) channelExclusions { + merged := channelExclusions{ + ids: make(map[string]struct{}, len(e.ids)+len(other.ids)), + kinds: make(map[string]struct{}, len(e.kinds)+len(other.kinds)), + } + for id := range e.ids { + merged.ids[id] = struct{}{} + } + for id := range other.ids { + merged.ids[id] = struct{}{} + } + for kind := range e.kinds { + merged.kinds[kind] = struct{}{} + } + for kind := range other.kinds { + merged.kinds[kind] = struct{}{} + } + return merged +} + +func (e channelExclusions) excludesID(channelID string) bool { + _, ok := e.ids[channelID] + return ok +} + +func (e channelExclusions) excludesKind(kind string) bool { + kind = strings.ToLower(strings.TrimSpace(kind)) + if _, ok := e.kinds[kind]; ok { + return true + } + switch kind { + case "news", "thread_news", "thread_announcement": + _, ok := e.kinds["announcement"] + return ok + default: + return false + } +} + +func (e channelExclusions) excludesDiscordChannel(channel *discordgo.Channel, channelByID map[string]*discordgo.Channel) bool { + return e.excludesDiscordChannelID(channel, channelByID, nil) +} + +func (e channelExclusions) excludesDiscordChannelID( + channel *discordgo.Channel, + channelByID map[string]*discordgo.Channel, + visiting map[string]struct{}, +) bool { + if channel == nil { + return false + } + if e.excludesID(channel.ID) || e.excludesKind(channelKind(channel)) { + return true + } + if channel.ParentID == "" { + return false + } + if e.excludesID(channel.ParentID) { + return true + } + if visiting == nil { + visiting = map[string]struct{}{} + } + if _, ok := visiting[channel.ID]; ok { + return false + } + visiting[channel.ID] = struct{}{} + defer delete(visiting, channel.ID) + parent := channelByID[channel.ParentID] + return parent != nil && e.excludesDiscordChannelID(parent, channelByID, visiting) +} + +func (e channelExclusions) excludesStoredChannel(channel store.ChannelRow, channelByID map[string]store.ChannelRow) bool { + return e.excludesStoredChannelID(channel, channelByID, nil) +} + +func (e channelExclusions) excludesStoredChannelID( + channel store.ChannelRow, + channelByID map[string]store.ChannelRow, + visiting map[string]struct{}, +) bool { + if e.excludesID(channel.ID) || e.excludesKind(channel.Kind) { + return true + } + parentID := storedChannelParentID(channel) + if parentID == "" { + return false + } + if e.excludesID(parentID) { + return true + } + if visiting == nil { + visiting = map[string]struct{}{} + } + if _, ok := visiting[channel.ID]; ok { + return false + } + visiting[channel.ID] = struct{}{} + defer delete(visiting, channel.ID) + parent, ok := channelByID[parentID] + return ok && e.excludesStoredChannelID(parent, channelByID, visiting) +} + +func storedChannelParentID(channel store.ChannelRow) string { + if channel.ThreadParentID != "" { + return channel.ThreadParentID + } + return channel.ParentID +} + +func (s *Syncer) effectiveChannelExclusions(opts SyncOptions) channelExclusions { + if s == nil { + return newChannelExclusions(opts.ExcludeChannelIDs, opts.ExcludeChannelKinds) + } + return s.channelExclusions.merged(newChannelExclusions(opts.ExcludeChannelIDs, opts.ExcludeChannelKinds)) +} + +func (s *Syncer) filterExcludedStoredChannelIDs(ctx context.Context, guildID string, channelIDs []string, opts SyncOptions) ([]string, error) { + exclusions := s.effectiveChannelExclusions(opts) + if len(exclusions.ids) == 0 && len(exclusions.kinds) == 0 { + return channelIDs, nil + } + channels, err := s.store.Channels(ctx, guildID) + if err != nil { + return nil, err + } + channelByID := make(map[string]store.ChannelRow, len(channels)) + for _, channel := range channels { + channelByID[channel.ID] = channel + } + out := make([]string, 0, len(channelIDs)) + for _, channelID := range channelIDs { + channel, ok := channelByID[channelID] + if ok && exclusions.excludesStoredChannel(channel, channelByID) { + continue + } + if !ok && exclusions.excludesID(channelID) { + continue + } + out = append(out, channelID) + } + return out, nil +} diff --git a/internal/syncer/errors.go b/internal/syncer/errors.go index 5ae26dce..e67e92dd 100644 --- a/internal/syncer/errors.go +++ b/internal/syncer/errors.go @@ -109,6 +109,19 @@ func isUnknownChannel(err error) bool { (strings.Contains(msg, "http 404") && strings.Contains(msg, `"code": 10003`)) } +func isUnknownMessage(err error) bool { + if err == nil { + return false + } + var restErr *discordgo.RESTError + if errors.As(err, &restErr) && restErr != nil && restErr.Message != nil { + return restErr.Message.Code == discordgo.ErrCodeUnknownMessage + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "http 404") && + (strings.Contains(msg, "unknown message") || strings.Contains(msg, `"code": 10008`)) +} + func isMissingAccess(err error) bool { if err == nil { return false diff --git a/internal/syncer/extracted_test.go b/internal/syncer/extracted_test.go index f6a1739e..35c836f4 100644 --- a/internal/syncer/extracted_test.go +++ b/internal/syncer/extracted_test.go @@ -51,11 +51,29 @@ func TestFilterMessageChannelsHonorsRequestedIDs(t *testing.T) { {ID: "t1", Type: discordgo.ChannelTypeGuildPublicThread}, } - filtered := filterMessageChannels(channels, []string{"t1"}) + filtered := filterMessageChannels(channels, []string{"t1"}, channelExclusions{}) require.Len(t, filtered, 1) require.Equal(t, "t1", filtered[0].ID) } +func TestFilterMessageChannelsHonorsCollectionExclusions(t *testing.T) { + t.Parallel() + + channels := []*discordgo.Channel{ + {ID: "general", Type: discordgo.ChannelTypeGuildText}, + {ID: "logs", Type: discordgo.ChannelTypeGuildText}, + {ID: "news", Type: discordgo.ChannelTypeGuildNews}, + {ID: "news-thread", ParentID: "news", Type: discordgo.ChannelTypeGuildPublicThread}, + } + exclusions := newChannelExclusions([]string{"logs"}, []string{"announcement"}) + + filtered := filterMessageChannels(channels, nil, exclusions) + require.Equal(t, []string{"general"}, channelIDs(filtered)) + + filtered = filterMessageChannels(channels, []string{"logs", "news"}, exclusions) + require.Empty(t, filtered) +} + func TestSeedChannelSyncStateUsesStoredBounds(t *testing.T) { t.Parallel() diff --git a/internal/syncer/failures.go b/internal/syncer/failures.go index 98de4992..9c485290 100644 --- a/internal/syncer/failures.go +++ b/internal/syncer/failures.go @@ -31,6 +31,9 @@ func (s *Syncer) recordChannelFailure(ctx context.Context, guildID, channelID st if s == nil || s.store == nil || failure == nil { return nil } + if errors.Is(failure, context.Canceled) && ctx != nil && errors.Is(ctx.Err(), context.Canceled) { + return nil + } ledgerCtx, cancel := failureLedgerContext(ctx) defer cancel() return s.store.RecordFailure(ledgerCtx, store.FailureRef{ @@ -292,6 +295,22 @@ func (t *tailHandler) resolveMessageFailure(ctx context.Context, guildID, channe ) } +func (t *tailHandler) resolveMessageUpdateFailuresByID(ctx context.Context, messageID string) error { + if t == nil || t.store == nil || messageID == "" { + return nil + } + discordclient.UpdateTailFailureStage(ctx, discordclient.TailFailureStageFailureResolution) + ledgerCtx, cancel := failureLedgerContext(ctx) + defer cancel() + return t.store.ResolveFailures(ledgerCtx, store.FailureRef{ + Operation: tailMessageFailureOperation, + Source: "discord", + MessageID: messageID, + RelatedKind: tailMessageFailureRelatedKind, + RelatedID: "update", + }) +} + func (s *Syncer) resolveTailMessageCreateFailuresForMessages(ctx context.Context, messages []*discordgo.Message, fallbackGuildID string) error { if s == nil || s.store == nil || len(messages) == 0 { return nil diff --git a/internal/syncer/failures_test.go b/internal/syncer/failures_test.go index 35b65119..a75e8238 100644 --- a/internal/syncer/failures_test.go +++ b/internal/syncer/failures_test.go @@ -3,6 +3,7 @@ package syncer import ( "context" "errors" + "fmt" "path/filepath" "testing" "time" @@ -367,3 +368,54 @@ func TestRecordTailFailureRejectsInvalidMessageMetadata(t *testing.T) { MessageID: "m1", }), "missing store") } + +func TestChannelFailureLedgerIgnoresOnlyParentCancellation(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = st.Close() }() + + svc := &Syncer{store: st} + canceledCtx, cancel := context.WithCancel(ctx) + cancel() + + require.NoError(t, svc.recordChannelFailure( + canceledCtx, + "g1", + "shutdown", + fmt.Errorf("request interrupted: %w", context.Canceled), + )) + require.NoError(t, svc.recordChannelFailure( + canceledCtx, + "g1", + "real-error", + errors.New("discord request failed"), + )) + require.NoError(t, svc.recordChannelFailure( + ctx, + "g1", + "unexpected-cancel", + context.Canceled, + )) + require.NoError(t, svc.recordChannelFailure( + ctx, + "g1", + "timeout", + context.DeadlineExceeded, + )) + + report, err := st.ListFailures(ctx, store.FailureListOptions{}, time.Now()) + require.NoError(t, err) + require.Len(t, report.Failures, 3) + + channelIDs := make(map[string]string, len(report.Failures)) + for _, failure := range report.Failures { + channelIDs[failure.ChannelID] = failure.ErrorClass + } + require.NotContains(t, channelIDs, "shutdown") + require.Equal(t, "errors.errorString", channelIDs["real-error"]) + require.Equal(t, "context_canceled", channelIDs["unexpected-cancel"]) + require.Equal(t, "deadline_exceeded", channelIDs["timeout"]) +} diff --git a/internal/syncer/message_sync.go b/internal/syncer/message_sync.go index 2571b59f..eb99a7d1 100644 --- a/internal/syncer/message_sync.go +++ b/internal/syncer/message_sync.go @@ -18,7 +18,7 @@ func (s *Syncer) syncMessageChannels( channels []*discordgo.Channel, opts SyncOptions, ) (int, error) { - messageChannels := filterMessageChannels(channels, opts.ChannelIDs) + messageChannels := filterMessageChannels(channels, opts.ChannelIDs, s.effectiveChannelExclusions(opts)) if len(messageChannels) == 0 { return 0, nil } @@ -38,7 +38,7 @@ func (s *Syncer) syncMessageChannels( return total, err } -func filterMessageChannels(channels []*discordgo.Channel, requested []string) []*discordgo.Channel { +func filterMessageChannels(channels []*discordgo.Channel, requested []string, exclusions channelExclusions) []*discordgo.Channel { requestedSet := makeGuildSet(requested) channelByID := make(map[string]*discordgo.Channel, len(channels)) for _, channel := range channels { @@ -51,6 +51,9 @@ func filterMessageChannels(channels []*discordgo.Channel, requested []string) [] if !isMessageChannel(channel) { continue } + if exclusions.excludesDiscordChannel(channel, channelByID) { + continue + } if len(requestedSet) > 0 && !requestedMessageTarget(channel, channelByID, requestedSet) { continue } @@ -80,7 +83,7 @@ func (s *Syncer) syncMessageChannelsSerial(ctx context.Context, guildID string, total := 0 for _, channel := range channels { progress.start(channel) - channelCtx, cancel := s.messageChannelContext(ctx) + channelCtx, cancel := s.messageChannelContext(ctx, opts) count, err := s.syncChannelMessages(channelCtx, guildID, channel, opts.Full, opts.Embeddings, opts.Since, opts.LatestOnly, progress) cancel() total += count @@ -135,7 +138,7 @@ func (s *Syncer) syncMessageChannelsConcurrent( return } progress.start(channel) - channelCtx, cancelChannel := s.messageChannelContext(ctx) + channelCtx, cancelChannel := s.messageChannelContext(ctx, opts) count, err := s.syncChannelMessages(channelCtx, guildID, channel, opts.Full, opts.Embeddings, opts.Since, opts.LatestOnly, progress) cancelChannel() succeeded := err == nil @@ -209,14 +212,22 @@ func (s *Syncer) clearUnavailableChannel(ctx context.Context, channelID string) return s.store.DeleteSyncState(ctx, channelMessageUnavailableScope(channelID)) } -func (s *Syncer) messageChannelContext(ctx context.Context) (context.Context, context.CancelFunc) { - if s == nil || s.messageChannelTimeout <= 0 { +func (s *Syncer) messageChannelContext(ctx context.Context, opts SyncOptions) (context.Context, context.CancelFunc) { + timeout := s.effectiveMessageChannelTimeout(opts) + if timeout <= 0 { return context.WithCancel(ctx) } - if _, ok := ctx.Deadline(); ok { - return context.WithCancel(ctx) + return context.WithTimeout(ctx, timeout) +} + +func (s *Syncer) effectiveMessageChannelTimeout(opts SyncOptions) time.Duration { + if opts.MessageChannelTimeout != 0 { + return opts.MessageChannelTimeout + } + if s == nil { + return 0 } - return context.WithTimeout(ctx, s.messageChannelTimeout) + return s.messageChannelTimeout } func (s *Syncer) syncChannelMessages(ctx context.Context, guildID string, channel *discordgo.Channel, full bool, embeddings bool, since time.Time, latestOnly bool, progress *messageSyncProgress) (int, error) { @@ -605,7 +616,7 @@ func newMessageSyncProgress(s *Syncer, guildID string, totalChannels int, opts S "channels", totalChannels, "full", opts.Full, "concurrency", max(1, opts.Concurrency), - "channel_timeout", timeoutLabel(s.messageChannelTimeout), + "channel_timeout", timeoutLabel(s.effectiveMessageChannelTimeout(opts)), ) go progress.runWaitHeartbeat() return progress diff --git a/internal/syncer/message_sync_helpers_test.go b/internal/syncer/message_sync_helpers_test.go index a02f6f5b..ad8b6a79 100644 --- a/internal/syncer/message_sync_helpers_test.go +++ b/internal/syncer/message_sync_helpers_test.go @@ -22,32 +22,43 @@ func TestMessageChannelSelectionAndTimeoutHelpers(t *testing.T) { text := &discordgo.Channel{ID: "text", GuildID: "g1", Name: "text", Type: discordgo.ChannelTypeGuildText} voice := &discordgo.Channel{ID: "voice", GuildID: "g1", Name: "voice", Type: discordgo.ChannelTypeGuildVoice} - rows := filterMessageChannels([]*discordgo.Channel{nil, parent, thread, text, voice}, []string{"forum"}) + rows := filterMessageChannels([]*discordgo.Channel{nil, parent, thread, text, voice}, []string{"forum"}, channelExclusions{}) require.Equal(t, []string{"thread"}, channelIDs(rows)) require.False(t, requestedMessageTarget(nil, nil, map[string]struct{}{})) require.True(t, requestedMessageTarget(text, map[string]*discordgo.Channel{"text": text}, map[string]struct{}{"text": {}})) require.False(t, requestedMessageTarget(thread, map[string]*discordgo.Channel{}, map[string]struct{}{"forum": {}})) - ctx, cancel := (*Syncer)(nil).messageChannelContext(context.Background()) + ctx, cancel := (*Syncer)(nil).messageChannelContext(context.Background(), SyncOptions{}) require.NoError(t, ctx.Err()) cancel() require.ErrorIs(t, ctx.Err(), context.Canceled) svc := New(&fakeClient{}, nil, nil) svc.messageChannelTimeout = time.Second - ctx, cancel = svc.messageChannelContext(context.Background()) + ctx, cancel = svc.messageChannelContext(context.Background(), SyncOptions{}) defer cancel() _, ok := ctx.Deadline() require.True(t, ok) - parentCtx, parentCancel := context.WithDeadline(context.Background(), time.Now().Add(time.Hour)) + startedAt := time.Now() + parentCtx, parentCancel := context.WithDeadline(context.Background(), startedAt.Add(time.Hour)) defer parentCancel() - ctx, cancel = svc.messageChannelContext(parentCtx) + ctx, cancel = svc.messageChannelContext(parentCtx, SyncOptions{MessageChannelTimeout: 2 * time.Second}) defer cancel() deadline, ok := ctx.Deadline() require.True(t, ok) parentDeadline, _ := parentCtx.Deadline() - require.Equal(t, parentDeadline, deadline) + require.WithinDuration(t, startedAt.Add(2*time.Second), deadline, 100*time.Millisecond) + require.True(t, deadline.Before(parentDeadline)) + + shortParentCtx, shortParentCancel := context.WithDeadline(context.Background(), time.Now().Add(2*time.Second)) + defer shortParentCancel() + ctx, cancel = svc.messageChannelContext(shortParentCtx, SyncOptions{MessageChannelTimeout: time.Hour}) + defer cancel() + deadline, ok = ctx.Deadline() + require.True(t, ok) + shortParentDeadline, _ := shortParentCtx.Deadline() + require.Equal(t, shortParentDeadline, deadline) } func TestChannelSyncStateHelpers(t *testing.T) { diff --git a/internal/syncer/message_sync_timeout_test.go b/internal/syncer/message_sync_timeout_test.go index 523956f4..1d700581 100644 --- a/internal/syncer/message_sync_timeout_test.go +++ b/internal/syncer/message_sync_timeout_test.go @@ -42,8 +42,7 @@ func TestSyncDefersSlowChannelAfterChannelTimeout(t *testing.T) { } svc := New(client, s, nil) - svc.messageChannelTimeout = 20 * time.Millisecond - stats, err := svc.Sync(ctx, SyncOptions{Full: true, Concurrency: 1}) + stats, err := svc.Sync(ctx, SyncOptions{Full: true, Concurrency: 1, MessageChannelTimeout: 20 * time.Millisecond}) require.NoError(t, err) require.Equal(t, 0, stats.Messages) diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go index 2564ae23..c6f69385 100644 --- a/internal/syncer/syncer.go +++ b/internal/syncer/syncer.go @@ -19,6 +19,7 @@ type Client interface { Guilds(context.Context) ([]*discordgo.UserGuild, error) Guild(context.Context, string) (*discordgo.Guild, error) GuildChannels(context.Context, string) ([]*discordgo.Channel, error) + Channel(context.Context, string) (*discordgo.Channel, error) ThreadsActive(context.Context, string) ([]*discordgo.Channel, error) GuildThreadsActive(context.Context, string) ([]*discordgo.Channel, error) ThreadsArchived(context.Context, string, bool) ([]*discordgo.Channel, error) @@ -43,28 +44,39 @@ type Syncer struct { messageSyncLogEvery time.Duration messageSyncWaitEvery time.Duration tailReady func(context.Context) error + tailShutdownTimeout time.Duration tailRepair func(context.Context, SyncOptions) (SyncStats, error) tailRepairJoinTimeout time.Duration - tailRepairMu sync.Mutex + tailRepairRunMu sync.Mutex + tailRepairOffsetMu sync.RWMutex + tailRepairOffset time.Duration + channelExclusions channelExclusions } type SyncOptions struct { - Full bool - GuildIDs []string - ChannelIDs []string - Concurrency int - Since time.Time - Embeddings bool - SkipMembers bool - RequireMembers bool - LatestOnly bool - RepairReason string + Full bool + GuildIDs []string + ChannelIDs []string + Concurrency int + Since time.Time + Embeddings bool + SkipMembers bool + RequireMembers bool + LatestOnly bool + MessageChannelTimeout time.Duration + ExcludeChannelIDs []string + ExcludeChannelKinds []string + RepairReason string } func (s *Syncer) SetTailReadyCallback(fn func(context.Context) error) { s.tailReady = fn } +func (s *Syncer) SetChannelExclusions(channelIDs, channelKinds []string) { + s.channelExclusions = newChannelExclusions(channelIDs, channelKinds) +} + type SyncStats struct { Guilds int `json:"guilds"` Channels int `json:"channels"` @@ -80,6 +92,7 @@ const ( defaultMessageChannelTimeout = 5 * time.Minute defaultMessageSyncLogEvery = 15 * time.Second defaultMessageSyncWaitEvery = 30 * time.Second + defaultTailShutdownTimeout = 90 * time.Second ) func New(client Client, store *store.Store, logger *slog.Logger) *Syncer { @@ -96,6 +109,7 @@ func New(client Client, store *store.Store, logger *slog.Logger) *Syncer { messageChannelTimeout: defaultMessageChannelTimeout, messageSyncLogEvery: defaultMessageSyncLogEvery, messageSyncWaitEvery: defaultMessageSyncWaitEvery, + tailShutdownTimeout: defaultTailShutdownTimeout, } } @@ -159,7 +173,13 @@ func (s *Syncer) syncGuild(ctx context.Context, guildID string, opts SyncOptions catalogMode = channelCatalogIncremental } } - channelList, targeted, err := s.channelList(ctx, guildID, opts.ChannelIDs, catalogMode) + channelList, targeted, err := s.channelList( + ctx, + guildID, + opts.ChannelIDs, + catalogMode, + s.effectiveChannelExclusions(opts), + ) if err != nil { return stats, err } @@ -240,6 +260,10 @@ func (s *Syncer) syncGuildIncompleteBatches(ctx context.Context, guildID string, if err != nil { return SyncStats{}, false, err } + incomplete, err = s.filterExcludedStoredChannelIDs(ctx, guildID, incomplete, opts) + if err != nil { + return SyncStats{}, false, err + } if len(incomplete) == 0 { return SyncStats{}, false, nil } @@ -305,9 +329,9 @@ func (s *Syncer) refreshGuildMembers(ctx context.Context, guildID string, force for _, member := range members { converted = append(converted, toMemberRecord(guildID, member)) } - if err := s.store.ReplaceMembers(ctx, guildID, converted); err != nil { - s.logger.Warn("member replace failed", "guild_id", guildID, "err", err) - return 0, fmt.Errorf("replace guild members: %w", err) + if err := s.store.MergeMembers(ctx, converted); err != nil { + s.logger.Warn("member merge failed", "guild_id", guildID, "err", err) + return 0, fmt.Errorf("merge guild members: %w", err) } if s.store != nil { if err := s.store.SetSyncState(ctx, guildMemberSyncSuccessScope(guildID), time.Now().UTC().Format(time.RFC3339Nano)); err != nil { diff --git a/internal/syncer/syncer_tail_test.go b/internal/syncer/syncer_tail_test.go index 44c4e55e..08f97bdd 100644 --- a/internal/syncer/syncer_tail_test.go +++ b/internal/syncer/syncer_tail_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "maps" "net/http" "net/http/httptest" "path/filepath" @@ -90,6 +91,12 @@ func TestTailHandlerWritesEvents(t *testing.T) { Timestamp: time.Now().UTC(), Author: &discordgo.User{ID: "u1", Username: "peter"}, } + require.NoError(t, handler.OnChannelUpsert(ctx, &discordgo.Channel{ + ID: "c1", + GuildID: "g1", + Name: "general", + Type: discordgo.ChannelTypeGuildText, + })) require.NoError(t, handler.OnMessageCreate(ctx, msg)) require.NoError(t, handler.OnMessageUpdate(ctx, msg)) require.NoError(t, handler.OnMessageDelete(ctx, &discordgo.MessageDelete{Message: &discordgo.Message{ @@ -97,12 +104,6 @@ func TestTailHandlerWritesEvents(t *testing.T) { GuildID: "g1", ChannelID: "c1", }})) - require.NoError(t, handler.OnChannelUpsert(ctx, &discordgo.Channel{ - ID: "c1", - GuildID: "g1", - Name: "general", - Type: discordgo.ChannelTypeGuildText, - })) require.NoError(t, handler.OnMemberUpsert(ctx, "g1", &discordgo.Member{ GuildID: "g1", Nick: "Peter", @@ -134,7 +135,7 @@ func TestTailHandlerWritesEvents(t *testing.T) { require.Equal(t, "10", cursor) } -func TestTailHandlerMessageUpdateFetchesFullMessageBeforeUpsert(t *testing.T) { +func TestTailHandlerReconcilesUnknownMessageUpdateAsDelete(t *testing.T) { t.Parallel() ctx := context.Background() @@ -142,230 +143,1290 @@ func TestTailHandlerMessageUpdateFetchesFullMessageBeforeUpsert(t *testing.T) { require.NoError(t, err) defer func() { _ = s.Close() }() - createdAt := time.Now().UTC() - original := &discordgo.Message{ - ID: "9", + message := &discordgo.Message{ + ID: "123", GuildID: "g1", ChannelID: "c1", - Content: "original <@u2>", - Timestamp: createdAt, + Content: "preserved before deletion", + Timestamp: time.Now().UTC(), Author: &discordgo.User{ID: "u1", Username: "peter"}, - Attachments: []*discordgo.MessageAttachment{{ - ID: "a1", - Filename: "trace.txt", - ContentType: "text/plain", - Size: 42, - }}, - Mentions: []*discordgo.User{{ID: "u2", Username: "shadow", GlobalName: "Shadow"}}, } - updated := *original - updated.Content = "edited <@u2>" - client := &fakeClient{ + channelsByID: map[string]*discordgo.Channel{ + "c1": { + ID: "c1", + GuildID: "g1", + Name: "general", + Type: discordgo.ChannelTypeGuildText, + }, + }, messages: map[string][]*discordgo.Message{ - "c1": {&updated}, + "c1": {message}, + }, + messageByIDErrs: map[string]error{ + "c1/123": errors.New(`HTTP 404 Not Found, {"message": "Unknown Message", "code": 10008}`), }, } - handler := &tailHandler{store: s, client: client} + handler := &tailHandler{ + guilds: makeGuildSet([]string{"g1"}), + store: s, + client: client, + channels: map[string]tailChannel{}, + kindExcludedChannelIDs: map[string]struct{}{}, + } + require.NoError(t, handler.OnChannelUpsert(ctx, client.channelsByID["c1"])) + require.NoError(t, handler.OnMessageCreate(ctx, message)) + require.NoError(t, s.RecordFailure( + ctx, + tailMessageFailureIdentity("g1", "c1", "123", "update"), + errors.New("prior update failure"), + )) + require.NoError(t, s.RecordFailure( + ctx, + tailMessageFailureIdentity("", "c1", "123", "update"), + errors.New("prior guildless update failure"), + )) - require.NoError(t, handler.OnMessageCreate(ctx, original)) require.NoError(t, handler.OnMessageUpdate(ctx, &discordgo.Message{ - ID: "9", + ID: "123", GuildID: "g1", ChannelID: "c1", - Content: "edited <@u2>", })) - messages, err := s.ListMessages(ctx, store.MessageListOptions{GuildIDs: []string{"g1"}, IncludeEmpty: true}) + var deletedAt string + require.NoError(t, s.DB().QueryRowContext( + ctx, + `select coalesce(deleted_at, '') from messages where id = '123'`, + ).Scan(&deletedAt)) + require.NotEmpty(t, deletedAt) + var deleteEvents int + require.NoError(t, s.DB().QueryRowContext( + ctx, + `select count(*) from message_events where message_id = '123' and event_type = 'delete'`, + ).Scan(&deleteEvents)) + require.Equal(t, 1, deleteEvents) + lastEvent, err := s.GetSyncState(ctx, "tail:last_event") require.NoError(t, err) - require.Len(t, messages, 1) - require.Equal(t, "edited <@u2>", messages[0].Content) - require.Equal(t, "u1", messages[0].AuthorID) - require.Equal(t, "peter", messages[0].AuthorName) - require.Equal(t, createdAt.Format(time.RFC3339Nano), messages[0].CreatedAt.Format(time.RFC3339Nano)) - require.True(t, messages[0].HasAttachments) - require.Equal(t, "trace.txt", messages[0].AttachmentNames) + require.Equal(t, "123", lastEvent) + report, err := s.ListFailures(ctx, store.FailureListOptions{}, time.Now().UTC()) + require.NoError(t, err) + require.Zero(t, report.UnresolvedCount) +} - attachments, err := s.ListAttachments(ctx, store.AttachmentListOptions{MessageID: "9"}) +func TestTailHandlerDoesNotDeleteExcludedMessageOnGuildlessUnknownUpdate(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) require.NoError(t, err) - require.Len(t, attachments, 1) - require.Equal(t, "a1", attachments[0].AttachmentID) - require.Equal(t, "trace.txt", attachments[0].Filename) - require.Equal(t, "u1", attachments[0].AuthorID) + defer func() { _ = s.Close() }() - mentions, err := s.ListMentions(ctx, store.MentionListOptions{Target: "u2"}) + channel := &discordgo.Channel{ + ID: "feed", + GuildID: "g1", + Name: "github", + Type: discordgo.ChannelTypeGuildText, + } + message := &discordgo.Message{ + ID: "125", + GuildID: "g1", + ChannelID: "feed", + Content: "preserved feed message", + Timestamp: time.Now().UTC(), + Author: &discordgo.User{ID: "u1", Username: "bot", Bot: true}, + } + client := &fakeClient{ + channelsByID: map[string]*discordgo.Channel{"feed": channel}, + messages: map[string][]*discordgo.Message{"feed": {message}}, + messageByIDErrs: map[string]error{ + "feed/125": errors.New(`HTTP 404 Not Found, {"message": "Unknown Message", "code": 10008}`), + }, + } + initialHandler := &tailHandler{ + guilds: makeGuildSet([]string{"g1"}), + store: s, + client: client, + channels: map[string]tailChannel{}, + kindExcludedChannelIDs: map[string]struct{}{}, + } + require.NoError(t, initialHandler.OnChannelUpsert(ctx, channel)) + require.NoError(t, initialHandler.OnMessageCreate(ctx, message)) + + handler := &tailHandler{ + guilds: makeGuildSet([]string{"g1"}), + store: s, + client: client, + exclusions: newChannelExclusions([]string{"feed"}, nil), + channels: map[string]tailChannel{}, + kindExcludedChannelIDs: map[string]struct{}{}, + } + require.NoError(t, handler.seedChannelExclusions(ctx)) + require.NoError(t, s.RecordFailure( + ctx, + tailMessageFailureIdentity("", "feed", "125", "update"), + errors.New("prior guildless update failure"), + )) + + require.NoError(t, handler.OnMessageUpdate(ctx, &discordgo.Message{ + ID: "125", + ChannelID: "feed", + })) + + var deletedAt string + require.NoError(t, s.DB().QueryRowContext( + ctx, + `select coalesce(deleted_at, '') from messages where id = '125'`, + ).Scan(&deletedAt)) + require.Empty(t, deletedAt) + var deleteEvents int + require.NoError(t, s.DB().QueryRowContext( + ctx, + `select count(*) from message_events where message_id = '125' and event_type = 'delete'`, + ).Scan(&deleteEvents)) + require.Zero(t, deleteEvents) + report, err := s.ListFailures(ctx, store.FailureListOptions{}, time.Now().UTC()) require.NoError(t, err) - require.Len(t, mentions, 1) - require.Equal(t, "u2", mentions[0].TargetID) - require.Equal(t, "Shadow", mentions[0].TargetName) + require.Zero(t, report.UnresolvedCount) } -func TestMessageUpdateSnapshotRejectsConflictingRefetchIdentity(t *testing.T) { +func TestTailHandlerDoesNotDeleteOutOfScopeMessageOnGuildlessUnknownUpdate(t *testing.T) { t.Parallel() - partial := &discordgo.Message{ - ID: "m1", - GuildID: "g1", - ChannelID: "c1", + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + channel := &discordgo.Channel{ + ID: "other-channel", + GuildID: "other-guild", + Name: "general", + Type: discordgo.ChannelTypeGuildText, } - tests := []struct { - name string - full *discordgo.Message - wantErr string - }{ - { - name: "message id", - full: &discordgo.Message{ID: "other", GuildID: "g1", ChannelID: "c1"}, - wantErr: "different message id", + message := &discordgo.Message{ + ID: "126", + GuildID: "other-guild", + ChannelID: "other-channel", + Content: "preserved outside scope", + Timestamp: time.Now().UTC(), + Author: &discordgo.User{ID: "u1", Username: "user"}, + } + client := &fakeClient{ + channelsByID: map[string]*discordgo.Channel{"other-channel": channel}, + messages: map[string][]*discordgo.Message{"other-channel": {message}}, + messageByIDErrs: map[string]error{ + "other-channel/126": errors.New(`HTTP 404 Not Found, {"message": "Unknown Message", "code": 10008}`), }, - { - name: "channel id", - full: &discordgo.Message{ID: "m1", GuildID: "g1", ChannelID: "other"}, - wantErr: "different channel id", + } + initialHandler := &tailHandler{ + store: s, + client: client, + channels: map[string]tailChannel{}, + kindExcludedChannelIDs: map[string]struct{}{}, + } + require.NoError(t, initialHandler.OnChannelUpsert(ctx, channel)) + require.NoError(t, initialHandler.OnMessageCreate(ctx, message)) + + handler := &tailHandler{ + guilds: makeGuildSet([]string{"g1"}), + store: s, + client: client, + channels: map[string]tailChannel{}, + kindExcludedChannelIDs: map[string]struct{}{}, + } + require.NoError(t, handler.seedChannelExclusions(ctx)) + require.NoError(t, s.RecordFailure( + ctx, + tailMessageFailureIdentity("", "other-channel", "126", "update"), + errors.New("prior guildless update failure"), + )) + + require.NoError(t, handler.OnMessageUpdate(ctx, &discordgo.Message{ + ID: "126", + ChannelID: "other-channel", + })) + + var deletedAt string + require.NoError(t, s.DB().QueryRowContext( + ctx, + `select coalesce(deleted_at, '') from messages where id = '126'`, + ).Scan(&deletedAt)) + require.Empty(t, deletedAt) + var deleteEvents int + require.NoError(t, s.DB().QueryRowContext( + ctx, + `select count(*) from message_events where message_id = '126' and event_type = 'delete'`, + ).Scan(&deleteEvents)) + require.Zero(t, deleteEvents) + report, err := s.ListFailures(ctx, store.FailureListOptions{}, time.Now().UTC()) + require.NoError(t, err) + require.Zero(t, report.UnresolvedCount) +} + +func TestTailHandlerDropsUnknownUpdateForUnseenMessageWithoutOrphanEvent(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + client := &fakeClient{ + channelsByID: map[string]*discordgo.Channel{ + "c1": { + ID: "c1", + GuildID: "g1", + Name: "general", + Type: discordgo.ChannelTypeGuildText, + }, }, - { - name: "guild id", - full: &discordgo.Message{ID: "m1", GuildID: "other", ChannelID: "c1"}, - wantErr: "different guild id", + messageByIDErrs: map[string]error{ + "c1/124": errors.New(`HTTP 404 Not Found, {"message": "Unknown Message", "code": 10008}`), }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - handler := &tailHandler{client: &tailSnapshotClient{message: tt.full}} - snapshot, err := handler.messageUpdateSnapshot(context.Background(), partial) - require.Nil(t, snapshot) - require.ErrorContains(t, err, tt.wantErr) - }) + handler := &tailHandler{ + guilds: makeGuildSet([]string{"g1"}), + store: s, + client: client, + channels: map[string]tailChannel{}, + kindExcludedChannelIDs: map[string]struct{}{}, } + require.NoError(t, handler.OnChannelUpsert(ctx, client.channelsByID["c1"])) + require.NoError(t, s.RecordFailure( + ctx, + tailMessageFailureIdentity("g1", "c1", "124", "update"), + errors.New("prior update failure"), + )) + + require.NoError(t, handler.OnMessageUpdate(ctx, &discordgo.Message{ + ID: "124", + GuildID: "g1", + ChannelID: "c1", + })) + + var eventCount int + require.NoError(t, s.DB().QueryRowContext( + ctx, + `select count(*) from message_events where message_id = '124'`, + ).Scan(&eventCount)) + require.Zero(t, eventCount) + report, err := s.ListFailures(ctx, store.FailureListOptions{}, time.Now().UTC()) + require.NoError(t, err) + require.Zero(t, report.UnresolvedCount) } -func TestTailHandlerMessageUpdateFailureUsesSyncerRefetchedMetadata(t *testing.T) { - tests := []struct { - name string - wantKind string - wantStage discordclient.TailFailureStage - }{ - { - name: "returned error", - wantKind: "returned_error", - wantStage: discordclient.TailFailureStageCanonicalWrite, - }, - { - name: "panic", - wantKind: "panic", - wantStage: discordclient.TailFailureStageMessageBuild, +func TestTailHandlerDoesNotRecordOrderlyShutdownCancellation(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + require.NoError(t, s.UpsertChannel(ctx, store.ChannelRecord{ + ID: "c1", + GuildID: "g1", + Kind: "text", + Name: "general", + RawJSON: `{}`, + })) + + canceledCtx, cancel := context.WithCancel(ctx) + cancel() + handler := &tailHandler{store: s} + require.NoError(t, handler.seedChannelExclusions(ctx)) + err = handler.OnMessageCreate(canceledCtx, &discordgo.Message{ + ID: "shutdown", + GuildID: "g1", + ChannelID: "c1", + Content: "tail event", + Timestamp: time.Now().UTC(), + Author: &discordgo.User{ID: "u1", Username: "peter"}, + }) + require.ErrorIs(t, err, context.Canceled) + + report, err := s.ListFailures(ctx, store.FailureListOptions{}, time.Now()) + require.NoError(t, err) + require.Empty(t, report.Failures) +} + +func TestTailHandlerExcludesConfiguredFeedChannels(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + require.NoError(t, s.UpsertChannel(ctx, store.ChannelRecord{ + ID: "news", + GuildID: "g1", + Kind: "announcement", + Name: "github", + RawJSON: `{}`, + })) + handler := &tailHandler{ + store: s, + client: &fakeClient{channelsByID: map[string]*discordgo.Channel{ + "logs": { + ID: "logs", + GuildID: "g1", + Name: "logs", + Type: discordgo.ChannelTypeGuildText, + }, + "general": { + ID: "general", + GuildID: "g1", + Name: "general", + Type: discordgo.ChannelTypeGuildText, + }, + }}, + exclusions: newChannelExclusions([]string{"logs"}, []string{"announcement"}), + kindExcludedChannelIDs: map[string]struct{}{}, + } + require.NoError(t, handler.seedChannelExclusions(ctx)) + + message := func(id, channelID string) *discordgo.Message { + return &discordgo.Message{ + ID: id, + GuildID: "g1", + ChannelID: channelID, + Content: "event", + Timestamp: time.Now().UTC(), + Author: &discordgo.User{ID: "u1", Username: "bot", Bot: true}, + } + } + require.NoError(t, handler.OnMessageCreate(ctx, message("1", "news"))) + require.NoError(t, handler.OnMessageCreate(ctx, message("2", "logs"))) + require.NoError(t, handler.OnMessageCreate(ctx, message("3", "general"))) + + require.NoError(t, handler.OnChannelUpsert(ctx, &discordgo.Channel{ + ID: "future-news", + GuildID: "g1", + Name: "future-feed", + Type: discordgo.ChannelTypeGuildNews, + })) + require.NoError(t, handler.OnMessageCreate(ctx, message("4", "future-news"))) + + status, err := s.Status(ctx, "db", "") + require.NoError(t, err) + require.Equal(t, 1, status.MessageCount) +} + +func TestTailHandlerReclassifiesDescendantsAfterParentUpdates(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + client := &fakeClient{channelsByID: map[string]*discordgo.Channel{ + "parent": { + ID: "parent", + GuildID: "g1", + Name: "feed", + Type: discordgo.ChannelTypeGuildNews, + }, + }} + handler := &tailHandler{ + store: s, + client: client, + exclusions: newChannelExclusions(nil, []string{"announcement"}), + channels: map[string]tailChannel{}, + kindExcludedChannelIDs: map[string]struct{}{}, + } + require.NoError(t, handler.OnChannelUpsert(ctx, &discordgo.Channel{ + ID: "thread", + GuildID: "g1", + ParentID: "parent", + Name: "feed-thread", + Type: discordgo.ChannelTypeGuildPublicThread, + })) + require.NoError(t, handler.OnChannelUpsert(ctx, &discordgo.Channel{ + ID: "parent", + GuildID: "g1", + Name: "feed", + Type: discordgo.ChannelTypeGuildNews, + })) + + message := func(id, content string) *discordgo.Message { + return &discordgo.Message{ + ID: id, + GuildID: "g1", + ChannelID: "thread", + Content: content, + Timestamp: time.Now().UTC(), + Author: &discordgo.User{ID: "u1", Username: "user"}, + } + } + require.NoError(t, handler.OnMessageCreate(ctx, message("1", "excluded"))) + + require.NoError(t, handler.OnChannelUpsert(ctx, &discordgo.Channel{ + ID: "parent", + GuildID: "g1", + Name: "discussion", + Type: discordgo.ChannelTypeGuildText, + })) + require.NoError(t, handler.OnMessageCreate(ctx, message("2", "allowed"))) + + require.NoError(t, handler.OnChannelUpsert(ctx, &discordgo.Channel{ + ID: "parent", + GuildID: "g1", + Name: "feed-again", + Type: discordgo.ChannelTypeGuildNews, + })) + require.NoError(t, handler.OnMessageUpdate(ctx, message("2", "must not update"))) + require.NoError(t, handler.OnMessageDelete(ctx, &discordgo.MessageDelete{Message: &discordgo.Message{ + ID: "2", + GuildID: "g1", + ChannelID: "thread", + }})) + + require.NoError(t, handler.OnChannelUpsert(ctx, &discordgo.Channel{ + ID: "parent", + GuildID: "g1", + Name: "discussion-again", + Type: discordgo.ChannelTypeGuildText, + })) + require.NoError(t, handler.OnMessageCreate(ctx, message("3", "allowed again"))) + + var count, deleted int + require.NoError(t, s.DB().QueryRowContext(ctx, ` + select count(*), coalesce(sum(case when deleted_at is not null then 1 else 0 end), 0) + from messages + where channel_id = 'thread' + `).Scan(&count, &deleted)) + require.Equal(t, 2, count) + require.Zero(t, deleted) + + var content string + require.NoError(t, s.DB().QueryRowContext(ctx, `select content from messages where id = '2'`).Scan(&content)) + require.Equal(t, "allowed", content) +} + +func TestTailHandlerMessageUpdateFetchesFullMessageBeforeUpsert(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + createdAt := time.Now().UTC() + original := &discordgo.Message{ + ID: "9", + GuildID: "g1", + ChannelID: "c1", + Content: "original <@u2>", + Timestamp: createdAt, + Author: &discordgo.User{ID: "u1", Username: "peter"}, + Attachments: []*discordgo.MessageAttachment{{ + ID: "a1", + Filename: "trace.txt", + ContentType: "text/plain", + Size: 42, + }}, + Mentions: []*discordgo.User{{ID: "u2", Username: "shadow", GlobalName: "Shadow"}}, + } + updated := *original + updated.Content = "edited <@u2>" + + client := &fakeClient{ + channelsByID: map[string]*discordgo.Channel{ + "c1": { + ID: "c1", + GuildID: "g1", + Name: "general", + Type: discordgo.ChannelTypeGuildText, + }, + }, + messages: map[string][]*discordgo.Message{ + "c1": {&updated}, + }, + } + handler := &tailHandler{store: s, client: client} + + require.NoError(t, handler.OnMessageCreate(ctx, original)) + require.NoError(t, handler.OnMessageUpdate(ctx, &discordgo.Message{ + ID: "9", + GuildID: "g1", + ChannelID: "c1", + Content: "edited <@u2>", + })) + + messages, err := s.ListMessages(ctx, store.MessageListOptions{GuildIDs: []string{"g1"}, IncludeEmpty: true}) + require.NoError(t, err) + require.Len(t, messages, 1) + require.Equal(t, "edited <@u2>", messages[0].Content) + require.Equal(t, "u1", messages[0].AuthorID) + require.Equal(t, "peter", messages[0].AuthorName) + require.Equal(t, createdAt.Format(time.RFC3339Nano), messages[0].CreatedAt.Format(time.RFC3339Nano)) + require.True(t, messages[0].HasAttachments) + require.Equal(t, "trace.txt", messages[0].AttachmentNames) + + attachments, err := s.ListAttachments(ctx, store.AttachmentListOptions{MessageID: "9"}) + require.NoError(t, err) + require.Len(t, attachments, 1) + require.Equal(t, "a1", attachments[0].AttachmentID) + require.Equal(t, "trace.txt", attachments[0].Filename) + require.Equal(t, "u1", attachments[0].AuthorID) + + mentions, err := s.ListMentions(ctx, store.MentionListOptions{Target: "u2"}) + require.NoError(t, err) + require.Len(t, mentions, 1) + require.Equal(t, "u2", mentions[0].TargetID) + require.Equal(t, "Shadow", mentions[0].TargetName) +} + +func TestMessageUpdateSnapshotRejectsConflictingRefetchIdentity(t *testing.T) { + t.Parallel() + + partial := &discordgo.Message{ + ID: "m1", + GuildID: "g1", + ChannelID: "c1", + } + tests := []struct { + name string + full *discordgo.Message + wantErr string + }{ + { + name: "message id", + full: &discordgo.Message{ID: "other", GuildID: "g1", ChannelID: "c1"}, + wantErr: "different message id", + }, + { + name: "channel id", + full: &discordgo.Message{ID: "m1", GuildID: "g1", ChannelID: "other"}, + wantErr: "different channel id", + }, + { + name: "guild id", + full: &discordgo.Message{ID: "m1", GuildID: "other", ChannelID: "c1"}, + wantErr: "different guild id", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + handler := &tailHandler{client: &tailSnapshotClient{message: tt.full}} + snapshot, err := handler.messageUpdateSnapshot(context.Background(), partial) + require.Nil(t, snapshot) + require.ErrorContains(t, err, tt.wantErr) + }) + } +} + +func TestTailHandlerMessageUpdateFailureUsesSyncerRefetchedMetadata(t *testing.T) { + tests := []struct { + name string + wantKind string + wantStage discordclient.TailFailureStage + }{ + { + name: "returned error", + wantKind: "returned_error", + wantStage: discordclient.TailFailureStageCanonicalWrite, + }, + { + name: "panic", + wantKind: "panic", + wantStage: discordclient.TailFailureStageMessageBuild, + }, + { + name: "cooperative timeout", + wantKind: "timeout", + wantStage: discordclient.TailFailureStageCanonicalWrite, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + var tailStore *store.Store + var onFetch func(context.Context) + switch tt.wantKind { + case "returned_error": + var err error + tailStore, err = store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + require.NoError(t, tailStore.Close()) + case "panic": + var err error + tailStore, err = store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + t.Cleanup(func() { _ = tailStore.Close() }) + case "timeout": + var err error + tailStore, err = store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + t.Cleanup(func() { _ = tailStore.Close() }) + tailStore.DB().SetMaxOpenConns(1) + heldConn, err := tailStore.DB().Conn(ctx) + require.NoError(t, err) + var releaseOnce sync.Once + release := func() { + releaseOnce.Do(func() { _ = heldConn.Close() }) + } + t.Cleanup(release) + onFetch = func(taskCtx context.Context) { + go func() { + <-taskCtx.Done() + time.Sleep(10 * time.Millisecond) + release() + }() + } + default: + t.Fatalf("unsupported failure kind %q", tt.wantKind) + } + + full := &discordgo.Message{ + ID: "m1", + GuildID: "g-refetched", + ChannelID: "c1", + Content: "full message", + Timestamp: time.Now().UTC(), + Author: &discordgo.User{ID: "u-refetched", Username: "test-user"}, + } + snapshotClient := &tailSnapshotClient{ + message: full, + onFetch: onFetch, + } + clientRefetches := &atomic.Int32{} + server := newSyncerTailMessageUpdateGateway(t, full, clientRefetches) + defer server.Close() + restore := patchSyncerTailDiscordEndpoints(server.URL + "/api/v10/") + defer restore() + + eventClient, err := discordclient.New("token") + require.NoError(t, err) + defer func() { _ = eventClient.Close() }() + setDiscordTailHandlerTimeout(t, eventClient, 25*time.Millisecond) + + handler := &capturingTailHandler{ + tailHandler: &tailHandler{ + store: tailStore, + client: snapshotClient, + attachmentTextEnabled: false, + channels: map[string]tailChannel{ + "c1": {kind: "text"}, + }, + kindExcludedChannelIDs: map[string]struct{}{}, + }, + cancel: cancel, + failures: make(chan discordclient.TailFailure, 1), + panicOnUpdate: tt.wantKind == "panic", + } + require.NoError(t, eventClient.Tail(ctx, handler)) + + var failure discordclient.TailFailure + select { + case failure = <-handler.failures: + default: + t.Fatalf("tail failure was not reported before Tail returned: %v", ctx.Err()) + } + require.Equal(t, "MESSAGE_UPDATE", failure.EventType) + require.Equal(t, tt.wantKind, failure.Kind) + require.Equal(t, "g-refetched", failure.GuildID) + require.Equal(t, "c1", failure.ChannelID) + require.Equal(t, "m1", failure.MessageID) + require.Equal(t, "u-refetched", failure.UserID) + require.Equal(t, tt.wantStage, failure.HandlerStage) + require.Positive(t, failure.HandlerStageElapsed) + require.Positive(t, failure.HandlerElapsed) + if tt.wantKind == "timeout" { + require.Equal(t, discordclient.TailFailureJoinJoined, failure.JoinOutcome) + require.Positive(t, failure.JoinElapsed) + } else { + require.Equal(t, discordclient.TailFailureJoinNotRequired, failure.JoinOutcome) + require.Zero(t, failure.JoinElapsed) + } + require.False(t, failure.ForceFallback) + require.EqualValues(t, 1, snapshotClient.calls.Load()) + require.Zero(t, clientRefetches.Load()) + }) + } +} + +func TestTailHandlerConsumesReconciledUnknownMessageRefetchFailure(t *testing.T) { + testCtx, cancelTest := context.WithTimeout(context.Background(), 5*time.Second) + defer cancelTest() + tailCtx, cancelTail := context.WithCancel(testCtx) + defer cancelTail() + + s, err := store.Open(testCtx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + message := &discordgo.Message{ + ID: "123456789012345678", + GuildID: "g1", + ChannelID: "c1", + Content: "preserved before deletion", + Timestamp: time.Now().UTC(), + Author: &discordgo.User{ID: "u1", Username: "user"}, + } + snapshotClient := &fakeClient{} + baseHandler := &tailHandler{ + guilds: makeGuildSet([]string{"g1"}), + store: s, + client: snapshotClient, + channels: map[string]tailChannel{ + "c1": {kind: "text"}, + }, + kindExcludedChannelIDs: map[string]struct{}{}, + } + require.NoError(t, baseHandler.OnChannelUpsert(testCtx, &discordgo.Channel{ + ID: "c1", + GuildID: "g1", + Name: "general", + Type: discordgo.ChannelTypeGuildText, + })) + require.NoError(t, baseHandler.OnMessageCreate(testCtx, message)) + require.NoError(t, s.RecordFailure( + testCtx, + tailMessageFailureIdentity(message.GuildID, message.ChannelID, message.ID, "update"), + errors.New("prior update failure"), + )) + + var clientRefetches atomic.Int32 + server := newSyncerTailUnknownMessageGateway(t, message.ID, 4, &clientRefetches) + defer server.Close() + restore := patchSyncerTailDiscordEndpoints(server.URL + "/api/v10/") + defer restore() + + eventClient, err := discordclient.New("token") + require.NoError(t, err) + defer func() { _ = eventClient.Close() }() + setDiscordTailHandlerTimeout(t, eventClient, time.Second) + + handler := &reconciledRefetchTailHandler{ + tailHandler: baseHandler, + drained: make(chan struct{}), + } + tailDone := make(chan error, 1) + go func() { + tailDone <- eventClient.Tail(tailCtx, handler) + }() + + select { + case <-handler.drained: + case err := <-tailDone: + t.Fatalf("Tail returned before the ordered sentinel: %v", err) + case <-testCtx.Done(): + t.Fatal("ordered sentinel was not handled") + } + require.Zero(t, handler.failureReports.Load()) + require.Zero(t, handler.failureRecords.Load()) + select { + case err := <-tailDone: + t.Fatalf("Tail returned before cancellation after reconciled updates: %v", err) + default: + } + + cancelTail() + select { + case err := <-tailDone: + require.NoError(t, err) + case <-testCtx.Done(): + t.Fatal("Tail did not stop after cancellation") + } + + require.EqualValues(t, 4, clientRefetches.Load()) + snapshotClient.mu.Lock() + snapshotRefetches := snapshotClient.exactMessageCalls + snapshotClient.mu.Unlock() + require.Zero(t, snapshotRefetches) + + var deletedAt string + require.NoError(t, s.DB().QueryRowContext( + context.Background(), + `select coalesce(deleted_at, '') from messages where id = ?`, + message.ID, + ).Scan(&deletedAt)) + require.NotEmpty(t, deletedAt) + var deleteEvents int + require.NoError(t, s.DB().QueryRowContext( + context.Background(), + `select count(*) from message_events where message_id = ? and event_type = 'delete'`, + message.ID, + ).Scan(&deleteEvents)) + require.Positive(t, deleteEvents) + report, err := s.ListFailures(context.Background(), store.FailureListOptions{}, time.Now().UTC()) + require.NoError(t, err) + require.Zero(t, report.UnresolvedCount) +} + +func TestTailHandlerResolvesUnknownThreadAndParentBeforeExclusion(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + client := &fakeClient{channelsByID: map[string]*discordgo.Channel{ + "thread": { + ID: "thread", + GuildID: "g1", + ParentID: "forum", + Name: "feed thread", + Type: discordgo.ChannelTypeGuildPublicThread, + }, + "forum": { + ID: "forum", + GuildID: "g1", + Name: "feed", + Type: discordgo.ChannelTypeGuildForum, + }, + }} + handler := &tailHandler{ + guilds: makeGuildSet([]string{"g1"}), + store: s, + client: client, + exclusions: newChannelExclusions(nil, []string{"forum"}), + channels: map[string]tailChannel{}, + kindExcludedChannelIDs: map[string]struct{}{}, + } + + const messageCount = 8 + var wg sync.WaitGroup + errs := make(chan error, messageCount) + for i := range messageCount { + wg.Go(func() { + errs <- handler.OnMessageCreate(ctx, &discordgo.Message{ + ID: string(rune('a' + i)), + GuildID: "g1", + ChannelID: "thread", + Content: "must stay excluded", + Timestamp: time.Now().UTC(), + Author: &discordgo.User{ID: "u1", Username: "user"}, + }) + }) + } + wg.Wait() + close(errs) + for err := range errs { + require.NoError(t, err) + } + + client.mu.Lock() + threadCalls := client.channelCalls["thread"] + parentCalls := client.channelCalls["forum"] + client.mu.Unlock() + require.Equal(t, 1, threadCalls) + require.Equal(t, 1, parentCalls) + + channels, err := s.Channels(ctx, "g1") + require.NoError(t, err) + require.Len(t, channels, 2) + status, err := s.Status(ctx, "db", "") + require.NoError(t, err) + require.Zero(t, status.MessageCount) + require.True(t, handler.excludeChannel("thread")) +} + +func TestTailHandlerResolvesUnknownChannelBeforeMessageWrite(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + client := &fakeClient{channelsByID: map[string]*discordgo.Channel{ + "c1": { + ID: "c1", + GuildID: "g1", + Name: "general", + Type: discordgo.ChannelTypeGuildText, + }, + }} + handler := &tailHandler{ + guilds: makeGuildSet([]string{"g1"}), + store: s, + client: client, + channels: map[string]tailChannel{}, + kindExcludedChannelIDs: map[string]struct{}{}, + } + + require.NoError(t, handler.OnMessageCreate(ctx, &discordgo.Message{ + ID: "1", + GuildID: "g1", + ChannelID: "c1", + Content: "archived", + Timestamp: time.Now().UTC(), + Author: &discordgo.User{ID: "u1", Username: "user"}, + })) + + status, err := s.Status(ctx, "db", "") + require.NoError(t, err) + require.Equal(t, 1, status.ChannelCount) + require.Equal(t, 1, status.MessageCount) + client.mu.Lock() + channelCalls := client.channelCalls["c1"] + client.mu.Unlock() + require.Equal(t, 1, channelCalls) +} + +func TestTailHandlerResolvesMissingSeededThreadParentBeforeExclusion(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + require.NoError(t, s.UpsertChannel(ctx, store.ChannelRecord{ + ID: "thread", + GuildID: "g1", + ParentID: "forum", + Kind: "thread_public", + Name: "seeded thread", + RawJSON: `{"id":"thread","parent_id":"forum"}`, + })) + client := &fakeClient{channelsByID: map[string]*discordgo.Channel{ + "thread": { + ID: "thread", + GuildID: "g1", + ParentID: "forum", + Name: "seeded thread", + Type: discordgo.ChannelTypeGuildPublicThread, + }, + "forum": { + ID: "forum", + GuildID: "g1", + Name: "feed", + Type: discordgo.ChannelTypeGuildForum, + }, + }} + handler := &tailHandler{ + guilds: makeGuildSet([]string{"g1"}), + store: s, + client: client, + exclusions: newChannelExclusions(nil, []string{"forum"}), + } + require.NoError(t, handler.seedChannelExclusions(ctx)) + require.False(t, handler.channelHierarchyKnown("thread")) + + require.NoError(t, handler.OnMessageCreate(ctx, &discordgo.Message{ + ID: "1", + GuildID: "g1", + ChannelID: "thread", + Content: "must stay excluded", + Timestamp: time.Now().UTC(), + Author: &discordgo.User{ID: "u1", Username: "user"}, + })) + + client.mu.Lock() + threadCalls := client.channelCalls["thread"] + parentCalls := client.channelCalls["forum"] + client.mu.Unlock() + require.Equal(t, 1, threadCalls) + require.Equal(t, 1, parentCalls) + require.True(t, handler.channelHierarchyKnown("thread")) + require.True(t, handler.excludeChannel("thread")) + + status, err := s.Status(ctx, "db", "") + require.NoError(t, err) + require.Equal(t, 2, status.ChannelCount) + require.Zero(t, status.MessageCount) +} + +func TestTailHandlerPersistsExplicitlyExcludedUnknownAncestor(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + client := &fakeClient{channelsByID: map[string]*discordgo.Channel{ + "feed": { + ID: "feed", + GuildID: "g1", + Name: "feed parent", + Type: discordgo.ChannelTypeGuildText, + }, + }} + handler := &tailHandler{ + guilds: makeGuildSet([]string{"g1"}), + store: s, + client: client, + exclusions: newChannelExclusions([]string{"feed"}, nil), + } + + require.NoError(t, handler.OnChannelUpsert(ctx, &discordgo.Channel{ + ID: "thread", + GuildID: "g1", + ParentID: "feed", + Name: "feed thread", + Type: discordgo.ChannelTypeGuildPublicThread, + })) + + client.mu.Lock() + parentCalls := client.channelCalls["feed"] + client.mu.Unlock() + require.Equal(t, 1, parentCalls) + require.True(t, handler.channelHierarchyKnown("thread")) + require.True(t, handler.excludeChannel("thread")) + + channels, err := s.Channels(ctx, "g1") + require.NoError(t, err) + require.Len(t, channels, 2) +} + +func TestTailHandlerResolvesUnknownChannelsForUpdateAndDelete(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + now := time.Now().UTC() + updated := &discordgo.Message{ + ID: "update", + GuildID: "g1", + ChannelID: "update-channel", + Content: "updated content", + Timestamp: now, + Author: &discordgo.User{ID: "u1", Username: "user"}, + } + client := &fakeClient{ + channelsByID: map[string]*discordgo.Channel{ + "update-channel": { + ID: "update-channel", + GuildID: "g1", + Name: "updates", + Type: discordgo.ChannelTypeGuildText, + }, + "delete-channel": { + ID: "delete-channel", + GuildID: "g1", + Name: "deletes", + Type: discordgo.ChannelTypeGuildText, + }, + }, + messages: map[string][]*discordgo.Message{ + "update-channel": {updated}, + }, + } + handler := &tailHandler{ + guilds: makeGuildSet([]string{"g1"}), + store: s, + client: client, + channels: map[string]tailChannel{}, + kindExcludedChannelIDs: map[string]struct{}{}, + } + + require.NoError(t, handler.OnMessageUpdate(ctx, &discordgo.Message{ + ID: updated.ID, + ChannelID: updated.ChannelID, + Content: updated.Content, + })) + + deleted := &discordgo.Message{ + ID: "delete", + GuildID: "g1", + ChannelID: "delete-channel", + Content: "delete me", + Timestamp: now, + Author: &discordgo.User{ID: "u1", Username: "user"}, + } + mutation, err := buildMessageMutation(ctx, deleted, "", "", false, false) + require.NoError(t, err) + require.NoError(t, s.UpsertMessages(ctx, []store.MessageMutation{mutation})) + require.NoError(t, handler.OnMessageDelete(ctx, &discordgo.MessageDelete{Message: deleted})) + + var content string + require.NoError(t, s.DB().QueryRowContext( + ctx, + `select content from messages where id = ?`, + updated.ID, + ).Scan(&content)) + require.Equal(t, updated.Content, content) + + var deletedAt string + require.NoError(t, s.DB().QueryRowContext( + ctx, + `select coalesce(deleted_at, '') from messages where id = ?`, + deleted.ID, + ).Scan(&deletedAt)) + require.NotEmpty(t, deletedAt) + + status, err := s.Status(ctx, "db", "") + require.NoError(t, err) + require.Equal(t, 2, status.ChannelCount) + require.Equal(t, 2, status.MessageCount) + client.mu.Lock() + updateCalls := client.channelCalls["update-channel"] + deleteCalls := client.channelCalls["delete-channel"] + client.mu.Unlock() + require.Equal(t, 1, updateCalls) + require.Equal(t, 1, deleteCalls) +} + +func TestTailHandlerIgnoresGuildlessPartialUpdateOutsideScope(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + full := &discordgo.Message{ + ID: "outside-message", + GuildID: "g2", + ChannelID: "outside-channel", + Content: "outside scope", + Timestamp: time.Now().UTC(), + Author: &discordgo.User{ID: "u2", Username: "outside"}, + } + client := &fakeClient{ + channelsByID: map[string]*discordgo.Channel{ + "outside-channel": { + ID: "outside-channel", + GuildID: "g2", + Name: "outside", + Type: discordgo.ChannelTypeGuildText, + }, + }, + messages: map[string][]*discordgo.Message{ + "outside-channel": {full}, + }, + } + handler := &tailHandler{ + guilds: makeGuildSet([]string{"g1"}), + store: s, + client: client, + channels: map[string]tailChannel{}, + kindExcludedChannelIDs: map[string]struct{}{}, + } + + require.NoError(t, handler.OnMessageUpdate(ctx, &discordgo.Message{ + ID: full.ID, + ChannelID: full.ChannelID, + Content: full.Content, + })) + + status, err := s.Status(ctx, "db", "") + require.NoError(t, err) + require.Zero(t, status.ChannelCount) + require.Zero(t, status.MessageCount) + client.mu.Lock() + channelCalls := client.channelCalls["outside-channel"] + client.mu.Unlock() + require.Zero(t, channelCalls) +} + +func TestTailHandlerPersistsThreadArchiveMetadata(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + client := &fakeClient{channelsByID: map[string]*discordgo.Channel{ + "parent": { + ID: "parent", + GuildID: "g1", + Name: "general", + Type: discordgo.ChannelTypeGuildText, }, - { - name: "cooperative timeout", - wantKind: "timeout", - wantStage: discordclient.TailFailureStageCanonicalWrite, + }} + handler := &tailHandler{ + guilds: makeGuildSet([]string{"g1"}), + store: s, + client: client, + } + archivedAt := time.Now().UTC().Truncate(time.Microsecond) + require.NoError(t, handler.OnChannelUpsert(ctx, &discordgo.Channel{ + ID: "thread", + GuildID: "g1", + ParentID: "parent", + Name: "archived thread", + Type: discordgo.ChannelTypeGuildPrivateThread, + ThreadMetadata: &discordgo.ThreadMetadata{ + Archived: true, + Locked: true, + ArchiveTimestamp: archivedAt, }, + })) + + channels, err := s.Channels(ctx, "g1") + require.NoError(t, err) + require.Len(t, channels, 2) + var thread store.ChannelRow + for _, channel := range channels { + if channel.ID == "thread" { + thread = channel + break + } } + require.Equal(t, "thread", thread.ID) + require.True(t, thread.IsArchived) + require.True(t, thread.IsLocked) + require.True(t, thread.IsPrivateThread) + require.Equal(t, "parent", thread.ThreadParentID) + require.Equal(t, archivedAt, thread.ArchiveTimestamp) +} - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() +func TestTailHandlerConcurrentResolutionErrorsFailClosed(t *testing.T) { + t.Parallel() - var tailStore *store.Store - var onFetch func(context.Context) - switch tt.wantKind { - case "returned_error": - var err error - tailStore, err = store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) - require.NoError(t, err) - require.NoError(t, tailStore.Close()) - case "panic": - var err error - tailStore, err = store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) - require.NoError(t, err) - t.Cleanup(func() { _ = tailStore.Close() }) - case "timeout": - var err error - tailStore, err = store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) - require.NoError(t, err) - t.Cleanup(func() { _ = tailStore.Close() }) - tailStore.DB().SetMaxOpenConns(1) - heldConn, err := tailStore.DB().Conn(ctx) - require.NoError(t, err) - var releaseOnce sync.Once - release := func() { - releaseOnce.Do(func() { _ = heldConn.Close() }) - } - t.Cleanup(release) - onFetch = func(taskCtx context.Context) { - go func() { - <-taskCtx.Done() - time.Sleep(10 * time.Millisecond) - release() - }() - } - default: - t.Fatalf("unsupported failure kind %q", tt.wantKind) - } + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() - full := &discordgo.Message{ - ID: "m1", - GuildID: "g-refetched", - ChannelID: "c1", - Content: "full message", + client := &fakeClient{channelErrors: map[string]error{ + "missing": errors.New("unavailable"), + }} + handler := &tailHandler{ + guilds: makeGuildSet([]string{"g1"}), + store: s, + client: client, + channels: map[string]tailChannel{}, + kindExcludedChannelIDs: map[string]struct{}{}, + } + + const messageCount = 8 + errs := make(chan error, messageCount) + var wg sync.WaitGroup + for i := range messageCount { + wg.Go(func() { + errs <- handler.OnMessageCreate(ctx, &discordgo.Message{ + ID: string(rune('a' + i)), + GuildID: "g1", + ChannelID: "missing", + Content: "must not be written", Timestamp: time.Now().UTC(), - Author: &discordgo.User{ID: "u-refetched", Username: "test-user"}, - } - snapshotClient := &tailSnapshotClient{ - message: full, - onFetch: onFetch, - } - clientRefetches := &atomic.Int32{} - server := newSyncerTailMessageUpdateGateway(t, full, clientRefetches) - defer server.Close() - restore := patchSyncerTailDiscordEndpoints(server.URL + "/api/v10/") - defer restore() - - eventClient, err := discordclient.New("token") - require.NoError(t, err) - defer func() { _ = eventClient.Close() }() - setDiscordTailHandlerTimeout(t, eventClient, 25*time.Millisecond) - - handler := &capturingTailHandler{ - tailHandler: &tailHandler{ - store: tailStore, - client: snapshotClient, - attachmentTextEnabled: false, - }, - cancel: cancel, - failures: make(chan discordclient.TailFailure, 1), - panicOnUpdate: tt.wantKind == "panic", - } - require.NoError(t, eventClient.Tail(ctx, handler)) - - var failure discordclient.TailFailure - select { - case failure = <-handler.failures: - default: - t.Fatalf("tail failure was not reported before Tail returned: %v", ctx.Err()) - } - require.Equal(t, "MESSAGE_UPDATE", failure.EventType) - require.Equal(t, tt.wantKind, failure.Kind) - require.Equal(t, "g-refetched", failure.GuildID) - require.Equal(t, "c1", failure.ChannelID) - require.Equal(t, "m1", failure.MessageID) - require.Equal(t, "u-refetched", failure.UserID) - require.Equal(t, tt.wantStage, failure.HandlerStage) - require.Positive(t, failure.HandlerStageElapsed) - require.Positive(t, failure.HandlerElapsed) - if tt.wantKind == "timeout" { - require.Equal(t, discordclient.TailFailureJoinJoined, failure.JoinOutcome) - require.Positive(t, failure.JoinElapsed) - } else { - require.Equal(t, discordclient.TailFailureJoinNotRequired, failure.JoinOutcome) - require.Zero(t, failure.JoinElapsed) - } - require.False(t, failure.ForceFallback) - require.EqualValues(t, 1, snapshotClient.calls.Load()) - require.Zero(t, clientRefetches.Load()) + Author: &discordgo.User{ID: "u1", Username: "user"}, + }) }) } + wg.Wait() + close(errs) + for err := range errs { + require.ErrorContains(t, err, "fetch channel missing: unavailable") + } + + status, err := s.Status(ctx, "db", "") + require.NoError(t, err) + require.Zero(t, status.ChannelCount) + require.Zero(t, status.MessageCount) + client.mu.Lock() + channelCalls := client.channelCalls["missing"] + client.mu.Unlock() + require.Equal(t, messageCount, channelCalls) + + report, err := s.ListFailures(ctx, store.FailureListOptions{}, time.Now()) + require.NoError(t, err) + require.Empty(t, report.Failures) } func TestHelpers(t *testing.T) { @@ -450,6 +1511,16 @@ func TestHelpers(t *testing.T) { require.Equal(t, "unknown_channel", unavailableReason(errors.New("HTTP 404 Not Found, {\"message\": \"Unknown Channel\", \"code\": 10003}"))) require.True(t, isUnknownChannel(errors.New("Unknown Channel"))) require.False(t, isUnknownChannel(errors.New("boom"))) + require.True(t, isUnknownMessage(&discordgo.RESTError{ + Response: &http.Response{Status: "404 Not Found"}, + Message: &discordgo.APIErrorMessage{ + Code: discordgo.ErrCodeUnknownMessage, + Message: "localized or empty server text", + }, + })) + require.True(t, isUnknownMessage(errors.New(`HTTP 404 Not Found, {"message": "Unknown Message", "code": 10008}`))) + require.False(t, isUnknownMessage(errors.New("unknown message encoding"))) + require.False(t, isUnknownMessage(errors.New("boom"))) require.True(t, isRetryableSyncError(context.Background(), context.DeadlineExceeded)) require.True(t, isRetryableSyncError(context.Background(), errors.New("HTTP 503 Service Unavailable"))) require.True(t, isRetryableSyncError(context.Background(), errors.New("stream error: stream ID 1; INTERNAL_ERROR"))) @@ -470,7 +1541,17 @@ func TestRunTail(t *testing.T) { defer func() { _ = s.Close() }() handled := make(chan struct{}, 1) - client := &fakeClient{tailHandled: handled} + client := &fakeClient{ + tailHandled: handled, + channelsByID: map[string]*discordgo.Channel{ + "c1": { + ID: "c1", + GuildID: "g1", + Name: "general", + Type: discordgo.ChannelTypeGuildText, + }, + }, + } svc := New(client, s, nil) go func() { select { @@ -496,17 +1577,48 @@ func TestRunTailWithRepairLoop(t *testing.T) { require.NoError(t, err) defer func() { _ = s.Close() }() + require.NoError(t, s.UpsertChannel(ctx, store.ChannelRecord{ + ID: "archived", + GuildID: "g1", + ParentID: "c1", + Kind: "thread_public", + Name: "archived-thread", + IsArchived: true, + RawJSON: `{"id":"archived"}`, + })) + require.NoError(t, s.SetSyncState(ctx, channelLatestScope("active"), "200")) + require.NoError(t, s.SetSyncState(ctx, channelLatestScope("archived"), "300")) + + messageStarted := make(chan string, 4) client := &fakeClient{ guilds: []*discordgo.UserGuild{{ID: "g1", Name: "Guild"}}, guildByID: map[string]*discordgo.Guild{ "g1": {ID: "g1", Name: "Guild"}, }, channels: map[string][]*discordgo.Channel{ - "g1": {{ID: "c1", GuildID: "g1", Name: "general", Type: discordgo.ChannelTypeGuildText}}, + "g1": {{ + ID: "c1", + GuildID: "g1", + Name: "general", + Type: discordgo.ChannelTypeGuildText, + LastMessageID: "3", + }}, + }, + guildThreads: map[string][]*discordgo.Channel{ + "g1": {{ + ID: "active", + GuildID: "g1", + ParentID: "c1", + Name: "active-thread", + Type: discordgo.ChannelTypeGuildPublicThread, + LastMessageID: "201", + }}, }, messages: map[string][]*discordgo.Message{ - "c1": {{ID: "10", GuildID: "g1", ChannelID: "c1", Content: "repair", Timestamp: time.Now().UTC(), Author: &discordgo.User{ID: "u1", Username: "user"}}}, + "active": {{ID: "201", GuildID: "g1", ChannelID: "active", Content: "repair", Timestamp: time.Now().UTC(), Author: &discordgo.User{ID: "u1", Username: "user"}}}, + "archived": {{ID: "301", GuildID: "g1", ChannelID: "archived", Content: "old history", Timestamp: time.Now().UTC(), Author: &discordgo.User{ID: "u1", Username: "user"}}}, }, + messageStarted: messageStarted, } require.NoError(t, s.RecordFailure(ctx, store.FailureRef{ Operation: tailMessageFailureOperation, @@ -516,16 +1628,40 @@ func TestRunTailWithRepairLoop(t *testing.T) { MessageID: "5", }, context.DeadlineExceeded)) svc := New(client, s, nil) + done := make(chan error, 1) go func() { - time.Sleep(40 * time.Millisecond) - cancel() + done <- svc.RunTail(ctx, []string{"g1"}, 10*time.Millisecond) }() - err = svc.RunTail(ctx, []string{"g1"}, 10*time.Millisecond) + + select { + case channelID := <-messageStarted: + require.Equal(t, "active", channelID) + case <-time.After(time.Second): + t.Fatal("periodic repair did not request the active thread") + } + require.Eventually(t, func() bool { + lastSuccess, getErr := s.GetSyncState(ctx, "sync:last_success") + return getErr == nil && lastSuccess != "" + }, time.Second, 5*time.Millisecond) + cancel() + err = <-done require.True(t, err == nil || errors.Is(err, context.Canceled)) + client.mu.Lock() + messageCalls := make(map[string]int, len(client.messageCalls)) + maps.Copy(messageCalls, client.messageCalls) + client.mu.Unlock() + + require.GreaterOrEqual(t, client.guildThreadCalls, 1) + require.Zero(t, client.threadCalls) + require.Zero(t, client.memberCalls) + require.Zero(t, messageCalls["c1"]) + require.Zero(t, messageCalls["archived"]) + require.GreaterOrEqual(t, messageCalls["active"], 1) + status, err := s.Status(context.Background(), "db", "") require.NoError(t, err) - require.GreaterOrEqual(t, status.MessageCount, 1) + require.GreaterOrEqual(t, status.MessageCount, 2) client.mu.Lock() exactMessageCalls := client.exactMessageCalls client.mu.Unlock() @@ -897,23 +2033,222 @@ func TestReplayTailMessageFailuresRetainsFetchAndIdentityFailures(t *testing.T) RelatedID: "update", }, context.DeadlineExceeded)) - client := &exactReplayClient{ - messages: map[string]*discordgo.Message{"c1/60": tt.message}, - errors: map[string]error{"c1/60": tt.err}, - } - stats, err := New(client, s, nil).replayTailMessageFailures(ctx, []string{"g1"}, 10) - require.NoError(t, err) - require.Equal(t, tailMessageReplayStats{Candidates: 1, Deferred: 1}, stats) + client := &exactReplayClient{ + messages: map[string]*discordgo.Message{"c1/60": tt.message}, + errors: map[string]error{"c1/60": tt.err}, + } + stats, err := New(client, s, nil).replayTailMessageFailures(ctx, []string{"g1"}, 10) + require.NoError(t, err) + require.Equal(t, tailMessageReplayStats{Candidates: 1, Deferred: 1}, stats) + + report, err := s.ListFailures(ctx, store.FailureListOptions{}, time.Now()) + require.NoError(t, err) + require.Equal(t, 1, report.UnresolvedCount) + require.Len(t, report.Failures, 1) + require.Equal(t, "60", report.Failures[0].MessageID) + require.Equal(t, 1, report.Failures[0].RetryCount) + var messageCount int + require.NoError(t, s.DB().QueryRowContext(ctx, `select count(*) from messages`).Scan(&messageCount)) + require.Zero(t, messageCount) + }) + } +} + +func TestRunTailDetectsGatewayExitWhileRepairIsBlocked(t *testing.T) { + ctx := t.Context() + + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + require.NoError(t, s.SetSyncState(ctx, channelLatestScope("c1"), "9")) + + client := &gatewayExitDuringRepairClient{ + fakeClient: &fakeClient{ + guilds: []*discordgo.UserGuild{{ID: "g1", Name: "Guild"}}, + guildByID: map[string]*discordgo.Guild{ + "g1": {ID: "g1", Name: "Guild"}, + }, + channels: map[string][]*discordgo.Channel{ + "g1": {{ + ID: "c1", + GuildID: "g1", + Name: "general", + Type: discordgo.ChannelTypeGuildText, + LastMessageID: "10", + }}, + }, + }, + tailStarted: make(chan struct{}), + tailExit: make(chan error, 1), + repairStarted: make(chan struct{}), + repairCanceled: make(chan struct{}), + } + svc := New(client, s, nil) + svc.tailShutdownTimeout = time.Second + done := make(chan error, 1) + go func() { + done <- svc.RunTail(ctx, []string{"g1"}, 10*time.Millisecond) + }() + + select { + case <-client.tailStarted: + case <-time.After(time.Second): + t.Fatal("tail did not start") + } + select { + case <-client.repairStarted: + case <-time.After(time.Second): + t.Fatal("repair did not start") + } + + gatewayErr := errors.New("gateway disconnected") + client.tailExit <- gatewayErr + select { + case err := <-done: + require.ErrorIs(t, err, gatewayErr) + case <-time.After(500 * time.Millisecond): + t.Fatal("RunTail did not notice Gateway exit while repair was blocked") + } + select { + case <-client.repairCanceled: + default: + t.Fatal("Gateway exit did not cancel the in-flight repair") + } +} + +func TestRunTailSchedulesNextRepairAfterOverrunCompletes(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + require.NoError(t, s.SetSyncState(ctx, channelLatestScope("c1"), "9")) + + block := make(chan struct{}) + started := make(chan string, 4) + client := &quietTailClient{fakeClient: &fakeClient{ + guilds: []*discordgo.UserGuild{{ID: "g1", Name: "Guild"}}, + guildByID: map[string]*discordgo.Guild{ + "g1": {ID: "g1", Name: "Guild"}, + }, + channels: map[string][]*discordgo.Channel{ + "g1": {{ + ID: "c1", + GuildID: "g1", + Name: "general", + Type: discordgo.ChannelTypeGuildText, + LastMessageID: "10", + }}, + }, + messageBlocks: map[string]chan struct{}{"c1": block}, + messageStarted: started, + }} + svc := New(client, s, nil) + done := make(chan error, 1) + go func() { + done <- svc.RunTail(ctx, []string{"g1"}, 100*time.Millisecond) + }() + + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("first repair did not start") + } + time.Sleep(250 * time.Millisecond) + close(block) + + select { + case <-started: + t.Fatal("overdue ticker triggered an immediate second repair") + case <-time.After(50 * time.Millisecond): + } + select { + case <-started: + case <-time.After(500 * time.Millisecond): + t.Fatal("next repair was not scheduled after the overrun completed") + } + + cancel() + require.NoError(t, <-done) +} + +func TestTailRepairSyncOptions(t *testing.T) { + t.Parallel() + + guildIDs := []string{"g1", "g2"} + opts := tailRepairSyncOptions(guildIDs) + require.Equal(t, guildIDs, opts.GuildIDs) + require.False(t, opts.Full) + require.True(t, opts.LatestOnly) + require.True(t, opts.SkipMembers) + require.Equal(t, "tail_repair", opts.RepairReason) + + guildIDs[0] = "changed" + require.Equal(t, []string{"g1", "g2"}, opts.GuildIDs) +} + +func TestNextTailRepairDelayForInitialSchedule(t *testing.T) { + t.Parallel() + + location := time.FixedZone("half-hour", 5*60*60+30*60) + now := time.Date(2026, 7, 9, 10, 10, 0, 0, location) + + require.Equal(t, time.Hour, nextTailRepairDelay(now, time.Hour, 0)) + require.Equal(t, time.Hour, nextTailRepairDelay(now, time.Hour, -15*time.Minute)) + require.Equal(t, 5*time.Minute, nextTailRepairDelay(now, time.Hour, 15*time.Minute)) + require.Equal(t, 5*time.Minute, nextTailRepairDelay(now, time.Hour, 75*time.Minute)) + require.Equal(t, 50*time.Minute, nextTailRepairDelay(now, time.Hour, time.Hour)) + require.Zero(t, nextTailRepairDelay(now, 0, 15*time.Minute)) + + onBoundary := time.Date(2026, 7, 9, 10, 15, 0, 0, location) + require.Equal(t, time.Hour, nextTailRepairDelay(onBoundary, time.Hour, 15*time.Minute)) +} + +func TestNextTailRepairDelayRealignsAfterRepair(t *testing.T) { + t.Parallel() + + location := time.FixedZone("half-hour", 5*60*60+30*60) + firstCheck := time.Date(2026, 7, 9, 10, 10, 0, 0, location) + require.Equal(t, 5*time.Minute, nextTailRepairDelay(firstCheck, time.Hour, 15*time.Minute)) + + shortRepairFinished := time.Date(2026, 7, 9, 10, 17, 0, 0, location) + require.Equal(t, 58*time.Minute, nextTailRepairDelay(shortRepairFinished, time.Hour, 15*time.Minute)) + + longRepairFinished := time.Date(2026, 7, 9, 13, 20, 0, 0, location) + require.Equal(t, 55*time.Minute, nextTailRepairDelay(longRepairFinished, time.Hour, 15*time.Minute)) +} - report, err := s.ListFailures(ctx, store.FailureListOptions{}, time.Now()) - require.NoError(t, err) - require.Equal(t, 1, report.UnresolvedCount) - require.Len(t, report.Failures, 1) - require.Equal(t, "60", report.Failures[0].MessageID) - require.Equal(t, 1, report.Failures[0].RetryCount) - var messageCount int - require.NoError(t, s.DB().QueryRowContext(ctx, `select count(*) from messages`).Scan(&messageCount)) - require.Zero(t, messageCount) +func TestNextTailRepairDelayKeepsEdmontonCivilPhaseAcrossDST(t *testing.T) { + t.Parallel() + + location, err := time.LoadLocation("America/Edmonton") + require.NoError(t, err) + + tests := []struct { + name string + now time.Time + want time.Time + wantDelay time.Duration + }{ + { + name: "spring forward", + now: time.Date(2026, time.March, 8, 0, 15, 0, 0, location), + want: time.Date(2026, time.March, 8, 6, 15, 0, 0, location), + wantDelay: 5 * time.Hour, + }, + { + name: "fall back", + now: time.Date(2026, time.November, 1, 0, 15, 0, 0, location), + want: time.Date(2026, time.November, 1, 6, 15, 0, 0, location), + wantDelay: 7 * time.Hour, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + delay := nextTailRepairDelay(test.now, 6*time.Hour, 15*time.Minute) + require.Equal(t, test.wantDelay, delay) + require.Equal(t, test.want, test.now.Add(delay)) }) } } @@ -1021,6 +2356,12 @@ func TestTailHandlersResolveOnlySuccessfulEventIdentity(t *testing.T) { )) } handler := &tailHandler{store: s} + require.NoError(t, handler.OnChannelUpsert(ctx, &discordgo.Channel{ + ID: "c1", + GuildID: "g1", + Name: "general", + Type: discordgo.ChannelTypeGuildText, + })) message := &discordgo.Message{ ID: "10", GuildID: "g1", @@ -1057,6 +2398,12 @@ func TestTailHandlerReturnedErrorIsRecordedOnlyByOuterOwner(t *testing.T) { require.NoError(t, err) handler := &tailHandler{store: s} + require.NoError(t, handler.OnChannelUpsert(ctx, &discordgo.Channel{ + ID: "c1", + GuildID: "g1", + Name: "general", + Type: discordgo.ChannelTypeGuildText, + })) message := &discordgo.Message{ ID: "m1", GuildID: "g1", @@ -1395,6 +2742,59 @@ func TestReplayTailMessageFailuresKeepsIncompleteIdentityVisible(t *testing.T) { require.Equal(t, 1, report.Failures[0].RetryCount) } +func TestNextTailRepairDelaySkipsNonexistentCivilSlot(t *testing.T) { + t.Parallel() + + location, err := time.LoadLocation("America/Edmonton") + require.NoError(t, err) + + now := time.Date(2026, time.March, 8, 1, 30, 0, 0, location) + delay := nextTailRepairDelay(now, time.Hour, 30*time.Minute) + + require.Equal(t, time.Hour, delay) + require.Equal(t, time.Date(2026, time.March, 8, 3, 30, 0, 0, location), now.Add(delay)) +} + +func TestRepairOffsetIsPerSyncer(t *testing.T) { + t.Parallel() + + first := New(&fakeClient{}, nil, nil) + second := New(&fakeClient{}, nil, nil) + first.SetRepairOffset(15 * time.Minute) + second.SetRepairOffset(45 * time.Minute) + + require.Equal(t, 15*time.Minute, first.repairOffset()) + require.Equal(t, 45*time.Minute, second.repairOffset()) + + first.SetRepairOffset(0) + require.Zero(t, first.repairOffset()) + require.Equal(t, 45*time.Minute, second.repairOffset()) +} + +func TestRepairOffsetConcurrentAccess(t *testing.T) { + t.Parallel() + + svc := New(&fakeClient{}, nil, nil) + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + for i := range 1_000 { + svc.SetRepairOffset(time.Duration(i%5) * time.Minute) + } + }() + go func() { + defer wg.Done() + for range 1_000 { + _ = svc.repairOffset() + } + }() + wg.Wait() + + svc.SetRepairOffset(15 * time.Minute) + require.Equal(t, 15*time.Minute, svc.repairOffset()) +} + func TestRunTailWithRepairLoopJoinsTailOnCancellation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) @@ -1685,6 +3085,98 @@ func TestRunTailPreservesGatewayOpenErrorWithoutMessageFailure(t *testing.T) { require.Empty(t, report.Failures) } +func TestRunTailBoundsNonCooperativeShutdown(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + client := &stuckTailClient{ + fakeClient: &fakeClient{}, + started: make(chan struct{}), + tailRelease: make(chan struct{}), + closeStarted: make(chan struct{}), + closeRelease: make(chan struct{}), + } + svc := New(client, s, nil) + svc.tailShutdownTimeout = 25 * time.Millisecond + done := make(chan error, 1) + go func() { + done <- svc.RunTail(ctx, nil, 0) + }() + + select { + case <-client.started: + case <-time.After(time.Second): + t.Fatal("tail did not start") + } + cancel() + select { + case <-client.closeStarted: + case <-time.After(time.Second): + t.Fatal("client close did not start") + } + + select { + case err := <-done: + require.ErrorContains(t, err, "tail shutdown timed out") + case <-time.After(time.Second): + t.Fatal("RunTail shutdown remained unbounded") + } + close(client.tailRelease) + close(client.closeRelease) +} + +func TestRunTailReturnsCloseFailureOnShutdown(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + closeErr := errors.New("close failed") + client := &failingCloseTailClient{ + fakeClient: &fakeClient{}, + started: make(chan struct{}), + closeErr: closeErr, + } + svc := New(client, s, nil) + done := make(chan error, 1) + go func() { + done <- svc.RunTail(ctx, nil, 0) + }() + + select { + case <-client.started: + case <-time.After(time.Second): + t.Fatal("tail did not start") + } + cancel() + require.ErrorIs(t, <-done, closeErr) +} + +func TestRunTailPreservesGatewayAndCloseFailures(t *testing.T) { + ctx := context.Background() + + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + gatewayErr := errors.New("gateway exited") + closeErr := errors.New("gateway close failed") + client := &exitingFailingCloseTailClient{ + fakeClient: &fakeClient{}, + tailErr: gatewayErr, + closeErr: closeErr, + } + svc := New(client, s, nil) + + err = svc.RunTail(ctx, nil, 0) + require.ErrorIs(t, err, gatewayErr) + require.ErrorIs(t, err, closeErr) +} + func TestTailReadyCallback(t *testing.T) { t.Parallel() @@ -1906,6 +3398,38 @@ func (h *capturingTailHandler) OnMessageUpdate(ctx context.Context, msg *discord panic("sensitive syncer message update panic") } +type reconciledRefetchTailHandler struct { + *tailHandler + drained chan struct{} + drainOnce sync.Once + failureReports atomic.Int32 + failureRecords atomic.Int32 +} + +func (h *reconciledRefetchTailHandler) OnMessageCreate( + ctx context.Context, + msg *discordgo.Message, +) error { + if err := h.tailHandler.OnMessageCreate(ctx, msg); err != nil { + return err + } + if msg != nil && msg.ID == syncerTailSentinelMessageID { + h.drainOnce.Do(func() { close(h.drained) }) + } + return nil +} + +func (h *reconciledRefetchTailHandler) OnTailFailure(discordclient.TailFailure) { + h.failureReports.Add(1) +} + +func (h *reconciledRefetchTailHandler) RecordTailFailure( + failure discordclient.TailFailure, +) error { + h.failureRecords.Add(1) + return h.tailHandler.RecordTailFailure(failure) +} + type panicReplayTailHandler struct { *tailHandler panicValue any @@ -2002,6 +3526,105 @@ func newSyncerTailMessageUpdateGateway( return httptest.NewServer(mux) } +func newSyncerTailUnknownMessageGateway( + t *testing.T, + messageID string, + updateCount int, + clientRefetches *atomic.Int32, +) *httptest.Server { + t.Helper() + + mux := http.NewServeMux() + mux.HandleFunc("/api/v10/channels/c1/messages/"+messageID, func(w http.ResponseWriter, _ *http.Request) { + clientRefetches.Add(1) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusNotFound) + _ = json.NewEncoder(w).Encode(map[string]any{ + "message": "Unknown Message", + "code": 10008, + }) + }) + mux.HandleFunc("/api/v10/gateway", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"url": "ws://" + r.Host + "/gateway"}) + }) + + upgrader := websocket.Upgrader{} + gatewayHandler := func(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + t.Errorf("upgrade gateway: %v", err) + return + } + defer func() { _ = conn.Close() }() + + if err := conn.WriteJSON(map[string]any{ + "op": 10, + "d": map[string]any{"heartbeat_interval": 1000}, + }); err != nil { + t.Errorf("write hello: %v", err) + return + } + if _, _, err := conn.ReadMessage(); err != nil { + t.Errorf("read identify: %v", err) + return + } + if err := conn.WriteJSON(map[string]any{ + "op": 0, + "t": "READY", + "s": 1, + "d": map[string]any{ + "session_id": "session", + "user": map[string]any{"id": "bot", "username": "bot"}, + }, + }); err != nil { + t.Errorf("write ready: %v", err) + return + } + for i := range updateCount { + if err := conn.WriteJSON(map[string]any{ + "op": 0, + "t": "MESSAGE_UPDATE", + "s": i + 2, + "d": map[string]any{ + "id": messageID, + "guild_id": "g1", + "channel_id": "c1", + }, + }); err != nil { + t.Errorf("write update: %v", err) + return + } + } + if err := conn.WriteJSON(map[string]any{ + "op": 0, + "t": "MESSAGE_CREATE", + "s": updateCount + 2, + "d": map[string]any{ + "id": syncerTailSentinelMessageID, + "guild_id": "g1", + "channel_id": "c1", + "content": "ordered sentinel", + "timestamp": time.Now().UTC().Format(time.RFC3339), + "author": map[string]any{"id": "u1", "username": "user"}, + }, + }); err != nil { + t.Errorf("write sentinel: %v", err) + return + } + for { + if _, _, err := conn.ReadMessage(); err != nil { + return + } + } + } + mux.HandleFunc("/gateway", gatewayHandler) + mux.HandleFunc("/gateway/", gatewayHandler) + return httptest.NewServer(mux) +} + +const syncerTailSentinelMessageID = "223456789012345678" + func newSyncerTailMessagePanicGateway( t *testing.T, exactMessage *discordgo.Message, @@ -2116,3 +3739,99 @@ func setDiscordTailHandlerTimeout(t *testing.T, client *discordclient.Client, ti // Keep the production client API unchanged while making the cooperative timeout test fast. reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())).Elem().SetInt(int64(timeout)) } + +type gatewayExitDuringRepairClient struct { + *fakeClient + tailStarted chan struct{} + tailExit chan error + repairStarted chan struct{} + repairCanceled chan struct{} + tailStartOnce sync.Once + repairOnce sync.Once + cancelOnce sync.Once +} + +func (c *gatewayExitDuringRepairClient) Tail(ctx context.Context, _ discordclient.EventHandler) error { + c.tailStartOnce.Do(func() { close(c.tailStarted) }) + select { + case err := <-c.tailExit: + return err + case <-ctx.Done(): + return nil + } +} + +func (c *gatewayExitDuringRepairClient) ChannelMessages( + ctx context.Context, + _ string, + _ int, + _ string, + _ string, +) ([]*discordgo.Message, error) { + c.repairOnce.Do(func() { close(c.repairStarted) }) + <-ctx.Done() + c.cancelOnce.Do(func() { close(c.repairCanceled) }) + return nil, ctx.Err() +} + +type quietTailClient struct { + *fakeClient +} + +func (c *quietTailClient) Tail(ctx context.Context, _ discordclient.EventHandler) error { + <-ctx.Done() + return nil +} + +type stuckTailClient struct { + *fakeClient + started chan struct{} + tailRelease chan struct{} + closeStarted chan struct{} + closeRelease chan struct{} + startOnce sync.Once + closeOnce sync.Once +} + +func (c *stuckTailClient) Tail(context.Context, discordclient.EventHandler) error { + c.startOnce.Do(func() { close(c.started) }) + <-c.tailRelease + return nil +} + +func (c *stuckTailClient) Close() error { + c.closeOnce.Do(func() { close(c.closeStarted) }) + <-c.closeRelease + return nil +} + +type failingCloseTailClient struct { + *fakeClient + started chan struct{} + closeErr error + startOnce sync.Once +} + +func (c *failingCloseTailClient) Tail(ctx context.Context, _ discordclient.EventHandler) error { + c.startOnce.Do(func() { close(c.started) }) + <-ctx.Done() + return nil +} + +func (c *failingCloseTailClient) Close() error { + return c.closeErr +} + +type exitingFailingCloseTailClient struct { + *fakeClient + tailErr error + closeErr error +} + +func (c *exitingFailingCloseTailClient) Tail(context.Context, discordclient.EventHandler) error { + return c.tailErr +} + +func (c *exitingFailingCloseTailClient) Close() error { + return c.closeErr +} diff --git a/internal/syncer/syncer_test.go b/internal/syncer/syncer_test.go index ac6e4ce4..21b2f7b1 100644 --- a/internal/syncer/syncer_test.go +++ b/internal/syncer/syncer_test.go @@ -19,6 +19,9 @@ type fakeClient struct { guilds []*discordgo.UserGuild guildByID map[string]*discordgo.Guild channels map[string][]*discordgo.Channel + channelsByID map[string]*discordgo.Channel + channelErrors map[string]error + channelCalls map[string]int activeThreads map[string][]*discordgo.Channel guildThreads map[string][]*discordgo.Channel threadErrors map[string]error @@ -30,6 +33,7 @@ type fakeClient struct { members map[string][]*discordgo.Member messages map[string][]*discordgo.Message messageErrors map[string]error + messageByIDErrs map[string]error messageCalls map[string]int exactMessageCalls int messageRequests []messageRequest @@ -73,6 +77,38 @@ func (f *fakeClient) GuildChannels(_ context.Context, guildID string) ([]*discor return f.channels[guildID], nil } +func (f *fakeClient) Channel(_ context.Context, channelID string) (*discordgo.Channel, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.channelCalls == nil { + f.channelCalls = make(map[string]int) + } + f.channelCalls[channelID]++ + if err := f.channelErrors[channelID]; err != nil { + return nil, err + } + if channel := f.channelsByID[channelID]; channel != nil { + return channel, nil + } + channelGroups := []map[string][]*discordgo.Channel{ + f.channels, + f.activeThreads, + f.guildThreads, + f.publicArchived, + f.privateArchive, + } + for _, groups := range channelGroups { + for _, channels := range groups { + for _, channel := range channels { + if channel != nil && channel.ID == channelID { + return channel, nil + } + } + } + } + return nil, nil +} + func (f *fakeClient) ThreadsActive(_ context.Context, channelID string) ([]*discordgo.Channel, error) { f.threadCalls++ if err := f.threadErrors[channelID]; err != nil { @@ -211,7 +247,11 @@ func (f *fakeClient) ChannelMessages(ctx context.Context, channelID string, limi func (f *fakeClient) ChannelMessage(_ context.Context, channelID, messageID string) (*discordgo.Message, error) { f.mu.Lock() f.exactMessageCalls++ + err := f.messageByIDErrs[channelID+"/"+messageID] f.mu.Unlock() + if err != nil { + return nil, err + } for _, msg := range f.messages[channelID] { if msg.ID == messageID { return msg, nil diff --git a/internal/syncer/tail.go b/internal/syncer/tail.go index 85e57e55..23ff8ff2 100644 --- a/internal/syncer/tail.go +++ b/internal/syncer/tail.go @@ -13,76 +13,205 @@ import ( "github.com/openclaw/discrawl/internal/store" ) +func (s *Syncer) SetRepairOffset(offset time.Duration) { + if s == nil { + return + } + if offset <= 0 { + offset = 0 + } + s.tailRepairOffsetMu.Lock() + s.tailRepairOffset = offset + s.tailRepairOffsetMu.Unlock() +} + +func (s *Syncer) repairOffset() time.Duration { + if s == nil { + return 0 + } + s.tailRepairOffsetMu.RLock() + defer s.tailRepairOffsetMu.RUnlock() + return s.tailRepairOffset +} + func (s *Syncer) RunTail(ctx context.Context, guildIDs []string, repairEvery time.Duration) error { if err := s.importTailMessageFailureFallbacks(ctx); err != nil { return err } handler := &tailHandler{ - guilds: makeGuildSet(guildIDs), - store: s.store, - client: s.client, - attachmentTextEnabled: s.attachmentTextEnabled, - onReady: s.tailReady, - logger: s.logger, - } - if repairEvery <= 0 { - return s.client.Tail(ctx, handler) + guilds: makeGuildSet(guildIDs), + store: s.store, + client: s.client, + attachmentTextEnabled: s.attachmentTextEnabled, + onReady: s.tailReady, + logger: s.logger, + exclusions: s.channelExclusions, + channels: map[string]tailChannel{}, + kindExcludedChannelIDs: map[string]struct{}{}, + } + if err := handler.seedChannelExclusions(ctx); err != nil { + return fmt.Errorf("seed tail channel exclusions: %w", err) } tailCtx, cancelTail := context.WithCancel(ctx) defer cancelTail() - var closeOnce sync.Once - closeClient := func() { - if closeable, ok := s.client.(closeableClient); ok { - _ = closeable.Close() - } - } tailDone := make(chan error, 1) go func() { tailDone <- s.client.Tail(tailCtx, handler) }() - ticker := time.NewTicker(repairEvery) - defer ticker.Stop() + + var repairTimer *time.Timer + var repairC <-chan time.Time + defer func() { + if repairTimer != nil { + repairTimer.Stop() + } + }() + repairOffset := s.repairOffset() + scheduleRepair := func() { + if repairEvery <= 0 { + repairC = nil + return + } + delay := nextTailRepairDelay(time.Now(), repairEvery, repairOffset) + if repairTimer == nil { + repairTimer = time.NewTimer(delay) + } else { + if !repairTimer.Stop() { + select { + case <-repairTimer.C: + default: + } + } + repairTimer.Reset(delay) + } + repairC = repairTimer.C + } + scheduleRepair() + var activeRepair *tailRepairRun var repairDone <-chan tailRepairResult for { select { case <-ctx.Done(): - cancelTail() - repairErr := s.joinTailRepair(activeRepair, "parent_shutdown") - closeOnce.Do(closeClient) - tailErr := <-tailDone - if discordclient.IsFatalTailError(tailErr) { - if repairErr != nil { - return errors.Join(tailErr, repairErr) - } - return tailErr - } - if repairErr != nil { - return repairErr - } - return nil + return s.finishTailRun( + cancelTail, + tailDone, + activeRepair, + nil, + false, + true, + "parent_shutdown", + ) case err := <-tailDone: - cancelTail() - repairErr := s.joinTailRepair(activeRepair, "tail_return") - closeOnce.Do(closeClient) - if repairErr != nil { - return errors.Join(err, repairErr) + return s.finishTailRun( + cancelTail, + nil, + activeRepair, + err, + true, + false, + "tail_return", + ) + case <-repairC: + repairC = nil + activeRepair = s.startTailRepair(tailCtx, guildIDs) + if activeRepair == nil { + scheduleRepair() + continue } - return err + repairDone = activeRepair.done case result := <-repairDone: s.logTailRepairResult(result) activeRepair = nil repairDone = nil - case <-ticker.C: - if activeRepair != nil { - continue - } - activeRepair = s.startTailRepair(ctx, guildIDs) - if activeRepair != nil { - repairDone = activeRepair.done + // Schedule from completion so an overrun cannot immediately replay stale ticks. + scheduleRepair() + } + } +} + +func (s *Syncer) finishTailRun( + cancelTail context.CancelFunc, + tailDone <-chan error, + repair *tailRepairRun, + tailErr error, + tailFinished bool, + requestedShutdown bool, + repairJoinReason string, +) error { + cancelTail() + + timeout := s.tailShutdownTimeout + if timeout <= 0 { + timeout = defaultTailShutdownTimeout + } + startedAt := time.Now() + repairErr := s.joinTailRepair(repair, repairJoinReason, timeout) + + var closeDone <-chan error + if closeable, ok := s.client.(closeableClient); ok { + done := make(chan error, 1) + closeDone = done + go func() { + done <- closeable.Close() + }() + } + + if tailFinished { + tailDone = nil + } + if tailDone == nil && closeDone == nil { + return tailRunResult(requestedShutdown, tailErr, repairErr, nil, nil) + } + + remaining := timeout - time.Since(startedAt) + if remaining <= 0 { + return tailRunResult( + requestedShutdown, + tailErr, + repairErr, + nil, + fmt.Errorf("tail shutdown timed out after %s", timeout), + ) + } + timer := time.NewTimer(remaining) + defer timer.Stop() + + var closeErr error + for tailDone != nil || closeDone != nil { + select { + case err := <-tailDone: + if tailErr == nil { + tailErr = err } + tailDone = nil + case err := <-closeDone: + closeErr = err + closeDone = nil + case <-timer.C: + return tailRunResult( + requestedShutdown, + tailErr, + repairErr, + closeErr, + fmt.Errorf("tail shutdown timed out after %s", timeout), + ) } } + return tailRunResult(requestedShutdown, tailErr, repairErr, closeErr, nil) +} + +func tailRunResult( + requestedShutdown bool, + tailErr error, + repairErr error, + closeErr error, + shutdownErr error, +) error { + if requestedShutdown && !discordclient.IsFatalTailError(tailErr) { + tailErr = nil + } + return errors.Join(tailErr, repairErr, closeErr, shutdownErr) } const defaultTailRepairJoinTimeout = 5 * time.Second @@ -112,7 +241,7 @@ func (s *Syncer) importTailMessageFailureFallbacks(ctx context.Context) error { } func (s *Syncer) startTailRepair(ctx context.Context, guildIDs []string) *tailRepairRun { - if s == nil || !s.tailRepairMu.TryLock() { + if s == nil || !s.tailRepairRunMu.TryLock() { if s != nil && s.logger != nil { s.logger.Warn("tail repair start skipped", "reason", "repair_already_running") } @@ -122,12 +251,8 @@ func (s *Syncer) startTailRepair(ctx context.Context, guildIDs []string) *tailRe done := make(chan tailRepairResult, 1) startedAt := time.Now() go func() { - defer s.tailRepairMu.Unlock() - _, err := s.runTailRepair(repairCtx, SyncOptions{ - GuildIDs: guildIDs, - Full: false, - RepairReason: "tail_repair", - }) + defer s.tailRepairRunMu.Unlock() + _, err := s.runTailRepair(repairCtx, tailRepairSyncOptions(guildIDs)) done <- tailRepairResult{err: err, elapsed: time.Since(startedAt)} }() return &tailRepairRun{cancel: cancel, done: done} @@ -140,7 +265,7 @@ func (s *Syncer) runTailRepair(ctx context.Context, opts SyncOptions) (SyncStats return s.Sync(ctx, opts) } -func (s *Syncer) joinTailRepair(repair *tailRepairRun, reason string) error { +func (s *Syncer) joinTailRepair(repair *tailRepairRun, reason string, maxWait time.Duration) error { if repair == nil { return nil } @@ -150,6 +275,9 @@ func (s *Syncer) joinTailRepair(repair *tailRepairRun, reason string) error { if timeout <= 0 { timeout = defaultTailRepairJoinTimeout } + if maxWait > 0 && timeout > maxWait { + timeout = maxWait + } timer := time.NewTimer(timeout) defer timer.Stop() outcome := "timed_out" @@ -189,14 +317,79 @@ func (s *Syncer) logTailRepairResult(result tailRepairResult) { ) } +func tailRepairSyncOptions(guildIDs []string) SyncOptions { + return SyncOptions{ + GuildIDs: append([]string(nil), guildIDs...), + Full: false, + SkipMembers: true, + LatestOnly: true, + RepairReason: "tail_repair", + } +} + +func nextTailRepairDelay(now time.Time, repairEvery, repairOffset time.Duration) time.Duration { + if repairEvery <= 0 { + return 0 + } + if repairOffset <= 0 { + return repairEvery + } + normalizedOffset := repairOffset % repairEvery + civilNow := civilTimelineTime(now) + nextCivil := civilNow.Truncate(repairEvery).Add(normalizedOffset) + if !nextCivil.After(civilNow) { + nextCivil = nextCivil.Add(repairEvery) + } + for { + // A candidate that does not round-trip is a nonexistent civil slot. + next := time.Date( + nextCivil.Year(), + nextCivil.Month(), + nextCivil.Day(), + nextCivil.Hour(), + nextCivil.Minute(), + nextCivil.Second(), + nextCivil.Nanosecond(), + now.Location(), + ) + if civilTimelineTime(next).Equal(nextCivil) && next.After(now) { + return next.Sub(now) + } + nextCivil = nextCivil.Add(repairEvery) + } +} + +func civilTimelineTime(value time.Time) time.Time { + return time.Date( + value.Year(), + value.Month(), + value.Day(), + value.Hour(), + value.Minute(), + value.Second(), + value.Nanosecond(), + time.UTC, + ) +} + type tailHandler struct { - guilds map[string]struct{} - store *store.Store - client Client - attachmentTextEnabled bool - failureLedgerTimeout time.Duration - onReady func(context.Context) error - logger *slog.Logger + guilds map[string]struct{} + store *store.Store + client Client + attachmentTextEnabled bool + failureLedgerTimeout time.Duration + onReady func(context.Context) error + logger *slog.Logger + exclusions channelExclusions + channelResolveMu sync.Mutex + exclusionMu sync.RWMutex + channels map[string]tailChannel + kindExcludedChannelIDs map[string]struct{} +} + +type tailChannel struct { + parentID string + kind string } func (t *tailHandler) OnTailReady(ctx context.Context) error { @@ -231,7 +424,14 @@ func (t *tailHandler) OnTailFailure(failure discordclient.TailFailure) { func (t *tailHandler) OnMessageCreate(ctx context.Context, msg *discordgo.Message) error { discordclient.UpdateTailFailureStage(ctx, discordclient.TailFailureStageHandler) - if !t.allowGuild(msg.GuildID) { + if msg == nil || !t.allowGuild(msg.GuildID) { + return nil + } + excluded, err := t.ensureMessageChannel(ctx, msg.GuildID, msg.ChannelID) + if err != nil { + return err + } + if excluded { return nil } discordclient.UpdateTailFailureStage(ctx, discordclient.TailFailureStageMessageBuild) @@ -266,12 +466,43 @@ func (t *tailHandler) OnMessageUpdate(ctx context.Context, msg *discordgo.Messag if msg.GuildID != "" && !t.allowGuild(msg.GuildID) { return nil } - var err error - msg, err = t.messageUpdateSnapshot(ctx, msg) + guildID, channelID, messageID := msg.GuildID, msg.ChannelID, msg.ID + if guildID == "" { + var err error + msg, err = t.messageUpdateSnapshot(ctx, msg) + if err != nil { + if isUnknownMessage(err) { + return t.reconcileUnknownMessageUpdate(ctx, messageID) + } + return err + } + if msg == nil || !t.allowGuild(msg.GuildID) { + return nil + } + } else { + excluded, err := t.ensureMessageChannel(ctx, guildID, channelID) + if err != nil { + return err + } + if excluded { + return nil + } + msg, err = t.messageUpdateSnapshot(ctx, msg) + if err != nil { + if isUnknownMessage(err) { + return t.reconcileUnknownMessageUpdate(ctx, messageID) + } + return err + } + } + if msg == nil || !t.allowGuild(msg.GuildID) { + return nil + } + excluded, err := t.ensureMessageChannel(ctx, msg.GuildID, msg.ChannelID) if err != nil { return err } - if msg == nil || !t.allowGuild(msg.GuildID) { + if excluded { return nil } discordclient.UpdateTailFailureStage(ctx, discordclient.TailFailureStageMessageBuild) @@ -294,6 +525,18 @@ func (t *tailHandler) OnMessageUpdate(ctx context.Context, msg *discordgo.Messag return t.resolveMessageFailure(ctx, msg.GuildID, msg.ChannelID, msg.ID, "update") } +func (t *tailHandler) OnMessageUpdateRefetchFailure( + ctx context.Context, + msg *discordgo.Message, + refetchErr error, +) (bool, error) { + if msg == nil || msg.ID == "" || !isUnknownMessage(refetchErr) { + return false, t.OnMessageUpdate(ctx, msg) + } + err := t.reconcileUnknownMessageUpdate(ctx, msg.ID) + return err == nil, err +} + func (t *tailHandler) messageUpdateSnapshot(ctx context.Context, msg *discordgo.Message) (*discordgo.Message, error) { if t.client == nil || msg.ChannelID == "" || msg.ID == "" { if isPartialMessageUpdate(msg) { @@ -359,9 +602,52 @@ func isPartialMessageUpdate(msg *discordgo.Message) bool { return msg == nil || msg.Author == nil || msg.Timestamp.IsZero() } +func (t *tailHandler) reconcileUnknownMessageUpdate( + ctx context.Context, + messageID string, +) error { + guildID, channelID, exists, err := t.store.MessageScope(ctx, messageID) + if err != nil { + return err + } + if !exists || !t.allowGuild(guildID) { + return t.resolveMessageUpdateFailuresByID(ctx, messageID) + } + excluded, err := t.ensureMessageChannel(ctx, guildID, channelID) + if err != nil { + return err + } + if excluded { + return t.resolveMessageUpdateFailuresByID(ctx, messageID) + } + payload := map[string]any{ + "id": messageID, + "guild_id": guildID, + "channel_id": channelID, + "reason": "unknown_message_during_update_fetch", + "discord_code": 10008, + } + discordclient.UpdateTailFailureStage(ctx, discordclient.TailFailureStageCanonicalDelete) + if err := t.store.MarkMessageDeleted(ctx, guildID, channelID, messageID, payload); err != nil { + return err + } + discordclient.UpdateTailFailureStage(ctx, discordclient.TailFailureStageStateUpdate) + if err := t.store.SetSyncState(ctx, "tail:last_event", messageID); err != nil { + return err + } + return t.resolveMessageUpdateFailuresByID(ctx, messageID) +} + func (t *tailHandler) OnMessageDelete(ctx context.Context, evt *discordgo.MessageDelete) error { discordclient.UpdateTailFailureStage(ctx, discordclient.TailFailureStageHandler) - if !t.allowGuild(evt.GuildID) { + if evt == nil || !t.allowGuild(evt.GuildID) { + return nil + } + excluded, err := t.ensureMessageChannel(ctx, evt.GuildID, evt.ChannelID) + if err != nil { + return err + } + if excluded { return nil } discordclient.UpdateTailFailureStage(ctx, discordclient.TailFailureStageCanonicalDelete) @@ -376,10 +662,12 @@ func (t *tailHandler) OnMessageDelete(ctx context.Context, evt *discordgo.Messag } func (t *tailHandler) OnChannelUpsert(ctx context.Context, channel *discordgo.Channel) error { - if !t.allowGuild(channel.GuildID) { + if channel == nil || !t.allowGuild(channel.GuildID) { return nil } - return t.store.UpsertChannel(ctx, toChannelRecord(channel, marshalJSONString(channel, "{}"))) + t.channelResolveMu.Lock() + defer t.channelResolveMu.Unlock() + return t.persistChannelHierarchyLocked(ctx, channel, map[string]struct{}{}) } func (t *tailHandler) OnMemberUpsert(ctx context.Context, guildID string, member *discordgo.Member) error { @@ -407,3 +695,225 @@ func (t *tailHandler) allowGuild(guildID string) bool { _, ok := t.guilds[guildID] return ok } + +func (t *tailHandler) ensureMessageChannel(ctx context.Context, guildID, channelID string) (bool, error) { + if channelID == "" { + return false, errors.New("message event missing channel id") + } + if t.channelHierarchyKnown(channelID) { + return t.excludeChannel(channelID), nil + } + + t.channelResolveMu.Lock() + defer t.channelResolveMu.Unlock() + if t.channelHierarchyKnown(channelID) { + return t.excludeChannel(channelID), nil + } + if t.client == nil { + return false, fmt.Errorf("resolve unknown channel %s: missing discord client", channelID) + } + if err := t.resolveUnknownChannelLocked(ctx, guildID, channelID, map[string]struct{}{}); err != nil { + return false, err + } + return t.excludeChannel(channelID), nil +} + +func (t *tailHandler) resolveUnknownChannelLocked( + ctx context.Context, + expectedGuildID string, + channelID string, + resolving map[string]struct{}, +) error { + if t.channelHierarchyKnown(channelID) { + return nil + } + if _, ok := resolving[channelID]; ok { + return fmt.Errorf("resolve channel hierarchy cycle at %s", channelID) + } + resolving[channelID] = struct{}{} + defer delete(resolving, channelID) + + channel, err := t.client.Channel(ctx, channelID) + if err != nil { + return fmt.Errorf("fetch channel %s: %w", channelID, err) + } + if channel == nil { + return fmt.Errorf("fetch channel %s: empty response", channelID) + } + resolved := *channel + channel = &resolved + if channel.ID == "" { + channel.ID = channelID + } + if channel.ID != channelID { + return fmt.Errorf("fetch channel %s: received channel %s", channelID, channel.ID) + } + if channel.GuildID == "" { + channel.GuildID = expectedGuildID + } + if expectedGuildID != "" && channel.GuildID != expectedGuildID { + return fmt.Errorf( + "fetch channel %s: expected guild %s, received %s", + channelID, + expectedGuildID, + channel.GuildID, + ) + } + if !t.allowGuild(channel.GuildID) { + return fmt.Errorf("fetch channel %s: guild %s is outside the tail scope", channelID, channel.GuildID) + } + return t.persistChannelHierarchyLocked(ctx, channel, resolving) +} + +func (t *tailHandler) persistChannelHierarchyLocked( + ctx context.Context, + channel *discordgo.Channel, + resolving map[string]struct{}, +) error { + if channel == nil || !t.allowGuild(channel.GuildID) { + return nil + } + if channel.ID == "" { + return errors.New("channel event missing channel id") + } + resolveParent := channel.ParentID != "" && !t.channelHierarchyKnown(channel.ParentID) + if resolveParent { + if t.client == nil { + return fmt.Errorf( + "resolve parent channel %s for %s: missing discord client", + channel.ParentID, + channel.ID, + ) + } + if err := t.resolveUnknownChannelLocked(ctx, channel.GuildID, channel.ParentID, resolving); err != nil { + return fmt.Errorf("resolve parent channel %s for %s: %w", channel.ParentID, channel.ID, err) + } + } + if err := t.store.UpsertChannel(ctx, toChannelRecord(channel, marshalJSONString(channel, "{}"))); err != nil { + return err + } + t.trackChannelExclusion(channel) + return nil +} + +func (t *tailHandler) channelHierarchyKnown(channelID string) bool { + t.exclusionMu.RLock() + defer t.exclusionMu.RUnlock() + return t.channelHierarchyKnownLocked(channelID, nil) +} + +func (t *tailHandler) channelHierarchyKnownLocked(channelID string, visiting map[string]struct{}) bool { + channel, ok := t.channels[channelID] + if !ok { + return false + } + if channel.parentID == "" { + return true + } + if visiting == nil { + visiting = map[string]struct{}{} + } + if _, ok := visiting[channelID]; ok { + return false + } + visiting[channelID] = struct{}{} + defer delete(visiting, channelID) + return t.channelHierarchyKnownLocked(channel.parentID, visiting) +} + +func (t *tailHandler) seedChannelExclusions(ctx context.Context) error { + if t.store == nil { + return nil + } + channels, err := t.store.Channels(ctx, "") + if err != nil { + return err + } + t.exclusionMu.Lock() + defer t.exclusionMu.Unlock() + t.channels = make(map[string]tailChannel, len(channels)) + if t.kindExcludedChannelIDs == nil { + t.kindExcludedChannelIDs = map[string]struct{}{} + } + for _, channel := range channels { + t.channels[channel.ID] = tailChannel{ + parentID: storedChannelParentID(channel), + kind: channel.Kind, + } + } + t.rebuildChannelExclusionsLocked() + return nil +} + +func (t *tailHandler) excludeChannel(channelID string) bool { + if t.exclusions.excludesID(channelID) { + return true + } + t.exclusionMu.RLock() + defer t.exclusionMu.RUnlock() + _, ok := t.kindExcludedChannelIDs[channelID] + return ok +} + +func (t *tailHandler) trackChannelExclusion(channel *discordgo.Channel) { + if channel == nil { + return + } + t.exclusionMu.Lock() + defer t.exclusionMu.Unlock() + if t.channels == nil { + t.channels = map[string]tailChannel{} + } + t.channels[channel.ID] = tailChannel{ + parentID: channel.ParentID, + kind: channelKind(channel), + } + t.rebuildChannelExclusionsLocked() +} + +func (t *tailHandler) rebuildChannelExclusionsLocked() { + if t.kindExcludedChannelIDs == nil { + t.kindExcludedChannelIDs = map[string]struct{}{} + } + clear(t.kindExcludedChannelIDs) + decisions := make(map[string]bool, len(t.channels)) + var excludes func(string, map[string]struct{}) bool + excludes = func(channelID string, visiting map[string]struct{}) bool { + if decision, ok := decisions[channelID]; ok { + return decision + } + if t.exclusions.excludesID(channelID) { + decisions[channelID] = true + return true + } + channel, ok := t.channels[channelID] + if !ok { + decisions[channelID] = false + return false + } + if t.exclusions.excludesKind(channel.kind) { + decisions[channelID] = true + return true + } + if channel.parentID == "" { + decisions[channelID] = false + return false + } + if visiting == nil { + visiting = map[string]struct{}{} + } + if _, ok := visiting[channelID]; ok { + return false + } + visiting[channelID] = struct{}{} + excluded := excludes(channel.parentID, visiting) + delete(visiting, channelID) + decisions[channelID] = excluded + return excluded + } + for channelID := range t.channels { + if excludes(channelID, nil) { + t.kindExcludedChannelIDs[channelID] = struct{}{} + } + } +}