From 3eff325e1c6c436948b4f7d21fb85814d8741f9b Mon Sep 17 00:00:00 2001 From: Hannes Rudolph <49103247+hannesrudolph@users.noreply.github.com> Date: Thu, 9 Jul 2026 13:18:31 -0600 Subject: [PATCH 1/7] feat: add hybrid local archive collection mode --- README.md | 2 + docs/commands/sync.md | 5 + docs/commands/tail.md | 7 + docs/commands/update.md | 6 + docs/configuration.md | 10 + docs/guides/git-snapshots.md | 5 + docs/guides/sync-sources.md | 8 + internal/cli/admin_commands.go | 66 +++-- internal/cli/cli.go | 25 +- internal/cli/cli_test.go | 42 +++ internal/cli/share_commands.go | 10 + internal/config/config.go | 35 ++- internal/config/config_test.go | 23 ++ internal/discorddesktop/import.go | 122 ++++++-- .../discorddesktop/import_helpers_test.go | 24 ++ internal/discorddesktop/import_run.go | 2 + internal/share/import_filter.go | 160 ++++++++++ internal/share/merge_test.go | 102 +++++++ internal/share/share.go | 34 ++- internal/store/coverage.go | 76 ++++- internal/store/coverage_test.go | 255 +++++++++++++++- internal/store/store_test.go | 33 +++ internal/store/write.go | 19 ++ internal/syncer/channel_exclusions.go | 138 +++++++++ internal/syncer/extracted_test.go | 20 +- internal/syncer/failures.go | 3 + internal/syncer/failures_test.go | 52 ++++ internal/syncer/message_sync.go | 30 +- internal/syncer/message_sync_helpers_test.go | 8 +- internal/syncer/message_sync_timeout_test.go | 3 +- internal/syncer/syncer.go | 40 ++- internal/syncer/syncer_tail_test.go | 277 +++++++++++++++++- internal/syncer/tail.go | 212 ++++++++++++-- 33 files changed, 1706 insertions(+), 148 deletions(-) create mode 100644 internal/share/import_filter.go create mode 100644 internal/syncer/channel_exclusions.go diff --git a/README.md b/README.md index abd5dcac..8e3c6217 100644 --- a/README.md +++ b/README.md @@ -797,6 +797,8 @@ source = "both" # use "discord" for bot-only sync or "wiretap" for desktop-cache concurrency = 16 repair_every = "6h" full_history = true +exclude_channel_ids = [] +exclude_channel_kinds = [] attachment_text = true attachment_media = false max_attachment_bytes = 104857600 diff --git a/docs/commands/sync.md b/docs/commands/sync.md index 4e330d8f..a7b6701c 100644 --- a/docs/commands/sync.md +++ b/docs/commands/sync.md @@ -27,6 +27,7 @@ discrawl sync --source discord # bot API only; aliases: key, bot, api discrawl sync --source wiretap # desktop cache only; aliases: desktop, cache discrawl sync --guild 123456789012345678 --all-channels discrawl sync --channels 111,222 --since 2026-03-01T00:00:00Z +discrawl sync --exclude-channels 333,444 --exclude-channel-kinds announcement discrawl sync --with-embeddings discrawl sync --with-media ``` @@ -59,6 +60,8 @@ discrawl sync --with-media - `--all` - ignore `default_guild_id` and fan out across every discovered guild - `--guild ` / `--guilds ` - target specific guilds - `--channels ` - target specific channels (forum ids expand to threads) +- `--exclude-channels ` - omit channel messages even if a target or repair would otherwise select them +- `--exclude-channel-kinds ` - omit message channels by stored Discord kind, such as `announcement` - `--since ` - limit initial history and `--full` backfill to messages at or after this timestamp - `--concurrency ` - override worker count (default auto-sized: floor 8, cap 32) - `--skip-members` - refresh guild/channel/message data without crawling members @@ -69,6 +72,8 @@ discrawl sync --with-media ## Notes - `--latest-only` is the default for untargeted `sync`. Use `--all-channels` to opt out without doing a full historical crawl. +- Collection exclusions apply to Discord API sync and the wiretap portion of `source=both`; exclusions win over explicit targets. +- With `--update=auto`, the same exclusions filter message-scoped rows from the safe upstream merge while retaining channel metadata and existing local rows. - `--with-media` records expired or removed Discord CDN URLs as failed fetches with the HTTP status, commonly `404`. - `--with-media` updates the local cache only; run `publish --push` afterward to include cached non-DM media in the Git backup as gzip-compressed files. - `--since` does not mark older history as complete, so a later `sync --full` without `--since` can continue the backfill. diff --git a/docs/commands/tail.md b/docs/commands/tail.md index d094aa11..96fdc9e4 100644 --- a/docs/commands/tail.md +++ b/docs/commands/tail.md @@ -9,6 +9,7 @@ discrawl tail discrawl tail --guild 123456789012345678 discrawl tail --repair-every 30m discrawl tail --replay-failures-only +discrawl tail --repair-every 6h --repair-offset 15m ``` ## What it does @@ -18,6 +19,8 @@ discrawl tail --replay-failures-only - periodically runs a repair pass to catch anything the live stream missed - can replay a bounded set of unresolved exact-message failures without starting the Gateway tail +- ignores messages, edits, and deletes from channels excluded by + `[sync].exclude_channel_ids` or `[sync].exclude_channel_kinds` ## Flags @@ -25,6 +28,7 @@ discrawl tail --replay-failures-only - `--repair-every ` - frequency of the repair sweep - `--replay-failures-only` - replay unresolved exact-message tail failures and exit - `--replay-limit ` - maximum failures to inspect in replay-only mode (default and maximum: `25`) +- `--repair-offset ` - for positive values, align repairs to that wall-clock offset within each repair period ## Notes @@ -33,6 +37,9 @@ discrawl tail --replay-failures-only - terminates cleanly on SIGINT / SIGTERM and treats cancellation as normal exit - replay-only mode uses the normal exclusive writer lock and does not create Gateway message events or update `tail:last_event` +- retains excluded channel metadata while omitting their message traffic +- positive-offset repairs skip missed aligned slots after a long repair rather than running catch-up sweeps +- separation from another scheduled job is only stable when that job also uses a fixed calendar schedule; a drifting `StartInterval` schedule cannot guarantee the offset ## See also diff --git a/docs/commands/update.md b/docs/commands/update.md index 954a91af..c411fe9c 100644 --- a/docs/commands/update.md +++ b/docs/commands/update.md @@ -37,6 +37,12 @@ discrawl update --force --ref backup-2026-06-19 Normal updates preserve rows learned from live Discord or the desktop cache, even when those rows are absent from the Git snapshot. Generated event history and local sync cursors are not replayed during routine merges. If a safe merge is impossible, Discrawl keeps the current database, marks the snapshot as needing attention in `status --json`, and asks you to rerun with `--force`. +Routine safe merges also honor `[sync].exclude_channel_ids` and +`[sync].exclude_channel_kinds`. Excluded channel metadata and existing local +rows remain in place, while new message-scoped snapshot rows for those channels +are skipped. `--force` is still an explicit exact-reconciliation path and does +not provide this preservation guarantee. + ## How `sync` interacts `discrawl sync` does **not** auto-import the share unless `--update=auto` (safe merge when stale) or `--update=force` (exact replacement before live deltas). Routine live refreshes stay fast; explicit imports happen via `update`. diff --git a/docs/configuration.md b/docs/configuration.md index 0212aa04..b37d7362 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -60,7 +60,10 @@ token_keyring_account = "discord_bot_token" source = "both" # "discord" for bot-only sync, "wiretap" for desktop-cache-only import concurrency = 16 repair_every = "6h" +repair_offset = "0s" full_history = true +exclude_channel_ids = [] +exclude_channel_kinds = [] # for example, ["announcement"] attachment_text = true attachment_media = false max_attachment_bytes = 104857600 @@ -104,6 +107,13 @@ token_env = "DISCRAWL_REMOTE_TOKEN" stale_after = "" ``` +`sync.exclude_channel_ids` and `sync.exclude_channel_kinds` apply to Discord API +sync, live tail events, periodic tail repair, Discord Desktop cache import, and +routine safe Git snapshot merges. Exclusions win over an explicit +`sync --channels` target. Channel metadata and existing local rows are retained, +but new message-scoped rows from excluded channels are omitted. Forced exact +snapshot replacement remains a separate destructive operation. + `concurrency` is auto-sized at `init` to `min(32, max(8, GOMAXPROCS*2))`. ## Token resolution diff --git a/docs/guides/git-snapshots.md b/docs/guides/git-snapshots.md index 10c44776..b7014520 100644 --- a/docs/guides/git-snapshots.md +++ b/docs/guides/git-snapshots.md @@ -65,6 +65,11 @@ discrawl subscribe --no-auto-update https://github.com/example/discord-archive.g `discrawl update` runs the same safe pull/merge step manually. `discrawl update --force --ref ` reads historical Git objects directly and leaves the share checkout unchanged. Snapshot imports are delta-planned from crawlkit shard fingerprints. Older manifests without those fields fall back to Git blob identity, so the common publish shape only imports changed canonical shards. Routine merges preserve destination-only rows and do not replay generated event history or remote sync cursors. +In hybrid mode, routine safe merges honor the collection exclusions under +`[sync]`. They keep excluded channel metadata and previously stored local rows, +but skip new message-scoped rows from excluded channels. Forced replacement is +an explicit destructive exception. + Discrawl does not silently fall back to a full import. Removed shards and incompatible table changes leave the current database intact and require `discrawl update --force`. Forced updates replace public snapshot tables and rebuild search indexes; local DM rows remain untouched. `discrawl sync` does **not** auto-import the share unless `--update=auto` or `--update=force` is provided. Auto mode uses the safe merge; force mode performs exact replacement before live deltas. diff --git a/docs/guides/sync-sources.md b/docs/guides/sync-sources.md index f451bbec..ea827012 100644 --- a/docs/guides/sync-sources.md +++ b/docs/guides/sync-sources.md @@ -35,8 +35,16 @@ Run one explicit `--full` pass when you want a complete historical guild archive - `--guilds 123,456` runs an explicit set - `--all` ignores `default_guild_id` and fans out across every discovered guild - `--channels 111,222` targets specific channels (forum ids expand to their threads) +- `--exclude-channels 333,444` omits specific channel messages +- `--exclude-channel-kinds announcement` omits message channels by Discord kind - `--since ` limits initial history and `--full` backfill to messages at or after the timestamp; older history is not marked complete, so a later `sync --full` without `--since` can continue the backfill +The same exclusions apply to routine sync, historical backfill, live tail, +periodic tail repair, Discord Desktop cache import, and safe upstream snapshot +merges. Exclusions win over explicit channel targets. Safe merges retain channel +metadata and existing local rows while omitting new message-scoped rows for +excluded channels. + ## Performance and resilience - Long runs emit periodic progress logs to stderr. diff --git a/internal/cli/admin_commands.go b/internal/cli/admin_commands.go index dd413718..58b88109 100644 --- a/internal/cli/admin_commands.go +++ b/internal/cli/admin_commands.go @@ -114,6 +114,8 @@ func (r *runtime) runSync(args []string) error { allChannels := fs.Bool("all-channels", false, "") since := fs.String("since", "", "") channels := fs.String("channels", "", "") + excludeChannels := fs.String("exclude-channels", strings.Join(r.cfg.Sync.ExcludeChannelIDs, ","), "") + excludeChannelKinds := fs.String("exclude-channel-kinds", strings.Join(r.cfg.Sync.ExcludeChannelKinds, ","), "") concurrency := fs.Int("concurrency", r.cfg.Sync.Concurrency, "") source := fs.String("source", r.cfg.Sync.Source, "") withEmbeddings := fs.Bool("with-embeddings", false, "") @@ -121,6 +123,7 @@ func (r *runtime) runSync(args []string) error { skipMembers := fs.Bool("skip-members", false, "") withMembers := fs.Bool("with-members", false, "") latestOnly := fs.Bool("latest-only", false, "") + channelTimeout := fs.String("channel-timeout", "", "") guildsFlag := fs.String("guilds", "", "") guildFlag := fs.String("guild", "", "") updateMode := fs.String("update", "", "") @@ -151,21 +154,32 @@ func (r *runtime) runSync(args []string) error { } sinceTime = parsed } + var messageChannelTimeout time.Duration + if strings.TrimSpace(*channelTimeout) != "" { + parsed, err := time.ParseDuration(strings.TrimSpace(*channelTimeout)) + if err != nil || parsed <= 0 { + return usageErr(fmt.Errorf("invalid --channel-timeout %q", *channelTimeout)) + } + messageChannelTimeout = parsed + } guildIDs, err := r.resolveSyncGuildsAll(*guildFlag, *guildsFlag, *all) if err != nil { return usageErr(err) } defaultLatest := defaultLatestSyncMode(*full, *allChannels, *since, *channels) opts := syncer.SyncOptions{ - Full: *full, - GuildIDs: guildIDs, - ChannelIDs: csvList(*channels), - Concurrency: *concurrency, - Since: sinceTime, - Embeddings: *withEmbeddings, - SkipMembers: syncSkipsMembers(*skipMembers, *withMembers, defaultLatest), - RequireMembers: *withMembers, - LatestOnly: syncLatestOnly(*latestOnly, defaultLatest), + Full: *full, + GuildIDs: guildIDs, + ChannelIDs: csvList(*channels), + Concurrency: *concurrency, + Since: sinceTime, + Embeddings: *withEmbeddings, + SkipMembers: syncSkipsMembers(*skipMembers, *withMembers, defaultLatest), + RequireMembers: *withMembers, + LatestOnly: syncLatestOnly(*latestOnly, defaultLatest), + MessageChannelTimeout: messageChannelTimeout, + ExcludeChannelIDs: csvList(*excludeChannels), + ExcludeChannelKinds: csvList(*excludeChannelKinds), } return r.withSyncLock(func() error { return r.runSyncLocked(sources, opts, *withMedia) @@ -193,10 +207,12 @@ func (r *runtime) runSyncLocked(sources syncSources, opts syncer.SyncOptions, wi if sources.wiretap { r.setSyncLockPhase("wiretap import") stats, err := discorddesktop.Import(r.ctx, r.store, discorddesktop.Options{ - Path: r.cfg.Desktop.Path, - MaxFileBytes: r.cfg.Desktop.MaxFileBytes, - FullCache: r.cfg.Desktop.FullCache, - Now: r.now, + Path: r.cfg.Desktop.Path, + MaxFileBytes: r.cfg.Desktop.MaxFileBytes, + FullCache: r.cfg.Desktop.FullCache, + ExcludeChannelIDs: opts.ExcludeChannelIDs, + ExcludeChannelKinds: opts.ExcludeChannelKinds, + Now: r.now, }) if err != nil { return err @@ -332,6 +348,15 @@ func (r *runtime) runTail(args []string) error { repairEvery := fs.Duration("repair-every", mustDuration(r.cfg.Sync.RepairEvery), "") replayFailuresOnly := fs.Bool("replay-failures-only", false, "") replayLimit := fs.Int("replay-limit", syncer.TailMessageReplayLimit, "") + repairOffsetDefault := time.Duration(0) + if raw := strings.TrimSpace(r.cfg.Sync.RepairOffset); raw != "" { + parsed, err := time.ParseDuration(raw) + if err != nil { + return configErr(fmt.Errorf("parse sync.repair_offset: %w", err)) + } + repairOffsetDefault = parsed + } + repairOffset := fs.Duration("repair-offset", repairOffsetDefault, "") guildsFlag := fs.String("guilds", "", "") guildFlag := fs.String("guild", "", "") if err := fs.Parse(args); err != nil { @@ -368,6 +393,9 @@ func (r *runtime) runTail(args []string) error { } ctx, stop := signal.NotifyContext(r.ctx, os.Interrupt, syscall.SIGTERM) defer stop() + if configurable, ok := r.syncer.(repairOffsetConfigurer); ok { + configurable.SetRepairOffset(*repairOffset) + } if configurable, ok := r.syncer.(tailReadyConfigurer); ok { configurable.SetTailReadyCallback(func(context.Context) error { return r.activateTailSyncLock() @@ -402,11 +430,13 @@ func (r *runtime) runWiretap(args []string) error { var previousCoverage *store.CoverageReport runOnce := func(ctx context.Context) error { stats, err := discorddesktop.Import(ctx, r.store, discorddesktop.Options{ - Path: *path, - MaxFileBytes: *maxFileBytes, - FullCache: *fullCache, - DryRun: *dryRun, - Now: r.now, + Path: *path, + MaxFileBytes: *maxFileBytes, + FullCache: *fullCache, + DryRun: *dryRun, + ExcludeChannelIDs: r.cfg.Sync.ExcludeChannelIDs, + ExcludeChannelKinds: r.cfg.Sync.ExcludeChannelKinds, + Now: r.now, }) if err != nil { return err diff --git a/internal/cli/cli.go b/internal/cli/cli.go index ade6d93c..494b9430 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -304,10 +304,18 @@ type tailMessageFailureReplayer interface { ReplayTailMessageFailures(context.Context, []string, int) (syncer.TailMessageReplayStats, error) } +type repairOffsetConfigurer interface { + SetRepairOffset(time.Duration) +} + type attachmentTextConfigurer interface { SetAttachmentTextEnabled(bool) } +type channelExclusionConfigurer interface { + SetChannelExclusions([]string, []string) +} + func (r *runtime) dispatch(rest []string) error { switch rest[0] { case "metadata": @@ -766,6 +774,9 @@ func (r *runtime) ensureDiscordServices() error { if configurable, ok := r.syncer.(attachmentTextConfigurer); ok { configurable.SetAttachmentTextEnabled(r.cfg.AttachmentTextEnabled()) } + if configurable, ok := r.syncer.(channelExclusionConfigurer); ok { + configurable.SetChannelExclusions(r.cfg.Sync.ExcludeChannelIDs, r.cfg.Sync.ExcludeChannelKinds) + } return nil } @@ -815,11 +826,13 @@ func (r *runtime) shareOptions() (share.Options, error) { return share.Options{}, configErr(err) } return share.Options{ - RepoPath: repoPath, - CacheDir: cacheDir, - Remote: r.cfg.Share.Remote, - Branch: r.cfg.Share.Branch, - IncludeMedia: r.cfg.ShareMediaEnabled(), - Progress: r.shareProgress, + RepoPath: repoPath, + CacheDir: cacheDir, + Remote: r.cfg.Share.Remote, + Branch: r.cfg.Share.Branch, + MergeExcludeChannelIDs: append([]string(nil), r.cfg.Sync.ExcludeChannelIDs...), + MergeExcludeChannelKinds: append([]string(nil), r.cfg.Sync.ExcludeChannelKinds...), + IncludeMedia: r.cfg.ShareMediaEnabled(), + Progress: r.shareProgress, }, nil } diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index 16fb86e1..51eb6e73 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -3661,7 +3661,10 @@ type fakeSyncService struct { replayHook func() replayStats syncer.TailMessageReplayStats replayErr error + repairOffset time.Duration attachmentTextEnabled bool + excludedChannelIDs []string + excludedChannelKinds []string callTailReady bool tailReadyCalls int tailReady func(context.Context) error @@ -3708,10 +3711,19 @@ func (f *fakeSyncService) SetTailReadyCallback(fn func(context.Context) error) { f.tailReady = fn } +func (f *fakeSyncService) SetRepairOffset(offset time.Duration) { + f.repairOffset = offset +} + func (f *fakeSyncService) SetAttachmentTextEnabled(enabled bool) { f.attachmentTextEnabled = enabled } +func (f *fakeSyncService) SetChannelExclusions(channelIDs, channelKinds []string) { + f.excludedChannelIDs = append([]string(nil), channelIDs...) + f.excludedChannelKinds = append([]string(nil), channelKinds...) +} + type hybridSyncService struct { store *store.Store sawGitMessage bool @@ -3797,6 +3809,9 @@ func TestRuntimeInitSyncTailAndDoctor(t *testing.T) { require.Equal(t, "g2", cfg.DefaultGuildID) require.True(t, cfg.Search.Embeddings.Enabled) cfg.Desktop.Path = filepath.Join(dir, "empty-discord") + cfg.Sync.RepairOffset = "15m" + cfg.Sync.ExcludeChannelIDs = []string{"feed"} + cfg.Sync.ExcludeChannelKinds = []string{"announcement"} require.NoError(t, os.MkdirAll(cfg.Desktop.Path, 0o755)) require.NoError(t, config.Write(cfgPath, cfg)) @@ -3806,6 +3821,10 @@ func TestRuntimeInitSyncTailAndDoctor(t *testing.T) { require.True(t, fakeSync.lastSync.LatestOnly) require.True(t, fakeSync.lastSync.SkipMembers) require.True(t, fakeSync.attachmentTextEnabled) + require.Equal(t, []string{"feed"}, fakeSync.lastSync.ExcludeChannelIDs) + require.Equal(t, []string{"announcement"}, fakeSync.lastSync.ExcludeChannelKinds) + require.Equal(t, []string{"feed"}, fakeSync.excludedChannelIDs) + require.Equal(t, []string{"announcement"}, fakeSync.excludedChannelKinds) rt = newRuntime() require.NoError(t, rt.withServices(true, func() error { return rt.runSync([]string{"--guilds", "g2", "--with-members"}) })) @@ -3826,10 +3845,25 @@ func TestRuntimeInitSyncTailAndDoctor(t *testing.T) { require.False(t, fakeSync.lastSync.LatestOnly) require.False(t, fakeSync.lastSync.SkipMembers) + rt = newRuntime() + require.NoError(t, rt.withServices(true, func() error { + return rt.runSync([]string{"--guilds", "g2", "--channels", "c1", "--channel-timeout", "20m"}) + })) + require.Equal(t, []string{"g2"}, fakeSync.lastSync.GuildIDs) + require.Equal(t, []string{"c1"}, fakeSync.lastSync.ChannelIDs) + require.Equal(t, 20*time.Minute, fakeSync.lastSync.MessageChannelTimeout) + rt = newRuntime() require.NoError(t, rt.withServices(true, func() error { return rt.runTail([]string{"--repair-every", "30s"}) })) require.Equal(t, []string{"g2"}, fakeSync.lastTail) require.Equal(t, 30*time.Second, fakeSync.lastRepair) + require.Equal(t, 15*time.Minute, fakeSync.repairOffset) + + rt = newRuntime() + require.NoError(t, rt.withServices(true, func() error { + return rt.runTail([]string{"--repair-every", "30s", "--repair-offset", "20m"}) + })) + require.Equal(t, 20*time.Minute, fakeSync.repairOffset) rt = newRuntime() var out bytes.Buffer @@ -4255,6 +4289,12 @@ func TestHelpers(t *testing.T) { require.NoError(t, err) require.Equal(t, "git@example.com:org/archive.git", opts.Remote) require.Equal(t, "main", opts.Branch) + cfg := config.Default() + cfg.Sync.ExcludeChannelIDs = []string{"feed"} + cfg.Sync.ExcludeChannelKinds = []string{"announcement"} + applyCollectionExclusionsToShareMerge(&opts, cfg) + require.Equal(t, []string{"feed"}, opts.MergeExcludeChannelIDs) + require.Equal(t, []string{"announcement"}, opts.MergeExcludeChannelKinds) var out bytes.Buffer require.NoError(t, printHuman(&out, syncer.SyncStats{Guilds: 1})) require.Contains(t, out.String(), "guilds=1") @@ -4737,6 +4777,8 @@ func TestCommandUsageErrors(t *testing.T) { require.Equal(t, 2, ExitCode(rt.runSync([]string{"--all", "--guild", "g1"}))) require.Equal(t, 2, ExitCode(rt.runSync([]string{"--update", "bogus"}))) require.Equal(t, 2, ExitCode(rt.runSync([]string{"--update=force", "--no-update"}))) + require.Equal(t, 2, ExitCode(rt.runSync([]string{"--channel-timeout", "0s"}))) + require.Equal(t, 2, ExitCode(rt.runTail([]string{"--repair-offset", "not-a-duration"}))) require.Equal(t, 2, ExitCode(rt.runChannels(nil))) require.Equal(t, 2, ExitCode(rt.runStatus([]string{"extra"}))) require.NoError(t, (&runtime{stdout: &bytes.Buffer{}}).runDoctor(nil)) diff --git a/internal/cli/share_commands.go b/internal/cli/share_commands.go index a0b558b8..671f0e39 100644 --- a/internal/cli/share_commands.go +++ b/internal/cli/share_commands.go @@ -222,6 +222,7 @@ func (r *runtime) runSubscribe(args []string) error { return configErr(err) } opts := share.Options{RepoPath: expandedRepo, Remote: cfg.Share.Remote, Branch: cfg.Share.Branch, Progress: r.shareProgress} + applyCollectionExclusionsToShareMerge(&opts, cfg) if err := applyMediaShareOptions(&opts, cfg, cfg.ShareMediaEnabled() && !*noMedia); err != nil { return err } @@ -283,6 +284,7 @@ func (r *runtime) runUpdate(args []string) error { if err != nil { return err } + applyCollectionExclusionsToShareMerge(&opts, r.cfg) opts.Progress = r.shareProgress if *withEmbeddings { applyEmbeddingShareOptions(&opts, r.cfg) @@ -344,6 +346,14 @@ func shareOptionsFromFlags(repoPath, remote, branch string) (share.Options, erro return share.Options{RepoPath: expandedRepo, Remote: remote, Branch: branch}, nil } +func applyCollectionExclusionsToShareMerge(opts *share.Options, cfg config.Config) { + if opts == nil { + return + } + opts.MergeExcludeChannelIDs = append([]string(nil), cfg.Sync.ExcludeChannelIDs...) + opts.MergeExcludeChannelKinds = append([]string(nil), cfg.Sync.ExcludeChannelKinds...) +} + func applyEmbeddingShareOptions(opts *share.Options, cfg config.Config) { opts.IncludeEmbeddings = true opts.EmbeddingProvider = cfg.Search.Embeddings.Provider diff --git a/internal/config/config.go b/internal/config/config.go index 2977692d..5ea432f8 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -51,13 +51,16 @@ type DesktopConfig struct { } type SyncConfig struct { - Source string `toml:"source"` - Concurrency int `toml:"concurrency"` - RepairEvery string `toml:"repair_every"` - FullHistory bool `toml:"full_history"` - AttachmentText *bool `toml:"attachment_text"` - AttachmentMedia *bool `toml:"attachment_media"` - MaxAttachmentBytes int64 `toml:"max_attachment_bytes"` + Source string `toml:"source"` + Concurrency int `toml:"concurrency"` + RepairEvery string `toml:"repair_every"` + RepairOffset string `toml:"repair_offset"` + FullHistory bool `toml:"full_history"` + ExcludeChannelIDs []string `toml:"exclude_channel_ids,omitempty"` + ExcludeChannelKinds []string `toml:"exclude_channel_kinds,omitempty"` + AttachmentText *bool `toml:"attachment_text"` + AttachmentMedia *bool `toml:"attachment_media"` + MaxAttachmentBytes int64 `toml:"max_attachment_bytes"` } type SearchConfig struct { @@ -138,6 +141,7 @@ func Default() Config { Source: "both", Concurrency: defaultSyncConcurrency(), RepairEvery: "6h", + RepairOffset: "0s", FullHistory: true, AttachmentText: new(true), AttachmentMedia: new(false), @@ -268,6 +272,15 @@ func (c *Config) Normalize() error { if c.Sync.RepairEvery == "" { c.Sync.RepairEvery = "6h" } + c.Sync.RepairOffset = strings.TrimSpace(c.Sync.RepairOffset) + if c.Sync.RepairOffset == "" { + c.Sync.RepairOffset = "0s" + } + if _, err := time.ParseDuration(c.Sync.RepairOffset); err != nil { + return fmt.Errorf("parse sync.repair_offset: %w", err) + } + c.Sync.ExcludeChannelIDs = uniqueStrings(c.Sync.ExcludeChannelIDs) + c.Sync.ExcludeChannelKinds = uniqueLowerStrings(c.Sync.ExcludeChannelKinds) if c.Sync.AttachmentText == nil { c.Sync.AttachmentText = new(true) } @@ -439,3 +452,11 @@ func uniqueStrings(in []string) []string { } return out } + +func uniqueLowerStrings(in []string) []string { + out := uniqueStrings(in) + for i := range out { + out[i] = strings.ToLower(out[i]) + } + return uniqueStrings(out) +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 5910032c..a979fb44 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -24,6 +24,10 @@ func TestNormalizeFillsDefaults(t *testing.T) { require.Equal(t, defaultSyncConcurrency(), cfg.Sync.Concurrency) require.GreaterOrEqual(t, cfg.Sync.Concurrency, 8) require.LessOrEqual(t, cfg.Sync.Concurrency, 32) + require.Equal(t, "6h", cfg.Sync.RepairEvery) + require.Equal(t, "0s", cfg.Sync.RepairOffset) + require.Empty(t, cfg.Sync.ExcludeChannelIDs) + require.Empty(t, cfg.Sync.ExcludeChannelKinds) require.NotNil(t, cfg.Sync.AttachmentText) require.True(t, *cfg.Sync.AttachmentText) require.Equal(t, "fts", cfg.Search.DefaultMode) @@ -325,6 +329,7 @@ func TestWriteAndLoadRoundTrip(t *testing.T) { cfg := Default() cfg.DefaultGuildID = "g1" cfg.GuildIDs = []string{"g1", "g2"} + cfg.Sync.RepairOffset = "15m" require.NoError(t, Write(path, cfg)) loaded, err := Load(path) @@ -332,10 +337,24 @@ func TestWriteAndLoadRoundTrip(t *testing.T) { require.Equal(t, "g1", loaded.EffectiveDefaultGuildID()) require.Equal(t, []string{"g1", "g2"}, loaded.GuildIDs) require.Equal(t, DefaultRemoteTokenEnv, loaded.Remote.TokenEnv) + require.Equal(t, "15m", loaded.Sync.RepairOffset) require.NotNil(t, loaded.Sync.AttachmentText) require.True(t, *loaded.Sync.AttachmentText) } +func TestNormalizeRejectsInvalidRepairOffset(t *testing.T) { + t.Parallel() + + cfg := Default() + cfg.Sync.RepairOffset = "not-a-duration" + require.ErrorContains(t, cfg.Normalize(), "parse sync.repair_offset") + + cfg = Default() + cfg.Sync.RepairOffset = "-15m" + require.NoError(t, cfg.Normalize()) + require.Equal(t, "-15m", cfg.Sync.RepairOffset) +} + func TestWriteRejectsNonPositiveEmbeddingTimeout(t *testing.T) { t.Parallel() @@ -489,9 +508,13 @@ func TestEffectiveDefaultGuildAndDirs(t *testing.T) { cfg := Default() cfg.GuildIDs = []string{"g1"} + cfg.Sync.ExcludeChannelIDs = []string{" c1 ", "", "c2", "c1"} + cfg.Sync.ExcludeChannelKinds = []string{" Announcement ", "announcement", "TEXT"} cfg.Share.Filter.IncludeChannelIDs = []string{" c1 ", "", "c2", "c1"} cfg.Share.Filter.ExcludeChannelIDs = []string{" c3 ", "c3"} require.NoError(t, cfg.Normalize()) + require.Equal(t, []string{"c1", "c2"}, cfg.Sync.ExcludeChannelIDs) + require.Equal(t, []string{"announcement", "text"}, cfg.Sync.ExcludeChannelKinds) require.Equal(t, []string{"c1", "c2"}, cfg.Share.Filter.IncludeChannelIDs) require.Equal(t, []string{"c3"}, cfg.Share.Filter.ExcludeChannelIDs) require.Equal(t, "g1", cfg.EffectiveDefaultGuildID()) diff --git a/internal/discorddesktop/import.go b/internal/discorddesktop/import.go index b304a36d..86265af2 100644 --- a/internal/discorddesktop/import.go +++ b/internal/discorddesktop/import.go @@ -38,11 +38,13 @@ var ( ) type Options struct { - Path string - MaxFileBytes int64 - DryRun bool - FullCache bool - Now func() time.Time + Path string + MaxFileBytes int64 + DryRun bool + FullCache bool + ExcludeChannelIDs []string + ExcludeChannelKinds []string + Now func() time.Time } type Stats struct { @@ -62,6 +64,8 @@ type Stats struct { GuildMessages int `json:"guild_messages"` SkippedMessages int `json:"skipped_messages"` SkippedChannels int `json:"skipped_channels"` + ExcludedMessages int `json:"excluded_messages"` + ExcludedChannels int `json:"excluded_channels"` Checkpoints int `json:"checkpoints"` DryRun bool `json:"dry_run,omitempty"` FullCache bool `json:"full_cache,omitempty"` @@ -106,14 +110,16 @@ type fileCandidate struct { } type scanTotals struct { - guilds map[string]struct{} - channels map[string]struct{} - messages map[string]struct{} - dmMessages map[string]struct{} - guildMessages map[string]struct{} - dmChannels map[string]struct{} - skippedMessages map[string]struct{} - skippedChannels map[string]struct{} + guilds map[string]struct{} + channels map[string]struct{} + messages map[string]struct{} + dmMessages map[string]struct{} + guildMessages map[string]struct{} + dmChannels map[string]struct{} + skippedMessages map[string]struct{} + skippedChannels map[string]struct{} + excludedMessages map[string]struct{} + excludedChannels map[string]struct{} } type unresolvedMessages map[string]string @@ -421,6 +427,7 @@ func scanFullCache(ctx context.Context, opts Options, state scanState) (Stats, s return stats, snap, err } totals := newScanTotals() + filterExcludedMessages(snap, state.channels, opts, totals, &stats) finalizeSnapshot(snap, state.channels, totals, &stats, true) stats.FinishedAt = now().UTC() return stats, snap, nil @@ -571,15 +578,88 @@ func collectCacheRouteHints(ctx context.Context, rootFS *os.Root, candidates []f func newScanTotals() scanTotals { return scanTotals{ - guilds: map[string]struct{}{}, - channels: map[string]struct{}{}, - messages: map[string]struct{}{}, - dmMessages: map[string]struct{}{}, - guildMessages: map[string]struct{}{}, - dmChannels: map[string]struct{}{}, - skippedMessages: map[string]struct{}{}, - skippedChannels: map[string]struct{}{}, + guilds: map[string]struct{}{}, + channels: map[string]struct{}{}, + messages: map[string]struct{}{}, + dmMessages: map[string]struct{}{}, + guildMessages: map[string]struct{}{}, + dmChannels: map[string]struct{}{}, + skippedMessages: map[string]struct{}{}, + skippedChannels: map[string]struct{}{}, + excludedMessages: map[string]struct{}{}, + excludedChannels: map[string]struct{}{}, + } +} + +func filterExcludedMessages(snap snapshot, channelLookup map[string]store.ChannelRecord, opts Options, totals scanTotals, stats *Stats) { + excludedIDs := normalizedCollectionSet(opts.ExcludeChannelIDs, false) + excludedKinds := normalizedCollectionSet(opts.ExcludeChannelKinds, true) + if len(excludedIDs) == 0 && len(excludedKinds) == 0 { + return + } + channelByID := func(channelID string) (store.ChannelRecord, bool) { + if channel, ok := channelLookup[channelID]; ok && strings.TrimSpace(channel.Kind) != "" { + return channel, true + } + channel, ok := snap.channels[channelID] + return channel, ok + } + excludesKind := func(kind string) bool { + kind = strings.ToLower(strings.TrimSpace(kind)) + if _, ok := excludedKinds[kind]; ok { + return true + } + if kind == "thread_announcement" { + _, ok := excludedKinds["announcement"] + return ok + } + return false } + excludesChannel := func(channelID string) bool { + if _, ok := excludedIDs[channelID]; ok { + return true + } + channel, ok := channelByID(channelID) + if !ok { + return false + } + if excludesKind(channel.Kind) { + return true + } + if channel.ParentID == "" { + return false + } + if _, ok := excludedIDs[channel.ParentID]; ok { + return true + } + parent, ok := channelByID(channel.ParentID) + return ok && excludesKind(parent.Kind) + } + for messageID, message := range snap.messages { + channelID := message.Record.ChannelID + if !excludesChannel(channelID) { + continue + } + totals.excludedMessages[messageID] = struct{}{} + totals.excludedChannels[channelID] = struct{}{} + delete(snap.messages, messageID) + } + stats.ExcludedMessages = len(totals.excludedMessages) + stats.ExcludedChannels = len(totals.excludedChannels) +} + +func normalizedCollectionSet(values []string, lower bool) map[string]struct{} { + out := make(map[string]struct{}, len(values)) + for _, value := range values { + value = strings.TrimSpace(value) + if lower { + value = strings.ToLower(value) + } + if value != "" { + out[value] = struct{}{} + } + } + return out } func finalizeSnapshot(snap snapshot, channelLookup map[string]store.ChannelRecord, totals scanTotals, stats *Stats, recordSkipped bool) unresolvedMessages { diff --git a/internal/discorddesktop/import_helpers_test.go b/internal/discorddesktop/import_helpers_test.go index 911705dc..5f62d9f5 100644 --- a/internal/discorddesktop/import_helpers_test.go +++ b/internal/discorddesktop/import_helpers_test.go @@ -69,6 +69,30 @@ func TestSnapshotWithoutMessageEvents(t *testing.T) { require.True(t, snap.messages["333333333333333346"].Options.AppendEvent) } +func TestFilterExcludedMessages(t *testing.T) { + snap := newSnapshot() + snap.messages["m-news"] = store.MessageMutation{Record: store.MessageRecord{ID: "m-news", ChannelID: "news"}} + snap.messages["m-logs"] = store.MessageMutation{Record: store.MessageRecord{ID: "m-logs", ChannelID: "logs"}} + snap.messages["m-general"] = store.MessageMutation{Record: store.MessageRecord{ID: "m-general", ChannelID: "general"}} + lookup := map[string]store.ChannelRecord{ + "news": {ID: "news", Kind: "announcement"}, + "logs": {ID: "logs", Kind: "text"}, + "general": {ID: "general", Kind: "text"}, + } + totals := newScanTotals() + stats := &Stats{} + + filterExcludedMessages(snap, lookup, Options{ + ExcludeChannelIDs: []string{"logs"}, + ExcludeChannelKinds: []string{"announcement"}, + }, totals, stats) + + require.Len(t, snap.messages, 1) + require.Contains(t, snap.messages, "m-general") + require.Equal(t, 2, stats.ExcludedMessages) + require.Equal(t, 2, stats.ExcludedChannels) +} + func TestRouteFilteredCacheHelpers(t *testing.T) { require.Equal(t, fileSourceCacheData, sourceForPath("/tmp/discord", "/tmp/discord/Cache/Cache_Data/entry", "Cache/Cache_Data/entry")) require.Equal(t, fileSourceCacheData, sourceForPath("/tmp/discord", "/tmp/discord/Service Worker/CacheStorage/cache/entry", "Service Worker/CacheStorage/cache/entry")) diff --git a/internal/discorddesktop/import_run.go b/internal/discorddesktop/import_run.go index f06fb3cd..01201a8f 100644 --- a/internal/discorddesktop/import_run.go +++ b/internal/discorddesktop/import_run.go @@ -64,6 +64,7 @@ func (r *importRun) scanCacheBatches(candidates []fileCandidate) error { } func (r *importRun) finalizeAndCommit(candidates []fileCandidate, snap snapshot, recordSkipped bool) error { + filterExcludedMessages(snap, r.channelLookup, r.opts, r.totals, r.stats) unresolved := finalizeSnapshot(snap, r.channelLookup, r.totals, r.stats, recordSkipped) checkpoint := len(unresolved) == 0 if !checkpoint { @@ -97,6 +98,7 @@ func (r *importRun) retryPending() error { if err := scanCandidates(r.ctx, r.rootFS, r.opts, r.pending, retry, r.channelLookup, r.stats); err != nil { return err } + filterExcludedMessages(retry, r.channelLookup, r.opts, r.totals, r.stats) unresolved := finalizeSnapshot(retry, r.channelLookup, r.totals, r.stats, true) checkpoint := len(unresolved) == 0 if err := commitSnapshot(r.ctx, r.st, r.opts, r.state, r.pending, retry, checkpoint, r.stats); err != nil { diff --git a/internal/share/import_filter.go b/internal/share/import_filter.go new file mode 100644 index 00000000..6049427e --- /dev/null +++ b/internal/share/import_filter.go @@ -0,0 +1,160 @@ +package share + +import ( + "context" + "database/sql" + "fmt" + "strings" +) + +type mergeImportChannel struct { + parentID string + kind string +} + +type mergeImportFilter struct { + excludedIDs map[string]struct{} + excludedKinds map[string]struct{} + channels map[string]mergeImportChannel + decisions map[string]bool +} + +func newMergeImportFilter( + ctx context.Context, + db *sql.DB, + opts Options, +) (func(string, map[string]any) (bool, error), error) { + filter := &mergeImportFilter{ + excludedIDs: normalizedImportSet(opts.MergeExcludeChannelIDs, false), + excludedKinds: normalizedImportSet(opts.MergeExcludeChannelKinds, true), + channels: map[string]mergeImportChannel{}, + decisions: map[string]bool{}, + } + if len(filter.excludedIDs) > 0 || len(filter.excludedKinds) > 0 { + if err := filter.loadChannels(ctx, db); err != nil { + return nil, err + } + } + return filter.allow, nil +} + +func (f *mergeImportFilter) loadChannels(ctx context.Context, db *sql.DB) error { + rows, err := db.QueryContext(ctx, ` + select id, coalesce(parent_id, ''), coalesce(thread_parent_id, ''), kind + from channels + `) + if err != nil { + return fmt.Errorf("load channels for snapshot merge exclusions: %w", err) + } + defer func() { _ = rows.Close() }() + for rows.Next() { + var id, parentID, threadParentID, kind string + if err := rows.Scan(&id, &parentID, &threadParentID, &kind); err != nil { + return fmt.Errorf("scan channel for snapshot merge exclusions: %w", err) + } + if threadParentID != "" { + parentID = threadParentID + } + f.channels[id] = mergeImportChannel{parentID: parentID, kind: kind} + } + if err := rows.Err(); err != nil { + return fmt.Errorf("iterate channels for snapshot merge exclusions: %w", err) + } + return nil +} + +func (f *mergeImportFilter) allow(table string, row map[string]any) (bool, error) { + if isDirectMessageSnapshotRow(table, row) { + return false, nil + } + switch table { + case "channels": + f.recordChannel(row) + return true, nil + case "messages", "message_events", "message_attachments", "mention_events": + return !f.excludesChannel(stringValue(row["channel_id"]), nil), nil + default: + return true, nil + } +} + +func (f *mergeImportFilter) recordChannel(row map[string]any) { + id := stringValue(row["id"]) + if id == "" { + return + } + parentID := stringValue(row["thread_parent_id"]) + if parentID == "" { + parentID = stringValue(row["parent_id"]) + } + f.channels[id] = mergeImportChannel{ + parentID: parentID, + kind: stringValue(row["kind"]), + } + clear(f.decisions) +} + +func (f *mergeImportFilter) excludesChannel(channelID string, visiting map[string]struct{}) bool { + if channelID == "" { + return false + } + if decision, ok := f.decisions[channelID]; ok { + return decision + } + if _, ok := f.excludedIDs[channelID]; ok { + f.decisions[channelID] = true + return true + } + channel, ok := f.channels[channelID] + if !ok { + f.decisions[channelID] = false + return false + } + if f.excludesKind(channel.kind) { + f.decisions[channelID] = true + return true + } + if channel.parentID == "" { + f.decisions[channelID] = false + return false + } + if visiting == nil { + visiting = map[string]struct{}{} + } + if _, ok := visiting[channelID]; ok { + return false + } + visiting[channelID] = struct{}{} + excluded := f.excludesChannel(channel.parentID, visiting) + delete(visiting, channelID) + f.decisions[channelID] = excluded + return excluded +} + +func (f *mergeImportFilter) excludesKind(kind string) bool { + kind = strings.ToLower(strings.TrimSpace(kind)) + if _, ok := f.excludedKinds[kind]; ok { + return true + } + switch kind { + case "news", "thread_news", "thread_announcement": + _, ok := f.excludedKinds["announcement"] + return ok + default: + return false + } +} + +func normalizedImportSet(values []string, lower bool) map[string]struct{} { + out := make(map[string]struct{}, len(values)) + for _, value := range values { + value = strings.TrimSpace(value) + if lower { + value = strings.ToLower(value) + } + if value != "" { + out[value] = struct{}{} + } + } + return out +} diff --git a/internal/share/merge_test.go b/internal/share/merge_test.go index 46593ca7..1c08fc24 100644 --- a/internal/share/merge_test.go +++ b/internal/share/merge_test.go @@ -132,6 +132,108 @@ func TestMergeIfChangedPreservesLocalRowsUntilForcedReplacement(t *testing.T) { require.Equal(t, "0", rows[0][0], "force must reconcile even when the manifest is unchanged") } +func TestMergeIfChangedExcludesConfiguredFeedChannelRows(t *testing.T) { + ctx := context.Background() + src, err := store.Open(ctx, filepath.Join(t.TempDir(), "src.db")) + require.NoError(t, err) + defer func() { _ = src.Close() }() + + require.NoError(t, src.UpsertGuild(ctx, store.GuildRecord{ID: "g1", Name: "Guild", RawJSON: `{}`})) + for _, channel := range []store.ChannelRecord{ + {ID: "general", GuildID: "g1", Kind: "text", Name: "general", RawJSON: `{}`}, + {ID: "github", GuildID: "g1", Kind: "announcement", Name: "github", RawJSON: `{}`}, + {ID: "github-thread", GuildID: "g1", ParentID: "github", Kind: "thread_public", Name: "thread", RawJSON: `{}`}, + {ID: "logs", GuildID: "g1", Kind: "text", Name: "logs", RawJSON: `{}`}, + } { + require.NoError(t, src.UpsertChannel(ctx, channel)) + } + now := time.Now().UTC().Format(time.RFC3339Nano) + message := func(id, channelID, content string) store.MessageRecord { + return store.MessageRecord{ + ID: id, + GuildID: "g1", + ChannelID: channelID, + ChannelName: channelID, + AuthorID: "u1", + AuthorName: "Bot", + CreatedAt: now, + Content: content, + NormalizedContent: content, + RawJSON: `{}`, + } + } + require.NoError(t, src.UpsertMessages(ctx, []store.MessageMutation{ + {Record: message("m-general", "general", "human discussion")}, + { + Record: message("m-github", "github", "automated feed"), + Attachments: []store.AttachmentRecord{{ + AttachmentID: "a-github", + MessageID: "m-github", + GuildID: "g1", + ChannelID: "github", + Filename: "payload.json", + }}, + Mentions: []store.MentionEventRecord{{ + MessageID: "m-github", + GuildID: "g1", + ChannelID: "github", + TargetType: "user", + TargetID: "u2", + EventAt: now, + }}, + }, + {Record: message("m-thread", "github-thread", "feed thread")}, + {Record: message("m-logs", "logs", "automated logs")}, + })) + + repo := filepath.Join(t.TempDir(), "share") + _, err = Export(ctx, src, Options{RepoPath: repo, Branch: "main"}) + require.NoError(t, err) + + dst, err := store.Open(ctx, filepath.Join(t.TempDir(), "dst.db")) + require.NoError(t, err) + defer func() { _ = dst.Close() }() + require.NoError(t, dst.UpsertGuild(ctx, store.GuildRecord{ID: "g1", Name: "Guild", RawJSON: `{}`})) + require.NoError(t, dst.UpsertChannel(ctx, store.ChannelRecord{ + ID: "github", GuildID: "g1", Kind: "announcement", Name: "github", RawJSON: `{}`, + })) + require.NoError(t, dst.UpsertMessage(ctx, message("local-feed", "github", "preserved existing feed row"))) + + opts := Options{ + RepoPath: repo, + Branch: "main", + MergeExcludeChannelIDs: []string{"logs"}, + MergeExcludeChannelKinds: []string{"announcement"}, + } + _, changed, err := MergeIfChanged(ctx, dst, opts) + require.NoError(t, err) + require.True(t, changed) + + _, rows, err := dst.ReadOnlyQuery(ctx, `select id from messages order by id`) + require.NoError(t, err) + require.Equal(t, [][]string{{"local-feed"}, {"m-general"}}, rows) + _, rows, err = dst.ReadOnlyQuery(ctx, `select id from channels order by id`) + require.NoError(t, err) + require.Equal(t, [][]string{{"general"}, {"github"}, {"github-thread"}, {"logs"}}, rows) + _, rows, err = dst.ReadOnlyQuery(ctx, `select count(*) from message_attachments`) + require.NoError(t, err) + require.Equal(t, "0", rows[0][0]) + _, rows, err = dst.ReadOnlyQuery(ctx, `select count(*) from mention_events`) + require.NoError(t, err) + require.Equal(t, "0", rows[0][0]) + + require.NoError(t, src.UpsertMessage(ctx, message("m-general-2", "general", "new human discussion"))) + require.NoError(t, src.UpsertMessage(ctx, message("m-github-2", "github", "new automated feed"))) + _, err = Export(ctx, src, Options{RepoPath: repo, Branch: "main"}) + require.NoError(t, err) + _, changed, err = MergeIfChanged(ctx, dst, opts) + require.NoError(t, err) + require.True(t, changed) + _, rows, err = dst.ReadOnlyQuery(ctx, `select id from messages order by id`) + require.NoError(t, err) + require.Equal(t, [][]string{{"local-feed"}, {"m-general"}, {"m-general-2"}}, rows) +} + func TestMergeIfChangedMarksReplacementPendingWithoutChangingRows(t *testing.T) { ctx := context.Background() src := seedStore(t, filepath.Join(t.TempDir(), "src.db")) diff --git a/internal/share/share.go b/internal/share/share.go index 03d57c58..7b271112 100644 --- a/internal/share/share.go +++ b/internal/share/share.go @@ -65,18 +65,20 @@ var SnapshotTables = []string{ } type Options struct { - RepoPath string - CacheDir string - Remote string - Branch string - Tag string - Filter FilterOptions - IncludeMedia bool - IncludeEmbeddings bool - EmbeddingProvider string - EmbeddingModel string - EmbeddingInputVersion string - Progress func(ImportProgress) + RepoPath string + CacheDir string + Remote string + Branch string + Tag string + Filter FilterOptions + MergeExcludeChannelIDs []string + MergeExcludeChannelKinds []string + IncludeMedia bool + IncludeEmbeddings bool + EmbeddingProvider string + EmbeddingModel string + EmbeddingInputVersion string + Progress func(ImportProgress) } type FilterOptions struct { @@ -628,6 +630,10 @@ func importMergePlan( } return manifest, copied > 0, nil } + rowFilter, err := newMergeImportFilter(ctx, s.DB(), opts) + if err != nil { + return Manifest{}, false, err + } opts.reportProgress(ImportProgress{Phase: "start", TotalRows: importPlanRowCount(plan)}) restorePragmas, err := applyImportPragmas(ctx, s.DB()) if err != nil { @@ -657,9 +663,7 @@ func importMergePlan( TotalRows: progress.TotalRows, }) }, - Filter: func(table string, row map[string]any) (bool, error) { - return !isDirectMessageSnapshotRow(table, row), nil - }, + Filter: rowFilter, BeforeImport: func(ctx context.Context, tx *sql.Tx) error { var err error existingMedia, err = attachmentMediaByID(ctx, tx) diff --git a/internal/store/coverage.go b/internal/store/coverage.go index e77bc8d7..aafaa82e 100644 --- a/internal/store/coverage.go +++ b/internal/store/coverage.go @@ -11,7 +11,50 @@ import ( const wiretapStatsScope = "wiretap:last_stats:v1" -const coverageQueryTimeout = 2 * time.Minute +const ( + coverageQueryTimeout = 2 * time.Minute + globalCoverageQueryTimeout = 4 * time.Minute +) + +const filteredCoverageChannelQuery = ` + select + c.id, c.guild_id, c.name, c.kind, + case when exists ( + select 1 from sync_state s + where s.scope = 'channel:' || c.id || ':history_complete' + ) then 1 else 0 end, + count(m.id), coalesce(min(m.created_at), ''), coalesce(max(m.created_at), '') + from channels c + left join messages m on m.channel_id = c.id and m.deleted_at is null + where c.guild_id = ? + group by c.id, c.guild_id, c.name, c.kind + order by c.guild_id, count(m.id) desc, lower(c.name), c.id +` + +const globalCoverageChannelQuery = ` + with message_coverage as materialized ( + select + channel_id, + count(*) as message_count, + coalesce(min(created_at), '') as earliest_message_at, + coalesce(max(created_at), '') as latest_message_at + from messages not indexed + where deleted_at is null + group by channel_id + ) + select + c.id, c.guild_id, c.name, c.kind, + case when exists ( + select 1 from sync_state s + where s.scope = 'channel:' || c.id || ':history_complete' + ) then 1 else 0 end, + coalesce(m.message_count, 0), + coalesce(m.earliest_message_at, ''), + coalesce(m.latest_message_at, '') + from channels c + left join message_coverage m on m.channel_id = c.id + order by c.guild_id, coalesce(m.message_count, 0) desc, lower(c.name), c.id +` var messageChannelKinds = map[string]struct{}{ "text": {}, "news": {}, "announcement": {}, "dm": {}, "group_dm": {}, @@ -102,7 +145,7 @@ func (s *Store) SetWiretapImportStats(ctx context.Context, stats WiretapImportSt func (s *Store) Coverage(ctx context.Context, guildID string, generatedAt time.Time) (CoverageReport, error) { report := CoverageReport{GeneratedAt: generatedAt.UTC(), Guilds: []CoverageGuild{}} - queryCtx, cancel := context.WithTimeout(ctx, coverageQueryTimeout) + queryCtx, cancel := withCoverageQueryTimeout(ctx, guildID) defer cancel() rows, err := s.db.QueryContext(queryCtx, ` @@ -137,20 +180,13 @@ func (s *Store) Coverage(ctx context.Context, guildID string, generatedAt time.T for i := range report.Guilds { guilds[report.Guilds[i].ID] = &report.Guilds[i] } - channelRows, err := s.db.QueryContext(queryCtx, ` - select - c.id, c.guild_id, c.name, c.kind, - case when exists ( - select 1 from sync_state s - where s.scope = 'channel:' || c.id || ':history_complete' - ) then 1 else 0 end, - count(m.id), coalesce(min(m.created_at), ''), coalesce(max(m.created_at), '') - from channels c - left join messages m on m.channel_id = c.id and m.deleted_at is null - where ? = '' or c.guild_id = ? - group by c.id, c.guild_id, c.name, c.kind - order by c.guild_id, count(m.id) desc, lower(c.name), c.id - `, guildID, guildID) + channelQuery := filteredCoverageChannelQuery + channelArgs := []any{guildID} + if guildID == "" { + channelQuery = globalCoverageChannelQuery + channelArgs = nil + } + channelRows, err := s.db.QueryContext(queryCtx, channelQuery, channelArgs...) if err != nil { return CoverageReport{}, fmt.Errorf("query channel coverage: %w", err) } @@ -226,6 +262,14 @@ func (s *Store) Coverage(ctx context.Context, guildID string, generatedAt time.T return report, nil } +func withCoverageQueryTimeout(ctx context.Context, guildID string) (context.Context, context.CancelFunc) { + timeout := coverageQueryTimeout + if guildID == "" { + timeout = globalCoverageQueryTimeout + } + return context.WithTimeout(ctx, timeout) +} + func (s *Store) loadKnownFailureCoverage(ctx context.Context, guildID string, report *CoverageReport) error { rows, err := s.db.QueryContext(ctx, ` select guild_id, channel_id, count(*) diff --git a/internal/store/coverage_test.go b/internal/store/coverage_test.go index 8af199c7..c8ceb792 100644 --- a/internal/store/coverage_test.go +++ b/internal/store/coverage_test.go @@ -2,14 +2,64 @@ package store import ( "context" + "database/sql" + "database/sql/driver" "errors" + "fmt" "path/filepath" + "strings" + "sync/atomic" "testing" "time" + "github.com/openclaw/discrawl/internal/store/storedb" "github.com/stretchr/testify/require" + moderncsqlite "modernc.org/sqlite" ) +var coverageObserverDriverID atomic.Uint64 + +type coverageQueryObservation struct { + contextErrAtStart error + contextErrAtReturn error + queryErr error +} + +type coverageObserverDriver struct { + driver.Driver + observations chan<- coverageQueryObservation +} + +func (d *coverageObserverDriver) Open(name string) (driver.Conn, error) { + conn, err := d.Driver.Open(name) + if err != nil { + return nil, err + } + return &coverageObserverConn{Conn: conn, observations: d.observations}, nil +} + +type coverageObserverConn struct { + driver.Conn + observations chan<- coverageQueryObservation +} + +func (c *coverageObserverConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + queryer, ok := c.Conn.(driver.QueryerContext) + if !ok { + return nil, driver.ErrSkip + } + if query != globalCoverageChannelQuery { + return queryer.QueryContext(ctx, query, args) + } + + observation := coverageQueryObservation{contextErrAtStart: ctx.Err()} + rows, err := queryer.QueryContext(ctx, query, args) + observation.contextErrAtReturn = ctx.Err() + observation.queryErr = err + c.observations <- observation + return rows, err +} + func TestCoverageReportsGuildChannelAndWiretapState(t *testing.T) { t.Parallel() ctx := context.Background() @@ -21,14 +71,19 @@ func TestCoverageReportsGuildChannelAndWiretapState(t *testing.T) { require.NoError(t, s.UpsertGuild(ctx, GuildRecord{ID: "g2", Name: "Guild Two", RawJSON: `{}`})) require.NoError(t, s.UpsertChannel(ctx, ChannelRecord{ID: "c1", GuildID: "g1", Kind: "text", Name: "general", RawJSON: `{}`})) require.NoError(t, s.UpsertChannel(ctx, ChannelRecord{ID: "c2", GuildID: "g1", Kind: "text", Name: "channel-c2", RawJSON: `{"source":"discord_desktop"}`})) + require.NoError(t, s.UpsertChannel(ctx, ChannelRecord{ID: "c3", GuildID: "g2", Kind: "text", Name: "empty", RawJSON: `{}`})) require.NoError(t, s.UpsertChannel(ctx, ChannelRecord{ID: "v1", GuildID: "g1", Kind: "voice", Name: "Voice", RawJSON: `{}`})) for _, message := range []MessageRecord{ + {ID: "deleted-early", GuildID: "g1", ChannelID: "c1", CreatedAt: "2026-05-01T10:00:00Z", Content: "deleted early", NormalizedContent: "deleted early", RawJSON: `{}`}, {ID: "m1", GuildID: "g1", ChannelID: "c1", CreatedAt: "2026-06-01T10:00:00Z", Content: "one", NormalizedContent: "one", RawJSON: `{}`}, {ID: "m2", GuildID: "g1", ChannelID: "c1", CreatedAt: "2026-06-02T10:00:00Z", Content: "two", NormalizedContent: "two", RawJSON: `{}`}, {ID: "m3", GuildID: "g1", ChannelID: "c2", CreatedAt: "2026-06-03T10:00:00Z", Content: "three", NormalizedContent: "three", RawJSON: `{}`}, + {ID: "deleted-late", GuildID: "g1", ChannelID: "c1", CreatedAt: "2026-07-01T10:00:00Z", Content: "deleted late", NormalizedContent: "deleted late", RawJSON: `{}`}, } { require.NoError(t, s.UpsertMessage(ctx, message)) } + require.NoError(t, s.MarkMessageDeleted(ctx, "g1", "c1", "deleted-early", nil)) + require.NoError(t, s.MarkMessageDeleted(ctx, "g1", "c1", "deleted-late", nil)) require.NoError(t, s.SetSyncState(ctx, "channel:c1:history_complete", "1")) require.NoError(t, s.SetSyncState(ctx, "sync:last_success", "2026-06-04T10:00:00Z")) require.NoError(t, s.SetSyncState(ctx, "wiretap:last_import", "2026-06-04T11:00:00Z")) @@ -44,8 +99,8 @@ func TestCoverageReportsGuildChannelAndWiretapState(t *testing.T) { require.NoError(t, err) require.Equal(t, generatedAt, report.GeneratedAt) require.Equal(t, CoverageTotals{ - GuildCount: 2, MessageCount: 3, ChannelCount: 3, MessageChannelCount: 2, - NamedChannelCount: 2, SyntheticChannelCount: 1, HistoryCompleteChannelCount: 1, + GuildCount: 2, MessageCount: 3, ChannelCount: 4, MessageChannelCount: 3, + NamedChannelCount: 3, SyntheticChannelCount: 1, HistoryCompleteChannelCount: 1, KnownFailureCount: 2, UnscopedKnownFailureCount: 1, }, report.Totals) require.Equal(t, time.Date(2026, 6, 4, 10, 0, 0, 0, time.UTC), report.LastBotSyncAt) @@ -63,15 +118,125 @@ func TestCoverageReportsGuildChannelAndWiretapState(t *testing.T) { require.Equal(t, 1, report.Guilds[0].KnownFailureCount) require.Equal(t, 1, report.Guilds[0].Channels[0].KnownFailureCount) require.True(t, report.Guilds[0].Channels[1].Synthetic) + require.Equal(t, 0, report.Guilds[0].Channels[2].MessageCount) + require.Equal(t, "v1", report.Guilds[0].Channels[2].ID) + + filtered, err := s.Coverage(ctx, "g1", generatedAt) + require.NoError(t, err) + require.Equal(t, 3, filtered.Totals.MessageCount) + require.Equal(t, 3, filtered.Totals.ChannelCount) + require.Equal(t, "g1", filtered.Guilds[0].ID) + require.Equal(t, report.Guilds[0].Channels, filtered.Guilds[0].Channels) - filtered, err := s.Coverage(ctx, "g2", generatedAt) + empty, err := s.Coverage(ctx, "g2", generatedAt) require.NoError(t, err) - require.Equal(t, CoverageTotals{GuildCount: 1}, filtered.Totals) - require.Equal(t, "g2", filtered.Guilds[0].ID) + require.Equal(t, CoverageTotals{ + GuildCount: 1, ChannelCount: 1, MessageChannelCount: 1, NamedChannelCount: 1, + }, empty.Totals) + require.Equal(t, "c3", empty.Guilds[0].Channels[0].ID) + require.Equal(t, 0, empty.Guilds[0].Channels[0].MessageCount) _, err = s.Coverage(ctx, "missing", generatedAt) require.ErrorContains(t, err, `guild "missing" not found`) } +func TestCoverageQueryTimeoutsAndCancellation(t *testing.T) { + t.Parallel() + + started := time.Now() + globalCtx, globalCancel := withCoverageQueryTimeout(context.Background(), "") + globalDeadline, ok := globalCtx.Deadline() + require.True(t, ok) + require.WithinDuration(t, started.Add(4*time.Minute), globalDeadline, time.Second) + globalCancel() + + started = time.Now() + filteredCtx, filteredCancel := withCoverageQueryTimeout(context.Background(), "g1") + filteredDeadline, ok := filteredCtx.Deadline() + require.True(t, ok) + require.WithinDuration(t, started.Add(2*time.Minute), filteredDeadline, time.Second) + filteredCancel() + + parentDeadline := time.Now().Add(time.Minute) + parentCtx, parentCancel := context.WithDeadline(context.Background(), parentDeadline) + defer parentCancel() + childCtx, childCancel := withCoverageQueryTimeout(parentCtx, "") + defer childCancel() + childDeadline, ok := childCtx.Deadline() + require.True(t, ok) + require.Equal(t, parentDeadline, childDeadline) + + ctx := context.Background() + s, err := Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + for _, guildID := range []string{"", "g1"} { + canceledCtx, cancel := context.WithCancel(ctx) + cancel() + _, err := s.Coverage(canceledCtx, guildID, time.Now()) + require.ErrorIs(t, err, context.Canceled) + } +} + +func TestCoverageGlobalQueryPlanMaterializesSequentialMessageScan(t *testing.T) { + ctx := context.Background() + s, err := Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { require.NoError(t, s.Close()) }() + + rows, err := s.DB().QueryContext(ctx, "explain query plan "+globalCoverageChannelQuery) + require.NoError(t, err) + defer func() { require.NoError(t, rows.Close()) }() + + var details []string + for rows.Next() { + var selectID, parentID, unused int + var detail string + require.NoError(t, rows.Scan(&selectID, &parentID, &unused, &detail)) + details = append(details, detail) + } + require.NoError(t, rows.Err()) + + plan := strings.ToLower(strings.Join(details, "\n")) + require.Contains(t, strings.ToLower(globalCoverageChannelQuery), "with message_coverage as materialized") + require.Contains(t, strings.ToLower(globalCoverageChannelQuery), "from messages not indexed") + require.Contains(t, plan, "materialize message_coverage") + require.Contains(t, plan, "scan messages") + require.NotContains(t, plan, "idx_messages_channel_id") + require.NotContains(t, plan, "idx_messages_channel_created_id") +} + +func TestCoverageGlobalQueryHonorsParentDeadlineInFlight(t *testing.T) { + const ( + messageCount = 300_000 + parentTimeout = 50 * time.Millisecond + ) + + s, observations := newCoverageDeadlineTestStore(t, messageCount) + parentCtx, cancel := context.WithTimeout(context.Background(), parentTimeout) + defer cancel() + parentDeadline, ok := parentCtx.Deadline() + require.True(t, ok) + + started := time.Now() + _, err := s.Coverage(parentCtx, "", time.Now()) + elapsed := time.Since(started) + require.ErrorIs(t, err, context.DeadlineExceeded) + require.ErrorContains(t, err, "query channel coverage") + require.NotContains(t, err.Error(), "list coverage guilds") + require.False(t, time.Now().Before(parentDeadline)) + + select { + case observation := <-observations: + require.NoError(t, observation.contextErrAtStart) + require.ErrorIs(t, observation.contextErrAtReturn, context.DeadlineExceeded) + require.ErrorIs(t, observation.queryErr, context.DeadlineExceeded) + default: + t.Fatalf("global channel query was not observed; Coverage returned %v", err) + } + t.Logf("canceled global channel query over %d generated messages after %s", messageCount, elapsed) +} + func TestCoverageDeltaSince(t *testing.T) { previous := CoverageReport{Totals: CoverageTotals{MessageCount: 4, ChannelCount: 3, NamedChannelCount: 2, SyntheticChannelCount: 1}} current := CoverageReport{Totals: CoverageTotals{MessageCount: 7, ChannelCount: 4, NamedChannelCount: 4, SyntheticChannelCount: 0}} @@ -84,3 +249,83 @@ func TestSyntheticChannelClassificationUsesGeneratedPlaceholder(t *testing.T) { require.False(t, isSyntheticChannel("123456123456", "general")) require.False(t, isSyntheticChannel("123456123456", "channel-other")) } + +func newCoverageDeadlineTestStore(t *testing.T, messageCount int) (*Store, <-chan coverageQueryObservation) { + t.Helper() + ctx := context.Background() + dbPath := filepath.Join(t.TempDir(), "coverage-deadline.db") + seedDB, err := sql.Open("sqlite", dbPath) + require.NoError(t, err) + _, err = seedDB.ExecContext(ctx, ` + create table guilds ( + id text primary key, + name text not null + ); + create table channels ( + id text primary key, + guild_id text not null, + name text not null, + kind text not null + ); + create table messages ( + id text primary key, + channel_id text not null, + created_at text not null, + deleted_at text + ); + create table sync_state ( + scope text primary key, + cursor text, + updated_at text + ); + create table failure_ledger ( + guild_id text not null default '', + channel_id text not null default '', + resolved_at text + ); + insert into guilds (id, name) values ('g1', 'Guild One'); + insert into channels (id, guild_id, name, kind) values ('c1', 'g1', 'general', 'text'); + `) + require.NoError(t, err) + _, err = seedDB.ExecContext(ctx, ` + with + digit(n) as ( + values (0), (1), (2), (3), (4), (5), (6), (7), (8), (9) + ), + generated(n) as ( + select 1 + a.n + 10*b.n + 100*c.n + 1000*d.n + 10000*e.n + 100000*f.n + from digit a + cross join digit b + cross join digit c + cross join digit d + cross join digit e + cross join digit f + limit ? + ) + insert into messages (id, channel_id, created_at) + select + printf('message-%06d', n), + printf('generated-channel-%06d', n), + '2026-01-01T00:00:00Z' + from generated + `, messageCount) + require.NoError(t, err) + require.NoError(t, seedDB.Close()) + + observations := make(chan coverageQueryObservation, 1) + driverName := fmt.Sprintf("coverage-observer-%d", coverageObserverDriverID.Add(1)) + sql.Register(driverName, &coverageObserverDriver{ + Driver: &moderncsqlite.Driver{}, + observations: observations, + }) + db, err := sql.Open(driverName, dbPath) + require.NoError(t, err) + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + require.NoError(t, db.PingContext(ctx)) + t.Cleanup(func() { + require.NoError(t, db.Close()) + }) + + return &Store{db: db, q: storedb.New(db), path: dbPath}, observations +} diff --git a/internal/store/store_test.go b/internal/store/store_test.go index deba64dc..f7e140d9 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -227,6 +227,7 @@ func TestClosedStoreOperationsReturnErrors(t *testing.T) { require.Error(t, s.UpsertGuild(ctx, GuildRecord{ID: "g1"})) require.Error(t, s.UpsertChannel(ctx, ChannelRecord{ID: "c1"})) require.Error(t, s.ReplaceMembers(ctx, "g1", nil)) + require.Error(t, s.MergeMembers(ctx, nil)) require.Error(t, s.UpsertMember(ctx, MemberRecord{GuildID: "g1", UserID: "u1"})) require.Error(t, s.DeleteMember(ctx, "g1", "u1")) require.Error(t, s.UpsertMessageWithOptions(ctx, MessageRecord{ID: "m1"}, WriteOptions{})) @@ -1649,6 +1650,38 @@ func TestUpsertAndDeleteMember(t *testing.T) { require.Equal(t, "Other bio", rows[0].Bio) } +func TestMergeMembersPreservesExistingRows(t *testing.T) { + t.Parallel() + + ctx := context.Background() + dbPath := filepath.Join(t.TempDir(), "discrawl.db") + s, err := Open(ctx, dbPath) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + require.NoError(t, s.ReplaceMembers(ctx, "g1", []MemberRecord{{ + GuildID: "g1", + UserID: "stale", + Username: "stale-user", + RoleIDsJSON: `[]`, + RawJSON: `{"source":"preserved"}`, + }})) + require.NoError(t, s.MergeMembers(ctx, []MemberRecord{{ + GuildID: "g1", + UserID: "live", + Username: "live-user", + RoleIDsJSON: `[]`, + RawJSON: `{"source":"live"}`, + }})) + + rows, err := s.Members(ctx, "g1", "", 10) + require.NoError(t, err) + require.Len(t, rows, 2) + ids := []string{rows[0].UserID, rows[1].UserID} + require.Contains(t, ids, "stale") + require.Contains(t, ids, "live") +} + func TestOpenTightensDBFilePerms(t *testing.T) { t.Parallel() diff --git a/internal/store/write.go b/internal/store/write.go index 1021f58e..d939e8cb 100644 --- a/internal/store/write.go +++ b/internal/store/write.go @@ -156,6 +156,25 @@ func (s *Store) ReplaceMembers(ctx context.Context, guildID string, members []Me return tx.Commit() } +func (s *Store) MergeMembers(ctx context.Context, members []MemberRecord) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer rollback(tx) + qtx := s.q.WithTx(tx) + now := time.Now().UTC().Format(timeLayout) + for _, member := range members { + if err := qtx.UpsertMember(ctx, upsertMemberParams(member, now)); err != nil { + return err + } + if err := upsertMemberFTSTx(ctx, tx, member); err != nil { + return err + } + } + return tx.Commit() +} + func (s *Store) UpsertMember(ctx context.Context, member MemberRecord) error { tx, err := s.db.BeginTx(ctx, nil) if err != nil { diff --git a/internal/syncer/channel_exclusions.go b/internal/syncer/channel_exclusions.go new file mode 100644 index 00000000..f05f5753 --- /dev/null +++ b/internal/syncer/channel_exclusions.go @@ -0,0 +1,138 @@ +package syncer + +import ( + "context" + "strings" + + "github.com/bwmarrin/discordgo" + + "github.com/openclaw/discrawl/internal/store" +) + +type channelExclusions struct { + ids map[string]struct{} + kinds map[string]struct{} +} + +func newChannelExclusions(ids, kinds []string) channelExclusions { + return channelExclusions{ + ids: normalizedStringSet(ids, false), + kinds: normalizedStringSet(kinds, true), + } +} + +func normalizedStringSet(values []string, lower bool) map[string]struct{} { + out := make(map[string]struct{}, len(values)) + for _, value := range values { + value = strings.TrimSpace(value) + if lower { + value = strings.ToLower(value) + } + if value != "" { + out[value] = struct{}{} + } + } + return out +} + +func (e channelExclusions) merged(other channelExclusions) channelExclusions { + merged := channelExclusions{ + ids: make(map[string]struct{}, len(e.ids)+len(other.ids)), + kinds: make(map[string]struct{}, len(e.kinds)+len(other.kinds)), + } + for id := range e.ids { + merged.ids[id] = struct{}{} + } + for id := range other.ids { + merged.ids[id] = struct{}{} + } + for kind := range e.kinds { + merged.kinds[kind] = struct{}{} + } + for kind := range other.kinds { + merged.kinds[kind] = struct{}{} + } + return merged +} + +func (e channelExclusions) excludesID(channelID string) bool { + _, ok := e.ids[channelID] + return ok +} + +func (e channelExclusions) excludesKind(kind string) bool { + kind = strings.ToLower(strings.TrimSpace(kind)) + if _, ok := e.kinds[kind]; ok { + return true + } + if kind == "thread_announcement" { + _, ok := e.kinds["announcement"] + return ok + } + return false +} + +func (e channelExclusions) excludesDiscordChannel(channel *discordgo.Channel, channelByID map[string]*discordgo.Channel) bool { + if channel == nil { + return false + } + if e.excludesID(channel.ID) || e.excludesKind(channelKind(channel)) { + return true + } + if channel.ParentID == "" { + return false + } + if e.excludesID(channel.ParentID) { + return true + } + parent := channelByID[channel.ParentID] + return parent != nil && e.excludesKind(channelKind(parent)) +} + +func (e channelExclusions) excludesStoredChannel(channel store.ChannelRow, channelByID map[string]store.ChannelRow) bool { + if e.excludesID(channel.ID) || e.excludesKind(channel.Kind) { + return true + } + if channel.ParentID == "" { + return false + } + if e.excludesID(channel.ParentID) { + return true + } + parent, ok := channelByID[channel.ParentID] + return ok && e.excludesKind(parent.Kind) +} + +func (s *Syncer) effectiveChannelExclusions(opts SyncOptions) channelExclusions { + if s == nil { + return newChannelExclusions(opts.ExcludeChannelIDs, opts.ExcludeChannelKinds) + } + return s.channelExclusions.merged(newChannelExclusions(opts.ExcludeChannelIDs, opts.ExcludeChannelKinds)) +} + +func (s *Syncer) filterExcludedStoredChannelIDs(ctx context.Context, guildID string, channelIDs []string, opts SyncOptions) ([]string, error) { + exclusions := s.effectiveChannelExclusions(opts) + if len(exclusions.ids) == 0 && len(exclusions.kinds) == 0 { + return channelIDs, nil + } + channels, err := s.store.Channels(ctx, guildID) + if err != nil { + return nil, err + } + channelByID := make(map[string]store.ChannelRow, len(channels)) + for _, channel := range channels { + channelByID[channel.ID] = channel + } + out := make([]string, 0, len(channelIDs)) + for _, channelID := range channelIDs { + channel, ok := channelByID[channelID] + if ok && exclusions.excludesStoredChannel(channel, channelByID) { + continue + } + if !ok && exclusions.excludesID(channelID) { + continue + } + out = append(out, channelID) + } + return out, nil +} diff --git a/internal/syncer/extracted_test.go b/internal/syncer/extracted_test.go index f6a1739e..35c836f4 100644 --- a/internal/syncer/extracted_test.go +++ b/internal/syncer/extracted_test.go @@ -51,11 +51,29 @@ func TestFilterMessageChannelsHonorsRequestedIDs(t *testing.T) { {ID: "t1", Type: discordgo.ChannelTypeGuildPublicThread}, } - filtered := filterMessageChannels(channels, []string{"t1"}) + filtered := filterMessageChannels(channels, []string{"t1"}, channelExclusions{}) require.Len(t, filtered, 1) require.Equal(t, "t1", filtered[0].ID) } +func TestFilterMessageChannelsHonorsCollectionExclusions(t *testing.T) { + t.Parallel() + + channels := []*discordgo.Channel{ + {ID: "general", Type: discordgo.ChannelTypeGuildText}, + {ID: "logs", Type: discordgo.ChannelTypeGuildText}, + {ID: "news", Type: discordgo.ChannelTypeGuildNews}, + {ID: "news-thread", ParentID: "news", Type: discordgo.ChannelTypeGuildPublicThread}, + } + exclusions := newChannelExclusions([]string{"logs"}, []string{"announcement"}) + + filtered := filterMessageChannels(channels, nil, exclusions) + require.Equal(t, []string{"general"}, channelIDs(filtered)) + + filtered = filterMessageChannels(channels, []string{"logs", "news"}, exclusions) + require.Empty(t, filtered) +} + func TestSeedChannelSyncStateUsesStoredBounds(t *testing.T) { t.Parallel() diff --git a/internal/syncer/failures.go b/internal/syncer/failures.go index 98de4992..4d22ba05 100644 --- a/internal/syncer/failures.go +++ b/internal/syncer/failures.go @@ -31,6 +31,9 @@ func (s *Syncer) recordChannelFailure(ctx context.Context, guildID, channelID st if s == nil || s.store == nil || failure == nil { return nil } + if errors.Is(failure, context.Canceled) && ctx != nil && errors.Is(ctx.Err(), context.Canceled) { + return nil + } ledgerCtx, cancel := failureLedgerContext(ctx) defer cancel() return s.store.RecordFailure(ledgerCtx, store.FailureRef{ diff --git a/internal/syncer/failures_test.go b/internal/syncer/failures_test.go index 35b65119..a75e8238 100644 --- a/internal/syncer/failures_test.go +++ b/internal/syncer/failures_test.go @@ -3,6 +3,7 @@ package syncer import ( "context" "errors" + "fmt" "path/filepath" "testing" "time" @@ -367,3 +368,54 @@ func TestRecordTailFailureRejectsInvalidMessageMetadata(t *testing.T) { MessageID: "m1", }), "missing store") } + +func TestChannelFailureLedgerIgnoresOnlyParentCancellation(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = st.Close() }() + + svc := &Syncer{store: st} + canceledCtx, cancel := context.WithCancel(ctx) + cancel() + + require.NoError(t, svc.recordChannelFailure( + canceledCtx, + "g1", + "shutdown", + fmt.Errorf("request interrupted: %w", context.Canceled), + )) + require.NoError(t, svc.recordChannelFailure( + canceledCtx, + "g1", + "real-error", + errors.New("discord request failed"), + )) + require.NoError(t, svc.recordChannelFailure( + ctx, + "g1", + "unexpected-cancel", + context.Canceled, + )) + require.NoError(t, svc.recordChannelFailure( + ctx, + "g1", + "timeout", + context.DeadlineExceeded, + )) + + report, err := st.ListFailures(ctx, store.FailureListOptions{}, time.Now()) + require.NoError(t, err) + require.Len(t, report.Failures, 3) + + channelIDs := make(map[string]string, len(report.Failures)) + for _, failure := range report.Failures { + channelIDs[failure.ChannelID] = failure.ErrorClass + } + require.NotContains(t, channelIDs, "shutdown") + require.Equal(t, "errors.errorString", channelIDs["real-error"]) + require.Equal(t, "context_canceled", channelIDs["unexpected-cancel"]) + require.Equal(t, "deadline_exceeded", channelIDs["timeout"]) +} diff --git a/internal/syncer/message_sync.go b/internal/syncer/message_sync.go index 2571b59f..2af843cf 100644 --- a/internal/syncer/message_sync.go +++ b/internal/syncer/message_sync.go @@ -18,7 +18,7 @@ func (s *Syncer) syncMessageChannels( channels []*discordgo.Channel, opts SyncOptions, ) (int, error) { - messageChannels := filterMessageChannels(channels, opts.ChannelIDs) + messageChannels := filterMessageChannels(channels, opts.ChannelIDs, s.effectiveChannelExclusions(opts)) if len(messageChannels) == 0 { return 0, nil } @@ -38,7 +38,7 @@ func (s *Syncer) syncMessageChannels( return total, err } -func filterMessageChannels(channels []*discordgo.Channel, requested []string) []*discordgo.Channel { +func filterMessageChannels(channels []*discordgo.Channel, requested []string, exclusions channelExclusions) []*discordgo.Channel { requestedSet := makeGuildSet(requested) channelByID := make(map[string]*discordgo.Channel, len(channels)) for _, channel := range channels { @@ -51,6 +51,9 @@ func filterMessageChannels(channels []*discordgo.Channel, requested []string) [] if !isMessageChannel(channel) { continue } + if exclusions.excludesDiscordChannel(channel, channelByID) { + continue + } if len(requestedSet) > 0 && !requestedMessageTarget(channel, channelByID, requestedSet) { continue } @@ -80,7 +83,7 @@ func (s *Syncer) syncMessageChannelsSerial(ctx context.Context, guildID string, total := 0 for _, channel := range channels { progress.start(channel) - channelCtx, cancel := s.messageChannelContext(ctx) + channelCtx, cancel := s.messageChannelContext(ctx, opts) count, err := s.syncChannelMessages(channelCtx, guildID, channel, opts.Full, opts.Embeddings, opts.Since, opts.LatestOnly, progress) cancel() total += count @@ -135,7 +138,7 @@ func (s *Syncer) syncMessageChannelsConcurrent( return } progress.start(channel) - channelCtx, cancelChannel := s.messageChannelContext(ctx) + channelCtx, cancelChannel := s.messageChannelContext(ctx, opts) count, err := s.syncChannelMessages(channelCtx, guildID, channel, opts.Full, opts.Embeddings, opts.Since, opts.LatestOnly, progress) cancelChannel() succeeded := err == nil @@ -209,14 +212,25 @@ func (s *Syncer) clearUnavailableChannel(ctx context.Context, channelID string) return s.store.DeleteSyncState(ctx, channelMessageUnavailableScope(channelID)) } -func (s *Syncer) messageChannelContext(ctx context.Context) (context.Context, context.CancelFunc) { - if s == nil || s.messageChannelTimeout <= 0 { +func (s *Syncer) messageChannelContext(ctx context.Context, opts SyncOptions) (context.Context, context.CancelFunc) { + timeout := s.effectiveMessageChannelTimeout(opts) + if timeout <= 0 { return context.WithCancel(ctx) } if _, ok := ctx.Deadline(); ok { return context.WithCancel(ctx) } - return context.WithTimeout(ctx, s.messageChannelTimeout) + return context.WithTimeout(ctx, timeout) +} + +func (s *Syncer) effectiveMessageChannelTimeout(opts SyncOptions) time.Duration { + if opts.MessageChannelTimeout != 0 { + return opts.MessageChannelTimeout + } + if s == nil { + return 0 + } + return s.messageChannelTimeout } func (s *Syncer) syncChannelMessages(ctx context.Context, guildID string, channel *discordgo.Channel, full bool, embeddings bool, since time.Time, latestOnly bool, progress *messageSyncProgress) (int, error) { @@ -605,7 +619,7 @@ func newMessageSyncProgress(s *Syncer, guildID string, totalChannels int, opts S "channels", totalChannels, "full", opts.Full, "concurrency", max(1, opts.Concurrency), - "channel_timeout", timeoutLabel(s.messageChannelTimeout), + "channel_timeout", timeoutLabel(s.effectiveMessageChannelTimeout(opts)), ) go progress.runWaitHeartbeat() return progress diff --git a/internal/syncer/message_sync_helpers_test.go b/internal/syncer/message_sync_helpers_test.go index a02f6f5b..6e2eb90a 100644 --- a/internal/syncer/message_sync_helpers_test.go +++ b/internal/syncer/message_sync_helpers_test.go @@ -22,27 +22,27 @@ func TestMessageChannelSelectionAndTimeoutHelpers(t *testing.T) { text := &discordgo.Channel{ID: "text", GuildID: "g1", Name: "text", Type: discordgo.ChannelTypeGuildText} voice := &discordgo.Channel{ID: "voice", GuildID: "g1", Name: "voice", Type: discordgo.ChannelTypeGuildVoice} - rows := filterMessageChannels([]*discordgo.Channel{nil, parent, thread, text, voice}, []string{"forum"}) + rows := filterMessageChannels([]*discordgo.Channel{nil, parent, thread, text, voice}, []string{"forum"}, channelExclusions{}) require.Equal(t, []string{"thread"}, channelIDs(rows)) require.False(t, requestedMessageTarget(nil, nil, map[string]struct{}{})) require.True(t, requestedMessageTarget(text, map[string]*discordgo.Channel{"text": text}, map[string]struct{}{"text": {}})) require.False(t, requestedMessageTarget(thread, map[string]*discordgo.Channel{}, map[string]struct{}{"forum": {}})) - ctx, cancel := (*Syncer)(nil).messageChannelContext(context.Background()) + ctx, cancel := (*Syncer)(nil).messageChannelContext(context.Background(), SyncOptions{}) require.NoError(t, ctx.Err()) cancel() require.ErrorIs(t, ctx.Err(), context.Canceled) svc := New(&fakeClient{}, nil, nil) svc.messageChannelTimeout = time.Second - ctx, cancel = svc.messageChannelContext(context.Background()) + ctx, cancel = svc.messageChannelContext(context.Background(), SyncOptions{}) defer cancel() _, ok := ctx.Deadline() require.True(t, ok) parentCtx, parentCancel := context.WithDeadline(context.Background(), time.Now().Add(time.Hour)) defer parentCancel() - ctx, cancel = svc.messageChannelContext(parentCtx) + ctx, cancel = svc.messageChannelContext(parentCtx, SyncOptions{MessageChannelTimeout: 2 * time.Second}) defer cancel() deadline, ok := ctx.Deadline() require.True(t, ok) diff --git a/internal/syncer/message_sync_timeout_test.go b/internal/syncer/message_sync_timeout_test.go index 523956f4..1d700581 100644 --- a/internal/syncer/message_sync_timeout_test.go +++ b/internal/syncer/message_sync_timeout_test.go @@ -42,8 +42,7 @@ func TestSyncDefersSlowChannelAfterChannelTimeout(t *testing.T) { } svc := New(client, s, nil) - svc.messageChannelTimeout = 20 * time.Millisecond - stats, err := svc.Sync(ctx, SyncOptions{Full: true, Concurrency: 1}) + stats, err := svc.Sync(ctx, SyncOptions{Full: true, Concurrency: 1, MessageChannelTimeout: 20 * time.Millisecond}) require.NoError(t, err) require.Equal(t, 0, stats.Messages) diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go index 2564ae23..0ad7e427 100644 --- a/internal/syncer/syncer.go +++ b/internal/syncer/syncer.go @@ -46,25 +46,35 @@ type Syncer struct { tailRepair func(context.Context, SyncOptions) (SyncStats, error) tailRepairJoinTimeout time.Duration tailRepairMu sync.Mutex + tailRepairOffsetMu sync.RWMutex + tailRepairOffset time.Duration + channelExclusions channelExclusions } type SyncOptions struct { - Full bool - GuildIDs []string - ChannelIDs []string - Concurrency int - Since time.Time - Embeddings bool - SkipMembers bool - RequireMembers bool - LatestOnly bool - RepairReason string + Full bool + GuildIDs []string + ChannelIDs []string + Concurrency int + Since time.Time + Embeddings bool + SkipMembers bool + RequireMembers bool + LatestOnly bool + MessageChannelTimeout time.Duration + ExcludeChannelIDs []string + ExcludeChannelKinds []string + RepairReason string } func (s *Syncer) SetTailReadyCallback(fn func(context.Context) error) { s.tailReady = fn } +func (s *Syncer) SetChannelExclusions(channelIDs, channelKinds []string) { + s.channelExclusions = newChannelExclusions(channelIDs, channelKinds) +} + type SyncStats struct { Guilds int `json:"guilds"` Channels int `json:"channels"` @@ -240,6 +250,10 @@ func (s *Syncer) syncGuildIncompleteBatches(ctx context.Context, guildID string, if err != nil { return SyncStats{}, false, err } + incomplete, err = s.filterExcludedStoredChannelIDs(ctx, guildID, incomplete, opts) + if err != nil { + return SyncStats{}, false, err + } if len(incomplete) == 0 { return SyncStats{}, false, nil } @@ -305,9 +319,9 @@ func (s *Syncer) refreshGuildMembers(ctx context.Context, guildID string, force for _, member := range members { converted = append(converted, toMemberRecord(guildID, member)) } - if err := s.store.ReplaceMembers(ctx, guildID, converted); err != nil { - s.logger.Warn("member replace failed", "guild_id", guildID, "err", err) - return 0, fmt.Errorf("replace guild members: %w", err) + if err := s.store.MergeMembers(ctx, converted); err != nil { + s.logger.Warn("member merge failed", "guild_id", guildID, "err", err) + return 0, fmt.Errorf("merge guild members: %w", err) } if s.store != nil { if err := s.store.SetSyncState(ctx, guildMemberSyncSuccessScope(guildID), time.Now().UTC().Format(time.RFC3339Nano)); err != nil { diff --git a/internal/syncer/syncer_tail_test.go b/internal/syncer/syncer_tail_test.go index 44c4e55e..7bef49e6 100644 --- a/internal/syncer/syncer_tail_test.go +++ b/internal/syncer/syncer_tail_test.go @@ -134,6 +134,81 @@ func TestTailHandlerWritesEvents(t *testing.T) { require.Equal(t, "10", cursor) } +func TestTailHandlerDoesNotRecordOrderlyShutdownCancellation(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + canceledCtx, cancel := context.WithCancel(ctx) + cancel() + handler := &tailHandler{store: s} + err = handler.OnMessageCreate(canceledCtx, &discordgo.Message{ + ID: "shutdown", + GuildID: "g1", + ChannelID: "c1", + Content: "tail event", + Timestamp: time.Now().UTC(), + Author: &discordgo.User{ID: "u1", Username: "peter"}, + }) + require.ErrorIs(t, err, context.Canceled) + + report, err := s.ListFailures(ctx, store.FailureListOptions{}, time.Now()) + require.NoError(t, err) + require.Empty(t, report.Failures) +} + +func TestTailHandlerExcludesConfiguredFeedChannels(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + require.NoError(t, s.UpsertChannel(ctx, store.ChannelRecord{ + ID: "news", + GuildID: "g1", + Kind: "announcement", + Name: "github", + RawJSON: `{}`, + })) + handler := &tailHandler{ + store: s, + exclusions: newChannelExclusions([]string{"logs"}, []string{"announcement"}), + kindExcludedChannelIDs: map[string]struct{}{}, + } + require.NoError(t, handler.seedChannelExclusions(ctx)) + + message := func(id, channelID string) *discordgo.Message { + return &discordgo.Message{ + ID: id, + GuildID: "g1", + ChannelID: channelID, + Content: "event", + Timestamp: time.Now().UTC(), + Author: &discordgo.User{ID: "u1", Username: "bot", Bot: true}, + } + } + require.NoError(t, handler.OnMessageCreate(ctx, message("1", "news"))) + require.NoError(t, handler.OnMessageCreate(ctx, message("2", "logs"))) + require.NoError(t, handler.OnMessageCreate(ctx, message("3", "general"))) + + require.NoError(t, handler.OnChannelUpsert(ctx, &discordgo.Channel{ + ID: "future-news", + GuildID: "g1", + Name: "future-feed", + Type: discordgo.ChannelTypeGuildNews, + })) + require.NoError(t, handler.OnMessageCreate(ctx, message("4", "future-news"))) + + status, err := s.Status(ctx, "db", "") + require.NoError(t, err) + require.Equal(t, 1, status.MessageCount) +} + func TestTailHandlerMessageUpdateFetchesFullMessageBeforeUpsert(t *testing.T) { t.Parallel() @@ -496,17 +571,48 @@ func TestRunTailWithRepairLoop(t *testing.T) { require.NoError(t, err) defer func() { _ = s.Close() }() + require.NoError(t, s.UpsertChannel(ctx, store.ChannelRecord{ + ID: "archived", + GuildID: "g1", + ParentID: "c1", + Kind: "thread_public", + Name: "archived-thread", + IsArchived: true, + RawJSON: `{"id":"archived"}`, + })) + require.NoError(t, s.SetSyncState(ctx, channelLatestScope("active"), "200")) + require.NoError(t, s.SetSyncState(ctx, channelLatestScope("archived"), "300")) + + messageStarted := make(chan string, 4) client := &fakeClient{ guilds: []*discordgo.UserGuild{{ID: "g1", Name: "Guild"}}, guildByID: map[string]*discordgo.Guild{ "g1": {ID: "g1", Name: "Guild"}, }, channels: map[string][]*discordgo.Channel{ - "g1": {{ID: "c1", GuildID: "g1", Name: "general", Type: discordgo.ChannelTypeGuildText}}, + "g1": {{ + ID: "c1", + GuildID: "g1", + Name: "general", + Type: discordgo.ChannelTypeGuildText, + LastMessageID: "3", + }}, + }, + guildThreads: map[string][]*discordgo.Channel{ + "g1": {{ + ID: "active", + GuildID: "g1", + ParentID: "c1", + Name: "active-thread", + Type: discordgo.ChannelTypeGuildPublicThread, + LastMessageID: "201", + }}, }, messages: map[string][]*discordgo.Message{ - "c1": {{ID: "10", GuildID: "g1", ChannelID: "c1", Content: "repair", Timestamp: time.Now().UTC(), Author: &discordgo.User{ID: "u1", Username: "user"}}}, + "active": {{ID: "201", GuildID: "g1", ChannelID: "active", Content: "repair", Timestamp: time.Now().UTC(), Author: &discordgo.User{ID: "u1", Username: "user"}}}, + "archived": {{ID: "301", GuildID: "g1", ChannelID: "archived", Content: "old history", Timestamp: time.Now().UTC(), Author: &discordgo.User{ID: "u1", Username: "user"}}}, }, + messageStarted: messageStarted, } require.NoError(t, s.RecordFailure(ctx, store.FailureRef{ Operation: tailMessageFailureOperation, @@ -516,16 +622,42 @@ func TestRunTailWithRepairLoop(t *testing.T) { MessageID: "5", }, context.DeadlineExceeded)) svc := New(client, s, nil) + done := make(chan error, 1) go func() { - time.Sleep(40 * time.Millisecond) - cancel() + done <- svc.RunTail(ctx, []string{"g1"}, 10*time.Millisecond) }() - err = svc.RunTail(ctx, []string{"g1"}, 10*time.Millisecond) + + select { + case channelID := <-messageStarted: + require.Equal(t, "active", channelID) + case <-time.After(time.Second): + t.Fatal("periodic repair did not request the active thread") + } + require.Eventually(t, func() bool { + lastSuccess, getErr := s.GetSyncState(ctx, "sync:last_success") + return getErr == nil && lastSuccess != "" + }, time.Second, 5*time.Millisecond) + cancel() + err = <-done require.True(t, err == nil || errors.Is(err, context.Canceled)) + client.mu.Lock() + messageCalls := make(map[string]int, len(client.messageCalls)) + for channelID, calls := range client.messageCalls { + messageCalls[channelID] = calls + } + client.mu.Unlock() + + require.GreaterOrEqual(t, client.guildThreadCalls, 1) + require.Zero(t, client.threadCalls) + require.Zero(t, client.memberCalls) + require.Zero(t, messageCalls["c1"]) + require.Zero(t, messageCalls["archived"]) + require.GreaterOrEqual(t, messageCalls["active"], 1) + status, err := s.Status(context.Background(), "db", "") require.NoError(t, err) - require.GreaterOrEqual(t, status.MessageCount, 1) + require.GreaterOrEqual(t, status.MessageCount, 2) client.mu.Lock() exactMessageCalls := client.exactMessageCalls client.mu.Unlock() @@ -918,6 +1050,86 @@ func TestReplayTailMessageFailuresRetainsFetchAndIdentityFailures(t *testing.T) } } +func TestTailRepairSyncOptions(t *testing.T) { + t.Parallel() + + guildIDs := []string{"g1", "g2"} + opts := tailRepairSyncOptions(guildIDs) + require.Equal(t, guildIDs, opts.GuildIDs) + require.False(t, opts.Full) + require.True(t, opts.LatestOnly) + require.True(t, opts.SkipMembers) + require.Equal(t, "tail_repair", opts.RepairReason) + + guildIDs[0] = "changed" + require.Equal(t, []string{"g1", "g2"}, opts.GuildIDs) +} + +func TestNextTailRepairDelayForInitialSchedule(t *testing.T) { + t.Parallel() + + location := time.FixedZone("half-hour", 5*60*60+30*60) + now := time.Date(2026, 7, 9, 10, 10, 0, 0, location) + + require.Equal(t, time.Hour, nextTailRepairDelay(now, time.Hour, 0)) + require.Equal(t, time.Hour, nextTailRepairDelay(now, time.Hour, -15*time.Minute)) + require.Equal(t, 5*time.Minute, nextTailRepairDelay(now, time.Hour, 15*time.Minute)) + require.Equal(t, 5*time.Minute, nextTailRepairDelay(now, time.Hour, 75*time.Minute)) + require.Equal(t, 50*time.Minute, nextTailRepairDelay(now, time.Hour, time.Hour)) + require.Zero(t, nextTailRepairDelay(now, 0, 15*time.Minute)) + + onBoundary := time.Date(2026, 7, 9, 10, 15, 0, 0, location) + require.Equal(t, time.Hour, nextTailRepairDelay(onBoundary, time.Hour, 15*time.Minute)) +} + +func TestNextTailRepairDelayRealignsAfterRepair(t *testing.T) { + t.Parallel() + + location := time.FixedZone("half-hour", 5*60*60+30*60) + firstCheck := time.Date(2026, 7, 9, 10, 10, 0, 0, location) + require.Equal(t, 5*time.Minute, nextTailRepairDelay(firstCheck, time.Hour, 15*time.Minute)) + + shortRepairFinished := time.Date(2026, 7, 9, 10, 17, 0, 0, location) + require.Equal(t, 58*time.Minute, nextTailRepairDelay(shortRepairFinished, time.Hour, 15*time.Minute)) + + longRepairFinished := time.Date(2026, 7, 9, 13, 20, 0, 0, location) + require.Equal(t, 55*time.Minute, nextTailRepairDelay(longRepairFinished, time.Hour, 15*time.Minute)) +} + +func TestNextTailRepairDelayKeepsEdmontonCivilPhaseAcrossDST(t *testing.T) { + t.Parallel() + + location, err := time.LoadLocation("America/Edmonton") + require.NoError(t, err) + + tests := []struct { + name string + now time.Time + want time.Time + wantDelay time.Duration + }{ + { + name: "spring forward", + now: time.Date(2026, time.March, 8, 0, 15, 0, 0, location), + want: time.Date(2026, time.March, 8, 6, 15, 0, 0, location), + wantDelay: 5 * time.Hour, + }, + { + name: "fall back", + now: time.Date(2026, time.November, 1, 0, 15, 0, 0, location), + want: time.Date(2026, time.November, 1, 6, 15, 0, 0, location), + wantDelay: 7 * time.Hour, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + delay := nextTailRepairDelay(test.now, 6*time.Hour, 15*time.Minute) + require.Equal(t, test.wantDelay, delay) + require.Equal(t, test.want, test.now.Add(delay)) + }) + } +} + func TestReplayTailMessageFailuresDoesNotRewriteLedgerOnParentCancellation(t *testing.T) { t.Parallel() @@ -1395,6 +1607,59 @@ func TestReplayTailMessageFailuresKeepsIncompleteIdentityVisible(t *testing.T) { require.Equal(t, 1, report.Failures[0].RetryCount) } +func TestNextTailRepairDelaySkipsNonexistentCivilSlot(t *testing.T) { + t.Parallel() + + location, err := time.LoadLocation("America/Edmonton") + require.NoError(t, err) + + now := time.Date(2026, time.March, 8, 1, 30, 0, 0, location) + delay := nextTailRepairDelay(now, time.Hour, 30*time.Minute) + + require.Equal(t, time.Hour, delay) + require.Equal(t, time.Date(2026, time.March, 8, 3, 30, 0, 0, location), now.Add(delay)) +} + +func TestRepairOffsetIsPerSyncer(t *testing.T) { + t.Parallel() + + first := New(&fakeClient{}, nil, nil) + second := New(&fakeClient{}, nil, nil) + first.SetRepairOffset(15 * time.Minute) + second.SetRepairOffset(45 * time.Minute) + + require.Equal(t, 15*time.Minute, first.repairOffset()) + require.Equal(t, 45*time.Minute, second.repairOffset()) + + first.SetRepairOffset(0) + require.Zero(t, first.repairOffset()) + require.Equal(t, 45*time.Minute, second.repairOffset()) +} + +func TestRepairOffsetConcurrentAccess(t *testing.T) { + t.Parallel() + + svc := New(&fakeClient{}, nil, nil) + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + for i := 0; i < 1_000; i++ { + svc.SetRepairOffset(time.Duration(i%5) * time.Minute) + } + }() + go func() { + defer wg.Done() + for i := 0; i < 1_000; i++ { + _ = svc.repairOffset() + } + }() + wg.Wait() + + svc.SetRepairOffset(15 * time.Minute) + require.Equal(t, 15*time.Minute, svc.repairOffset()) +} + func TestRunTailWithRepairLoopJoinsTailOnCancellation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) diff --git a/internal/syncer/tail.go b/internal/syncer/tail.go index 85e57e55..19a71b1a 100644 --- a/internal/syncer/tail.go +++ b/internal/syncer/tail.go @@ -13,17 +13,43 @@ import ( "github.com/openclaw/discrawl/internal/store" ) +func (s *Syncer) SetRepairOffset(offset time.Duration) { + if s == nil { + return + } + if offset <= 0 { + offset = 0 + } + s.tailRepairOffsetMu.Lock() + s.tailRepairOffset = offset + s.tailRepairOffsetMu.Unlock() +} + +func (s *Syncer) repairOffset() time.Duration { + if s == nil { + return 0 + } + s.tailRepairOffsetMu.RLock() + defer s.tailRepairOffsetMu.RUnlock() + return s.tailRepairOffset +} + func (s *Syncer) RunTail(ctx context.Context, guildIDs []string, repairEvery time.Duration) error { if err := s.importTailMessageFailureFallbacks(ctx); err != nil { return err } handler := &tailHandler{ - guilds: makeGuildSet(guildIDs), - store: s.store, - client: s.client, - attachmentTextEnabled: s.attachmentTextEnabled, - onReady: s.tailReady, - logger: s.logger, + guilds: makeGuildSet(guildIDs), + store: s.store, + client: s.client, + attachmentTextEnabled: s.attachmentTextEnabled, + onReady: s.tailReady, + logger: s.logger, + exclusions: s.channelExclusions, + kindExcludedChannelIDs: map[string]struct{}{}, + } + if err := handler.seedChannelExclusions(ctx); err != nil { + return fmt.Errorf("seed tail channel exclusions: %w", err) } if repairEvery <= 0 { return s.client.Tail(ctx, handler) @@ -40,8 +66,16 @@ func (s *Syncer) RunTail(ctx context.Context, guildIDs []string, repairEvery tim go func() { tailDone <- s.client.Tail(tailCtx, handler) }() - ticker := time.NewTicker(repairEvery) - defer ticker.Stop() + repairOffset := s.repairOffset() + repairTimer := time.NewTimer(nextTailRepairDelay(time.Now(), repairEvery, repairOffset)) + defer repairTimer.Stop() + repairC := repairTimer.C + var repairTicker *time.Ticker + defer func() { + if repairTicker != nil { + repairTicker.Stop() + } + }() var activeRepair *tailRepairRun var repairDone <-chan tailRepairResult for { @@ -73,7 +107,15 @@ func (s *Syncer) RunTail(ctx context.Context, guildIDs []string, repairEvery tim s.logTailRepairResult(result) activeRepair = nil repairDone = nil - case <-ticker.C: + if repairOffset > 0 { + repairTimer.Reset(nextTailRepairDelay(time.Now(), repairEvery, repairOffset)) + repairC = repairTimer.C + } + case <-repairC: + if repairOffset <= 0 && repairTicker == nil { + repairTicker = time.NewTicker(repairEvery) + repairC = repairTicker.C + } if activeRepair != nil { continue } @@ -81,6 +123,14 @@ func (s *Syncer) RunTail(ctx context.Context, guildIDs []string, repairEvery tim if activeRepair != nil { repairDone = activeRepair.done } + if repairOffset > 0 { + if activeRepair == nil { + repairTimer.Reset(nextTailRepairDelay(time.Now(), repairEvery, repairOffset)) + repairC = repairTimer.C + } else { + repairC = nil + } + } } } } @@ -123,11 +173,7 @@ func (s *Syncer) startTailRepair(ctx context.Context, guildIDs []string) *tailRe startedAt := time.Now() go func() { defer s.tailRepairMu.Unlock() - _, err := s.runTailRepair(repairCtx, SyncOptions{ - GuildIDs: guildIDs, - Full: false, - RepairReason: "tail_repair", - }) + _, err := s.runTailRepair(repairCtx, tailRepairSyncOptions(guildIDs)) done <- tailRepairResult{err: err, elapsed: time.Since(startedAt)} }() return &tailRepairRun{cancel: cancel, done: done} @@ -189,14 +235,72 @@ func (s *Syncer) logTailRepairResult(result tailRepairResult) { ) } +func tailRepairSyncOptions(guildIDs []string) SyncOptions { + return SyncOptions{ + GuildIDs: append([]string(nil), guildIDs...), + Full: false, + SkipMembers: true, + LatestOnly: true, + RepairReason: "tail_repair", + } +} + +func nextTailRepairDelay(now time.Time, repairEvery, repairOffset time.Duration) time.Duration { + if repairEvery <= 0 { + return 0 + } + if repairOffset <= 0 { + return repairEvery + } + normalizedOffset := repairOffset % repairEvery + civilNow := civilTimelineTime(now) + nextCivil := civilNow.Truncate(repairEvery).Add(normalizedOffset) + if !nextCivil.After(civilNow) { + nextCivil = nextCivil.Add(repairEvery) + } + for { + // A candidate that does not round-trip is a nonexistent civil slot. + next := time.Date( + nextCivil.Year(), + nextCivil.Month(), + nextCivil.Day(), + nextCivil.Hour(), + nextCivil.Minute(), + nextCivil.Second(), + nextCivil.Nanosecond(), + now.Location(), + ) + if civilTimelineTime(next).Equal(nextCivil) && next.After(now) { + return next.Sub(now) + } + nextCivil = nextCivil.Add(repairEvery) + } +} + +func civilTimelineTime(value time.Time) time.Time { + return time.Date( + value.Year(), + value.Month(), + value.Day(), + value.Hour(), + value.Minute(), + value.Second(), + value.Nanosecond(), + time.UTC, + ) +} + type tailHandler struct { - guilds map[string]struct{} - store *store.Store - client Client - attachmentTextEnabled bool - failureLedgerTimeout time.Duration - onReady func(context.Context) error - logger *slog.Logger + guilds map[string]struct{} + store *store.Store + client Client + attachmentTextEnabled bool + failureLedgerTimeout time.Duration + onReady func(context.Context) error + logger *slog.Logger + exclusions channelExclusions + exclusionMu sync.RWMutex + kindExcludedChannelIDs map[string]struct{} } func (t *tailHandler) OnTailReady(ctx context.Context) error { @@ -231,7 +335,7 @@ func (t *tailHandler) OnTailFailure(failure discordclient.TailFailure) { func (t *tailHandler) OnMessageCreate(ctx context.Context, msg *discordgo.Message) error { discordclient.UpdateTailFailureStage(ctx, discordclient.TailFailureStageHandler) - if !t.allowGuild(msg.GuildID) { + if msg == nil || !t.allowGuild(msg.GuildID) || t.excludeChannel(msg.ChannelID) { return nil } discordclient.UpdateTailFailureStage(ctx, discordclient.TailFailureStageMessageBuild) @@ -263,7 +367,7 @@ func (t *tailHandler) OnMessageUpdate(ctx context.Context, msg *discordgo.Messag if msg == nil { return nil } - if msg.GuildID != "" && !t.allowGuild(msg.GuildID) { + if t.excludeChannel(msg.ChannelID) || (msg.GuildID != "" && !t.allowGuild(msg.GuildID)) { return nil } var err error @@ -271,7 +375,7 @@ func (t *tailHandler) OnMessageUpdate(ctx context.Context, msg *discordgo.Messag if err != nil { return err } - if msg == nil || !t.allowGuild(msg.GuildID) { + if msg == nil || !t.allowGuild(msg.GuildID) || t.excludeChannel(msg.ChannelID) { return nil } discordclient.UpdateTailFailureStage(ctx, discordclient.TailFailureStageMessageBuild) @@ -361,7 +465,7 @@ func isPartialMessageUpdate(msg *discordgo.Message) bool { func (t *tailHandler) OnMessageDelete(ctx context.Context, evt *discordgo.MessageDelete) error { discordclient.UpdateTailFailureStage(ctx, discordclient.TailFailureStageHandler) - if !t.allowGuild(evt.GuildID) { + if evt == nil || !t.allowGuild(evt.GuildID) || t.excludeChannel(evt.ChannelID) { return nil } discordclient.UpdateTailFailureStage(ctx, discordclient.TailFailureStageCanonicalDelete) @@ -376,9 +480,10 @@ func (t *tailHandler) OnMessageDelete(ctx context.Context, evt *discordgo.Messag } func (t *tailHandler) OnChannelUpsert(ctx context.Context, channel *discordgo.Channel) error { - if !t.allowGuild(channel.GuildID) { + if channel == nil || !t.allowGuild(channel.GuildID) { return nil } + t.trackChannelExclusion(channel) return t.store.UpsertChannel(ctx, toChannelRecord(channel, marshalJSONString(channel, "{}"))) } @@ -407,3 +512,58 @@ func (t *tailHandler) allowGuild(guildID string) bool { _, ok := t.guilds[guildID] return ok } + +func (t *tailHandler) seedChannelExclusions(ctx context.Context) error { + if t.store == nil { + return nil + } + channels, err := t.store.Channels(ctx, "") + if err != nil { + return err + } + channelByID := make(map[string]store.ChannelRow, len(channels)) + for _, channel := range channels { + channelByID[channel.ID] = channel + } + t.exclusionMu.Lock() + defer t.exclusionMu.Unlock() + if t.kindExcludedChannelIDs == nil { + t.kindExcludedChannelIDs = map[string]struct{}{} + } + for _, channel := range channels { + if t.exclusions.excludesStoredChannel(channel, channelByID) { + t.kindExcludedChannelIDs[channel.ID] = struct{}{} + } + } + return nil +} + +func (t *tailHandler) excludeChannel(channelID string) bool { + if t.exclusions.excludesID(channelID) { + return true + } + t.exclusionMu.RLock() + defer t.exclusionMu.RUnlock() + _, ok := t.kindExcludedChannelIDs[channelID] + return ok +} + +func (t *tailHandler) trackChannelExclusion(channel *discordgo.Channel) { + if channel == nil { + return + } + excluded := t.exclusions.excludesKind(channelKind(channel)) + if !excluded && channel.ParentID != "" { + excluded = t.excludeChannel(channel.ParentID) + } + t.exclusionMu.Lock() + defer t.exclusionMu.Unlock() + if t.kindExcludedChannelIDs == nil { + t.kindExcludedChannelIDs = map[string]struct{}{} + } + if excluded { + t.kindExcludedChannelIDs[channel.ID] = struct{}{} + return + } + delete(t.kindExcludedChannelIDs, channel.ID) +} From 9612267f7c822a34e7b8f400fd5fe0bc1ca1b5c5 Mon Sep 17 00:00:00 2001 From: Hannes Rudolph <49103247+hannesrudolph@users.noreply.github.com> Date: Thu, 9 Jul 2026 13:38:34 -0600 Subject: [PATCH 2/7] fix: harden hybrid archive collection safeguards --- docs/commands/sync.md | 1 + internal/cli/admin_commands.go | 12 + internal/cli/cli.go | 62 ++- internal/cli/cli_test.go | 194 +++++++++ internal/discorddesktop/import.go | 132 ++++-- .../discorddesktop/import_helpers_test.go | 34 +- .../discorddesktop/import_pipeline_test.go | 165 ++++++++ internal/share/import_filter.go | 28 +- internal/share/merge.go | 78 +++- internal/share/share.go | 167 ++++++-- internal/share/share_test.go | 239 ++++++++++- internal/syncer/channel_catalog.go | 59 ++- internal/syncer/channel_catalog_test.go | 55 +++ internal/syncer/channel_exclusions.go | 56 ++- internal/syncer/message_sync.go | 3 - internal/syncer/message_sync_helpers_test.go | 15 +- internal/syncer/syncer.go | 13 +- internal/syncer/syncer_tail_test.go | 391 ++++++++++++++++++ internal/syncer/tail.go | 282 +++++++++---- 19 files changed, 1808 insertions(+), 178 deletions(-) diff --git a/docs/commands/sync.md b/docs/commands/sync.md index a7b6701c..4a8ea0b2 100644 --- a/docs/commands/sync.md +++ b/docs/commands/sync.md @@ -64,6 +64,7 @@ discrawl sync --with-media - `--exclude-channel-kinds ` - omit message channels by stored Discord kind, such as `announcement` - `--since ` - limit initial history and `--full` backfill to messages at or after this timestamp - `--concurrency ` - override worker count (default auto-sized: floor 8, cap 32) +- `--channel-timeout ` - bound each channel crawl; the shorter parent deadline still wins - `--skip-members` - refresh guild/channel/message data without crawling members - `--with-members` - refresh guild members even during the default latest-only sync; fail if the member crawl cannot complete - `--with-embeddings` - also enqueue changed messages into `embedding_jobs` diff --git a/internal/cli/admin_commands.go b/internal/cli/admin_commands.go index 58b88109..5b0c8b2e 100644 --- a/internal/cli/admin_commands.go +++ b/internal/cli/admin_commands.go @@ -107,6 +107,10 @@ func (r *runtime) runInit(args []string) error { } func (r *runtime) runSync(args []string) error { + return r.runSyncWithShareUpdate(args, shareUpdateNever) +} + +func (r *runtime) runSyncWithShareUpdate(args []string, shareUpdate shareUpdateMode) error { fs := flag.NewFlagSet("sync", flag.ContinueOnError) fs.SetOutput(io.Discard) full := fs.Bool("full", false, "") @@ -182,6 +186,14 @@ func (r *runtime) runSync(args []string) error { ExcludeChannelKinds: csvList(*excludeChannelKinds), } return r.withSyncLock(func() error { + if shareUpdate != shareUpdateNever && os.Getenv("DISCRAWL_NO_AUTO_UPDATE") != "1" { + if err := r.autoUpdateShareWithExclusions(shareUpdate, &shareMergeExclusions{ + channelIDs: opts.ExcludeChannelIDs, + channelKinds: opts.ExcludeChannelKinds, + }); err != nil { + return err + } + } return r.runSyncLocked(sources, opts, *withMedia) }) } diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 494b9430..f9eb929a 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -8,6 +8,7 @@ import ( "log/slog" "os" "strings" + "sync" "time" "github.com/alecthomas/kong" @@ -290,6 +291,48 @@ type discordClient interface { Guilds(context.Context) ([]*discordgo.UserGuild, error) } +// coordinatedDiscordClient lets the tail shutdown path own the first Close +// while making outer service cleanup idempotent and non-blocking if that close +// is still in progress. +type coordinatedDiscordClient struct { + discordClient + closeMu sync.Mutex + closeStarted bool + closeDone chan struct{} + closeErr error +} + +func (c *coordinatedDiscordClient) Close() error { + if c == nil || c.discordClient == nil { + return nil + } + c.closeMu.Lock() + if c.closeStarted { + done := c.closeDone + c.closeMu.Unlock() + select { + case <-done: + c.closeMu.Lock() + err := c.closeErr + c.closeMu.Unlock() + return err + default: + return nil + } + } + c.closeStarted = true + c.closeDone = make(chan struct{}) + done := c.closeDone + c.closeMu.Unlock() + + err := c.discordClient.Close() + c.closeMu.Lock() + c.closeErr = err + close(done) + c.closeMu.Unlock() + return err +} + type syncService interface { DiscoverGuilds(context.Context) ([]*discordgo.UserGuild, error) Sync(context.Context, syncer.SyncOptions) (syncer.SyncStats, error) @@ -329,7 +372,9 @@ func (r *runtime) dispatch(rest []string) error { if err != nil { return usageErr(err) } - return r.withLocalStoreUpdateLocked(updateMode, true, func() error { return r.runSync(rest[1:]) }) + return r.withLocalStoreUpdateLocked(shareUpdateNever, true, func() error { + return r.runSyncWithShareUpdate(rest[1:], updateMode) + }) case "tail": operation := "tail-starting" if tailReplayOnlyEnabled(rest[1:]) { @@ -763,7 +808,7 @@ func (r *runtime) ensureDiscordServices() error { if err != nil { return authErr(err) } - r.client = client + r.client = &coordinatedDiscordClient{discordClient: client} syncerFactory := r.newSyncer if syncerFactory == nil { syncerFactory = func(client syncer.Client, s *store.Store, logger *slog.Logger) syncService { @@ -780,7 +825,16 @@ func (r *runtime) ensureDiscordServices() error { return nil } +type shareMergeExclusions struct { + channelIDs []string + channelKinds []string +} + func (r *runtime) autoUpdateShare(mode shareUpdateMode) error { + return r.autoUpdateShareWithExclusions(mode, nil) +} + +func (r *runtime) autoUpdateShareWithExclusions(mode shareUpdateMode, exclusions *shareMergeExclusions) error { if !r.cfg.ShareEnabled() || (mode == shareUpdateConfigured && !r.cfg.Share.AutoUpdate) { return nil } @@ -795,6 +849,10 @@ func (r *runtime) autoUpdateShare(mode shareUpdateMode) error { if err != nil { return err } + if exclusions != nil { + opts.MergeExcludeChannelIDs = append([]string(nil), exclusions.channelIDs...) + opts.MergeExcludeChannelKinds = append([]string(nil), exclusions.channelKinds...) + } r.setSyncLockPhase("share pull") r.logger.Info("share update pulling", "repo_path", opts.RepoPath, "remote", opts.Remote) if err := share.Pull(r.ctx, opts); err != nil { diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index 51eb6e73..367dc291 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -18,6 +18,7 @@ import ( "path/filepath" goruntime "runtime" "strings" + "sync" "testing" "time" @@ -2418,6 +2419,84 @@ func TestSyncSkipsGitShareByDefaultAndCanImportBeforeLiveDiscord(t *testing.T) { require.Contains(t, contents, "live discord filled the delta") } +func TestSyncUpdateUsesCommandExclusionsForPrecedingShareMerge(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + remoteRepo := filepath.Join(dir, "remote.git") + runGit(t, dir, "init", "--bare", remoteRepo) + + publisher := seedCLIStore(t, filepath.Join(dir, "publisher.db")) + defer func() { _ = publisher.Close() }() + now := time.Now().UTC().Format(time.RFC3339Nano) + for _, channel := range []store.ChannelRecord{ + {ID: "feed", GuildID: "g1", Kind: "text", Name: "feed", RawJSON: `{}`}, + {ID: "announcements", GuildID: "g1", Kind: "announcement", Name: "announcements", RawJSON: `{}`}, + } { + require.NoError(t, publisher.UpsertChannel(ctx, channel)) + } + for _, message := range []store.MessageRecord{ + { + ID: "m-feed", GuildID: "g1", ChannelID: "feed", ChannelName: "feed", + AuthorID: "bot", AuthorName: "Bot", CreatedAt: now, + Content: "command excluded feed", NormalizedContent: "command excluded feed", RawJSON: `{}`, + }, + { + ID: "m-announcement", GuildID: "g1", ChannelID: "announcements", ChannelName: "announcements", + AuthorID: "bot", AuthorName: "Bot", CreatedAt: now, + Content: "command excluded announcement", NormalizedContent: "command excluded announcement", RawJSON: `{}`, + }, + } { + require.NoError(t, publisher.UpsertMessage(ctx, message)) + } + publisherOpts := share.Options{ + RepoPath: filepath.Join(dir, "publisher-share"), + Remote: remoteRepo, + Branch: "main", + } + publishSnapshot(t, ctx, publisher, publisherOpts, "test: command exclusions") + + cfgPath := filepath.Join(dir, "reader.toml") + cfg := config.Default() + cfg.DBPath = filepath.Join(dir, "reader.db") + cfg.Share.Remote = remoteRepo + cfg.Share.RepoPath = filepath.Join(dir, "reader-share") + cfg.Share.StaleAfter = "15m" + require.NoError(t, config.Write(cfgPath, cfg)) + + fakeSync := &fakeSyncService{} + rt := &runtime{ + ctx: ctx, + configPath: cfgPath, + stdout: &bytes.Buffer{}, + stderr: &bytes.Buffer{}, + logger: discardLogger(), + openStore: store.Open, + newDiscord: func(config.Config) (discordClient, error) { + return &fakeDiscordClient{guilds: []*discordgo.UserGuild{{ID: "g1"}}, self: &discordgo.User{ID: "bot"}}, nil + }, + newSyncer: func(syncer.Client, *store.Store, *slog.Logger) syncService { + return fakeSync + }, + } + + require.NoError(t, rt.dispatch([]string{ + "sync", + "--all", + "--update=auto", + "--exclude-channels", "feed", + "--exclude-channel-kinds", "announcement", + })) + require.Equal(t, []string{"feed"}, fakeSync.lastSync.ExcludeChannelIDs) + require.Equal(t, []string{"announcement"}, fakeSync.lastSync.ExcludeChannelKinds) + + reader, err := store.Open(ctx, cfg.DBPath) + require.NoError(t, err) + defer func() { _ = reader.Close() }() + _, rows, err := reader.ReadOnlyQuery(ctx, `select id from messages order by id`) + require.NoError(t, err) + require.Equal(t, [][]string{{"m100"}}, rows) +} + func TestSyncLockSerializesConcurrentRuns(t *testing.T) { if goruntime.GOOS == "windows" { t.Skip("sync lock is currently a no-op on Windows") @@ -2869,6 +2948,77 @@ func TestTailReadyPromotesOnceAndCleansUpOnCancellation(t *testing.T) { require.Empty(t, syncLockOwnerFiles(t, lockPath)) } +func TestTailShutdownTimeoutDoesNotRecloseClientOrHoldWriterLock(t *testing.T) { + if goruntime.GOOS == "windows" { + t.Skip("sync lock is currently a no-op on Windows") + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + dir := t.TempDir() + cfg, cfgPath := writeTestConfig(t, dir) + s := seedCLIStore(t, cfg.DBPath) + require.NoError(t, s.Close()) + t.Setenv(config.DefaultTokenEnv, "env-token") + + client := &blockingCloseDiscordClient{ + closeStarted: make(chan struct{}), + closeRelease: make(chan struct{}), + closeFinished: make(chan struct{}), + } + tailService := &boundaryTimeoutTailService{started: make(chan struct{})} + rt := &runtime{ + ctx: ctx, + configPath: cfgPath, + stdout: &bytes.Buffer{}, + stderr: &bytes.Buffer{}, + logger: discardLogger(), + openStore: store.Open, + newDiscord: func(config.Config) (discordClient, error) { + return client, nil + }, + newSyncer: func(syncClient syncer.Client, _ *store.Store, _ *slog.Logger) syncService { + tailService.client = syncClient.(interface{ Close() error }) + tailService.closeStarted = client.closeStarted + return tailService + }, + } + done := make(chan error, 1) + go func() { + done <- rt.dispatch([]string{"tail"}) + }() + + select { + case <-tailService.started: + case <-time.After(time.Second): + t.Fatal("tail did not start") + } + cancel() + select { + case <-client.closeStarted: + case <-time.After(time.Second): + t.Fatal("tail close did not start") + } + select { + case err := <-done: + require.ErrorContains(t, err, "tail shutdown timed out") + case <-time.After(time.Second): + t.Fatal("CLI cleanup repeated the blocked client close") + } + + lockPath := filepath.Join(dir, ".discrawl-sync.lock") + release, err := acquireSyncLock(context.Background(), lockPath) + require.NoError(t, err) + require.NoError(t, release()) + require.Empty(t, syncLockOwnerFiles(t, lockPath)) + + close(client.closeRelease) + select { + case <-client.closeFinished: + case <-time.After(time.Second): + t.Fatal("blocked client close did not finish") + } +} + func TestSyncLockHelperEdges(t *testing.T) { ctx := context.Background() dir := t.TempDir() @@ -3648,6 +3798,23 @@ func (f *fakeDiscordClient) Tail(context.Context, discordclient.EventHandler) er return nil } +type blockingCloseDiscordClient struct { + fakeDiscordClient + closeStarted chan struct{} + closeRelease chan struct{} + closeFinished chan struct{} + closeOnce sync.Once +} + +func (c *blockingCloseDiscordClient) Close() error { + c.closeOnce.Do(func() { + close(c.closeStarted) + <-c.closeRelease + close(c.closeFinished) + }) + return nil +} + type fakeSyncService struct { discovered []*discordgo.UserGuild lastSync syncer.SyncOptions @@ -3724,6 +3891,33 @@ func (f *fakeSyncService) SetChannelExclusions(channelIDs, channelKinds []string f.excludedChannelKinds = append([]string(nil), channelKinds...) } +type boundaryTimeoutTailService struct { + fakeSyncService + client interface{ Close() error } + started chan struct{} + closeStarted <-chan struct{} +} + +func (f *boundaryTimeoutTailService) RunTail(ctx context.Context, guildIDs []string, repairEvery time.Duration) error { + f.lastTail = guildIDs + f.lastRepair = repairEvery + if f.tailReady != nil { + f.tailReadyCalls++ + if err := f.tailReady(ctx); err != nil { + return err + } + } + close(f.started) + <-ctx.Done() + go func() { _ = f.client.Close() }() + select { + case <-f.closeStarted: + case <-time.After(time.Second): + return errors.New("tail client close did not start") + } + return errors.New("tail shutdown timed out after 25ms") +} + type hybridSyncService struct { store *store.Store sawGitMessage bool diff --git a/internal/discorddesktop/import.go b/internal/discorddesktop/import.go index 86265af2..4fbb974e 100644 --- a/internal/discorddesktop/import.go +++ b/internal/discorddesktop/import.go @@ -209,16 +209,18 @@ func loadScanState(ctx context.Context, st *store.Store, opts Options) (scanStat current: map[string]fileFingerprint{}, channels: map[string]store.ChannelRecord{}, } - if st == nil || opts.DryRun { + if st == nil { return state, nil } - raw, err := st.GetSyncState(ctx, fileIndexScope(opts)) - if err != nil { - return state, err - } - if strings.TrimSpace(raw) != "" { - if err := json.Unmarshal([]byte(raw), &state.previous); err != nil { - state.previous = map[string]fileFingerprint{} + if !opts.DryRun { + raw, err := st.GetSyncState(ctx, fileIndexScope(opts)) + if err != nil { + return state, err + } + if strings.TrimSpace(raw) != "" { + if err := json.Unmarshal([]byte(raw), &state.previous); err != nil { + state.previous = map[string]fileFingerprint{} + } } } channels, err := st.Channels(ctx, "") @@ -227,10 +229,13 @@ func loadScanState(ctx context.Context, st *store.Store, opts Options) (scanStat } for _, channel := range channels { state.channels[channel.ID] = store.ChannelRecord{ - ID: channel.ID, - GuildID: channel.GuildID, - Kind: channel.Kind, - Name: channel.Name, + ID: channel.ID, + GuildID: channel.GuildID, + ParentID: channel.ParentID, + Kind: channel.Kind, + Name: channel.Name, + IsPrivateThread: channel.IsPrivateThread, + ThreadParentID: channel.ThreadParentID, } } return state, nil @@ -609,35 +614,53 @@ func filterExcludedMessages(snap snapshot, channelLookup map[string]store.Channe if _, ok := excludedKinds[kind]; ok { return true } - if kind == "thread_announcement" { + switch kind { + case "news", "thread_news", "thread_announcement": _, ok := excludedKinds["announcement"] return ok + default: + return false } - return false } - excludesChannel := func(channelID string) bool { + decisions := map[string]bool{} + var excludesChannel func(string, map[string]struct{}) bool + excludesChannel = func(channelID string, visiting map[string]struct{}) bool { + if decision, ok := decisions[channelID]; ok { + return decision + } if _, ok := excludedIDs[channelID]; ok { + decisions[channelID] = true return true } channel, ok := channelByID(channelID) if !ok { + decisions[channelID] = false return false } if excludesKind(channel.Kind) { + decisions[channelID] = true return true } - if channel.ParentID == "" { + parentID := inheritedChannelParentID(channel) + if parentID == "" { + decisions[channelID] = false return false } - if _, ok := excludedIDs[channel.ParentID]; ok { - return true + if visiting == nil { + visiting = map[string]struct{}{} + } + if _, ok := visiting[channelID]; ok { + return false } - parent, ok := channelByID(channel.ParentID) - return ok && excludesKind(parent.Kind) + visiting[channelID] = struct{}{} + excluded := excludesChannel(parentID, visiting) + delete(visiting, channelID) + decisions[channelID] = excluded + return excluded } for messageID, message := range snap.messages { channelID := message.Record.ChannelID - if !excludesChannel(channelID) { + if !excludesChannel(channelID, nil) { continue } totals.excludedMessages[messageID] = struct{}{} @@ -648,6 +671,13 @@ func filterExcludedMessages(snap snapshot, channelLookup map[string]store.Channe stats.ExcludedChannels = len(totals.excludedChannels) } +func inheritedChannelParentID(channel store.ChannelRecord) string { + if channel.ThreadParentID != "" { + return channel.ThreadParentID + } + return channel.ParentID +} + func normalizedCollectionSet(values []string, lower bool) map[string]struct{} { out := make(map[string]struct{}, len(values)) for _, value := range values { @@ -980,6 +1010,9 @@ func collectValue(snap snapshot, channelLookup map[string]store.ChannelRecord, v collectUserLabel(snap, typed) collectSelectedDirectMessageRoutes(snap, typed) if channel, ok := parseChannel(typed); ok { + if existing, exists := channelLookup[channel.ID]; exists { + channel = mergeCachedChannelParentMetadata(existing, channel, typed) + } snap.channels[channel.ID] = channel channelLookup[channel.ID] = channel if channel.GuildID == DirectMessageGuildID { @@ -1083,14 +1116,42 @@ func parseChannel(raw map[string]any) (store.ChannelRecord, bool) { name = "channel-" + shortID(id) } } - rawJSON := channelRawJSON(raw, id, guildID, name, kindForChannelType(typeValue, isDM)) - return store.ChannelRecord{ - ID: id, - GuildID: guildID, - Kind: kindForChannelType(typeValue, isDM), - Name: name, - RawJSON: rawJSON, - }, true + kind := kindForChannelType(typeValue, isDM) + parentID := stringField(raw, "parent_id") + channel := store.ChannelRecord{ + ID: id, + GuildID: guildID, + ParentID: parentID, + Kind: kind, + Name: name, + RawJSON: channelRawJSON(raw, id, guildID, name, kind), + } + if isThreadChannelKind(kind) { + channel.ThreadParentID = parentID + channel.IsPrivateThread = kind == "thread_private" + } + return channel, true +} + +func mergeCachedChannelParentMetadata(existing, cached store.ChannelRecord, raw map[string]any) store.ChannelRecord { + if _, hasType := raw["type"]; !hasType && isThreadChannelKind(existing.Kind) { + cached.Kind = existing.Kind + cached.IsPrivateThread = existing.IsPrivateThread + } + if _, hasParentID := raw["parent_id"]; !hasParentID { + cached.ParentID = existing.ParentID + } + if cached.ThreadParentID == "" && (isThreadChannelKind(cached.Kind) || isThreadChannelKind(existing.Kind)) { + cached.ThreadParentID = firstNonEmpty(cached.ParentID, existing.ThreadParentID, existing.ParentID) + } + if cached.ParentID == "" && cached.ThreadParentID != "" { + cached.ParentID = cached.ThreadParentID + } + return cached +} + +func isThreadChannelKind(kind string) bool { + return strings.HasPrefix(strings.ToLower(strings.TrimSpace(kind)), "thread_") } func parseMessage(raw map[string]any, fallbackTime time.Time, channels map[string]store.ChannelRecord) (store.MessageMutation, bool) { @@ -1515,12 +1576,13 @@ func kindForChannelType(typeValue int, dm bool) string { func channelRawJSON(raw map[string]any, id, guildID, name, kind string) string { return marshalJSONString(map[string]any{ - "id": id, - "guild_id": guildID, - "name": name, - "kind": kind, - "source": "discord_desktop", - "type": raw["type"], + "id": id, + "guild_id": guildID, + "parent_id": raw["parent_id"], + "name": name, + "kind": kind, + "source": "discord_desktop", + "type": raw["type"], }, "{}") } diff --git a/internal/discorddesktop/import_helpers_test.go b/internal/discorddesktop/import_helpers_test.go index 5f62d9f5..97480ad2 100644 --- a/internal/discorddesktop/import_helpers_test.go +++ b/internal/discorddesktop/import_helpers_test.go @@ -93,6 +93,28 @@ func TestFilterExcludedMessages(t *testing.T) { require.Equal(t, 2, stats.ExcludedChannels) } +func TestFilterExcludedMessagesInheritsLegacyAnnouncementExclusions(t *testing.T) { + snap := newSnapshot() + snap.messages["m-news"] = store.MessageMutation{Record: store.MessageRecord{ID: "m-news", ChannelID: "news"}} + snap.messages["m-thread-news"] = store.MessageMutation{Record: store.MessageRecord{ID: "m-thread-news", ChannelID: "thread-news"}} + snap.messages["m-child"] = store.MessageMutation{Record: store.MessageRecord{ID: "m-child", ChannelID: "child"}} + lookup := map[string]store.ChannelRecord{ + "news": {ID: "news", Kind: "news"}, + "thread-news": {ID: "thread-news", Kind: "thread_news"}, + "child": {ID: "child", Kind: "thread_public", ThreadParentID: "news"}, + } + totals := newScanTotals() + stats := &Stats{} + + filterExcludedMessages(snap, lookup, Options{ + ExcludeChannelKinds: []string{"announcement"}, + }, totals, stats) + + require.Empty(t, snap.messages) + require.Equal(t, 3, stats.ExcludedMessages) + require.Equal(t, 3, stats.ExcludedChannels) +} + func TestRouteFilteredCacheHelpers(t *testing.T) { require.Equal(t, fileSourceCacheData, sourceForPath("/tmp/discord", "/tmp/discord/Cache/Cache_Data/entry", "Cache/Cache_Data/entry")) require.Equal(t, fileSourceCacheData, sourceForPath("/tmp/discord", "/tmp/discord/Service Worker/CacheStorage/cache/entry", "Service Worker/CacheStorage/cache/entry")) @@ -151,11 +173,21 @@ func TestImportAndStateEdgeBranches(t *testing.T) { require.True(t, stats.FullCache) require.NoError(t, s.SetSyncState(ctx, fileIndexScope(Options{}), "{not-json")) - require.NoError(t, s.UpsertChannel(ctx, store.ChannelRecord{ID: "c1", GuildID: "g1", Kind: "text", Name: "general", RawJSON: `{}`})) + require.NoError(t, s.UpsertChannel(ctx, store.ChannelRecord{ + ID: "c1", + GuildID: "g1", + ParentID: "parent", + Kind: "thread_public", + Name: "general", + ThreadParentID: "parent", + RawJSON: `{}`, + })) state, err := loadScanState(ctx, s, Options{}) require.NoError(t, err) require.Empty(t, state.previous) require.Equal(t, "general", state.channels["c1"].Name) + require.Equal(t, "parent", state.channels["c1"].ParentID) + require.Equal(t, "parent", state.channels["c1"].ThreadParentID) } func TestSnapshotFinalizeAndCommitBranches(t *testing.T) { diff --git a/internal/discorddesktop/import_pipeline_test.go b/internal/discorddesktop/import_pipeline_test.go index d466f8d4..f3b6f6af 100644 --- a/internal/discorddesktop/import_pipeline_test.go +++ b/internal/discorddesktop/import_pipeline_test.go @@ -363,6 +363,171 @@ func TestImportFastCachePreservesKnownChannelMetadataAcrossBatches(t *testing.T) require.Contains(t, rows[0][0], `"type":11`) } +func TestImportExcludesCachedThreadRowsThroughParent(t *testing.T) { + ctx := context.Background() + guildID := "999999999999999996" + parentID := "111111111111111120" + threadID := "111111111111111121" + + tests := []struct { + name string + parentType int + options Options + }{ + { + name: "parent id", + parentType: 0, + options: Options{ExcludeChannelIDs: []string{parentID}}, + }, + { + name: "parent kind", + parentType: 5, + options: Options{ExcludeChannelKinds: []string{"announcement"}}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + cachePath := filepath.Join(dir, "Cache", "Cache_Data") + require.NoError(t, os.MkdirAll(cachePath, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(cachePath, "entry_0"), bytesf(`https://discord.com/api/v9/channels/%s/messages?limit=50 +{"id":"%s","guild_id":"%s","type":%d,"name":"excluded-parent"} +{"id":"%s","guild_id":"%s","parent_id":"%s","type":11,"name":"cached-thread"} +{"id":"333333333333333346","channel_id":"%s","content":"excluded cached thread payload","timestamp":"2026-04-23T18:20:43Z","author":{"id":"222222222222222232","username":"alice"},"attachments":[{"id":"444444444444444445","filename":"trace.txt"}],"mentions":[{"id":"555555555555555556","username":"bob"}]} +`, threadID, parentID, guildID, tt.parentType, threadID, guildID, parentID, threadID), 0o600)) + + st, err := store.Open(ctx, filepath.Join(dir, "discrawl.db")) + require.NoError(t, err) + defer func() { _ = st.Close() }() + + opts := tt.options + opts.Path = dir + stats, err := Import(ctx, st, opts) + require.NoError(t, err) + require.Zero(t, stats.Messages) + require.Equal(t, 1, stats.ExcludedMessages) + require.Equal(t, 1, stats.ExcludedChannels) + for _, table := range []string{"messages", "message_events", "message_attachments", "mention_events"} { + requireMessageCount(t, ctx, st, table, 0) + } + + channels, err := st.Channels(ctx, guildID) + require.NoError(t, err) + channelsByID := make(map[string]store.ChannelRow, len(channels)) + for _, channel := range channels { + channelsByID[channel.ID] = channel + } + require.Equal(t, parentID, channelsByID[threadID].ParentID) + require.Equal(t, parentID, channelsByID[threadID].ThreadParentID) + }) + } +} + +func TestImportDoesNotEraseStoredThreadParentFromPoorerCacheChannel(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + cachePath := filepath.Join(dir, "Cache", "Cache_Data") + require.NoError(t, os.MkdirAll(cachePath, 0o755)) + + guildID := "999999999999999996" + parentID := "111111111111111120" + threadID := "111111111111111121" + require.NoError(t, os.WriteFile(filepath.Join(cachePath, "entry_0"), bytesf(`https://discord.com/api/v9/channels/%s/messages?limit=50 +{"id":"%s","guild_id":"%s","type":11,"name":"poorer-cached-thread"} +{"id":"333333333333333346","channel_id":"%s","content":"stored parent must still exclude","timestamp":"2026-04-23T18:20:43Z","author":{"id":"222222222222222232","username":"alice"},"attachments":[{"id":"444444444444444445","filename":"trace.txt"}],"mentions":[{"id":"555555555555555556","username":"bob"}]} +`, threadID, threadID, guildID, threadID), 0o600)) + + st, err := store.Open(ctx, filepath.Join(dir, "discrawl.db")) + require.NoError(t, err) + defer func() { _ = st.Close() }() + require.NoError(t, st.UpsertGuild(ctx, store.GuildRecord{ID: guildID, Name: "Guild", RawJSON: `{}`})) + require.NoError(t, st.UpsertChannel(ctx, store.ChannelRecord{ + ID: parentID, + GuildID: guildID, + Kind: "news", + Name: "legacy-announcement", + RawJSON: `{}`, + })) + require.NoError(t, st.UpsertChannel(ctx, store.ChannelRecord{ + ID: threadID, + GuildID: guildID, + ParentID: parentID, + Kind: "thread_public", + Name: "stored-thread", + ThreadParentID: parentID, + RawJSON: `{}`, + })) + + stats, err := Import(ctx, st, Options{ + Path: dir, + ExcludeChannelKinds: []string{"announcement"}, + }) + require.NoError(t, err) + require.Zero(t, stats.Messages) + require.Equal(t, 1, stats.ExcludedMessages) + require.Equal(t, 1, stats.ExcludedChannels) + for _, table := range []string{"messages", "message_events", "message_attachments", "mention_events"} { + requireMessageCount(t, ctx, st, table, 0) + } + + channels, err := st.Channels(ctx, guildID) + require.NoError(t, err) + channelsByID := make(map[string]store.ChannelRow, len(channels)) + for _, channel := range channels { + channelsByID[channel.ID] = channel + } + require.Equal(t, parentID, channelsByID[threadID].ParentID) + require.Equal(t, parentID, channelsByID[threadID].ThreadParentID) +} + +func TestImportDryRunUsesStoredThreadParentForExclusions(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + cachePath := filepath.Join(dir, "Cache", "Cache_Data") + require.NoError(t, os.MkdirAll(cachePath, 0o755)) + + guildID := "999999999999999996" + parentID := "111111111111111120" + threadID := "111111111111111121" + require.NoError(t, os.WriteFile(filepath.Join(cachePath, "entry_0"), bytesf(`https://discord.com/api/v9/channels/%s/messages?limit=50 +{"id":"%s","guild_id":"%s","type":11,"name":"poorer-cached-thread"} +{"id":"333333333333333346","channel_id":"%s","content":"dry run must inherit stored parent","timestamp":"2026-04-23T18:20:43Z","author":{"id":"222222222222222232","username":"alice"}} +`, threadID, threadID, guildID, threadID), 0o600)) + + st, err := store.Open(ctx, filepath.Join(dir, "discrawl.db")) + require.NoError(t, err) + defer func() { _ = st.Close() }() + require.NoError(t, st.UpsertGuild(ctx, store.GuildRecord{ID: guildID, Name: "Guild", RawJSON: `{}`})) + require.NoError(t, st.UpsertChannel(ctx, store.ChannelRecord{ + ID: parentID, + GuildID: guildID, + Kind: "news", + Name: "legacy-announcement", + RawJSON: `{}`, + })) + require.NoError(t, st.UpsertChannel(ctx, store.ChannelRecord{ + ID: threadID, + GuildID: guildID, + ParentID: parentID, + Kind: "thread_public", + Name: "stored-thread", + ThreadParentID: parentID, + RawJSON: `{}`, + })) + + stats, err := Import(ctx, st, Options{ + Path: dir, + DryRun: true, + ExcludeChannelKinds: []string{"announcement"}, + }) + require.NoError(t, err) + require.True(t, stats.DryRun) + require.Zero(t, stats.Messages) + require.Equal(t, 1, stats.ExcludedMessages) + require.Equal(t, 1, stats.ExcludedChannels) + requireMessageCount(t, ctx, st, "messages", 0) +} + func TestImportFastCacheRouteFiltersServiceWorkerCacheStorage(t *testing.T) { ctx := context.Background() dir := t.TempDir() diff --git a/internal/share/import_filter.go b/internal/share/import_filter.go index 6049427e..6264889d 100644 --- a/internal/share/import_filter.go +++ b/internal/share/import_filter.go @@ -3,6 +3,7 @@ package share import ( "context" "database/sql" + "errors" "fmt" "strings" ) @@ -19,11 +20,15 @@ type mergeImportFilter struct { decisions map[string]bool } +type mergeImportQuerier interface { + QueryContext(context.Context, string, ...any) (*sql.Rows, error) +} + func newMergeImportFilter( ctx context.Context, - db *sql.DB, + db mergeImportQuerier, opts Options, -) (func(string, map[string]any) (bool, error), error) { +) (*mergeImportFilter, error) { filter := &mergeImportFilter{ excludedIDs: normalizedImportSet(opts.MergeExcludeChannelIDs, false), excludedKinds: normalizedImportSet(opts.MergeExcludeChannelKinds, true), @@ -35,10 +40,10 @@ func newMergeImportFilter( return nil, err } } - return filter.allow, nil + return filter, nil } -func (f *mergeImportFilter) loadChannels(ctx context.Context, db *sql.DB) error { +func (f *mergeImportFilter) loadChannels(ctx context.Context, db mergeImportQuerier) error { rows, err := db.QueryContext(ctx, ` select id, coalesce(parent_id, ''), coalesce(thread_parent_id, ''), kind from channels @@ -63,6 +68,21 @@ func (f *mergeImportFilter) loadChannels(ctx context.Context, db *sql.DB) error return nil } +func (f *mergeImportFilter) allowMessageID(ctx context.Context, lookup *sql.Stmt, messageID string) (bool, error) { + var guildID, channelID string + err := lookup.QueryRowContext(ctx, messageID).Scan(&guildID, &channelID) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("load embedding message %s: %w", messageID, err) + } + return f.allow("messages", map[string]any{ + "guild_id": guildID, + "channel_id": channelID, + }) +} + func (f *mergeImportFilter) allow(table string, row map[string]any) (bool, error) { if isDirectMessageSnapshotRow(table, row) { return false, nil diff --git a/internal/share/merge.go b/internal/share/merge.go index 12347d90..89f32193 100644 --- a/internal/share/merge.go +++ b/internal/share/merge.go @@ -15,11 +15,12 @@ import ( ) const ( - LastCheckSyncScope = "share:last_check_at" - LastMergeSyncScope = "share:last_merge_at" - LastMergeManifestSyncScope = "share:last_merge_manifest_generated_at" - LastMergeManifestJSONScope = "share:last_merge_manifest_json" - PendingReplacementSyncScope = "share:pending_replacement" + LastCheckSyncScope = "share:last_check_at" + LastMergeSyncScope = "share:last_merge_at" + LastMergeManifestSyncScope = "share:last_merge_manifest_generated_at" + LastMergeManifestJSONScope = "share:last_merge_manifest_json" + LastMergeEmbeddingsSyncScope = "share:last_merge_embeddings_state_v1" + PendingReplacementSyncScope = "share:pending_replacement" ) var ErrReplacementRequired = errors.New("share snapshot replacement required") @@ -192,6 +193,73 @@ func PreviousMergedManifest(ctx context.Context, s *store.Store, opts Options) ( return PreviousImportedManifest(ctx, s, opts) } +type mergeEmbeddingsState struct { + Version int `json:"version"` + GeneratedAt string `json:"generated_at"` + Provider string `json:"provider"` + Model string `json:"model"` + InputVersion string `json:"input_version"` + ExcludeChannelIDs []string `json:"exclude_channel_ids"` + ExcludeChannelKinds []string `json:"exclude_channel_kinds"` + Manifests []EmbeddingManifest `json:"manifests"` +} + +func mergedEmbeddingsStateValue(manifest Manifest, opts Options) (string, error) { + inputVersion := strings.TrimSpace(opts.EmbeddingInputVersion) + if inputVersion == "" { + inputVersion = store.EmbeddingInputVersion + } + state := mergeEmbeddingsState{ + Version: 1, + GeneratedAt: manifest.GeneratedAt.Format(time.RFC3339Nano), + Provider: strings.ToLower(strings.TrimSpace(opts.EmbeddingProvider)), + Model: strings.TrimSpace(opts.EmbeddingModel), + InputVersion: inputVersion, + ExcludeChannelIDs: sortedNormalizedImportValues(opts.MergeExcludeChannelIDs, false), + ExcludeChannelKinds: sortedNormalizedImportValues(opts.MergeExcludeChannelKinds, true), + Manifests: []EmbeddingManifest{}, + } + for _, embeddingManifest := range manifest.Embeddings { + if embeddingManifestMatches(opts, embeddingManifest) { + state.Manifests = append(state.Manifests, embeddingManifest) + } + } + body, err := json.Marshal(state) + if err != nil { + return "", fmt.Errorf("marshal merged embedding state: %w", err) + } + return string(body), nil +} + +func sortedNormalizedImportValues(values []string, lower bool) []string { + set := normalizedImportSet(values, lower) + out := make([]string, 0, len(set)) + for value := range set { + out = append(out, value) + } + sort.Strings(out) + return out +} + +func mergedEmbeddingsNeedImport(ctx context.Context, s *store.Store, manifest Manifest, opts Options) (string, bool, error) { + state, err := mergedEmbeddingsStateValue(manifest, opts) + if err != nil { + return "", false, err + } + previous, err := s.GetSyncState(ctx, LastMergeEmbeddingsSyncScope) + if err != nil { + return "", false, fmt.Errorf("read merged embedding state: %w", err) + } + return state, strings.TrimSpace(previous) != state, nil +} + +func markMergedEmbeddings(ctx context.Context, s *store.Store, state string) error { + if err := s.SetSyncState(ctx, LastMergeEmbeddingsSyncScope, state); err != nil { + return fmt.Errorf("write merged embedding state: %w", err) + } + return nil +} + func MarkChecked(ctx context.Context, s *store.Store) error { return s.SetSyncState(ctx, LastCheckSyncScope, time.Now().UTC().Format(time.RFC3339Nano)) } diff --git a/internal/share/share.go b/internal/share/share.go index 7b271112..8af999a4 100644 --- a/internal/share/share.go +++ b/internal/share/share.go @@ -538,7 +538,11 @@ func Import(ctx context.Context, s *store.Store, opts Options) (Manifest, error) return err } if opts.IncludeEmbeddings { - return importEmbeddings(ctx, tx, opts, manifest.Embeddings) + embeddingOpts := opts + embeddingOpts.MergeExcludeChannelIDs = nil + embeddingOpts.MergeExcludeChannelKinds = nil + _, err := importEmbeddings(ctx, tx, embeddingOpts, manifest.Embeddings) + return err } return nil }, @@ -554,6 +558,18 @@ func Import(ctx context.Context, s *store.Store, opts Options) (Manifest, error) return Manifest{}, err } } + if opts.IncludeEmbeddings { + embeddingOpts := opts + embeddingOpts.MergeExcludeChannelIDs = nil + embeddingOpts.MergeExcludeChannelKinds = nil + state, err := mergedEmbeddingsStateValue(manifest, embeddingOpts) + if err != nil { + return Manifest{}, err + } + if err := markMergedEmbeddings(ctx, s, state); err != nil { + return Manifest{}, err + } + } if err := MarkImported(ctx, s, manifest); err != nil { return Manifest{}, err } @@ -612,10 +628,21 @@ func importMergePlan( plan snapshot.ImportPlan, ) (Manifest, bool, error) { if !plan.Changed() { + embeddingRows := 0 if opts.IncludeEmbeddings { - if err := ImportEmbeddings(ctx, s, opts, manifest); err != nil { + state, needed, err := mergedEmbeddingsNeedImport(ctx, s, manifest, opts) + if err != nil { return Manifest{}, false, err } + if needed { + embeddingRows, err = importEmbeddingsIntoStore(ctx, s, opts, manifest) + if err != nil { + return Manifest{}, false, err + } + if err := markMergedEmbeddings(ctx, s, state); err != nil { + return Manifest{}, false, err + } + } } copied := 0 if opts.IncludeMedia { @@ -628,7 +655,7 @@ func importMergePlan( if err := MarkMerged(ctx, s, manifest); err != nil { return Manifest{}, false, err } - return manifest, copied > 0, nil + return manifest, embeddingRows > 0 || copied > 0, nil } rowFilter, err := newMergeImportFilter(ctx, s.DB(), opts) if err != nil { @@ -663,7 +690,7 @@ func importMergePlan( TotalRows: progress.TotalRows, }) }, - Filter: rowFilter, + Filter: rowFilter.allow, BeforeImport: func(ctx context.Context, tx *sql.Tx) error { var err error existingMedia, err = attachmentMediaByID(ctx, tx) @@ -685,7 +712,8 @@ func importMergePlan( return err } if opts.IncludeEmbeddings { - return importEmbeddings(ctx, tx, opts, manifest.Embeddings) + _, err := importEmbeddingsWithFilter(ctx, tx, opts, manifest.Embeddings, rowFilter) + return err } return nil }, @@ -710,6 +738,15 @@ func importMergePlan( return Manifest{}, false, err } } + if opts.IncludeEmbeddings { + state, err := mergedEmbeddingsStateValue(manifest, opts) + if err != nil { + return Manifest{}, false, err + } + if err := markMergedEmbeddings(ctx, s, state); err != nil { + return Manifest{}, false, err + } + } if err := MarkMerged(ctx, s, manifest); err != nil { return Manifest{}, false, err } @@ -760,24 +797,37 @@ func importPlanRowCount(plan snapshot.ImportPlan) int { } func ImportEmbeddings(ctx context.Context, s *store.Store, opts Options, manifest Manifest) error { - tx, err := s.DB().BeginTx(ctx, nil) + _, err := importEmbeddingsIntoStore(ctx, s, opts, manifest) if err != nil { return err } + state, err := mergedEmbeddingsStateValue(manifest, opts) + if err != nil { + return err + } + return markMergedEmbeddings(ctx, s, state) +} + +func importEmbeddingsIntoStore(ctx context.Context, s *store.Store, opts Options, manifest Manifest) (int, error) { + tx, err := s.DB().BeginTx(ctx, nil) + if err != nil { + return 0, err + } committed := false defer func() { if !committed { _ = tx.Rollback() } }() - if err := importEmbeddings(ctx, tx, opts, manifest.Embeddings); err != nil { - return err + rows, err := importEmbeddings(ctx, tx, opts, manifest.Embeddings) + if err != nil { + return 0, err } if err := tx.Commit(); err != nil { - return err + return 0, err } committed = true - return nil + return rows, nil } func ManifestAlreadyImported(ctx context.Context, s *store.Store, manifest Manifest) bool { @@ -2201,9 +2251,28 @@ func isLocalOnlyGuildID(guildID string) bool { return guildID == directMessageGuildID } -func importEmbeddings(ctx context.Context, tx *sql.Tx, opts Options, manifests []EmbeddingManifest) error { +func importEmbeddings( + ctx context.Context, + tx *sql.Tx, + opts Options, + manifests []EmbeddingManifest, +) (int, error) { + filter, err := newMergeImportFilter(ctx, tx, opts) + if err != nil { + return 0, err + } + return importEmbeddingsWithFilter(ctx, tx, opts, manifests, filter) +} + +func importEmbeddingsWithFilter( + ctx context.Context, + tx *sql.Tx, + opts Options, + manifests []EmbeddingManifest, + filter *mergeImportFilter, +) (int, error) { if len(manifests) == 0 { - return nil + return 0, nil } stmt, err := tx.PrepareContext(ctx, ` insert into message_embeddings( @@ -2213,50 +2282,72 @@ func importEmbeddings(ctx context.Context, tx *sql.Tx, opts Options, manifests [ dimensions = excluded.dimensions, embedding_blob = excluded.embedding_blob, embedded_at = excluded.embedded_at + where message_embeddings.dimensions is not excluded.dimensions + or message_embeddings.embedding_blob is not excluded.embedding_blob + or message_embeddings.embedded_at is not excluded.embedded_at `) if err != nil { - return fmt.Errorf("prepare import message_embeddings: %w", err) + return 0, fmt.Errorf("prepare import message_embeddings: %w", err) } defer func() { _ = stmt.Close() }() + messageLookup, err := tx.PrepareContext(ctx, ` + select guild_id, channel_id + from messages + where id = ? + `) + if err != nil { + return 0, fmt.Errorf("prepare embedding message lookup: %w", err) + } + defer func() { _ = messageLookup.Close() }() + changed := 0 for _, manifest := range manifests { if err := ctx.Err(); err != nil { - return err + return 0, err } if !embeddingManifestMatches(opts, manifest) { continue } files := manifest.Files if len(files) == 0 { - return fmt.Errorf("embedding manifest %s/%s/%s has no files", manifest.Provider, manifest.Model, manifest.InputVersion) + return 0, fmt.Errorf("embedding manifest %s/%s/%s has no files", manifest.Provider, manifest.Model, manifest.InputVersion) } for _, rel := range files { if err := ctx.Err(); err != nil { - return err + return 0, err } - if err := importEmbeddingFile(ctx, stmt, opts.RepoPath, rel); err != nil { - return err + rows, err := importEmbeddingFile(ctx, stmt, messageLookup, filter, opts.RepoPath, rel) + if err != nil { + return 0, err } + changed += rows } } - return nil + return changed, nil } -func importEmbeddingFile(ctx context.Context, stmt *sql.Stmt, repoPath, rel string) error { +func importEmbeddingFile( + ctx context.Context, + stmt *sql.Stmt, + messageLookup *sql.Stmt, + filter *mergeImportFilter, + repoPath, + rel string, +) (int, error) { path, err := embeddingRepoPath(repoPath, rel) if err != nil { - return err + return 0, err } if _, err := regularFileInRoot(filepath.Join(repoPath, "embeddings"), path, rel, "embedding"); err != nil { - return err + return 0, err } file, err := os.Open(path) // #nosec G304 -- path is confined by embeddingRepoPath and regularFileInRoot. if err != nil { - return fmt.Errorf("open %s: %w", rel, err) + return 0, fmt.Errorf("open %s: %w", rel, err) } defer func() { _ = file.Close() }() gz, err := gzip.NewReader(file) if err != nil { - return fmt.Errorf("read gzip %s: %w", rel, err) + return 0, fmt.Errorf("read gzip %s: %w", rel, err) } defer func() { _ = gz.Close() }() dec := json.NewDecoder(&limitedReader{ @@ -2265,9 +2356,10 @@ func importEmbeddingFile(ctx context.Context, stmt *sql.Stmt, repoPath, rel stri label: "embedding", }) dec.UseNumber() + changed := 0 for { if err := ctx.Err(); err != nil { - return err + return 0, err } var row struct { MessageID string `json:"message_id"` @@ -2283,21 +2375,34 @@ func importEmbeddingFile(ctx context.Context, stmt *sql.Stmt, repoPath, rel stri break } if err != nil { - return fmt.Errorf("decode %s: %w", rel, err) + return 0, fmt.Errorf("decode %s: %w", rel, err) } dimensions, err := strconv.Atoi(row.Dimensions.String()) if err != nil { - return fmt.Errorf("decode dimensions in %s: %w", rel, err) + return 0, fmt.Errorf("decode dimensions in %s: %w", rel, err) } blob, err := base64.StdEncoding.DecodeString(row.EmbeddingBlob) if err != nil { - return fmt.Errorf("decode embedding blob in %s: %w", rel, err) + return 0, fmt.Errorf("decode embedding blob in %s: %w", rel, err) } - if _, err := stmt.ExecContext(ctx, row.MessageID, row.Provider, row.Model, row.InputVersion, dimensions, blob, row.EmbeddedAt); err != nil { - return fmt.Errorf("insert message_embeddings: %w", err) + allowed, err := filter.allowMessageID(ctx, messageLookup, row.MessageID) + if err != nil { + return 0, err + } + if !allowed { + continue + } + result, err := stmt.ExecContext(ctx, row.MessageID, row.Provider, row.Model, row.InputVersion, dimensions, blob, row.EmbeddedAt) + if err != nil { + return 0, fmt.Errorf("insert message_embeddings: %w", err) + } + rows, err := result.RowsAffected() + if err != nil { + return 0, fmt.Errorf("count imported message_embeddings: %w", err) } + changed += int(rows) } - return nil + return changed, nil } func embeddingRepoPath(repoPath, rel string) (string, error) { diff --git a/internal/share/share_test.go b/internal/share/share_test.go index b4921aa9..a0988af2 100644 --- a/internal/share/share_test.go +++ b/internal/share/share_test.go @@ -1535,6 +1535,213 @@ func TestImportEmbeddingsFiltersByConfiguredIdentity(t *testing.T) { require.Equal(t, "0", rows[0][0]) } +func TestMergeIfChangedFiltersExcludedFeedEmbeddings(t *testing.T) { + ctx := context.Background() + src := seedStore(t, filepath.Join(t.TempDir(), "src.db")) + defer func() { _ = src.Close() }() + require.NoError(t, src.UpsertChannel(ctx, store.ChannelRecord{ + ID: "feed", GuildID: "g1", Kind: "announcement", Name: "feed", RawJSON: `{}`, + })) + now := time.Now().UTC().Format(time.RFC3339Nano) + require.NoError(t, src.UpsertMessage(ctx, store.MessageRecord{ + ID: "m-feed", GuildID: "g1", ChannelID: "feed", ChannelName: "feed", + AuthorID: "bot", AuthorName: "Bot", CreatedAt: now, + Content: "automated feed", NormalizedContent: "automated feed", RawJSON: `{}`, + })) + insertShareTestEmbedding(t, ctx, src, "m1", []float32{1, 0}) + insertShareTestEmbedding(t, ctx, src, "m-feed", []float32{0, 1}) + + opts := Options{ + RepoPath: filepath.Join(t.TempDir(), "share"), + Branch: "main", + IncludeEmbeddings: true, + EmbeddingProvider: "openai", + EmbeddingModel: "text-embedding-3-small", + EmbeddingInputVersion: store.EmbeddingInputVersion, + } + _, err := Export(ctx, src, opts) + require.NoError(t, err) + + dst, err := store.Open(ctx, filepath.Join(t.TempDir(), "dst.db")) + require.NoError(t, err) + defer func() { _ = dst.Close() }() + opts.MergeExcludeChannelKinds = []string{"announcement"} + _, changed, err := MergeIfChanged(ctx, dst, opts) + require.NoError(t, err) + require.True(t, changed) + + _, rows, err := dst.ReadOnlyQuery(ctx, `select message_id from message_embeddings order by message_id`) + require.NoError(t, err) + require.Equal(t, [][]string{{"m1"}}, rows) + _, rows, err = dst.ReadOnlyQuery(ctx, `select id from messages order by id`) + require.NoError(t, err) + require.Equal(t, [][]string{{"m1"}}, rows) +} + +func TestExactImportDoesNotApplySafeMergeEmbeddingExclusions(t *testing.T) { + ctx := context.Background() + src := seedStore(t, filepath.Join(t.TempDir(), "src.db")) + defer func() { _ = src.Close() }() + require.NoError(t, src.UpsertChannel(ctx, store.ChannelRecord{ + ID: "feed", GuildID: "g1", Kind: "announcement", Name: "feed", RawJSON: `{}`, + })) + now := time.Now().UTC().Format(time.RFC3339Nano) + require.NoError(t, src.UpsertMessage(ctx, store.MessageRecord{ + ID: "m-feed", GuildID: "g1", ChannelID: "feed", ChannelName: "feed", + AuthorID: "bot", AuthorName: "Bot", CreatedAt: now, + Content: "exact feed row", NormalizedContent: "exact feed row", RawJSON: `{}`, + })) + insertShareTestEmbedding(t, ctx, src, "m-feed", []float32{0, 1}) + + opts := Options{ + RepoPath: filepath.Join(t.TempDir(), "share"), + Branch: "main", + IncludeEmbeddings: true, + EmbeddingProvider: "openai", + EmbeddingModel: "text-embedding-3-small", + EmbeddingInputVersion: store.EmbeddingInputVersion, + MergeExcludeChannelKinds: []string{"announcement"}, + } + _, err := Export(ctx, src, opts) + require.NoError(t, err) + + dst, err := store.Open(ctx, filepath.Join(t.TempDir(), "dst.db")) + require.NoError(t, err) + defer func() { _ = dst.Close() }() + _, err = Import(ctx, dst, opts) + require.NoError(t, err) + + _, rows, err := dst.ReadOnlyQuery(ctx, `select id from messages where id = 'm-feed'`) + require.NoError(t, err) + require.Equal(t, [][]string{{"m-feed"}}, rows) + _, rows, err = dst.ReadOnlyQuery(ctx, `select message_id from message_embeddings where message_id = 'm-feed'`) + require.NoError(t, err) + require.Equal(t, [][]string{{"m-feed"}}, rows) +} + +func TestImportEmbeddingsSkipsOrphans(t *testing.T) { + ctx := context.Background() + src := seedStore(t, filepath.Join(t.TempDir(), "src.db")) + defer func() { _ = src.Close() }() + insertShareTestEmbedding(t, ctx, src, "m1", []float32{1, 0}) + + opts := Options{ + RepoPath: filepath.Join(t.TempDir(), "share"), + Branch: "main", + IncludeEmbeddings: true, + EmbeddingProvider: "openai", + EmbeddingModel: "text-embedding-3-small", + EmbeddingInputVersion: store.EmbeddingInputVersion, + } + manifest, err := Export(ctx, src, opts) + require.NoError(t, err) + validLine := strings.TrimSpace(snapshotFilesText(t, opts.RepoPath, manifest.Embeddings[0].Files)) + var orphanRow map[string]any + require.NoError(t, json.Unmarshal([]byte(validLine), &orphanRow)) + orphanRow["message_id"] = "orphan" + orphanLine, err := json.Marshal(orphanRow) + require.NoError(t, err) + writeGzipJSONLines( + t, + filepath.Join(opts.RepoPath, filepath.FromSlash(manifest.Embeddings[0].Files[0])), + []string{validLine, string(orphanLine)}, + ) + manifest.Embeddings[0].Rows = 2 + require.Equal(t, 2, manifest.Embeddings[0].Rows) + + dst := seedStore(t, filepath.Join(t.TempDir(), "dst.db")) + defer func() { _ = dst.Close() }() + require.NoError(t, ImportEmbeddings(ctx, dst, opts, manifest)) + _, rows, err := dst.ReadOnlyQuery(ctx, `select message_id from message_embeddings order by message_id`) + require.NoError(t, err) + require.Equal(t, [][]string{{"m1"}}, rows) +} + +func TestMergeIfChangedImportsEmbeddingsWhenCanonicalTablesUnchanged(t *testing.T) { + ctx := context.Background() + src := seedStore(t, filepath.Join(t.TempDir(), "src.db")) + defer func() { _ = src.Close() }() + insertShareTestEmbedding(t, ctx, src, "m1", []float32{1, 0}) + + opts := Options{ + RepoPath: filepath.Join(t.TempDir(), "share"), + Branch: "main", + IncludeEmbeddings: true, + EmbeddingProvider: "openai", + EmbeddingModel: "text-embedding-3-small", + EmbeddingInputVersion: store.EmbeddingInputVersion, + } + manifest, err := Export(ctx, src, opts) + require.NoError(t, err) + + dst, err := store.Open(ctx, filepath.Join(t.TempDir(), "dst.db")) + require.NoError(t, err) + defer func() { _ = dst.Close() }() + canonicalOnly := opts + canonicalOnly.IncludeEmbeddings = false + _, changed, err := MergeIfChanged(ctx, dst, canonicalOnly) + require.NoError(t, err) + require.True(t, changed) + _, rows, err := dst.ReadOnlyQuery(ctx, `select count(*) from message_embeddings`) + require.NoError(t, err) + require.Equal(t, "0", rows[0][0]) + + _, changed, err = MergeIfChanged(ctx, dst, opts) + require.NoError(t, err) + require.True(t, changed) + _, rows, err = dst.ReadOnlyQuery(ctx, `select message_id from message_embeddings`) + require.NoError(t, err) + require.Equal(t, [][]string{{"m1"}}, rows) + + require.NoError(t, dst.SetSyncState(ctx, LastMergeEmbeddingsSyncScope, "")) + _, changed, err = MergeIfChanged(ctx, dst, opts) + require.NoError(t, err) + require.False(t, changed) + + require.NoError(t, os.Remove(filepath.Join(opts.RepoPath, filepath.FromSlash(manifest.Embeddings[0].Files[0])))) + _, changed, err = MergeIfChanged(ctx, dst, opts) + require.NoError(t, err) + require.False(t, changed) +} + +func TestImportEmbeddingsPreservesLocalDirectMessageRows(t *testing.T) { + ctx := context.Background() + src := seedStore(t, filepath.Join(t.TempDir(), "src.db")) + defer func() { _ = src.Close() }() + require.NoError(t, src.UpsertMessage(ctx, store.MessageRecord{ + ID: "dm1", GuildID: "g1", ChannelID: "c1", ChannelName: "general", + AuthorID: "u1", AuthorName: "Peter", CreatedAt: time.Now().UTC().Format(time.RFC3339Nano), + Content: "public collision", NormalizedContent: "public collision", RawJSON: `{}`, + })) + insertShareTestEmbedding(t, ctx, src, "dm1", []float32{1, 0}) + opts := Options{ + RepoPath: filepath.Join(t.TempDir(), "share"), + Branch: "main", + IncludeEmbeddings: true, + EmbeddingProvider: "openai", + EmbeddingModel: "text-embedding-3-small", + EmbeddingInputVersion: store.EmbeddingInputVersion, + } + manifest, err := Export(ctx, src, opts) + require.NoError(t, err) + + dst := seedStore(t, filepath.Join(t.TempDir(), "dst.db")) + defer func() { _ = dst.Close() }() + seedDirectMessageData(t, ctx, dst) + insertShareTestEmbedding(t, ctx, dst, "dm1", []float32{9, 9}) + require.NoError(t, ImportEmbeddings(ctx, dst, opts, manifest)) + + var blob []byte + require.NoError(t, dst.DB().QueryRowContext(ctx, ` + select embedding_blob + from message_embeddings + where message_id = 'dm1' + `).Scan(&blob)) + vector, err := store.DecodeEmbeddingVector(blob) + require.NoError(t, err) + require.Equal(t, []float32{9, 9}, vector) +} + func TestMergeIfChangedSkipsSameManifest(t *testing.T) { ctx := context.Background() src := seedStore(t, filepath.Join(t.TempDir(), "src.db")) @@ -2243,7 +2450,7 @@ func TestLegacyManifestFileImportAndEmbeddingDecodeErrors(t *testing.T) { }) tx, err = s.DB().BeginTx(ctx, nil) require.NoError(t, err) - require.ErrorContains(t, importEmbeddings(ctx, tx, Options{ + _, err = importEmbeddings(ctx, tx, Options{ RepoPath: repo, EmbeddingProvider: "openai", EmbeddingModel: "model", @@ -2253,7 +2460,8 @@ func TestLegacyManifestFileImportAndEmbeddingDecodeErrors(t *testing.T) { Model: "model", InputVersion: store.EmbeddingInputVersion, Files: []string{embeddingRel}, - }}), "decode dimensions") + }}) + require.ErrorContains(t, err, "decode dimensions") require.NoError(t, tx.Rollback()) writeGzipJSONLines(t, filepath.Join(repo, filepath.FromSlash(embeddingRel)), []string{ @@ -2261,7 +2469,7 @@ func TestLegacyManifestFileImportAndEmbeddingDecodeErrors(t *testing.T) { }) tx, err = s.DB().BeginTx(ctx, nil) require.NoError(t, err) - require.ErrorContains(t, importEmbeddings(ctx, tx, Options{ + _, err = importEmbeddings(ctx, tx, Options{ RepoPath: repo, EmbeddingProvider: "openai", EmbeddingModel: "model", @@ -2271,7 +2479,8 @@ func TestLegacyManifestFileImportAndEmbeddingDecodeErrors(t *testing.T) { Model: "model", InputVersion: store.EmbeddingInputVersion, Files: []string{embeddingRel}, - }}), "decode embedding blob") + }}) + require.ErrorContains(t, err, "decode embedding blob") require.NoError(t, tx.Rollback()) } @@ -2330,7 +2539,7 @@ func TestImportEmbeddingsRejectsUnsafeManifestFiles(t *testing.T) { } { tx, err := s.DB().BeginTx(ctx, nil) require.NoError(t, err) - require.ErrorContains(t, importEmbeddings(ctx, tx, Options{ + _, err = importEmbeddings(ctx, tx, Options{ RepoPath: repo, EmbeddingProvider: "openai", EmbeddingModel: "model", @@ -2340,7 +2549,8 @@ func TestImportEmbeddingsRejectsUnsafeManifestFiles(t *testing.T) { Model: "model", InputVersion: store.EmbeddingInputVersion, Files: []string{rel}, - }}), "invalid embedding manifest path") + }}) + require.ErrorContains(t, err, "invalid embedding manifest path") require.NoError(t, tx.Rollback()) } } @@ -2364,7 +2574,7 @@ func TestImportEmbeddingsBoundsDecompressedInput(t *testing.T) { tx, err := s.DB().BeginTx(ctx, nil) require.NoError(t, err) - require.ErrorContains(t, importEmbeddings(ctx, tx, Options{ + _, err = importEmbeddings(ctx, tx, Options{ RepoPath: repo, EmbeddingProvider: "openai", EmbeddingModel: "model", @@ -2374,7 +2584,8 @@ func TestImportEmbeddingsBoundsDecompressedInput(t *testing.T) { Model: "model", InputVersion: store.EmbeddingInputVersion, Files: []string{embeddingRel}, - }}), "embedding decompressed size exceeds") + }}) + require.ErrorContains(t, err, "embedding decompressed size exceeds") require.NoError(t, tx.Rollback()) } @@ -2493,6 +2704,18 @@ func seedStore(t *testing.T, path string) *store.Store { return s } +func insertShareTestEmbedding(t *testing.T, ctx context.Context, s *store.Store, messageID string, vector []float32) { + t.Helper() + blob, err := store.EncodeEmbeddingVector(vector) + require.NoError(t, err) + _, err = s.DB().ExecContext(ctx, ` + insert into message_embeddings( + message_id, provider, model, input_version, dimensions, embedding_blob, embedded_at + ) values (?, 'openai', 'text-embedding-3-small', ?, ?, ?, ?) + `, messageID, store.EmbeddingInputVersion, len(vector), blob, time.Now().UTC().Format(time.RFC3339Nano)) + require.NoError(t, err) +} + func addCachedAttachment(ctx context.Context, s *store.Store, mediaPath, hash string, size int64) error { now := time.Now().UTC().Format(time.RFC3339Nano) if err := s.UpsertMessages(ctx, []store.MessageMutation{{ diff --git a/internal/syncer/channel_catalog.go b/internal/syncer/channel_catalog.go index 64ace597..ee114b1d 100644 --- a/internal/syncer/channel_catalog.go +++ b/internal/syncer/channel_catalog.go @@ -18,7 +18,13 @@ const ( channelCatalogIncremental ) -func (s *Syncer) channelList(ctx context.Context, guildID string, requested []string, mode channelCatalogMode) ([]*discordgo.Channel, bool, error) { +func (s *Syncer) channelList( + ctx context.Context, + guildID string, + requested []string, + mode channelCatalogMode, + exclusions channelExclusions, +) ([]*discordgo.Channel, bool, error) { if len(requested) == 0 { channels, err := s.liveChannelList(ctx, guildID, mode) if err != nil { @@ -34,6 +40,10 @@ func (s *Syncer) channelList(ctx context.Context, guildID string, requested []st if err != nil { return nil, false, err } + requestedSet = filterExcludedStoredTargets(rows, requestedSet, exclusions) + if len(requestedSet) == 0 { + return nil, true, nil + } storedByID = selectStoredChannels(rows, requestedSet) if canUseStoredTargets(storedByID, requestedSet) { selected := selectRequestedChannels(nil, storedByID, requestedSet) @@ -65,9 +75,54 @@ func (s *Syncer) channelList(ctx context.Context, guildID string, requested []st } selected = selectRequestedChannels(allChannels, storedByID, requestedSet) } + selected = filterExcludedDiscordTargets(selected, allChannels, exclusions) return selected, true, nil } +func filterExcludedStoredTargets( + rows []store.ChannelRow, + requested map[string]struct{}, + exclusions channelExclusions, +) map[string]struct{} { + if len(requested) == 0 || (len(exclusions.ids) == 0 && len(exclusions.kinds) == 0) { + return requested + } + channelByID := make(map[string]store.ChannelRow, len(rows)) + for _, row := range rows { + channelByID[row.ID] = row + } + out := make(map[string]struct{}, len(requested)) + for channelID := range requested { + channel, ok := channelByID[channelID] + if ok && exclusions.excludesStoredChannel(channel, channelByID) { + continue + } + if !ok && exclusions.excludesID(channelID) { + continue + } + out[channelID] = struct{}{} + } + return out +} + +func filterExcludedDiscordTargets( + channels []*discordgo.Channel, + channelByID map[string]*discordgo.Channel, + exclusions channelExclusions, +) []*discordgo.Channel { + if len(channels) == 0 || (len(exclusions.ids) == 0 && len(exclusions.kinds) == 0) { + return channels + } + out := make([]*discordgo.Channel, 0, len(channels)) + for _, channel := range channels { + if exclusions.excludesDiscordChannel(channel, channelByID) { + continue + } + out = append(out, channel) + } + return out +} + func (s *Syncer) liveChannelList(ctx context.Context, guildID string, mode channelCatalogMode) ([]*discordgo.Channel, error) { channels, err := s.client.GuildChannels(ctx, guildID) if err != nil { @@ -244,7 +299,7 @@ func channelFromRow(row store.ChannelRow) *discordgo.Channel { return &discordgo.Channel{ ID: row.ID, GuildID: row.GuildID, - ParentID: row.ParentID, + ParentID: storedChannelParentID(row), Name: row.Name, Topic: row.Topic, Position: row.Position, diff --git a/internal/syncer/channel_catalog_test.go b/internal/syncer/channel_catalog_test.go index 1272251d..7111ea32 100644 --- a/internal/syncer/channel_catalog_test.go +++ b/internal/syncer/channel_catalog_test.go @@ -396,6 +396,61 @@ func TestSyncChannelSubsetMergesStoredAndLiveTargets(t *testing.T) { require.Equal(t, "20", latest) } +func TestSyncTargetedStoredThreadInheritsExcludedParentKind(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + require.NoError(t, s.UpsertGuild(ctx, store.GuildRecord{ID: "g1", Name: "Guild", RawJSON: `{}`})) + require.NoError(t, s.UpsertChannel(ctx, store.ChannelRecord{ + ID: "news", + GuildID: "g1", + Kind: "announcement", + Name: "feed", + RawJSON: `{}`, + })) + require.NoError(t, s.UpsertChannel(ctx, store.ChannelRecord{ + ID: "thread", + GuildID: "g1", + Kind: "thread_public", + Name: "feed-thread", + ThreadParentID: "news", + RawJSON: `{}`, + })) + + client := &fakeClient{ + guilds: []*discordgo.UserGuild{{ID: "g1", Name: "Guild"}}, + guildByID: map[string]*discordgo.Guild{ + "g1": {ID: "g1", Name: "Guild"}, + }, + messages: map[string][]*discordgo.Message{ + "thread": {{ + ID: "10", + GuildID: "g1", + ChannelID: "thread", + Content: "must remain excluded", + Timestamp: time.Now().UTC(), + Author: &discordgo.User{ID: "u1", Username: "user"}, + }}, + }, + } + + svc := New(client, s, nil) + stats, err := svc.Sync(ctx, SyncOptions{ + Full: true, + GuildIDs: []string{"g1"}, + ChannelIDs: []string{"thread"}, + ExcludeChannelKinds: []string{"announcement"}, + }) + require.NoError(t, err) + require.Zero(t, stats.Messages) + require.Zero(t, client.guildChanCalls) + require.Zero(t, client.messageCalls["thread"]) +} + func TestSyncSkipsUnchangedThreadsWhenHistoryComplete(t *testing.T) { t.Parallel() diff --git a/internal/syncer/channel_exclusions.go b/internal/syncer/channel_exclusions.go index f05f5753..bb0d3b99 100644 --- a/internal/syncer/channel_exclusions.go +++ b/internal/syncer/channel_exclusions.go @@ -65,14 +65,24 @@ func (e channelExclusions) excludesKind(kind string) bool { if _, ok := e.kinds[kind]; ok { return true } - if kind == "thread_announcement" { + switch kind { + case "news", "thread_news", "thread_announcement": _, ok := e.kinds["announcement"] return ok + default: + return false } - return false } func (e channelExclusions) excludesDiscordChannel(channel *discordgo.Channel, channelByID map[string]*discordgo.Channel) bool { + return e.excludesDiscordChannelID(channel, channelByID, nil) +} + +func (e channelExclusions) excludesDiscordChannelID( + channel *discordgo.Channel, + channelByID map[string]*discordgo.Channel, + visiting map[string]struct{}, +) bool { if channel == nil { return false } @@ -85,22 +95,54 @@ func (e channelExclusions) excludesDiscordChannel(channel *discordgo.Channel, ch if e.excludesID(channel.ParentID) { return true } + if visiting == nil { + visiting = map[string]struct{}{} + } + if _, ok := visiting[channel.ID]; ok { + return false + } + visiting[channel.ID] = struct{}{} + defer delete(visiting, channel.ID) parent := channelByID[channel.ParentID] - return parent != nil && e.excludesKind(channelKind(parent)) + return parent != nil && e.excludesDiscordChannelID(parent, channelByID, visiting) } func (e channelExclusions) excludesStoredChannel(channel store.ChannelRow, channelByID map[string]store.ChannelRow) bool { + return e.excludesStoredChannelID(channel, channelByID, nil) +} + +func (e channelExclusions) excludesStoredChannelID( + channel store.ChannelRow, + channelByID map[string]store.ChannelRow, + visiting map[string]struct{}, +) bool { if e.excludesID(channel.ID) || e.excludesKind(channel.Kind) { return true } - if channel.ParentID == "" { + parentID := storedChannelParentID(channel) + if parentID == "" { return false } - if e.excludesID(channel.ParentID) { + if e.excludesID(parentID) { return true } - parent, ok := channelByID[channel.ParentID] - return ok && e.excludesKind(parent.Kind) + if visiting == nil { + visiting = map[string]struct{}{} + } + if _, ok := visiting[channel.ID]; ok { + return false + } + visiting[channel.ID] = struct{}{} + defer delete(visiting, channel.ID) + parent, ok := channelByID[parentID] + return ok && e.excludesStoredChannelID(parent, channelByID, visiting) +} + +func storedChannelParentID(channel store.ChannelRow) string { + if channel.ThreadParentID != "" { + return channel.ThreadParentID + } + return channel.ParentID } func (s *Syncer) effectiveChannelExclusions(opts SyncOptions) channelExclusions { diff --git a/internal/syncer/message_sync.go b/internal/syncer/message_sync.go index 2af843cf..eb99a7d1 100644 --- a/internal/syncer/message_sync.go +++ b/internal/syncer/message_sync.go @@ -217,9 +217,6 @@ func (s *Syncer) messageChannelContext(ctx context.Context, opts SyncOptions) (c if timeout <= 0 { return context.WithCancel(ctx) } - if _, ok := ctx.Deadline(); ok { - return context.WithCancel(ctx) - } return context.WithTimeout(ctx, timeout) } diff --git a/internal/syncer/message_sync_helpers_test.go b/internal/syncer/message_sync_helpers_test.go index 6e2eb90a..ad8b6a79 100644 --- a/internal/syncer/message_sync_helpers_test.go +++ b/internal/syncer/message_sync_helpers_test.go @@ -40,14 +40,25 @@ func TestMessageChannelSelectionAndTimeoutHelpers(t *testing.T) { _, ok := ctx.Deadline() require.True(t, ok) - parentCtx, parentCancel := context.WithDeadline(context.Background(), time.Now().Add(time.Hour)) + startedAt := time.Now() + parentCtx, parentCancel := context.WithDeadline(context.Background(), startedAt.Add(time.Hour)) defer parentCancel() ctx, cancel = svc.messageChannelContext(parentCtx, SyncOptions{MessageChannelTimeout: 2 * time.Second}) defer cancel() deadline, ok := ctx.Deadline() require.True(t, ok) parentDeadline, _ := parentCtx.Deadline() - require.Equal(t, parentDeadline, deadline) + require.WithinDuration(t, startedAt.Add(2*time.Second), deadline, 100*time.Millisecond) + require.True(t, deadline.Before(parentDeadline)) + + shortParentCtx, shortParentCancel := context.WithDeadline(context.Background(), time.Now().Add(2*time.Second)) + defer shortParentCancel() + ctx, cancel = svc.messageChannelContext(shortParentCtx, SyncOptions{MessageChannelTimeout: time.Hour}) + defer cancel() + deadline, ok = ctx.Deadline() + require.True(t, ok) + shortParentDeadline, _ := shortParentCtx.Deadline() + require.Equal(t, shortParentDeadline, deadline) } func TestChannelSyncStateHelpers(t *testing.T) { diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go index 0ad7e427..5ed525e2 100644 --- a/internal/syncer/syncer.go +++ b/internal/syncer/syncer.go @@ -43,9 +43,10 @@ type Syncer struct { messageSyncLogEvery time.Duration messageSyncWaitEvery time.Duration tailReady func(context.Context) error + tailShutdownTimeout time.Duration tailRepair func(context.Context, SyncOptions) (SyncStats, error) tailRepairJoinTimeout time.Duration - tailRepairMu sync.Mutex + tailRepairRunMu sync.Mutex tailRepairOffsetMu sync.RWMutex tailRepairOffset time.Duration channelExclusions channelExclusions @@ -90,6 +91,7 @@ const ( defaultMessageChannelTimeout = 5 * time.Minute defaultMessageSyncLogEvery = 15 * time.Second defaultMessageSyncWaitEvery = 30 * time.Second + defaultTailShutdownTimeout = 90 * time.Second ) func New(client Client, store *store.Store, logger *slog.Logger) *Syncer { @@ -106,6 +108,7 @@ func New(client Client, store *store.Store, logger *slog.Logger) *Syncer { messageChannelTimeout: defaultMessageChannelTimeout, messageSyncLogEvery: defaultMessageSyncLogEvery, messageSyncWaitEvery: defaultMessageSyncWaitEvery, + tailShutdownTimeout: defaultTailShutdownTimeout, } } @@ -169,7 +172,13 @@ func (s *Syncer) syncGuild(ctx context.Context, guildID string, opts SyncOptions catalogMode = channelCatalogIncremental } } - channelList, targeted, err := s.channelList(ctx, guildID, opts.ChannelIDs, catalogMode) + channelList, targeted, err := s.channelList( + ctx, + guildID, + opts.ChannelIDs, + catalogMode, + s.effectiveChannelExclusions(opts), + ) if err != nil { return stats, err } diff --git a/internal/syncer/syncer_tail_test.go b/internal/syncer/syncer_tail_test.go index 7bef49e6..bb48b3c2 100644 --- a/internal/syncer/syncer_tail_test.go +++ b/internal/syncer/syncer_tail_test.go @@ -209,6 +209,89 @@ func TestTailHandlerExcludesConfiguredFeedChannels(t *testing.T) { require.Equal(t, 1, status.MessageCount) } +func TestTailHandlerReclassifiesDescendantsAfterParentUpdates(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + handler := &tailHandler{ + store: s, + exclusions: newChannelExclusions(nil, []string{"announcement"}), + channels: map[string]tailChannel{}, + kindExcludedChannelIDs: map[string]struct{}{}, + } + require.NoError(t, handler.OnChannelUpsert(ctx, &discordgo.Channel{ + ID: "thread", + GuildID: "g1", + ParentID: "parent", + Name: "feed-thread", + Type: discordgo.ChannelTypeGuildPublicThread, + })) + require.NoError(t, handler.OnChannelUpsert(ctx, &discordgo.Channel{ + ID: "parent", + GuildID: "g1", + Name: "feed", + Type: discordgo.ChannelTypeGuildNews, + })) + + message := func(id, content string) *discordgo.Message { + return &discordgo.Message{ + ID: id, + GuildID: "g1", + ChannelID: "thread", + Content: content, + Timestamp: time.Now().UTC(), + Author: &discordgo.User{ID: "u1", Username: "user"}, + } + } + require.NoError(t, handler.OnMessageCreate(ctx, message("1", "excluded"))) + + require.NoError(t, handler.OnChannelUpsert(ctx, &discordgo.Channel{ + ID: "parent", + GuildID: "g1", + Name: "discussion", + Type: discordgo.ChannelTypeGuildText, + })) + require.NoError(t, handler.OnMessageCreate(ctx, message("2", "allowed"))) + + require.NoError(t, handler.OnChannelUpsert(ctx, &discordgo.Channel{ + ID: "parent", + GuildID: "g1", + Name: "feed-again", + Type: discordgo.ChannelTypeGuildNews, + })) + require.NoError(t, handler.OnMessageUpdate(ctx, message("2", "must not update"))) + require.NoError(t, handler.OnMessageDelete(ctx, &discordgo.MessageDelete{Message: &discordgo.Message{ + ID: "2", + GuildID: "g1", + ChannelID: "thread", + }})) + + require.NoError(t, handler.OnChannelUpsert(ctx, &discordgo.Channel{ + ID: "parent", + GuildID: "g1", + Name: "discussion-again", + Type: discordgo.ChannelTypeGuildText, + })) + require.NoError(t, handler.OnMessageCreate(ctx, message("3", "allowed again"))) + + var count, deleted int + require.NoError(t, s.DB().QueryRowContext(ctx, ` + select count(*), coalesce(sum(case when deleted_at is not null then 1 else 0 end), 0) + from messages + where channel_id = 'thread' + `).Scan(&count, &deleted)) + require.Equal(t, 2, count) + require.Zero(t, deleted) + + var content string + require.NoError(t, s.DB().QueryRowContext(ctx, `select content from messages where id = '2'`).Scan(&content)) + require.Equal(t, "allowed", content) +} + func TestTailHandlerMessageUpdateFetchesFullMessageBeforeUpsert(t *testing.T) { t.Parallel() @@ -1050,6 +1133,126 @@ func TestReplayTailMessageFailuresRetainsFetchAndIdentityFailures(t *testing.T) } } +func TestRunTailDetectsGatewayExitWhileRepairIsBlocked(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + require.NoError(t, s.SetSyncState(ctx, channelLatestScope("c1"), "9")) + + client := &gatewayExitDuringRepairClient{ + fakeClient: &fakeClient{ + guilds: []*discordgo.UserGuild{{ID: "g1", Name: "Guild"}}, + guildByID: map[string]*discordgo.Guild{ + "g1": {ID: "g1", Name: "Guild"}, + }, + channels: map[string][]*discordgo.Channel{ + "g1": {{ + ID: "c1", + GuildID: "g1", + Name: "general", + Type: discordgo.ChannelTypeGuildText, + LastMessageID: "10", + }}, + }, + }, + tailStarted: make(chan struct{}), + tailExit: make(chan error, 1), + repairStarted: make(chan struct{}), + repairCanceled: make(chan struct{}), + } + svc := New(client, s, nil) + svc.tailShutdownTimeout = time.Second + done := make(chan error, 1) + go func() { + done <- svc.RunTail(ctx, []string{"g1"}, 10*time.Millisecond) + }() + + select { + case <-client.tailStarted: + case <-time.After(time.Second): + t.Fatal("tail did not start") + } + select { + case <-client.repairStarted: + case <-time.After(time.Second): + t.Fatal("repair did not start") + } + + gatewayErr := errors.New("gateway disconnected") + client.tailExit <- gatewayErr + select { + case err := <-done: + require.ErrorIs(t, err, gatewayErr) + case <-time.After(500 * time.Millisecond): + t.Fatal("RunTail did not notice Gateway exit while repair was blocked") + } + select { + case <-client.repairCanceled: + default: + t.Fatal("Gateway exit did not cancel the in-flight repair") + } +} + +func TestRunTailSchedulesNextRepairAfterOverrunCompletes(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + require.NoError(t, s.SetSyncState(ctx, channelLatestScope("c1"), "9")) + + block := make(chan struct{}) + started := make(chan string, 4) + client := &quietTailClient{fakeClient: &fakeClient{ + guilds: []*discordgo.UserGuild{{ID: "g1", Name: "Guild"}}, + guildByID: map[string]*discordgo.Guild{ + "g1": {ID: "g1", Name: "Guild"}, + }, + channels: map[string][]*discordgo.Channel{ + "g1": {{ + ID: "c1", + GuildID: "g1", + Name: "general", + Type: discordgo.ChannelTypeGuildText, + LastMessageID: "10", + }}, + }, + messageBlocks: map[string]chan struct{}{"c1": block}, + messageStarted: started, + }} + svc := New(client, s, nil) + done := make(chan error, 1) + go func() { + done <- svc.RunTail(ctx, []string{"g1"}, 100*time.Millisecond) + }() + + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("first repair did not start") + } + time.Sleep(250 * time.Millisecond) + close(block) + + select { + case <-started: + t.Fatal("overdue ticker triggered an immediate second repair") + case <-time.After(50 * time.Millisecond): + } + select { + case <-started: + case <-time.After(500 * time.Millisecond): + t.Fatal("next repair was not scheduled after the overrun completed") + } + + cancel() + require.NoError(t, <-done) +} + func TestTailRepairSyncOptions(t *testing.T) { t.Parallel() @@ -1950,6 +2153,98 @@ func TestRunTailPreservesGatewayOpenErrorWithoutMessageFailure(t *testing.T) { require.Empty(t, report.Failures) } +func TestRunTailBoundsNonCooperativeShutdown(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + client := &stuckTailClient{ + fakeClient: &fakeClient{}, + started: make(chan struct{}), + tailRelease: make(chan struct{}), + closeStarted: make(chan struct{}), + closeRelease: make(chan struct{}), + } + svc := New(client, s, nil) + svc.tailShutdownTimeout = 25 * time.Millisecond + done := make(chan error, 1) + go func() { + done <- svc.RunTail(ctx, nil, 0) + }() + + select { + case <-client.started: + case <-time.After(time.Second): + t.Fatal("tail did not start") + } + cancel() + select { + case <-client.closeStarted: + case <-time.After(time.Second): + t.Fatal("client close did not start") + } + + select { + case err := <-done: + require.ErrorContains(t, err, "tail shutdown timed out") + case <-time.After(time.Second): + t.Fatal("RunTail shutdown remained unbounded") + } + close(client.tailRelease) + close(client.closeRelease) +} + +func TestRunTailReturnsCloseFailureOnShutdown(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + closeErr := errors.New("close failed") + client := &failingCloseTailClient{ + fakeClient: &fakeClient{}, + started: make(chan struct{}), + closeErr: closeErr, + } + svc := New(client, s, nil) + done := make(chan error, 1) + go func() { + done <- svc.RunTail(ctx, nil, 0) + }() + + select { + case <-client.started: + case <-time.After(time.Second): + t.Fatal("tail did not start") + } + cancel() + require.ErrorIs(t, <-done, closeErr) +} + +func TestRunTailPreservesGatewayAndCloseFailures(t *testing.T) { + ctx := context.Background() + + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + gatewayErr := errors.New("gateway exited") + closeErr := errors.New("gateway close failed") + client := &exitingFailingCloseTailClient{ + fakeClient: &fakeClient{}, + tailErr: gatewayErr, + closeErr: closeErr, + } + svc := New(client, s, nil) + + err = svc.RunTail(ctx, nil, 0) + require.ErrorIs(t, err, gatewayErr) + require.ErrorIs(t, err, closeErr) +} + func TestTailReadyCallback(t *testing.T) { t.Parallel() @@ -2381,3 +2676,99 @@ func setDiscordTailHandlerTimeout(t *testing.T, client *discordclient.Client, ti // Keep the production client API unchanged while making the cooperative timeout test fast. reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())).Elem().SetInt(int64(timeout)) } + +type gatewayExitDuringRepairClient struct { + *fakeClient + tailStarted chan struct{} + tailExit chan error + repairStarted chan struct{} + repairCanceled chan struct{} + tailStartOnce sync.Once + repairOnce sync.Once + cancelOnce sync.Once +} + +func (c *gatewayExitDuringRepairClient) Tail(ctx context.Context, _ discordclient.EventHandler) error { + c.tailStartOnce.Do(func() { close(c.tailStarted) }) + select { + case err := <-c.tailExit: + return err + case <-ctx.Done(): + return nil + } +} + +func (c *gatewayExitDuringRepairClient) ChannelMessages( + ctx context.Context, + _ string, + _ int, + _ string, + _ string, +) ([]*discordgo.Message, error) { + c.repairOnce.Do(func() { close(c.repairStarted) }) + <-ctx.Done() + c.cancelOnce.Do(func() { close(c.repairCanceled) }) + return nil, ctx.Err() +} + +type quietTailClient struct { + *fakeClient +} + +func (c *quietTailClient) Tail(ctx context.Context, _ discordclient.EventHandler) error { + <-ctx.Done() + return nil +} + +type stuckTailClient struct { + *fakeClient + started chan struct{} + tailRelease chan struct{} + closeStarted chan struct{} + closeRelease chan struct{} + startOnce sync.Once + closeOnce sync.Once +} + +func (c *stuckTailClient) Tail(context.Context, discordclient.EventHandler) error { + c.startOnce.Do(func() { close(c.started) }) + <-c.tailRelease + return nil +} + +func (c *stuckTailClient) Close() error { + c.closeOnce.Do(func() { close(c.closeStarted) }) + <-c.closeRelease + return nil +} + +type failingCloseTailClient struct { + *fakeClient + started chan struct{} + closeErr error + startOnce sync.Once +} + +func (c *failingCloseTailClient) Tail(ctx context.Context, _ discordclient.EventHandler) error { + c.startOnce.Do(func() { close(c.started) }) + <-ctx.Done() + return nil +} + +func (c *failingCloseTailClient) Close() error { + return c.closeErr +} + +type exitingFailingCloseTailClient struct { + *fakeClient + tailErr error + closeErr error +} + +func (c *exitingFailingCloseTailClient) Tail(context.Context, discordclient.EventHandler) error { + return c.tailErr +} + +func (c *exitingFailingCloseTailClient) Close() error { + return c.closeErr +} diff --git a/internal/syncer/tail.go b/internal/syncer/tail.go index 19a71b1a..fc4e9b64 100644 --- a/internal/syncer/tail.go +++ b/internal/syncer/tail.go @@ -46,93 +46,172 @@ func (s *Syncer) RunTail(ctx context.Context, guildIDs []string, repairEvery tim onReady: s.tailReady, logger: s.logger, exclusions: s.channelExclusions, + channels: map[string]tailChannel{}, kindExcludedChannelIDs: map[string]struct{}{}, } if err := handler.seedChannelExclusions(ctx); err != nil { return fmt.Errorf("seed tail channel exclusions: %w", err) } - if repairEvery <= 0 { - return s.client.Tail(ctx, handler) - } tailCtx, cancelTail := context.WithCancel(ctx) defer cancelTail() - var closeOnce sync.Once - closeClient := func() { - if closeable, ok := s.client.(closeableClient); ok { - _ = closeable.Close() - } - } tailDone := make(chan error, 1) go func() { tailDone <- s.client.Tail(tailCtx, handler) }() - repairOffset := s.repairOffset() - repairTimer := time.NewTimer(nextTailRepairDelay(time.Now(), repairEvery, repairOffset)) - defer repairTimer.Stop() - repairC := repairTimer.C - var repairTicker *time.Ticker + + var repairTimer *time.Timer + var repairC <-chan time.Time defer func() { - if repairTicker != nil { - repairTicker.Stop() + if repairTimer != nil { + repairTimer.Stop() } }() + repairOffset := s.repairOffset() + scheduleRepair := func() { + if repairEvery <= 0 { + repairC = nil + return + } + delay := nextTailRepairDelay(time.Now(), repairEvery, repairOffset) + if repairTimer == nil { + repairTimer = time.NewTimer(delay) + } else { + if !repairTimer.Stop() { + select { + case <-repairTimer.C: + default: + } + } + repairTimer.Reset(delay) + } + repairC = repairTimer.C + } + scheduleRepair() + var activeRepair *tailRepairRun var repairDone <-chan tailRepairResult for { select { case <-ctx.Done(): - cancelTail() - repairErr := s.joinTailRepair(activeRepair, "parent_shutdown") - closeOnce.Do(closeClient) - tailErr := <-tailDone - if discordclient.IsFatalTailError(tailErr) { - if repairErr != nil { - return errors.Join(tailErr, repairErr) - } - return tailErr - } - if repairErr != nil { - return repairErr - } - return nil + return s.finishTailRun( + cancelTail, + tailDone, + activeRepair, + nil, + false, + true, + "parent_shutdown", + ) case err := <-tailDone: - cancelTail() - repairErr := s.joinTailRepair(activeRepair, "tail_return") - closeOnce.Do(closeClient) - if repairErr != nil { - return errors.Join(err, repairErr) + return s.finishTailRun( + cancelTail, + nil, + activeRepair, + err, + true, + false, + "tail_return", + ) + case <-repairC: + repairC = nil + activeRepair = s.startTailRepair(tailCtx, guildIDs) + if activeRepair == nil { + scheduleRepair() + continue } - return err + repairDone = activeRepair.done case result := <-repairDone: s.logTailRepairResult(result) activeRepair = nil repairDone = nil - if repairOffset > 0 { - repairTimer.Reset(nextTailRepairDelay(time.Now(), repairEvery, repairOffset)) - repairC = repairTimer.C - } - case <-repairC: - if repairOffset <= 0 && repairTicker == nil { - repairTicker = time.NewTicker(repairEvery) - repairC = repairTicker.C - } - if activeRepair != nil { - continue - } - activeRepair = s.startTailRepair(ctx, guildIDs) - if activeRepair != nil { - repairDone = activeRepair.done - } - if repairOffset > 0 { - if activeRepair == nil { - repairTimer.Reset(nextTailRepairDelay(time.Now(), repairEvery, repairOffset)) - repairC = repairTimer.C - } else { - repairC = nil - } + // Schedule from completion so an overrun cannot immediately replay stale ticks. + scheduleRepair() + } + } +} + +func (s *Syncer) finishTailRun( + cancelTail context.CancelFunc, + tailDone <-chan error, + repair *tailRepairRun, + tailErr error, + tailFinished bool, + requestedShutdown bool, + repairJoinReason string, +) error { + cancelTail() + + timeout := s.tailShutdownTimeout + if timeout <= 0 { + timeout = defaultTailShutdownTimeout + } + startedAt := time.Now() + repairErr := s.joinTailRepair(repair, repairJoinReason, timeout) + + var closeDone <-chan error + if closeable, ok := s.client.(closeableClient); ok { + done := make(chan error, 1) + closeDone = done + go func() { + done <- closeable.Close() + }() + } + + if tailFinished { + tailDone = nil + } + if tailDone == nil && closeDone == nil { + return tailRunResult(requestedShutdown, tailErr, repairErr, nil, nil) + } + + remaining := timeout - time.Since(startedAt) + if remaining <= 0 { + return tailRunResult( + requestedShutdown, + tailErr, + repairErr, + nil, + fmt.Errorf("tail shutdown timed out after %s", timeout), + ) + } + timer := time.NewTimer(remaining) + defer timer.Stop() + + var closeErr error + for tailDone != nil || closeDone != nil { + select { + case err := <-tailDone: + if tailErr == nil { + tailErr = err } + tailDone = nil + case err := <-closeDone: + closeErr = err + closeDone = nil + case <-timer.C: + return tailRunResult( + requestedShutdown, + tailErr, + repairErr, + closeErr, + fmt.Errorf("tail shutdown timed out after %s", timeout), + ) } } + return tailRunResult(requestedShutdown, tailErr, repairErr, closeErr, nil) +} + +func tailRunResult( + requestedShutdown bool, + tailErr error, + repairErr error, + closeErr error, + shutdownErr error, +) error { + if requestedShutdown && !discordclient.IsFatalTailError(tailErr) { + tailErr = nil + } + return errors.Join(tailErr, repairErr, closeErr, shutdownErr) } const defaultTailRepairJoinTimeout = 5 * time.Second @@ -162,7 +241,7 @@ func (s *Syncer) importTailMessageFailureFallbacks(ctx context.Context) error { } func (s *Syncer) startTailRepair(ctx context.Context, guildIDs []string) *tailRepairRun { - if s == nil || !s.tailRepairMu.TryLock() { + if s == nil || !s.tailRepairRunMu.TryLock() { if s != nil && s.logger != nil { s.logger.Warn("tail repair start skipped", "reason", "repair_already_running") } @@ -172,7 +251,7 @@ func (s *Syncer) startTailRepair(ctx context.Context, guildIDs []string) *tailRe done := make(chan tailRepairResult, 1) startedAt := time.Now() go func() { - defer s.tailRepairMu.Unlock() + defer s.tailRepairRunMu.Unlock() _, err := s.runTailRepair(repairCtx, tailRepairSyncOptions(guildIDs)) done <- tailRepairResult{err: err, elapsed: time.Since(startedAt)} }() @@ -186,7 +265,7 @@ func (s *Syncer) runTailRepair(ctx context.Context, opts SyncOptions) (SyncStats return s.Sync(ctx, opts) } -func (s *Syncer) joinTailRepair(repair *tailRepairRun, reason string) error { +func (s *Syncer) joinTailRepair(repair *tailRepairRun, reason string, maxWait time.Duration) error { if repair == nil { return nil } @@ -196,6 +275,9 @@ func (s *Syncer) joinTailRepair(repair *tailRepairRun, reason string) error { if timeout <= 0 { timeout = defaultTailRepairJoinTimeout } + if maxWait > 0 && timeout > maxWait { + timeout = maxWait + } timer := time.NewTimer(timeout) defer timer.Stop() outcome := "timed_out" @@ -300,9 +382,15 @@ type tailHandler struct { logger *slog.Logger exclusions channelExclusions exclusionMu sync.RWMutex + channels map[string]tailChannel kindExcludedChannelIDs map[string]struct{} } +type tailChannel struct { + parentID string + kind string +} + func (t *tailHandler) OnTailReady(ctx context.Context) error { if t.onReady == nil { return nil @@ -521,20 +609,19 @@ func (t *tailHandler) seedChannelExclusions(ctx context.Context) error { if err != nil { return err } - channelByID := make(map[string]store.ChannelRow, len(channels)) - for _, channel := range channels { - channelByID[channel.ID] = channel - } t.exclusionMu.Lock() defer t.exclusionMu.Unlock() + t.channels = make(map[string]tailChannel, len(channels)) if t.kindExcludedChannelIDs == nil { t.kindExcludedChannelIDs = map[string]struct{}{} } for _, channel := range channels { - if t.exclusions.excludesStoredChannel(channel, channelByID) { - t.kindExcludedChannelIDs[channel.ID] = struct{}{} + t.channels[channel.ID] = tailChannel{ + parentID: storedChannelParentID(channel), + kind: channel.Kind, } } + t.rebuildChannelExclusionsLocked() return nil } @@ -552,18 +639,61 @@ func (t *tailHandler) trackChannelExclusion(channel *discordgo.Channel) { if channel == nil { return } - excluded := t.exclusions.excludesKind(channelKind(channel)) - if !excluded && channel.ParentID != "" { - excluded = t.excludeChannel(channel.ParentID) - } t.exclusionMu.Lock() defer t.exclusionMu.Unlock() + if t.channels == nil { + t.channels = map[string]tailChannel{} + } + t.channels[channel.ID] = tailChannel{ + parentID: channel.ParentID, + kind: channelKind(channel), + } + t.rebuildChannelExclusionsLocked() +} + +func (t *tailHandler) rebuildChannelExclusionsLocked() { if t.kindExcludedChannelIDs == nil { t.kindExcludedChannelIDs = map[string]struct{}{} } - if excluded { - t.kindExcludedChannelIDs[channel.ID] = struct{}{} - return + clear(t.kindExcludedChannelIDs) + decisions := make(map[string]bool, len(t.channels)) + var excludes func(string, map[string]struct{}) bool + excludes = func(channelID string, visiting map[string]struct{}) bool { + if decision, ok := decisions[channelID]; ok { + return decision + } + if t.exclusions.excludesID(channelID) { + decisions[channelID] = true + return true + } + channel, ok := t.channels[channelID] + if !ok { + decisions[channelID] = false + return false + } + if t.exclusions.excludesKind(channel.kind) { + decisions[channelID] = true + return true + } + if channel.parentID == "" { + decisions[channelID] = false + return false + } + if visiting == nil { + visiting = map[string]struct{}{} + } + if _, ok := visiting[channelID]; ok { + return false + } + visiting[channelID] = struct{}{} + excluded := excludes(channel.parentID, visiting) + delete(visiting, channelID) + decisions[channelID] = excluded + return excluded + } + for channelID := range t.channels { + if excludes(channelID, nil) { + t.kindExcludedChannelIDs[channelID] = struct{}{} + } } - delete(t.kindExcludedChannelIDs, channel.ID) } From e7209a835442f85b07354e2f2ec19714a7ed25b1 Mon Sep 17 00:00:00 2001 From: Hannes Rudolph <49103247+hannesrudolph@users.noreply.github.com> Date: Thu, 9 Jul 2026 19:54:10 -0600 Subject: [PATCH 3/7] fix: preserve tail channel ancestry ordering --- internal/cli/cli_test.go | 4 + internal/discord/client.go | 34 ++ internal/discord/client_test.go | 226 ++++++++++- internal/syncer/syncer.go | 1 + internal/syncer/syncer_tail_test.go | 555 +++++++++++++++++++++++++++- internal/syncer/syncer_test.go | 35 ++ internal/syncer/tail.go | 182 ++++++++- 7 files changed, 1017 insertions(+), 20 deletions(-) diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index 367dc291..6d7ebf5f 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -3770,6 +3770,10 @@ func (f *fakeDiscordClient) GuildChannels(context.Context, string) ([]*discordgo return nil, nil } +func (f *fakeDiscordClient) Channel(context.Context, string) (*discordgo.Channel, error) { + return nil, nil +} + func (f *fakeDiscordClient) ThreadsActive(context.Context, string) ([]*discordgo.Channel, error) { return nil, nil } diff --git a/internal/discord/client.go b/internal/discord/client.go index 91e4d817..64265617 100644 --- a/internal/discord/client.go +++ b/internal/discord/client.go @@ -294,6 +294,12 @@ func (c *Client) GuildChannels(ctx context.Context, guildID string) ([]*discordg return c.session.GuildChannels(guildID, discordgo.WithContext(reqCtx)) } +func (c *Client) Channel(ctx context.Context, channelID string) (*discordgo.Channel, error) { + reqCtx, cancel := c.requestContext(ctx) + defer cancel() + return c.session.Channel(channelID, discordgo.WithContext(reqCtx)) +} + func (c *Client) ThreadsActive(ctx context.Context, channelID string) ([]*discordgo.Channel, error) { reqCtx, cancel := c.requestContext(ctx) defer cancel() @@ -603,6 +609,34 @@ func (c *Client) Tail(ctx context.Context, handler EventHandler) error { before, )) }) + addHandler(func(_ *discordgo.Session, evt *discordgo.ThreadCreate) { + var channel *discordgo.Channel + if evt != nil { + channel = evt.Channel + } + c.enqueueTailTask(tailCtx, orderedWorkCh, fatal, newChannelTailTask( + "THREAD_CREATE", + func(taskCtx context.Context) error { + return handler.OnChannelUpsert(taskCtx, channel) + }, + channel, + )) + }) + addHandler(func(_ *discordgo.Session, evt *discordgo.ThreadUpdate) { + var channel, before *discordgo.Channel + if evt != nil { + channel = evt.Channel + before = evt.BeforeUpdate + } + c.enqueueTailTask(tailCtx, orderedWorkCh, fatal, newChannelTailTask( + "THREAD_UPDATE", + func(taskCtx context.Context) error { + return handler.OnChannelUpsert(taskCtx, channel) + }, + channel, + before, + )) + }) addHandler(func(_ *discordgo.Session, evt *discordgo.GuildMemberAdd) { var member *discordgo.Member if evt != nil { diff --git a/internal/discord/client_test.go b/internal/discord/client_test.go index 66331fb8..d04dd7a7 100644 --- a/internal/discord/client_test.go +++ b/internal/discord/client_test.go @@ -34,6 +34,9 @@ func TestClientRESTWrappers(t *testing.T) { mux.HandleFunc("/api/v10/guilds/g1/channels", writeJSON([]map[string]any{ {"id": "c1", "guild_id": "g1", "name": "general", "type": 0}, })) + mux.HandleFunc("/api/v10/channels/c1", writeJSON(map[string]any{ + "id": "c1", "guild_id": "g1", "name": "general", "type": 0, + })) mux.HandleFunc("/api/v10/guilds/g1/threads/active", writeJSON(map[string]any{ "threads": []map[string]any{ {"id": "tg1", "guild_id": "g1", "parent_id": "c1", "name": "guild-thread", "type": 11}, @@ -140,6 +143,10 @@ func TestClientRESTWrappers(t *testing.T) { require.NoError(t, err) require.Len(t, channels, 1) + channel, err := client.Channel(ctx, "c1") + require.NoError(t, err) + require.Equal(t, "c1", channel.ID) + members, err := client.GuildMembers(ctx, "g1") require.NoError(t, err) require.Len(t, members, 1) @@ -1099,8 +1106,10 @@ func TestTailReceivesGatewayEvents(t *testing.T) { {"op": 0, "t": "MESSAGE_UPDATE", "s": 3, "d": map[string]any{"id": "m1", "guild_id": "g1", "channel_id": "c1", "content": "hello 2", "timestamp": now, "author": map[string]any{"id": "u1", "username": "user"}}}, {"op": 0, "t": "MESSAGE_DELETE", "s": 4, "d": map[string]any{"id": "m1", "guild_id": "g1", "channel_id": "c1"}}, {"op": 0, "t": "CHANNEL_CREATE", "s": 5, "d": map[string]any{"id": "c1", "guild_id": "g1", "name": "general", "type": 0}}, - {"op": 0, "t": "GUILD_MEMBER_ADD", "s": 6, "d": map[string]any{"guild_id": "g1", "user": map[string]any{"id": "u1", "username": "user"}, "roles": []string{}}}, - {"op": 0, "t": "GUILD_MEMBER_REMOVE", "s": 7, "d": map[string]any{"guild_id": "g1", "user": map[string]any{"id": "u1", "username": "user"}}}, + {"op": 0, "t": "THREAD_CREATE", "s": 6, "d": map[string]any{"id": "t1", "guild_id": "g1", "parent_id": "c1", "name": "thread", "type": 11, "newly_created": true}}, + {"op": 0, "t": "THREAD_UPDATE", "s": 7, "d": map[string]any{"id": "t1", "guild_id": "g1", "parent_id": "c1", "name": "thread updated", "type": 11}}, + {"op": 0, "t": "GUILD_MEMBER_ADD", "s": 8, "d": map[string]any{"guild_id": "g1", "user": map[string]any{"id": "u1", "username": "user"}, "roles": []string{}}}, + {"op": 0, "t": "GUILD_MEMBER_REMOVE", "s": 9, "d": map[string]any{"guild_id": "g1", "user": map[string]any{"id": "u1", "username": "user"}}}, } for _, event := range events { if err := conn.WriteJSON(event); err != nil { @@ -1120,6 +1129,7 @@ func TestTailReceivesGatewayEvents(t *testing.T) { require.NoError(t, err) defer func() { _ = client.Close() }() client.session.ShouldReconnectOnError = false + client.tailWorkerCount = 1 require.Zero(t, discordSessionHandlerCount(client.session)) ctx, cancel := context.WithCancel(context.Background()) @@ -1133,7 +1143,7 @@ func TestTailReceivesGatewayEvents(t *testing.T) { require.Equal(t, 1, handler.creates) require.Equal(t, 1, handler.updates) require.Equal(t, 1, handler.deletes) - require.Equal(t, 1, handler.channels) + require.Equal(t, 3, handler.channels) require.Equal(t, 1, handler.memberUpserts) require.Equal(t, 1, handler.memberDeletes) require.Equal(t, 1, handler.ready) @@ -2665,7 +2675,6 @@ func TestTailMessageUpdateRefetchTimeoutReportsRefetchStage(t *testing.T) { }, ) defer server.Close() - restore := patchDiscordEndpoints(server.URL + "/api/v10/") defer restore() @@ -3018,6 +3027,137 @@ func TestTailReadyHandlerFailureRemainsTerminal(t *testing.T) { require.ErrorIs(t, err, readyErr) } +func TestTailSerializesThreadEventsInGatewayOrder(t *testing.T) { + mux := http.NewServeMux() + upgrader := websocket.Upgrader{} + gatewayURL := "" + serverRelease := make(chan struct{}) + mux.HandleFunc("/api/v10/gateway", func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"url": gatewayURL}) + }) + server := httptest.NewServer(mux) + defer server.Close() + defer close(serverRelease) + + gatewayURL = "ws" + server.URL[len("http"):] + "/gateway" + gatewayHandler := func(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + t.Errorf("upgrade gateway: %v", err) + return + } + defer func() { _ = conn.Close() }() + + if err := conn.WriteJSON(map[string]any{ + "op": 10, + "d": map[string]any{"heartbeat_interval": 1000}, + }); err != nil { + t.Errorf("write hello: %v", err) + return + } + if _, _, err := conn.ReadMessage(); err != nil { + t.Errorf("read identify: %v", err) + return + } + events := []map[string]any{ + { + "op": 0, + "t": "READY", + "s": 1, + "d": map[string]any{ + "session_id": "session", + "user": map[string]any{"id": "bot", "username": "bot"}, + }, + }, + { + "op": 0, + "t": "THREAD_CREATE", + "s": 2, + "d": map[string]any{ + "id": "t1", "guild_id": "g1", "parent_id": "c1", + "name": "created", "type": 11, "newly_created": true, + }, + }, + { + "op": 0, + "t": "THREAD_UPDATE", + "s": 3, + "d": map[string]any{ + "id": "t1", "guild_id": "g1", "parent_id": "c1", + "name": "updated", "type": 11, + }, + }, + { + "op": 0, + "t": "MESSAGE_CREATE", + "s": 4, + "d": map[string]any{ + "id": "m1", "guild_id": "g1", "channel_id": "t1", + "content": "after update", "timestamp": time.Now().UTC().Format(time.RFC3339), + "author": map[string]any{"id": "u1", "username": "user"}, + }, + }, + } + for _, event := range events { + if err := conn.WriteJSON(event); err != nil { + t.Errorf("write event %v: %v", event["t"], err) + return + } + } + select { + case <-serverRelease: + case <-r.Context().Done(): + } + } + mux.HandleFunc("/gateway", gatewayHandler) + mux.HandleFunc("/gateway/", gatewayHandler) + + restore := patchDiscordEndpoints(server.URL + "/api/v10/") + defer restore() + + client, err := New("token") + require.NoError(t, err) + defer func() { _ = client.Close() }() + client.session.ShouldReconnectOnError = false + client.tailWorkerCount = 4 + client.tailQueueSize = 8 + + handler := newOrderedChannelHandler() + ctx, cancel := context.WithCancel(context.Background()) + tailDone := make(chan error, 1) + go func() { + tailDone <- client.Tail(ctx, handler) + }() + + select { + case <-handler.firstStarted: + case <-time.After(time.Second): + t.Fatal("first thread event did not start") + } + require.True(t, client.session.SyncEvents) + select { + case <-handler.secondStarted: + t.Fatal("second thread event overtook the first") + case <-time.After(50 * time.Millisecond): + } + select { + case <-handler.messageStarted: + t.Fatal("message event overtook preceding thread events") + case <-time.After(50 * time.Millisecond): + } + close(handler.releaseFirst) + select { + case <-handler.done: + case <-time.After(time.Second): + t.Fatal("ordered thread events did not complete") + } + cancel() + require.NoError(t, <-tailDone) + require.Equal(t, []string{"created", "updated", "message"}, handler.names()) + require.True(t, client.session.SyncEvents) +} + func TestTailFailsFastWhenWorkerQueueFills(t *testing.T) { mux := http.NewServeMux() upgrader := websocket.Upgrader{} @@ -3997,3 +4137,81 @@ func (s *slowHandler) OnMemberUpsert(context.Context, string, *discordgo.Member) func (s *slowHandler) OnMemberDelete(context.Context, string, string) error { return nil } + +type orderedChannelHandler struct { + mu sync.Mutex + firstOnce sync.Once + secondOnce sync.Once + doneOnce sync.Once + firstStarted chan struct{} + secondStarted chan struct{} + messageStarted chan struct{} + releaseFirst chan struct{} + done chan struct{} + channelNames []string +} + +func newOrderedChannelHandler() *orderedChannelHandler { + return &orderedChannelHandler{ + firstStarted: make(chan struct{}), + secondStarted: make(chan struct{}), + messageStarted: make(chan struct{}), + releaseFirst: make(chan struct{}), + done: make(chan struct{}), + } +} + +func (h *orderedChannelHandler) OnMessageCreate(context.Context, *discordgo.Message) error { + close(h.messageStarted) + h.record("message") + return nil +} + +func (h *orderedChannelHandler) OnMessageUpdate(context.Context, *discordgo.Message) error { + return nil +} + +func (h *orderedChannelHandler) OnMessageDelete(context.Context, *discordgo.MessageDelete) error { + return nil +} + +func (h *orderedChannelHandler) OnChannelUpsert(ctx context.Context, channel *discordgo.Channel) error { + switch channel.Name { + case "created": + h.firstOnce.Do(func() { close(h.firstStarted) }) + select { + case <-ctx.Done(): + return ctx.Err() + case <-h.releaseFirst: + } + case "updated": + h.secondOnce.Do(func() { close(h.secondStarted) }) + } + + h.record(channel.Name) + return nil +} + +func (h *orderedChannelHandler) OnMemberUpsert(context.Context, string, *discordgo.Member) error { + return nil +} + +func (h *orderedChannelHandler) OnMemberDelete(context.Context, string, string) error { + return nil +} + +func (h *orderedChannelHandler) names() []string { + h.mu.Lock() + defer h.mu.Unlock() + return append([]string(nil), h.channelNames...) +} + +func (h *orderedChannelHandler) record(name string) { + h.mu.Lock() + h.channelNames = append(h.channelNames, name) + count := len(h.channelNames) + h.mu.Unlock() + if count == 3 { + h.doneOnce.Do(func() { close(h.done) }) + } +} diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go index 5ed525e2..c6f69385 100644 --- a/internal/syncer/syncer.go +++ b/internal/syncer/syncer.go @@ -19,6 +19,7 @@ type Client interface { Guilds(context.Context) ([]*discordgo.UserGuild, error) Guild(context.Context, string) (*discordgo.Guild, error) GuildChannels(context.Context, string) ([]*discordgo.Channel, error) + Channel(context.Context, string) (*discordgo.Channel, error) ThreadsActive(context.Context, string) ([]*discordgo.Channel, error) GuildThreadsActive(context.Context, string) ([]*discordgo.Channel, error) ThreadsArchived(context.Context, string, bool) ([]*discordgo.Channel, error) diff --git a/internal/syncer/syncer_tail_test.go b/internal/syncer/syncer_tail_test.go index bb48b3c2..8c347515 100644 --- a/internal/syncer/syncer_tail_test.go +++ b/internal/syncer/syncer_tail_test.go @@ -90,6 +90,12 @@ func TestTailHandlerWritesEvents(t *testing.T) { Timestamp: time.Now().UTC(), Author: &discordgo.User{ID: "u1", Username: "peter"}, } + require.NoError(t, handler.OnChannelUpsert(ctx, &discordgo.Channel{ + ID: "c1", + GuildID: "g1", + Name: "general", + Type: discordgo.ChannelTypeGuildText, + })) require.NoError(t, handler.OnMessageCreate(ctx, msg)) require.NoError(t, handler.OnMessageUpdate(ctx, msg)) require.NoError(t, handler.OnMessageDelete(ctx, &discordgo.MessageDelete{Message: &discordgo.Message{ @@ -97,12 +103,6 @@ func TestTailHandlerWritesEvents(t *testing.T) { GuildID: "g1", ChannelID: "c1", }})) - require.NoError(t, handler.OnChannelUpsert(ctx, &discordgo.Channel{ - ID: "c1", - GuildID: "g1", - Name: "general", - Type: discordgo.ChannelTypeGuildText, - })) require.NoError(t, handler.OnMemberUpsert(ctx, "g1", &discordgo.Member{ GuildID: "g1", Nick: "Peter", @@ -141,10 +141,18 @@ func TestTailHandlerDoesNotRecordOrderlyShutdownCancellation(t *testing.T) { s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) require.NoError(t, err) defer func() { _ = s.Close() }() + require.NoError(t, s.UpsertChannel(ctx, store.ChannelRecord{ + ID: "c1", + GuildID: "g1", + Kind: "text", + Name: "general", + RawJSON: `{}`, + })) canceledCtx, cancel := context.WithCancel(ctx) cancel() handler := &tailHandler{store: s} + require.NoError(t, handler.seedChannelExclusions(ctx)) err = handler.OnMessageCreate(canceledCtx, &discordgo.Message{ ID: "shutdown", GuildID: "g1", @@ -176,7 +184,21 @@ func TestTailHandlerExcludesConfiguredFeedChannels(t *testing.T) { RawJSON: `{}`, })) handler := &tailHandler{ - store: s, + store: s, + client: &fakeClient{channelsByID: map[string]*discordgo.Channel{ + "logs": { + ID: "logs", + GuildID: "g1", + Name: "logs", + Type: discordgo.ChannelTypeGuildText, + }, + "general": { + ID: "general", + GuildID: "g1", + Name: "general", + Type: discordgo.ChannelTypeGuildText, + }, + }}, exclusions: newChannelExclusions([]string{"logs"}, []string{"announcement"}), kindExcludedChannelIDs: map[string]struct{}{}, } @@ -217,8 +239,17 @@ func TestTailHandlerReclassifiesDescendantsAfterParentUpdates(t *testing.T) { require.NoError(t, err) defer func() { _ = s.Close() }() + client := &fakeClient{channelsByID: map[string]*discordgo.Channel{ + "parent": { + ID: "parent", + GuildID: "g1", + Name: "feed", + Type: discordgo.ChannelTypeGuildNews, + }, + }} handler := &tailHandler{ store: s, + client: client, exclusions: newChannelExclusions(nil, []string{"announcement"}), channels: map[string]tailChannel{}, kindExcludedChannelIDs: map[string]struct{}{}, @@ -320,6 +351,14 @@ func TestTailHandlerMessageUpdateFetchesFullMessageBeforeUpsert(t *testing.T) { updated.Content = "edited <@u2>" client := &fakeClient{ + channelsByID: map[string]*discordgo.Channel{ + "c1": { + ID: "c1", + GuildID: "g1", + Name: "general", + Type: discordgo.ChannelTypeGuildText, + }, + }, messages: map[string][]*discordgo.Message{ "c1": {&updated}, }, @@ -490,6 +529,10 @@ func TestTailHandlerMessageUpdateFailureUsesSyncerRefetchedMetadata(t *testing.T store: tailStore, client: snapshotClient, attachmentTextEnabled: false, + channels: map[string]tailChannel{ + "c1": {kind: "text"}, + }, + kindExcludedChannelIDs: map[string]struct{}{}, }, cancel: cancel, failures: make(chan discordclient.TailFailure, 1), @@ -526,6 +569,480 @@ func TestTailHandlerMessageUpdateFailureUsesSyncerRefetchedMetadata(t *testing.T } } +func TestTailHandlerResolvesUnknownThreadAndParentBeforeExclusion(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + client := &fakeClient{channelsByID: map[string]*discordgo.Channel{ + "thread": { + ID: "thread", + GuildID: "g1", + ParentID: "forum", + Name: "feed thread", + Type: discordgo.ChannelTypeGuildPublicThread, + }, + "forum": { + ID: "forum", + GuildID: "g1", + Name: "feed", + Type: discordgo.ChannelTypeGuildForum, + }, + }} + handler := &tailHandler{ + guilds: makeGuildSet([]string{"g1"}), + store: s, + client: client, + exclusions: newChannelExclusions(nil, []string{"forum"}), + channels: map[string]tailChannel{}, + kindExcludedChannelIDs: map[string]struct{}{}, + } + + const messageCount = 8 + var wg sync.WaitGroup + errs := make(chan error, messageCount) + for i := range messageCount { + wg.Add(1) + go func() { + defer wg.Done() + errs <- handler.OnMessageCreate(ctx, &discordgo.Message{ + ID: string(rune('a' + i)), + GuildID: "g1", + ChannelID: "thread", + Content: "must stay excluded", + Timestamp: time.Now().UTC(), + Author: &discordgo.User{ID: "u1", Username: "user"}, + }) + }() + } + wg.Wait() + close(errs) + for err := range errs { + require.NoError(t, err) + } + + client.mu.Lock() + threadCalls := client.channelCalls["thread"] + parentCalls := client.channelCalls["forum"] + client.mu.Unlock() + require.Equal(t, 1, threadCalls) + require.Equal(t, 1, parentCalls) + + channels, err := s.Channels(ctx, "g1") + require.NoError(t, err) + require.Len(t, channels, 2) + status, err := s.Status(ctx, "db", "") + require.NoError(t, err) + require.Zero(t, status.MessageCount) + require.True(t, handler.excludeChannel("thread")) +} + +func TestTailHandlerResolvesUnknownChannelBeforeMessageWrite(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + client := &fakeClient{channelsByID: map[string]*discordgo.Channel{ + "c1": { + ID: "c1", + GuildID: "g1", + Name: "general", + Type: discordgo.ChannelTypeGuildText, + }, + }} + handler := &tailHandler{ + guilds: makeGuildSet([]string{"g1"}), + store: s, + client: client, + channels: map[string]tailChannel{}, + kindExcludedChannelIDs: map[string]struct{}{}, + } + + require.NoError(t, handler.OnMessageCreate(ctx, &discordgo.Message{ + ID: "1", + GuildID: "g1", + ChannelID: "c1", + Content: "archived", + Timestamp: time.Now().UTC(), + Author: &discordgo.User{ID: "u1", Username: "user"}, + })) + + status, err := s.Status(ctx, "db", "") + require.NoError(t, err) + require.Equal(t, 1, status.ChannelCount) + require.Equal(t, 1, status.MessageCount) + client.mu.Lock() + channelCalls := client.channelCalls["c1"] + client.mu.Unlock() + require.Equal(t, 1, channelCalls) +} + +func TestTailHandlerResolvesMissingSeededThreadParentBeforeExclusion(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + require.NoError(t, s.UpsertChannel(ctx, store.ChannelRecord{ + ID: "thread", + GuildID: "g1", + ParentID: "forum", + Kind: "thread_public", + Name: "seeded thread", + RawJSON: `{"id":"thread","parent_id":"forum"}`, + })) + client := &fakeClient{channelsByID: map[string]*discordgo.Channel{ + "thread": { + ID: "thread", + GuildID: "g1", + ParentID: "forum", + Name: "seeded thread", + Type: discordgo.ChannelTypeGuildPublicThread, + }, + "forum": { + ID: "forum", + GuildID: "g1", + Name: "feed", + Type: discordgo.ChannelTypeGuildForum, + }, + }} + handler := &tailHandler{ + guilds: makeGuildSet([]string{"g1"}), + store: s, + client: client, + exclusions: newChannelExclusions(nil, []string{"forum"}), + } + require.NoError(t, handler.seedChannelExclusions(ctx)) + require.False(t, handler.channelHierarchyKnown("thread")) + + require.NoError(t, handler.OnMessageCreate(ctx, &discordgo.Message{ + ID: "1", + GuildID: "g1", + ChannelID: "thread", + Content: "must stay excluded", + Timestamp: time.Now().UTC(), + Author: &discordgo.User{ID: "u1", Username: "user"}, + })) + + client.mu.Lock() + threadCalls := client.channelCalls["thread"] + parentCalls := client.channelCalls["forum"] + client.mu.Unlock() + require.Equal(t, 1, threadCalls) + require.Equal(t, 1, parentCalls) + require.True(t, handler.channelHierarchyKnown("thread")) + require.True(t, handler.excludeChannel("thread")) + + status, err := s.Status(ctx, "db", "") + require.NoError(t, err) + require.Equal(t, 2, status.ChannelCount) + require.Zero(t, status.MessageCount) +} + +func TestTailHandlerPersistsExplicitlyExcludedUnknownAncestor(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + client := &fakeClient{channelsByID: map[string]*discordgo.Channel{ + "feed": { + ID: "feed", + GuildID: "g1", + Name: "feed parent", + Type: discordgo.ChannelTypeGuildText, + }, + }} + handler := &tailHandler{ + guilds: makeGuildSet([]string{"g1"}), + store: s, + client: client, + exclusions: newChannelExclusions([]string{"feed"}, nil), + } + + require.NoError(t, handler.OnChannelUpsert(ctx, &discordgo.Channel{ + ID: "thread", + GuildID: "g1", + ParentID: "feed", + Name: "feed thread", + Type: discordgo.ChannelTypeGuildPublicThread, + })) + + client.mu.Lock() + parentCalls := client.channelCalls["feed"] + client.mu.Unlock() + require.Equal(t, 1, parentCalls) + require.True(t, handler.channelHierarchyKnown("thread")) + require.True(t, handler.excludeChannel("thread")) + + channels, err := s.Channels(ctx, "g1") + require.NoError(t, err) + require.Len(t, channels, 2) +} + +func TestTailHandlerResolvesUnknownChannelsForUpdateAndDelete(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + now := time.Now().UTC() + updated := &discordgo.Message{ + ID: "update", + GuildID: "g1", + ChannelID: "update-channel", + Content: "updated content", + Timestamp: now, + Author: &discordgo.User{ID: "u1", Username: "user"}, + } + client := &fakeClient{ + channelsByID: map[string]*discordgo.Channel{ + "update-channel": { + ID: "update-channel", + GuildID: "g1", + Name: "updates", + Type: discordgo.ChannelTypeGuildText, + }, + "delete-channel": { + ID: "delete-channel", + GuildID: "g1", + Name: "deletes", + Type: discordgo.ChannelTypeGuildText, + }, + }, + messages: map[string][]*discordgo.Message{ + "update-channel": {updated}, + }, + } + handler := &tailHandler{ + guilds: makeGuildSet([]string{"g1"}), + store: s, + client: client, + channels: map[string]tailChannel{}, + kindExcludedChannelIDs: map[string]struct{}{}, + } + + require.NoError(t, handler.OnMessageUpdate(ctx, &discordgo.Message{ + ID: updated.ID, + ChannelID: updated.ChannelID, + Content: updated.Content, + })) + + deleted := &discordgo.Message{ + ID: "delete", + GuildID: "g1", + ChannelID: "delete-channel", + Content: "delete me", + Timestamp: now, + Author: &discordgo.User{ID: "u1", Username: "user"}, + } + mutation, err := buildMessageMutation(ctx, deleted, "", "", false, false) + require.NoError(t, err) + require.NoError(t, s.UpsertMessages(ctx, []store.MessageMutation{mutation})) + require.NoError(t, handler.OnMessageDelete(ctx, &discordgo.MessageDelete{Message: deleted})) + + var content string + require.NoError(t, s.DB().QueryRowContext( + ctx, + `select content from messages where id = ?`, + updated.ID, + ).Scan(&content)) + require.Equal(t, updated.Content, content) + + var deletedAt string + require.NoError(t, s.DB().QueryRowContext( + ctx, + `select coalesce(deleted_at, '') from messages where id = ?`, + deleted.ID, + ).Scan(&deletedAt)) + require.NotEmpty(t, deletedAt) + + status, err := s.Status(ctx, "db", "") + require.NoError(t, err) + require.Equal(t, 2, status.ChannelCount) + require.Equal(t, 2, status.MessageCount) + client.mu.Lock() + updateCalls := client.channelCalls["update-channel"] + deleteCalls := client.channelCalls["delete-channel"] + client.mu.Unlock() + require.Equal(t, 1, updateCalls) + require.Equal(t, 1, deleteCalls) +} + +func TestTailHandlerIgnoresGuildlessPartialUpdateOutsideScope(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + full := &discordgo.Message{ + ID: "outside-message", + GuildID: "g2", + ChannelID: "outside-channel", + Content: "outside scope", + Timestamp: time.Now().UTC(), + Author: &discordgo.User{ID: "u2", Username: "outside"}, + } + client := &fakeClient{ + channelsByID: map[string]*discordgo.Channel{ + "outside-channel": { + ID: "outside-channel", + GuildID: "g2", + Name: "outside", + Type: discordgo.ChannelTypeGuildText, + }, + }, + messages: map[string][]*discordgo.Message{ + "outside-channel": {full}, + }, + } + handler := &tailHandler{ + guilds: makeGuildSet([]string{"g1"}), + store: s, + client: client, + channels: map[string]tailChannel{}, + kindExcludedChannelIDs: map[string]struct{}{}, + } + + require.NoError(t, handler.OnMessageUpdate(ctx, &discordgo.Message{ + ID: full.ID, + ChannelID: full.ChannelID, + Content: full.Content, + })) + + status, err := s.Status(ctx, "db", "") + require.NoError(t, err) + require.Zero(t, status.ChannelCount) + require.Zero(t, status.MessageCount) + client.mu.Lock() + channelCalls := client.channelCalls["outside-channel"] + client.mu.Unlock() + require.Zero(t, channelCalls) +} + +func TestTailHandlerPersistsThreadArchiveMetadata(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + client := &fakeClient{channelsByID: map[string]*discordgo.Channel{ + "parent": { + ID: "parent", + GuildID: "g1", + Name: "general", + Type: discordgo.ChannelTypeGuildText, + }, + }} + handler := &tailHandler{ + guilds: makeGuildSet([]string{"g1"}), + store: s, + client: client, + } + archivedAt := time.Now().UTC().Truncate(time.Microsecond) + require.NoError(t, handler.OnChannelUpsert(ctx, &discordgo.Channel{ + ID: "thread", + GuildID: "g1", + ParentID: "parent", + Name: "archived thread", + Type: discordgo.ChannelTypeGuildPrivateThread, + ThreadMetadata: &discordgo.ThreadMetadata{ + Archived: true, + Locked: true, + ArchiveTimestamp: archivedAt, + }, + })) + + channels, err := s.Channels(ctx, "g1") + require.NoError(t, err) + require.Len(t, channels, 2) + var thread store.ChannelRow + for _, channel := range channels { + if channel.ID == "thread" { + thread = channel + break + } + } + require.Equal(t, "thread", thread.ID) + require.True(t, thread.IsArchived) + require.True(t, thread.IsLocked) + require.True(t, thread.IsPrivateThread) + require.Equal(t, "parent", thread.ThreadParentID) + require.Equal(t, archivedAt, thread.ArchiveTimestamp) +} + +func TestTailHandlerConcurrentResolutionErrorsFailClosed(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + client := &fakeClient{channelErrors: map[string]error{ + "missing": errors.New("unavailable"), + }} + handler := &tailHandler{ + guilds: makeGuildSet([]string{"g1"}), + store: s, + client: client, + channels: map[string]tailChannel{}, + kindExcludedChannelIDs: map[string]struct{}{}, + } + + const messageCount = 8 + errs := make(chan error, messageCount) + var wg sync.WaitGroup + for i := range messageCount { + wg.Add(1) + go func() { + defer wg.Done() + errs <- handler.OnMessageCreate(ctx, &discordgo.Message{ + ID: string(rune('a' + i)), + GuildID: "g1", + ChannelID: "missing", + Content: "must not be written", + Timestamp: time.Now().UTC(), + Author: &discordgo.User{ID: "u1", Username: "user"}, + }) + }() + } + wg.Wait() + close(errs) + for err := range errs { + require.ErrorContains(t, err, "fetch channel missing: unavailable") + } + + status, err := s.Status(ctx, "db", "") + require.NoError(t, err) + require.Zero(t, status.ChannelCount) + require.Zero(t, status.MessageCount) + client.mu.Lock() + channelCalls := client.channelCalls["missing"] + client.mu.Unlock() + require.Equal(t, messageCount, channelCalls) + + report, err := s.ListFailures(ctx, store.FailureListOptions{}, time.Now()) + require.NoError(t, err) + require.Empty(t, report.Failures) +} + func TestHelpers(t *testing.T) { t.Parallel() @@ -628,7 +1145,17 @@ func TestRunTail(t *testing.T) { defer func() { _ = s.Close() }() handled := make(chan struct{}, 1) - client := &fakeClient{tailHandled: handled} + client := &fakeClient{ + tailHandled: handled, + channelsByID: map[string]*discordgo.Channel{ + "c1": { + ID: "c1", + GuildID: "g1", + Name: "general", + Type: discordgo.ChannelTypeGuildText, + }, + }, + } svc := New(client, s, nil) go func() { select { @@ -1436,6 +1963,12 @@ func TestTailHandlersResolveOnlySuccessfulEventIdentity(t *testing.T) { )) } handler := &tailHandler{store: s} + require.NoError(t, handler.OnChannelUpsert(ctx, &discordgo.Channel{ + ID: "c1", + GuildID: "g1", + Name: "general", + Type: discordgo.ChannelTypeGuildText, + })) message := &discordgo.Message{ ID: "10", GuildID: "g1", @@ -1472,6 +2005,12 @@ func TestTailHandlerReturnedErrorIsRecordedOnlyByOuterOwner(t *testing.T) { require.NoError(t, err) handler := &tailHandler{store: s} + require.NoError(t, handler.OnChannelUpsert(ctx, &discordgo.Channel{ + ID: "c1", + GuildID: "g1", + Name: "general", + Type: discordgo.ChannelTypeGuildText, + })) message := &discordgo.Message{ ID: "m1", GuildID: "g1", diff --git a/internal/syncer/syncer_test.go b/internal/syncer/syncer_test.go index ac6e4ce4..e7d914c9 100644 --- a/internal/syncer/syncer_test.go +++ b/internal/syncer/syncer_test.go @@ -19,6 +19,9 @@ type fakeClient struct { guilds []*discordgo.UserGuild guildByID map[string]*discordgo.Guild channels map[string][]*discordgo.Channel + channelsByID map[string]*discordgo.Channel + channelErrors map[string]error + channelCalls map[string]int activeThreads map[string][]*discordgo.Channel guildThreads map[string][]*discordgo.Channel threadErrors map[string]error @@ -73,6 +76,38 @@ func (f *fakeClient) GuildChannels(_ context.Context, guildID string) ([]*discor return f.channels[guildID], nil } +func (f *fakeClient) Channel(_ context.Context, channelID string) (*discordgo.Channel, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.channelCalls == nil { + f.channelCalls = make(map[string]int) + } + f.channelCalls[channelID]++ + if err := f.channelErrors[channelID]; err != nil { + return nil, err + } + if channel := f.channelsByID[channelID]; channel != nil { + return channel, nil + } + channelGroups := []map[string][]*discordgo.Channel{ + f.channels, + f.activeThreads, + f.guildThreads, + f.publicArchived, + f.privateArchive, + } + for _, groups := range channelGroups { + for _, channels := range groups { + for _, channel := range channels { + if channel != nil && channel.ID == channelID { + return channel, nil + } + } + } + } + return nil, nil +} + func (f *fakeClient) ThreadsActive(_ context.Context, channelID string) ([]*discordgo.Channel, error) { f.threadCalls++ if err := f.threadErrors[channelID]; err != nil { diff --git a/internal/syncer/tail.go b/internal/syncer/tail.go index fc4e9b64..10a9a722 100644 --- a/internal/syncer/tail.go +++ b/internal/syncer/tail.go @@ -381,6 +381,7 @@ type tailHandler struct { onReady func(context.Context) error logger *slog.Logger exclusions channelExclusions + channelResolveMu sync.Mutex exclusionMu sync.RWMutex channels map[string]tailChannel kindExcludedChannelIDs map[string]struct{} @@ -423,7 +424,14 @@ func (t *tailHandler) OnTailFailure(failure discordclient.TailFailure) { func (t *tailHandler) OnMessageCreate(ctx context.Context, msg *discordgo.Message) error { discordclient.UpdateTailFailureStage(ctx, discordclient.TailFailureStageHandler) - if msg == nil || !t.allowGuild(msg.GuildID) || t.excludeChannel(msg.ChannelID) { + if msg == nil || !t.allowGuild(msg.GuildID) { + return nil + } + excluded, err := t.ensureMessageChannel(ctx, msg.GuildID, msg.ChannelID) + if err != nil { + return err + } + if excluded { return nil } discordclient.UpdateTailFailureStage(ctx, discordclient.TailFailureStageMessageBuild) @@ -455,15 +463,40 @@ func (t *tailHandler) OnMessageUpdate(ctx context.Context, msg *discordgo.Messag if msg == nil { return nil } - if t.excludeChannel(msg.ChannelID) || (msg.GuildID != "" && !t.allowGuild(msg.GuildID)) { + if msg.GuildID != "" && !t.allowGuild(msg.GuildID) { + return nil + } + guildID, channelID := msg.GuildID, msg.ChannelID + if guildID == "" { + var err error + msg, err = t.messageUpdateSnapshot(ctx, msg) + if err != nil { + return err + } + if msg == nil || !t.allowGuild(msg.GuildID) { + return nil + } + } else { + excluded, err := t.ensureMessageChannel(ctx, guildID, channelID) + if err != nil { + return err + } + if excluded { + return nil + } + msg, err = t.messageUpdateSnapshot(ctx, msg) + if err != nil { + return err + } + } + if msg == nil || !t.allowGuild(msg.GuildID) { return nil } - var err error - msg, err = t.messageUpdateSnapshot(ctx, msg) + excluded, err := t.ensureMessageChannel(ctx, msg.GuildID, msg.ChannelID) if err != nil { return err } - if msg == nil || !t.allowGuild(msg.GuildID) || t.excludeChannel(msg.ChannelID) { + if excluded { return nil } discordclient.UpdateTailFailureStage(ctx, discordclient.TailFailureStageMessageBuild) @@ -553,7 +586,14 @@ func isPartialMessageUpdate(msg *discordgo.Message) bool { func (t *tailHandler) OnMessageDelete(ctx context.Context, evt *discordgo.MessageDelete) error { discordclient.UpdateTailFailureStage(ctx, discordclient.TailFailureStageHandler) - if evt == nil || !t.allowGuild(evt.GuildID) || t.excludeChannel(evt.ChannelID) { + if evt == nil || !t.allowGuild(evt.GuildID) { + return nil + } + excluded, err := t.ensureMessageChannel(ctx, evt.GuildID, evt.ChannelID) + if err != nil { + return err + } + if excluded { return nil } discordclient.UpdateTailFailureStage(ctx, discordclient.TailFailureStageCanonicalDelete) @@ -571,8 +611,9 @@ func (t *tailHandler) OnChannelUpsert(ctx context.Context, channel *discordgo.Ch if channel == nil || !t.allowGuild(channel.GuildID) { return nil } - t.trackChannelExclusion(channel) - return t.store.UpsertChannel(ctx, toChannelRecord(channel, marshalJSONString(channel, "{}"))) + t.channelResolveMu.Lock() + defer t.channelResolveMu.Unlock() + return t.persistChannelHierarchyLocked(ctx, channel, map[string]struct{}{}) } func (t *tailHandler) OnMemberUpsert(ctx context.Context, guildID string, member *discordgo.Member) error { @@ -601,6 +642,131 @@ func (t *tailHandler) allowGuild(guildID string) bool { return ok } +func (t *tailHandler) ensureMessageChannel(ctx context.Context, guildID, channelID string) (bool, error) { + if channelID == "" { + return false, errors.New("message event missing channel id") + } + if t.channelHierarchyKnown(channelID) { + return t.excludeChannel(channelID), nil + } + + t.channelResolveMu.Lock() + defer t.channelResolveMu.Unlock() + if t.channelHierarchyKnown(channelID) { + return t.excludeChannel(channelID), nil + } + if t.client == nil { + return false, fmt.Errorf("resolve unknown channel %s: missing discord client", channelID) + } + if err := t.resolveUnknownChannelLocked(ctx, guildID, channelID, map[string]struct{}{}); err != nil { + return false, err + } + return t.excludeChannel(channelID), nil +} + +func (t *tailHandler) resolveUnknownChannelLocked( + ctx context.Context, + expectedGuildID string, + channelID string, + resolving map[string]struct{}, +) error { + if t.channelHierarchyKnown(channelID) { + return nil + } + if _, ok := resolving[channelID]; ok { + return fmt.Errorf("resolve channel hierarchy cycle at %s", channelID) + } + resolving[channelID] = struct{}{} + defer delete(resolving, channelID) + + channel, err := t.client.Channel(ctx, channelID) + if err != nil { + return fmt.Errorf("fetch channel %s: %w", channelID, err) + } + if channel == nil { + return fmt.Errorf("fetch channel %s: empty response", channelID) + } + resolved := *channel + channel = &resolved + if channel.ID == "" { + channel.ID = channelID + } + if channel.ID != channelID { + return fmt.Errorf("fetch channel %s: received channel %s", channelID, channel.ID) + } + if channel.GuildID == "" { + channel.GuildID = expectedGuildID + } + if expectedGuildID != "" && channel.GuildID != expectedGuildID { + return fmt.Errorf( + "fetch channel %s: expected guild %s, received %s", + channelID, + expectedGuildID, + channel.GuildID, + ) + } + if !t.allowGuild(channel.GuildID) { + return fmt.Errorf("fetch channel %s: guild %s is outside the tail scope", channelID, channel.GuildID) + } + return t.persistChannelHierarchyLocked(ctx, channel, resolving) +} + +func (t *tailHandler) persistChannelHierarchyLocked( + ctx context.Context, + channel *discordgo.Channel, + resolving map[string]struct{}, +) error { + if channel == nil || !t.allowGuild(channel.GuildID) { + return nil + } + if channel.ID == "" { + return errors.New("channel event missing channel id") + } + resolveParent := channel.ParentID != "" && !t.channelHierarchyKnown(channel.ParentID) + if resolveParent { + if t.client == nil { + return fmt.Errorf( + "resolve parent channel %s for %s: missing discord client", + channel.ParentID, + channel.ID, + ) + } + if err := t.resolveUnknownChannelLocked(ctx, channel.GuildID, channel.ParentID, resolving); err != nil { + return fmt.Errorf("resolve parent channel %s for %s: %w", channel.ParentID, channel.ID, err) + } + } + if err := t.store.UpsertChannel(ctx, toChannelRecord(channel, marshalJSONString(channel, "{}"))); err != nil { + return err + } + t.trackChannelExclusion(channel) + return nil +} + +func (t *tailHandler) channelHierarchyKnown(channelID string) bool { + t.exclusionMu.RLock() + defer t.exclusionMu.RUnlock() + return t.channelHierarchyKnownLocked(channelID, nil) +} + +func (t *tailHandler) channelHierarchyKnownLocked(channelID string, visiting map[string]struct{}) bool { + channel, ok := t.channels[channelID] + if !ok { + return false + } + if channel.parentID == "" { + return true + } + if visiting == nil { + visiting = map[string]struct{}{} + } + if _, ok := visiting[channelID]; ok { + return false + } + visiting[channelID] = struct{}{} + defer delete(visiting, channelID) + return t.channelHierarchyKnownLocked(channel.parentID, visiting) +} + func (t *tailHandler) seedChannelExclusions(ctx context.Context) error { if t.store == nil { return nil From 8866a68484be801d799c1933de5e90e943a91154 Mon Sep 17 00:00:00 2001 From: Hannes Rudolph <49103247+hannesrudolph@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:36:19 -0600 Subject: [PATCH 4/7] fix: guard writer resources after lock acquisition --- internal/cli/cli.go | 1 + internal/cli/cli_test.go | 58 ++++++++++++++++ internal/cli/locked_resource_guard.go | 76 +++++++++++++++++++++ internal/cli/locked_resource_guard_other.go | 9 +++ internal/cli/locked_resource_guard_test.go | 44 ++++++++++++ internal/cli/locked_resource_guard_unix.go | 17 +++++ internal/cli/sync_lock.go | 13 ++++ 7 files changed, 218 insertions(+) create mode 100644 internal/cli/locked_resource_guard.go create mode 100644 internal/cli/locked_resource_guard_other.go create mode 100644 internal/cli/locked_resource_guard_test.go create mode 100644 internal/cli/locked_resource_guard_unix.go diff --git a/internal/cli/cli.go b/internal/cli/cli.go index f9eb929a..c4f6964a 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -265,6 +265,7 @@ type runtime struct { lockOperation string lockToken string lockTokenFree func() error + postLockGuard func() error openStore func(context.Context, string) (*store.Store, error) newDiscord func(config.Config) (discordClient, error) newRemote func(config.Config) (remoteArchiveClient, error) diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index 6d7ebf5f..0de5115e 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -2530,6 +2530,64 @@ func TestSyncLockSerializesConcurrentRuns(t *testing.T) { require.ErrorIs(t, err, context.DeadlineExceeded) } +func TestPostLockGuardRunsWhileArchiveLockIsHeld(t *testing.T) { + if goruntime.GOOS == "windows" { + t.Skip("sync lock timing is flaky on Windows") + } + ctx := context.Background() + dir := t.TempDir() + cfg := config.Default() + cfg.DBPath = filepath.Join(dir, "discrawl.db") + lockPath := filepath.Join(dir, ".discrawl-sync.lock") + guardErr := errors.New("guard stopped writer") + callbackCalled := false + rt := &runtime{ + ctx: ctx, + cfg: cfg, + postLockGuard: func() error { + held, known, err := syncLockState(lockPath) + require.NoError(t, err) + require.True(t, known) + require.True(t, held) + return guardErr + }, + } + + err := rt.withSyncLock(func() error { + callbackCalled = true + return nil + }) + + require.ErrorIs(t, err, guardErr) + require.False(t, callbackCalled) + held, known, err := syncLockState(lockPath) + require.NoError(t, err) + require.True(t, known) + require.False(t, held) +} + +func TestLockedResourceGuardBlocksBeforeWriterCallback(t *testing.T) { + if goruntime.GOOS == "windows" { + t.Skip("locked free-space guard is unsupported on Windows") + } + ctx := context.Background() + dir := t.TempDir() + cfg := config.Default() + cfg.DBPath = filepath.Join(dir, "discrawl.db") + t.Setenv(lockedMinFreeKiBEnv, "18446744073709551615") + t.Setenv(lockedMaxWALBytesEnv, "4294967296") + callbackCalled := false + rt := &runtime{ctx: ctx, cfg: cfg} + + err := rt.withSyncLock(func() error { + callbackCalled = true + return nil + }) + + require.ErrorContains(t, err, "free space at or below threshold") + require.False(t, callbackCalled) +} + func TestReadCommandsDoNotWaitForSyncLock(t *testing.T) { if goruntime.GOOS == "windows" { t.Skip("sync lock timing is flaky on Windows") diff --git a/internal/cli/locked_resource_guard.go b/internal/cli/locked_resource_guard.go new file mode 100644 index 00000000..5b500238 --- /dev/null +++ b/internal/cli/locked_resource_guard.go @@ -0,0 +1,76 @@ +package cli + +import ( + "errors" + "fmt" + "os" + "strconv" + "strings" + + "github.com/openclaw/discrawl/internal/config" +) + +const ( + lockedMinFreeKiBEnv = "DISCRAWL_LOCKED_MIN_FREE_KIB" + lockedMaxWALBytesEnv = "DISCRAWL_LOCKED_MAX_WAL_BYTES" +) + +func lockedResourceGuard(rawDBPath string) error { + rawMinFree := strings.TrimSpace(os.Getenv(lockedMinFreeKiBEnv)) + rawMaxWAL := strings.TrimSpace(os.Getenv(lockedMaxWALBytesEnv)) + if rawMinFree == "" && rawMaxWAL == "" { + return nil + } + if rawMinFree == "" || rawMaxWAL == "" { + return errors.New("both locked resource guard thresholds are required") + } + minFreeKiB, err := strconv.ParseUint(rawMinFree, 10, 64) + if err != nil || minFreeKiB == 0 { + return fmt.Errorf("invalid %s value %q", lockedMinFreeKiBEnv, rawMinFree) + } + maxWALBytes, err := strconv.ParseUint(rawMaxWAL, 10, 64) + if err != nil || maxWALBytes == 0 { + return fmt.Errorf("invalid %s value %q", lockedMaxWALBytesEnv, rawMaxWAL) + } + dbPath, err := config.ExpandPath(rawDBPath) + if err != nil { + return fmt.Errorf("expand database path: %w", err) + } + freeKiB, err := lockedFreeKiB(dbPath) + if err != nil { + return fmt.Errorf("probe free space: %w", err) + } + walBytes, err := lockedWALBytes(dbPath + "-wal") + if err != nil { + return fmt.Errorf("probe WAL: %w", err) + } + if freeKiB <= minFreeKiB { + return fmt.Errorf( + "free space at or below threshold: free_kib=%d threshold_kib=%d", + freeKiB, + minFreeKiB, + ) + } + if walBytes >= maxWALBytes { + return fmt.Errorf( + "WAL at or above threshold: wal_bytes=%d threshold_bytes=%d", + walBytes, + maxWALBytes, + ) + } + return nil +} + +func lockedWALBytes(path string) (uint64, error) { + info, err := os.Lstat(path) + if errors.Is(err, os.ErrNotExist) { + return 0, nil + } + if err != nil { + return 0, err + } + if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() { + return 0, fmt.Errorf("WAL path is not a regular file: %s", path) + } + return uint64(info.Size()), nil +} diff --git a/internal/cli/locked_resource_guard_other.go b/internal/cli/locked_resource_guard_other.go new file mode 100644 index 00000000..ded851e0 --- /dev/null +++ b/internal/cli/locked_resource_guard_other.go @@ -0,0 +1,9 @@ +//go:build !unix + +package cli + +import "errors" + +func lockedFreeKiB(string) (uint64, error) { + return 0, errors.New("locked free-space guard is unsupported on this platform") +} diff --git a/internal/cli/locked_resource_guard_test.go b/internal/cli/locked_resource_guard_test.go new file mode 100644 index 00000000..aa48a904 --- /dev/null +++ b/internal/cli/locked_resource_guard_test.go @@ -0,0 +1,44 @@ +package cli + +import ( + "os" + "path/filepath" + goruntime "runtime" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestLockedResourceGuardRejectsWALAtThreshold(t *testing.T) { + if goruntime.GOOS == "windows" { + t.Skip("locked free-space guard is unsupported on Windows") + } + dir := t.TempDir() + dbPath := filepath.Join(dir, "discrawl.db") + require.NoError(t, os.WriteFile(dbPath, nil, 0o600)) + require.NoError(t, os.WriteFile(dbPath+"-wal", []byte{1}, 0o600)) + t.Setenv(lockedMinFreeKiBEnv, "1") + t.Setenv(lockedMaxWALBytesEnv, "1") + + err := lockedResourceGuard(dbPath) + + require.ErrorContains(t, err, "WAL at or above threshold") +} + +func TestLockedResourceGuardRejectsSymlinkedWAL(t *testing.T) { + if goruntime.GOOS == "windows" { + t.Skip("locked free-space guard is unsupported on Windows") + } + dir := t.TempDir() + dbPath := filepath.Join(dir, "discrawl.db") + target := filepath.Join(dir, "target") + require.NoError(t, os.WriteFile(dbPath, nil, 0o600)) + require.NoError(t, os.WriteFile(target, nil, 0o600)) + require.NoError(t, os.Symlink(target, dbPath+"-wal")) + t.Setenv(lockedMinFreeKiBEnv, "1") + t.Setenv(lockedMaxWALBytesEnv, "4294967296") + + err := lockedResourceGuard(dbPath) + + require.ErrorContains(t, err, "WAL path is not a regular file") +} diff --git a/internal/cli/locked_resource_guard_unix.go b/internal/cli/locked_resource_guard_unix.go new file mode 100644 index 00000000..71778cf3 --- /dev/null +++ b/internal/cli/locked_resource_guard_unix.go @@ -0,0 +1,17 @@ +//go:build unix + +package cli + +import ( + "path/filepath" + + "golang.org/x/sys/unix" +) + +func lockedFreeKiB(dbPath string) (uint64, error) { + var details unix.Statfs_t + if err := unix.Statfs(filepath.Dir(dbPath), &details); err != nil { + return 0, err + } + return uint64(details.Bavail) * uint64(details.Bsize) / 1024, nil +} diff --git a/internal/cli/sync_lock.go b/internal/cli/sync_lock.go index 88955d0c..45f7aa97 100644 --- a/internal/cli/sync_lock.go +++ b/internal/cli/sync_lock.go @@ -124,9 +124,22 @@ func (r *runtime) runWithHeldSyncLock(lockPath string, release func() error, ope } _ = release() }() + if err := r.runPostLockGuard(); err != nil { + return err + } return fn() } +func (r *runtime) runPostLockGuard() error { + if r.postLockGuard != nil { + return r.postLockGuard() + } + if err := lockedResourceGuard(r.cfg.DBPath); err != nil { + return dbErr(fmt.Errorf("post-lock resource guard: %w", err)) + } + return nil +} + func (r *runtime) activateTailSyncLock() error { if !r.dbLockHeld { return nil From ddce3fee7c86d14fae929e805bb7f7e333f903e9 Mon Sep 17 00:00:00 2001 From: Hannes Rudolph <49103247+hannesrudolph@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:19:08 -0600 Subject: [PATCH 5/7] fix: reconcile deleted message update races --- internal/store/write.go | 17 ++++ internal/syncer/errors.go | 13 +++ internal/syncer/failures.go | 16 ++++ internal/syncer/syncer_tail_test.go | 135 ++++++++++++++++++++++++++++ internal/syncer/syncer_test.go | 5 ++ internal/syncer/tail.go | 44 ++++++++- 6 files changed, 229 insertions(+), 1 deletion(-) diff --git a/internal/store/write.go b/internal/store/write.go index d939e8cb..d6360c15 100644 --- a/internal/store/write.go +++ b/internal/store/write.go @@ -462,6 +462,23 @@ func (s *Store) markMessageDeleted( return tx.Commit() } +func (s *Store) MessageScope(ctx context.Context, messageID string) (string, string, bool, error) { + var guildID string + var channelID string + err := s.db.QueryRowContext( + ctx, + `select guild_id, channel_id from messages where id = ?`, + strings.TrimSpace(messageID), + ).Scan(&guildID, &channelID) + if errors.Is(err, sql.ErrNoRows) { + return "", "", false, nil + } + if err != nil { + return "", "", false, err + } + return guildID, channelID, true, nil +} + func (s *Store) AppendMessageEvent(ctx context.Context, guildID, channelID, messageID, eventType string, payload any) error { body, err := json.Marshal(payload) if err != nil { diff --git a/internal/syncer/errors.go b/internal/syncer/errors.go index 5ae26dce..e67e92dd 100644 --- a/internal/syncer/errors.go +++ b/internal/syncer/errors.go @@ -109,6 +109,19 @@ func isUnknownChannel(err error) bool { (strings.Contains(msg, "http 404") && strings.Contains(msg, `"code": 10003`)) } +func isUnknownMessage(err error) bool { + if err == nil { + return false + } + var restErr *discordgo.RESTError + if errors.As(err, &restErr) && restErr != nil && restErr.Message != nil { + return restErr.Message.Code == discordgo.ErrCodeUnknownMessage + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "http 404") && + (strings.Contains(msg, "unknown message") || strings.Contains(msg, `"code": 10008`)) +} + func isMissingAccess(err error) bool { if err == nil { return false diff --git a/internal/syncer/failures.go b/internal/syncer/failures.go index 4d22ba05..9c485290 100644 --- a/internal/syncer/failures.go +++ b/internal/syncer/failures.go @@ -295,6 +295,22 @@ func (t *tailHandler) resolveMessageFailure(ctx context.Context, guildID, channe ) } +func (t *tailHandler) resolveMessageUpdateFailuresByID(ctx context.Context, messageID string) error { + if t == nil || t.store == nil || messageID == "" { + return nil + } + discordclient.UpdateTailFailureStage(ctx, discordclient.TailFailureStageFailureResolution) + ledgerCtx, cancel := failureLedgerContext(ctx) + defer cancel() + return t.store.ResolveFailures(ledgerCtx, store.FailureRef{ + Operation: tailMessageFailureOperation, + Source: "discord", + MessageID: messageID, + RelatedKind: tailMessageFailureRelatedKind, + RelatedID: "update", + }) +} + func (s *Syncer) resolveTailMessageCreateFailuresForMessages(ctx context.Context, messages []*discordgo.Message, fallbackGuildID string) error { if s == nil || s.store == nil || len(messages) == 0 { return nil diff --git a/internal/syncer/syncer_tail_test.go b/internal/syncer/syncer_tail_test.go index 8c347515..40aaaaa2 100644 --- a/internal/syncer/syncer_tail_test.go +++ b/internal/syncer/syncer_tail_test.go @@ -134,6 +134,131 @@ func TestTailHandlerWritesEvents(t *testing.T) { require.Equal(t, "10", cursor) } +func TestTailHandlerReconcilesUnknownMessageUpdateAsDelete(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + message := &discordgo.Message{ + ID: "123", + GuildID: "g1", + ChannelID: "c1", + Content: "preserved before deletion", + Timestamp: time.Now().UTC(), + Author: &discordgo.User{ID: "u1", Username: "peter"}, + } + client := &fakeClient{ + channelsByID: map[string]*discordgo.Channel{ + "c1": { + ID: "c1", + GuildID: "g1", + Name: "general", + Type: discordgo.ChannelTypeGuildText, + }, + }, + messages: map[string][]*discordgo.Message{ + "c1": {message}, + }, + messageByIDErrs: map[string]error{ + "c1/123": errors.New(`HTTP 404 Not Found, {"message": "Unknown Message", "code": 10008}`), + }, + } + handler := &tailHandler{ + guilds: makeGuildSet([]string{"g1"}), + store: s, + client: client, + channels: map[string]tailChannel{}, + kindExcludedChannelIDs: map[string]struct{}{}, + } + require.NoError(t, handler.OnChannelUpsert(ctx, client.channelsByID["c1"])) + require.NoError(t, handler.OnMessageCreate(ctx, message)) + require.NoError(t, s.RecordFailure( + ctx, + tailMessageFailureIdentity("g1", "c1", "123", "update"), + errors.New("prior update failure"), + )) + + require.NoError(t, handler.OnMessageUpdate(ctx, &discordgo.Message{ + ID: "123", + GuildID: "g1", + ChannelID: "c1", + })) + + var deletedAt string + require.NoError(t, s.DB().QueryRowContext( + ctx, + `select coalesce(deleted_at, '') from messages where id = '123'`, + ).Scan(&deletedAt)) + require.NotEmpty(t, deletedAt) + var deleteEvents int + require.NoError(t, s.DB().QueryRowContext( + ctx, + `select count(*) from message_events where message_id = '123' and event_type = 'delete'`, + ).Scan(&deleteEvents)) + require.Equal(t, 1, deleteEvents) + lastEvent, err := s.GetSyncState(ctx, "tail:last_event") + require.NoError(t, err) + require.Equal(t, "123", lastEvent) + report, err := s.ListFailures(ctx, store.FailureListOptions{}, time.Now().UTC()) + require.NoError(t, err) + require.Zero(t, report.UnresolvedCount) +} + +func TestTailHandlerDropsUnknownUpdateForUnseenMessageWithoutOrphanEvent(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + client := &fakeClient{ + channelsByID: map[string]*discordgo.Channel{ + "c1": { + ID: "c1", + GuildID: "g1", + Name: "general", + Type: discordgo.ChannelTypeGuildText, + }, + }, + messageByIDErrs: map[string]error{ + "c1/124": errors.New(`HTTP 404 Not Found, {"message": "Unknown Message", "code": 10008}`), + }, + } + handler := &tailHandler{ + guilds: makeGuildSet([]string{"g1"}), + store: s, + client: client, + channels: map[string]tailChannel{}, + kindExcludedChannelIDs: map[string]struct{}{}, + } + require.NoError(t, handler.OnChannelUpsert(ctx, client.channelsByID["c1"])) + require.NoError(t, s.RecordFailure( + ctx, + tailMessageFailureIdentity("g1", "c1", "124", "update"), + errors.New("prior update failure"), + )) + + require.NoError(t, handler.OnMessageUpdate(ctx, &discordgo.Message{ + ID: "124", + GuildID: "g1", + ChannelID: "c1", + })) + + var eventCount int + require.NoError(t, s.DB().QueryRowContext( + ctx, + `select count(*) from message_events where message_id = '124'`, + ).Scan(&eventCount)) + require.Zero(t, eventCount) + report, err := s.ListFailures(ctx, store.FailureListOptions{}, time.Now().UTC()) + require.NoError(t, err) + require.Zero(t, report.UnresolvedCount) +} + func TestTailHandlerDoesNotRecordOrderlyShutdownCancellation(t *testing.T) { t.Parallel() @@ -1125,6 +1250,16 @@ func TestHelpers(t *testing.T) { require.Equal(t, "unknown_channel", unavailableReason(errors.New("HTTP 404 Not Found, {\"message\": \"Unknown Channel\", \"code\": 10003}"))) require.True(t, isUnknownChannel(errors.New("Unknown Channel"))) require.False(t, isUnknownChannel(errors.New("boom"))) + require.True(t, isUnknownMessage(&discordgo.RESTError{ + Response: &http.Response{Status: "404 Not Found"}, + Message: &discordgo.APIErrorMessage{ + Code: discordgo.ErrCodeUnknownMessage, + Message: "localized or empty server text", + }, + })) + require.True(t, isUnknownMessage(errors.New(`HTTP 404 Not Found, {"message": "Unknown Message", "code": 10008}`))) + require.False(t, isUnknownMessage(errors.New("unknown message encoding"))) + require.False(t, isUnknownMessage(errors.New("boom"))) require.True(t, isRetryableSyncError(context.Background(), context.DeadlineExceeded)) require.True(t, isRetryableSyncError(context.Background(), errors.New("HTTP 503 Service Unavailable"))) require.True(t, isRetryableSyncError(context.Background(), errors.New("stream error: stream ID 1; INTERNAL_ERROR"))) diff --git a/internal/syncer/syncer_test.go b/internal/syncer/syncer_test.go index e7d914c9..21b2f7b1 100644 --- a/internal/syncer/syncer_test.go +++ b/internal/syncer/syncer_test.go @@ -33,6 +33,7 @@ type fakeClient struct { members map[string][]*discordgo.Member messages map[string][]*discordgo.Message messageErrors map[string]error + messageByIDErrs map[string]error messageCalls map[string]int exactMessageCalls int messageRequests []messageRequest @@ -246,7 +247,11 @@ func (f *fakeClient) ChannelMessages(ctx context.Context, channelID string, limi func (f *fakeClient) ChannelMessage(_ context.Context, channelID, messageID string) (*discordgo.Message, error) { f.mu.Lock() f.exactMessageCalls++ + err := f.messageByIDErrs[channelID+"/"+messageID] f.mu.Unlock() + if err != nil { + return nil, err + } for _, msg := range f.messages[channelID] { if msg.ID == messageID { return msg, nil diff --git a/internal/syncer/tail.go b/internal/syncer/tail.go index 10a9a722..9088d9cd 100644 --- a/internal/syncer/tail.go +++ b/internal/syncer/tail.go @@ -466,11 +466,14 @@ func (t *tailHandler) OnMessageUpdate(ctx context.Context, msg *discordgo.Messag if msg.GuildID != "" && !t.allowGuild(msg.GuildID) { return nil } - guildID, channelID := msg.GuildID, msg.ChannelID + guildID, channelID, messageID := msg.GuildID, msg.ChannelID, msg.ID if guildID == "" { var err error msg, err = t.messageUpdateSnapshot(ctx, msg) if err != nil { + if isUnknownMessage(err) { + return t.reconcileUnknownMessageUpdate(ctx, messageID) + } return err } if msg == nil || !t.allowGuild(msg.GuildID) { @@ -486,6 +489,9 @@ func (t *tailHandler) OnMessageUpdate(ctx context.Context, msg *discordgo.Messag } msg, err = t.messageUpdateSnapshot(ctx, msg) if err != nil { + if isUnknownMessage(err) { + return t.reconcileUnknownMessageUpdate(ctx, messageID) + } return err } } @@ -584,6 +590,42 @@ func isPartialMessageUpdate(msg *discordgo.Message) bool { return msg == nil || msg.Author == nil || msg.Timestamp.IsZero() } +func (t *tailHandler) reconcileUnknownMessageUpdate( + ctx context.Context, + messageID string, +) error { + guildID, channelID, exists, err := t.store.MessageScope(ctx, messageID) + if err != nil { + return err + } + if !exists || !t.allowGuild(guildID) { + return t.resolveMessageUpdateFailuresByID(ctx, messageID) + } + excluded, err := t.ensureMessageChannel(ctx, guildID, channelID) + if err != nil { + return err + } + if excluded { + return t.resolveMessageUpdateFailuresByID(ctx, messageID) + } + payload := map[string]any{ + "id": messageID, + "guild_id": guildID, + "channel_id": channelID, + "reason": "unknown_message_during_update_fetch", + "discord_code": 10008, + } + discordclient.UpdateTailFailureStage(ctx, discordclient.TailFailureStageCanonicalDelete) + if err := t.store.MarkMessageDeleted(ctx, guildID, channelID, messageID, payload); err != nil { + return err + } + discordclient.UpdateTailFailureStage(ctx, discordclient.TailFailureStageStateUpdate) + if err := t.store.SetSyncState(ctx, "tail:last_event", messageID); err != nil { + return err + } + return t.resolveMessageUpdateFailuresByID(ctx, messageID) +} + func (t *tailHandler) OnMessageDelete(ctx context.Context, evt *discordgo.MessageDelete) error { discordclient.UpdateTailFailureStage(ctx, discordclient.TailFailureStageHandler) if evt == nil || !t.allowGuild(evt.GuildID) { From 802fccb181f20e8948d9e7335aba6b1398e09388 Mon Sep 17 00:00:00 2001 From: Hannes Rudolph <49103247+hannesrudolph@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:22:27 -0600 Subject: [PATCH 6/7] fix: preserve excluded tail messages during reconciliation --- internal/syncer/syncer_tail_test.go | 155 ++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) diff --git a/internal/syncer/syncer_tail_test.go b/internal/syncer/syncer_tail_test.go index 40aaaaa2..e2537ade 100644 --- a/internal/syncer/syncer_tail_test.go +++ b/internal/syncer/syncer_tail_test.go @@ -180,6 +180,11 @@ func TestTailHandlerReconcilesUnknownMessageUpdateAsDelete(t *testing.T) { tailMessageFailureIdentity("g1", "c1", "123", "update"), errors.New("prior update failure"), )) + require.NoError(t, s.RecordFailure( + ctx, + tailMessageFailureIdentity("", "c1", "123", "update"), + errors.New("prior guildless update failure"), + )) require.NoError(t, handler.OnMessageUpdate(ctx, &discordgo.Message{ ID: "123", @@ -207,6 +212,156 @@ func TestTailHandlerReconcilesUnknownMessageUpdateAsDelete(t *testing.T) { require.Zero(t, report.UnresolvedCount) } +func TestTailHandlerDoesNotDeleteExcludedMessageOnGuildlessUnknownUpdate(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + channel := &discordgo.Channel{ + ID: "feed", + GuildID: "g1", + Name: "github", + Type: discordgo.ChannelTypeGuildText, + } + message := &discordgo.Message{ + ID: "125", + GuildID: "g1", + ChannelID: "feed", + Content: "preserved feed message", + Timestamp: time.Now().UTC(), + Author: &discordgo.User{ID: "u1", Username: "bot", Bot: true}, + } + client := &fakeClient{ + channelsByID: map[string]*discordgo.Channel{"feed": channel}, + messages: map[string][]*discordgo.Message{"feed": {message}}, + messageByIDErrs: map[string]error{ + "feed/125": errors.New(`HTTP 404 Not Found, {"message": "Unknown Message", "code": 10008}`), + }, + } + initialHandler := &tailHandler{ + guilds: makeGuildSet([]string{"g1"}), + store: s, + client: client, + channels: map[string]tailChannel{}, + kindExcludedChannelIDs: map[string]struct{}{}, + } + require.NoError(t, initialHandler.OnChannelUpsert(ctx, channel)) + require.NoError(t, initialHandler.OnMessageCreate(ctx, message)) + + handler := &tailHandler{ + guilds: makeGuildSet([]string{"g1"}), + store: s, + client: client, + exclusions: newChannelExclusions([]string{"feed"}, nil), + channels: map[string]tailChannel{}, + kindExcludedChannelIDs: map[string]struct{}{}, + } + require.NoError(t, handler.seedChannelExclusions(ctx)) + require.NoError(t, s.RecordFailure( + ctx, + tailMessageFailureIdentity("", "feed", "125", "update"), + errors.New("prior guildless update failure"), + )) + + require.NoError(t, handler.OnMessageUpdate(ctx, &discordgo.Message{ + ID: "125", + ChannelID: "feed", + })) + + var deletedAt string + require.NoError(t, s.DB().QueryRowContext( + ctx, + `select coalesce(deleted_at, '') from messages where id = '125'`, + ).Scan(&deletedAt)) + require.Empty(t, deletedAt) + var deleteEvents int + require.NoError(t, s.DB().QueryRowContext( + ctx, + `select count(*) from message_events where message_id = '125' and event_type = 'delete'`, + ).Scan(&deleteEvents)) + require.Zero(t, deleteEvents) + report, err := s.ListFailures(ctx, store.FailureListOptions{}, time.Now().UTC()) + require.NoError(t, err) + require.Zero(t, report.UnresolvedCount) +} + +func TestTailHandlerDoesNotDeleteOutOfScopeMessageOnGuildlessUnknownUpdate(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + channel := &discordgo.Channel{ + ID: "other-channel", + GuildID: "other-guild", + Name: "general", + Type: discordgo.ChannelTypeGuildText, + } + message := &discordgo.Message{ + ID: "126", + GuildID: "other-guild", + ChannelID: "other-channel", + Content: "preserved outside scope", + Timestamp: time.Now().UTC(), + Author: &discordgo.User{ID: "u1", Username: "user"}, + } + client := &fakeClient{ + channelsByID: map[string]*discordgo.Channel{"other-channel": channel}, + messages: map[string][]*discordgo.Message{"other-channel": {message}}, + messageByIDErrs: map[string]error{ + "other-channel/126": errors.New(`HTTP 404 Not Found, {"message": "Unknown Message", "code": 10008}`), + }, + } + initialHandler := &tailHandler{ + store: s, + client: client, + channels: map[string]tailChannel{}, + kindExcludedChannelIDs: map[string]struct{}{}, + } + require.NoError(t, initialHandler.OnChannelUpsert(ctx, channel)) + require.NoError(t, initialHandler.OnMessageCreate(ctx, message)) + + handler := &tailHandler{ + guilds: makeGuildSet([]string{"g1"}), + store: s, + client: client, + channels: map[string]tailChannel{}, + kindExcludedChannelIDs: map[string]struct{}{}, + } + require.NoError(t, handler.seedChannelExclusions(ctx)) + require.NoError(t, s.RecordFailure( + ctx, + tailMessageFailureIdentity("", "other-channel", "126", "update"), + errors.New("prior guildless update failure"), + )) + + require.NoError(t, handler.OnMessageUpdate(ctx, &discordgo.Message{ + ID: "126", + ChannelID: "other-channel", + })) + + var deletedAt string + require.NoError(t, s.DB().QueryRowContext( + ctx, + `select coalesce(deleted_at, '') from messages where id = '126'`, + ).Scan(&deletedAt)) + require.Empty(t, deletedAt) + var deleteEvents int + require.NoError(t, s.DB().QueryRowContext( + ctx, + `select count(*) from message_events where message_id = '126' and event_type = 'delete'`, + ).Scan(&deleteEvents)) + require.Zero(t, deleteEvents) + report, err := s.ListFailures(ctx, store.FailureListOptions{}, time.Now().UTC()) + require.NoError(t, err) + require.Zero(t, report.UnresolvedCount) +} + func TestTailHandlerDropsUnknownUpdateForUnseenMessageWithoutOrphanEvent(t *testing.T) { t.Parallel() From d7c07e25e60b7749c4a47de72349830099c52c1a Mon Sep 17 00:00:00 2001 From: Hannes Rudolph <49103247+hannesrudolph@users.noreply.github.com> Date: Fri, 17 Jul 2026 18:08:54 -0600 Subject: [PATCH 7/7] fix: reconcile unknown message update failures --- go.mod | 2 +- internal/cli/locked_resource_guard_unix.go | 2 +- internal/discord/client.go | 29 ++- internal/discord/client_test.go | 2 + internal/syncer/syncer_tail_test.go | 264 +++++++++++++++++++-- internal/syncer/tail.go | 12 + 6 files changed, 291 insertions(+), 20 deletions(-) diff --git a/go.mod b/go.mod index bf8fd1c4..9029db1e 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/zalando/go-keyring v0.2.8 golang.org/x/sys v0.47.0 golang.org/x/text v0.40.0 + modernc.org/sqlite v1.54.0 ) require ( @@ -17,7 +18,6 @@ require ( github.com/clipperhouse/displaywidth v0.11.0 // indirect github.com/clipperhouse/uax29/v2 v2.7.0 // indirect github.com/pelletier/go-toml/v2 v2.4.3 // indirect - modernc.org/sqlite v1.54.0 // indirect ) require ( diff --git a/internal/cli/locked_resource_guard_unix.go b/internal/cli/locked_resource_guard_unix.go index 71778cf3..8b6ef3dc 100644 --- a/internal/cli/locked_resource_guard_unix.go +++ b/internal/cli/locked_resource_guard_unix.go @@ -13,5 +13,5 @@ func lockedFreeKiB(dbPath string) (uint64, error) { if err := unix.Statfs(filepath.Dir(dbPath), &details); err != nil { return 0, err } - return uint64(details.Bavail) * uint64(details.Bsize) / 1024, nil + return details.Bavail * uint64(details.Bsize) / 1024, nil } diff --git a/internal/discord/client.go b/internal/discord/client.go index 64265617..95a1e397 100644 --- a/internal/discord/client.go +++ b/internal/discord/client.go @@ -26,6 +26,10 @@ type tailGuildFilter interface { TailAllowsGuild(string) bool } +type messageUpdateRefetchFailureHandler interface { + OnMessageUpdateRefetchFailure(context.Context, *discordgo.Message, error) (bool, error) +} + type TailReadyHandler interface { OnTailReady(context.Context) error } @@ -555,10 +559,29 @@ func (c *Client) Tail(ctx context.Context, handler EventHandler) error { if msg == nil { return refetchErr } - // A failed refetch does not suppress the partial update, but it remains - // an event failure even when the handler accepts that recovery input. + // A failed refetch does not suppress the partial update. It remains an + // event failure unless an explicitly capable handler reconciles it. UpdateTailFailureStage(taskCtx, TailFailureStageHandler) - handlerErr := handler.OnMessageUpdate(taskCtx, msg) + var ( + refetchConsumed bool + handlerErr error + ) + if refetchErr != nil { + if refetchHandler, ok := handler.(messageUpdateRefetchFailureHandler); ok { + refetchConsumed, handlerErr = refetchHandler.OnMessageUpdateRefetchFailure( + taskCtx, + msg, + refetchErr, + ) + } else { + handlerErr = handler.OnMessageUpdate(taskCtx, msg) + } + } else { + handlerErr = handler.OnMessageUpdate(taskCtx, msg) + } + if refetchConsumed && handlerErr == nil { + refetchErr = nil + } if refetchErr != nil { UpdateTailFailureStage(taskCtx, TailFailureStageMessageUpdateRefetch) } diff --git a/internal/discord/client_test.go b/internal/discord/client_test.go index d04dd7a7..5149c4a7 100644 --- a/internal/discord/client_test.go +++ b/internal/discord/client_test.go @@ -2729,6 +2729,8 @@ func TestTailMessageUpdateImmediateHTTPRefetchFailureStillInvokesHandlerAndRecor recorded: make(chan TailFailure, 1), updates: make(chan *discordgo.Message, 1), } + _, consumesRefetchFailures := any(handler).(messageUpdateRefetchFailureHandler) + require.False(t, consumesRefetchFailures) var refetchCalls atomic.Int32 now := time.Now().UTC().Format(time.RFC3339) server := newTailTestGatewayWithRoutes( diff --git a/internal/syncer/syncer_tail_test.go b/internal/syncer/syncer_tail_test.go index e2537ade..08f97bdd 100644 --- a/internal/syncer/syncer_tail_test.go +++ b/internal/syncer/syncer_tail_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "maps" "net/http" "net/http/httptest" "path/filepath" @@ -849,6 +850,115 @@ func TestTailHandlerMessageUpdateFailureUsesSyncerRefetchedMetadata(t *testing.T } } +func TestTailHandlerConsumesReconciledUnknownMessageRefetchFailure(t *testing.T) { + testCtx, cancelTest := context.WithTimeout(context.Background(), 5*time.Second) + defer cancelTest() + tailCtx, cancelTail := context.WithCancel(testCtx) + defer cancelTail() + + s, err := store.Open(testCtx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + message := &discordgo.Message{ + ID: "123456789012345678", + GuildID: "g1", + ChannelID: "c1", + Content: "preserved before deletion", + Timestamp: time.Now().UTC(), + Author: &discordgo.User{ID: "u1", Username: "user"}, + } + snapshotClient := &fakeClient{} + baseHandler := &tailHandler{ + guilds: makeGuildSet([]string{"g1"}), + store: s, + client: snapshotClient, + channels: map[string]tailChannel{ + "c1": {kind: "text"}, + }, + kindExcludedChannelIDs: map[string]struct{}{}, + } + require.NoError(t, baseHandler.OnChannelUpsert(testCtx, &discordgo.Channel{ + ID: "c1", + GuildID: "g1", + Name: "general", + Type: discordgo.ChannelTypeGuildText, + })) + require.NoError(t, baseHandler.OnMessageCreate(testCtx, message)) + require.NoError(t, s.RecordFailure( + testCtx, + tailMessageFailureIdentity(message.GuildID, message.ChannelID, message.ID, "update"), + errors.New("prior update failure"), + )) + + var clientRefetches atomic.Int32 + server := newSyncerTailUnknownMessageGateway(t, message.ID, 4, &clientRefetches) + defer server.Close() + restore := patchSyncerTailDiscordEndpoints(server.URL + "/api/v10/") + defer restore() + + eventClient, err := discordclient.New("token") + require.NoError(t, err) + defer func() { _ = eventClient.Close() }() + setDiscordTailHandlerTimeout(t, eventClient, time.Second) + + handler := &reconciledRefetchTailHandler{ + tailHandler: baseHandler, + drained: make(chan struct{}), + } + tailDone := make(chan error, 1) + go func() { + tailDone <- eventClient.Tail(tailCtx, handler) + }() + + select { + case <-handler.drained: + case err := <-tailDone: + t.Fatalf("Tail returned before the ordered sentinel: %v", err) + case <-testCtx.Done(): + t.Fatal("ordered sentinel was not handled") + } + require.Zero(t, handler.failureReports.Load()) + require.Zero(t, handler.failureRecords.Load()) + select { + case err := <-tailDone: + t.Fatalf("Tail returned before cancellation after reconciled updates: %v", err) + default: + } + + cancelTail() + select { + case err := <-tailDone: + require.NoError(t, err) + case <-testCtx.Done(): + t.Fatal("Tail did not stop after cancellation") + } + + require.EqualValues(t, 4, clientRefetches.Load()) + snapshotClient.mu.Lock() + snapshotRefetches := snapshotClient.exactMessageCalls + snapshotClient.mu.Unlock() + require.Zero(t, snapshotRefetches) + + var deletedAt string + require.NoError(t, s.DB().QueryRowContext( + context.Background(), + `select coalesce(deleted_at, '') from messages where id = ?`, + message.ID, + ).Scan(&deletedAt)) + require.NotEmpty(t, deletedAt) + var deleteEvents int + require.NoError(t, s.DB().QueryRowContext( + context.Background(), + `select count(*) from message_events where message_id = ? and event_type = 'delete'`, + message.ID, + ).Scan(&deleteEvents)) + require.Positive(t, deleteEvents) + report, err := s.ListFailures(context.Background(), store.FailureListOptions{}, time.Now().UTC()) + require.NoError(t, err) + require.Zero(t, report.UnresolvedCount) +} + func TestTailHandlerResolvesUnknownThreadAndParentBeforeExclusion(t *testing.T) { t.Parallel() @@ -885,9 +995,7 @@ func TestTailHandlerResolvesUnknownThreadAndParentBeforeExclusion(t *testing.T) var wg sync.WaitGroup errs := make(chan error, messageCount) for i := range messageCount { - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { errs <- handler.OnMessageCreate(ctx, &discordgo.Message{ ID: string(rune('a' + i)), GuildID: "g1", @@ -896,7 +1004,7 @@ func TestTailHandlerResolvesUnknownThreadAndParentBeforeExclusion(t *testing.T) Timestamp: time.Now().UTC(), Author: &discordgo.User{ID: "u1", Username: "user"}, }) - }() + }) } wg.Wait() close(errs) @@ -1290,9 +1398,7 @@ func TestTailHandlerConcurrentResolutionErrorsFailClosed(t *testing.T) { errs := make(chan error, messageCount) var wg sync.WaitGroup for i := range messageCount { - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { errs <- handler.OnMessageCreate(ctx, &discordgo.Message{ ID: string(rune('a' + i)), GuildID: "g1", @@ -1301,7 +1407,7 @@ func TestTailHandlerConcurrentResolutionErrorsFailClosed(t *testing.T) { Timestamp: time.Now().UTC(), Author: &discordgo.User{ID: "u1", Username: "user"}, }) - }() + }) } wg.Wait() close(errs) @@ -1543,9 +1649,7 @@ func TestRunTailWithRepairLoop(t *testing.T) { client.mu.Lock() messageCalls := make(map[string]int, len(client.messageCalls)) - for channelID, calls := range client.messageCalls { - messageCalls[channelID] = calls - } + maps.Copy(messageCalls, client.messageCalls) client.mu.Unlock() require.GreaterOrEqual(t, client.guildThreadCalls, 1) @@ -1951,8 +2055,7 @@ func TestReplayTailMessageFailuresRetainsFetchAndIdentityFailures(t *testing.T) } func TestRunTailDetectsGatewayExitWhileRepairIsBlocked(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) require.NoError(t, err) @@ -2676,13 +2779,13 @@ func TestRepairOffsetConcurrentAccess(t *testing.T) { wg.Add(2) go func() { defer wg.Done() - for i := 0; i < 1_000; i++ { + for i := range 1_000 { svc.SetRepairOffset(time.Duration(i%5) * time.Minute) } }() go func() { defer wg.Done() - for i := 0; i < 1_000; i++ { + for range 1_000 { _ = svc.repairOffset() } }() @@ -3295,6 +3398,38 @@ func (h *capturingTailHandler) OnMessageUpdate(ctx context.Context, msg *discord panic("sensitive syncer message update panic") } +type reconciledRefetchTailHandler struct { + *tailHandler + drained chan struct{} + drainOnce sync.Once + failureReports atomic.Int32 + failureRecords atomic.Int32 +} + +func (h *reconciledRefetchTailHandler) OnMessageCreate( + ctx context.Context, + msg *discordgo.Message, +) error { + if err := h.tailHandler.OnMessageCreate(ctx, msg); err != nil { + return err + } + if msg != nil && msg.ID == syncerTailSentinelMessageID { + h.drainOnce.Do(func() { close(h.drained) }) + } + return nil +} + +func (h *reconciledRefetchTailHandler) OnTailFailure(discordclient.TailFailure) { + h.failureReports.Add(1) +} + +func (h *reconciledRefetchTailHandler) RecordTailFailure( + failure discordclient.TailFailure, +) error { + h.failureRecords.Add(1) + return h.tailHandler.RecordTailFailure(failure) +} + type panicReplayTailHandler struct { *tailHandler panicValue any @@ -3391,6 +3526,105 @@ func newSyncerTailMessageUpdateGateway( return httptest.NewServer(mux) } +func newSyncerTailUnknownMessageGateway( + t *testing.T, + messageID string, + updateCount int, + clientRefetches *atomic.Int32, +) *httptest.Server { + t.Helper() + + mux := http.NewServeMux() + mux.HandleFunc("/api/v10/channels/c1/messages/"+messageID, func(w http.ResponseWriter, _ *http.Request) { + clientRefetches.Add(1) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusNotFound) + _ = json.NewEncoder(w).Encode(map[string]any{ + "message": "Unknown Message", + "code": 10008, + }) + }) + mux.HandleFunc("/api/v10/gateway", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"url": "ws://" + r.Host + "/gateway"}) + }) + + upgrader := websocket.Upgrader{} + gatewayHandler := func(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + t.Errorf("upgrade gateway: %v", err) + return + } + defer func() { _ = conn.Close() }() + + if err := conn.WriteJSON(map[string]any{ + "op": 10, + "d": map[string]any{"heartbeat_interval": 1000}, + }); err != nil { + t.Errorf("write hello: %v", err) + return + } + if _, _, err := conn.ReadMessage(); err != nil { + t.Errorf("read identify: %v", err) + return + } + if err := conn.WriteJSON(map[string]any{ + "op": 0, + "t": "READY", + "s": 1, + "d": map[string]any{ + "session_id": "session", + "user": map[string]any{"id": "bot", "username": "bot"}, + }, + }); err != nil { + t.Errorf("write ready: %v", err) + return + } + for i := range updateCount { + if err := conn.WriteJSON(map[string]any{ + "op": 0, + "t": "MESSAGE_UPDATE", + "s": i + 2, + "d": map[string]any{ + "id": messageID, + "guild_id": "g1", + "channel_id": "c1", + }, + }); err != nil { + t.Errorf("write update: %v", err) + return + } + } + if err := conn.WriteJSON(map[string]any{ + "op": 0, + "t": "MESSAGE_CREATE", + "s": updateCount + 2, + "d": map[string]any{ + "id": syncerTailSentinelMessageID, + "guild_id": "g1", + "channel_id": "c1", + "content": "ordered sentinel", + "timestamp": time.Now().UTC().Format(time.RFC3339), + "author": map[string]any{"id": "u1", "username": "user"}, + }, + }); err != nil { + t.Errorf("write sentinel: %v", err) + return + } + for { + if _, _, err := conn.ReadMessage(); err != nil { + return + } + } + } + mux.HandleFunc("/gateway", gatewayHandler) + mux.HandleFunc("/gateway/", gatewayHandler) + return httptest.NewServer(mux) +} + +const syncerTailSentinelMessageID = "223456789012345678" + func newSyncerTailMessagePanicGateway( t *testing.T, exactMessage *discordgo.Message, diff --git a/internal/syncer/tail.go b/internal/syncer/tail.go index 9088d9cd..23ff8ff2 100644 --- a/internal/syncer/tail.go +++ b/internal/syncer/tail.go @@ -525,6 +525,18 @@ func (t *tailHandler) OnMessageUpdate(ctx context.Context, msg *discordgo.Messag return t.resolveMessageFailure(ctx, msg.GuildID, msg.ChannelID, msg.ID, "update") } +func (t *tailHandler) OnMessageUpdateRefetchFailure( + ctx context.Context, + msg *discordgo.Message, + refetchErr error, +) (bool, error) { + if msg == nil || msg.ID == "" || !isUnknownMessage(refetchErr) { + return false, t.OnMessageUpdate(ctx, msg) + } + err := t.reconcileUnknownMessageUpdate(ctx, msg.ID) + return err == nil, err +} + func (t *tailHandler) messageUpdateSnapshot(ctx context.Context, msg *discordgo.Message) (*discordgo.Message, error) { if t.client == nil || msg.ChannelID == "" || msg.ID == "" { if isPartialMessageUpdate(msg) {