From 3e8b93d12eadf8e77e3dc0ce95d9ebc52a16bdf0 Mon Sep 17 00:00:00 2001 From: Hannes Rudolph <49103247+hannesrudolph@users.noreply.github.com> Date: Sat, 25 Jul 2026 11:27:31 -0600 Subject: [PATCH 1/3] feat: scope Discord collection by category --- internal/cli/cli.go | 25 ++ internal/config/config.go | 37 ++- internal/discord/client.go | 28 +++ internal/discord/client_test.go | 8 +- internal/syncer/channel_catalog.go | 26 ++- internal/syncer/channel_exclusions.go | 255 +++++++++++++++++++++ internal/syncer/channel_exclusions_test.go | 77 +++++++ internal/syncer/message_sync.go | 11 +- internal/syncer/syncer.go | 64 +++++- internal/syncer/tail.go | 198 +++++++++++++--- 10 files changed, 675 insertions(+), 54 deletions(-) create mode 100644 internal/syncer/channel_exclusions.go create mode 100644 internal/syncer/channel_exclusions_test.go diff --git a/internal/cli/cli.go b/internal/cli/cli.go index ade6d93c..085e021b 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -308,6 +308,18 @@ type attachmentTextConfigurer interface { SetAttachmentTextEnabled(bool) } +type channelExclusionConfigurer interface { + SetChannelExclusions([]string, []string) +} + +type categoryInclusionConfigurer interface { + SetIncludedCategories([]string) +} + +type repairOffsetConfigurer interface { + SetRepairOffset(time.Duration) +} + func (r *runtime) dispatch(rest []string) error { switch rest[0] { case "metadata": @@ -766,6 +778,19 @@ func (r *runtime) ensureDiscordServices() error { if configurable, ok := r.syncer.(attachmentTextConfigurer); ok { configurable.SetAttachmentTextEnabled(r.cfg.AttachmentTextEnabled()) } + if configurable, ok := r.syncer.(channelExclusionConfigurer); ok { + configurable.SetChannelExclusions(r.cfg.Sync.ExcludeChannelIDs, r.cfg.Sync.ExcludeChannelKinds) + } + if configurable, ok := r.syncer.(categoryInclusionConfigurer); ok { + configurable.SetIncludedCategories(r.cfg.Sync.IncludeCategoryIDs) + } + if configurable, ok := r.syncer.(repairOffsetConfigurer); ok { + offset, err := time.ParseDuration(r.cfg.Sync.RepairOffset) + if err != nil { + return configErr(fmt.Errorf("parse sync.repair_offset: %w", err)) + } + configurable.SetRepairOffset(offset) + } return nil } diff --git a/internal/config/config.go b/internal/config/config.go index 2977692d..52a1566d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -51,13 +51,17 @@ type DesktopConfig struct { } type SyncConfig struct { - Source string `toml:"source"` - Concurrency int `toml:"concurrency"` - RepairEvery string `toml:"repair_every"` - FullHistory bool `toml:"full_history"` - AttachmentText *bool `toml:"attachment_text"` - AttachmentMedia *bool `toml:"attachment_media"` - MaxAttachmentBytes int64 `toml:"max_attachment_bytes"` + Source string `toml:"source"` + Concurrency int `toml:"concurrency"` + RepairEvery string `toml:"repair_every"` + RepairOffset string `toml:"repair_offset"` + FullHistory bool `toml:"full_history"` + ExcludeChannelIDs []string `toml:"exclude_channel_ids,omitempty"` + ExcludeChannelKinds []string `toml:"exclude_channel_kinds,omitempty"` + IncludeCategoryIDs []string `toml:"include_category_ids,omitempty"` + AttachmentText *bool `toml:"attachment_text"` + AttachmentMedia *bool `toml:"attachment_media"` + MaxAttachmentBytes int64 `toml:"max_attachment_bytes"` } type SearchConfig struct { @@ -138,6 +142,7 @@ func Default() Config { Source: "both", Concurrency: defaultSyncConcurrency(), RepairEvery: "6h", + RepairOffset: "0s", FullHistory: true, AttachmentText: new(true), AttachmentMedia: new(false), @@ -268,6 +273,16 @@ func (c *Config) Normalize() error { if c.Sync.RepairEvery == "" { c.Sync.RepairEvery = "6h" } + c.Sync.RepairOffset = strings.TrimSpace(c.Sync.RepairOffset) + if c.Sync.RepairOffset == "" { + c.Sync.RepairOffset = "0s" + } + if _, err := time.ParseDuration(c.Sync.RepairOffset); err != nil { + return fmt.Errorf("parse sync.repair_offset: %w", err) + } + c.Sync.ExcludeChannelIDs = uniqueStrings(c.Sync.ExcludeChannelIDs) + c.Sync.ExcludeChannelKinds = uniqueLowerStrings(c.Sync.ExcludeChannelKinds) + c.Sync.IncludeCategoryIDs = uniqueStrings(c.Sync.IncludeCategoryIDs) if c.Sync.AttachmentText == nil { c.Sync.AttachmentText = new(true) } @@ -439,3 +454,11 @@ func uniqueStrings(in []string) []string { } return out } + +func uniqueLowerStrings(in []string) []string { + out := uniqueStrings(in) + for i := range out { + out[i] = strings.ToLower(out[i]) + } + return uniqueStrings(out) +} diff --git a/internal/discord/client.go b/internal/discord/client.go index 3488b870..0fef474c 100644 --- a/internal/discord/client.go +++ b/internal/discord/client.go @@ -608,6 +608,34 @@ func (c *Client) Tail(ctx context.Context, handler EventHandler) error { before, )) }) + addHandler(func(_ *discordgo.Session, evt *discordgo.ThreadCreate) { + var channel *discordgo.Channel + if evt != nil { + channel = evt.Channel + } + c.enqueueTailTask(tailCtx, orderedWorkCh, fatal, newChannelTailTask( + "THREAD_CREATE", + func(taskCtx context.Context) error { + return handler.OnChannelUpsert(taskCtx, channel) + }, + channel, + )) + }) + addHandler(func(_ *discordgo.Session, evt *discordgo.ThreadUpdate) { + var channel, before *discordgo.Channel + if evt != nil { + channel = evt.Channel + before = evt.BeforeUpdate + } + c.enqueueTailTask(tailCtx, orderedWorkCh, fatal, newChannelTailTask( + "THREAD_UPDATE", + func(taskCtx context.Context) error { + return handler.OnChannelUpsert(taskCtx, channel) + }, + channel, + before, + )) + }) addHandler(func(_ *discordgo.Session, evt *discordgo.GuildCreate) { guildHandler, ok := handler.(guildEventHandler) if !ok { diff --git a/internal/discord/client_test.go b/internal/discord/client_test.go index b562f8c4..94c01cf7 100644 --- a/internal/discord/client_test.go +++ b/internal/discord/client_test.go @@ -1099,8 +1099,10 @@ func TestTailReceivesGatewayEvents(t *testing.T) { {"op": 0, "t": "MESSAGE_UPDATE", "s": 3, "d": map[string]any{"id": "m1", "guild_id": "g1", "channel_id": "c1", "content": "hello 2", "timestamp": now, "author": map[string]any{"id": "u1", "username": "user"}}}, {"op": 0, "t": "MESSAGE_DELETE", "s": 4, "d": map[string]any{"id": "m1", "guild_id": "g1", "channel_id": "c1"}}, {"op": 0, "t": "CHANNEL_CREATE", "s": 5, "d": map[string]any{"id": "c1", "guild_id": "g1", "name": "general", "type": 0}}, - {"op": 0, "t": "GUILD_MEMBER_ADD", "s": 6, "d": map[string]any{"guild_id": "g1", "user": map[string]any{"id": "u1", "username": "user"}, "roles": []string{}}}, - {"op": 0, "t": "GUILD_MEMBER_REMOVE", "s": 7, "d": map[string]any{"guild_id": "g1", "user": map[string]any{"id": "u1", "username": "user"}}}, + {"op": 0, "t": "THREAD_CREATE", "s": 6, "d": map[string]any{"id": "thread-1", "guild_id": "g1", "parent_id": "c1", "name": "support request", "type": 11, "newly_created": true}}, + {"op": 0, "t": "THREAD_UPDATE", "s": 7, "d": map[string]any{"id": "thread-1", "guild_id": "g1", "parent_id": "c1", "name": "support request", "type": 11}}, + {"op": 0, "t": "GUILD_MEMBER_ADD", "s": 8, "d": map[string]any{"guild_id": "g1", "user": map[string]any{"id": "u1", "username": "user"}, "roles": []string{}}}, + {"op": 0, "t": "GUILD_MEMBER_REMOVE", "s": 9, "d": map[string]any{"guild_id": "g1", "user": map[string]any{"id": "u1", "username": "user"}}}, } for _, event := range events { if err := conn.WriteJSON(event); err != nil { @@ -1133,7 +1135,7 @@ func TestTailReceivesGatewayEvents(t *testing.T) { require.Equal(t, 1, handler.creates) require.Equal(t, 1, handler.updates) require.Equal(t, 1, handler.deletes) - require.Equal(t, 1, handler.channels) + require.Equal(t, 3, handler.channels) require.Equal(t, 1, handler.memberUpserts) require.Equal(t, 1, handler.memberDeletes) require.Equal(t, 1, handler.ready) diff --git a/internal/syncer/channel_catalog.go b/internal/syncer/channel_catalog.go index 64ace597..cfe3ae9d 100644 --- a/internal/syncer/channel_catalog.go +++ b/internal/syncer/channel_catalog.go @@ -18,9 +18,9 @@ const ( channelCatalogIncremental ) -func (s *Syncer) channelList(ctx context.Context, guildID string, requested []string, mode channelCatalogMode) ([]*discordgo.Channel, bool, error) { +func (s *Syncer) channelList(ctx context.Context, guildID string, requested []string, mode channelCatalogMode, exclusions channelExclusions) ([]*discordgo.Channel, bool, error) { if len(requested) == 0 { - channels, err := s.liveChannelList(ctx, guildID, mode) + channels, err := s.liveChannelList(ctx, guildID, mode, exclusions) if err != nil { return nil, false, err } @@ -68,7 +68,7 @@ func (s *Syncer) channelList(ctx context.Context, guildID string, requested []st return selected, true, nil } -func (s *Syncer) liveChannelList(ctx context.Context, guildID string, mode channelCatalogMode) ([]*discordgo.Channel, error) { +func (s *Syncer) liveChannelList(ctx context.Context, guildID string, mode channelCatalogMode, exclusions channelExclusions) ([]*discordgo.Channel, error) { channels, err := s.client.GuildChannels(ctx, guildID) if err != nil { return nil, fmt.Errorf("fetch channels for guild %s: %w", guildID, err) @@ -78,7 +78,7 @@ func (s *Syncer) liveChannelList(ctx context.Context, guildID string, mode chann allChannels[channel.ID] = channel } if mode == channelCatalogIncremental { - if err := s.appendActiveThreadCatalog(ctx, allChannels, guildID, threadParentIDs(channels)); err != nil { + if err := s.appendActiveThreadCatalog(ctx, allChannels, guildID, scopedThreadParentIDs(channels, exclusions)); err != nil { return nil, err } return mapsToSlice(allChannels), nil @@ -92,7 +92,7 @@ func (s *Syncer) liveChannelList(ctx context.Context, guildID string, mode chann storedRows = rows mergeStoredThreadChannels(allChannels, rows) } - parentIDs := threadParentIDs(channels) + parentIDs := scopedThreadParentIDs(channels, exclusions) if len(storedThreadParentIDs(storedRows)) == 0 { if err := s.appendThreadCatalog(ctx, allChannels, parentIDs); err != nil { return nil, err @@ -103,6 +103,22 @@ func (s *Syncer) liveChannelList(ctx context.Context, guildID string, mode chann return mapsToSlice(allChannels), nil } +func scopedThreadParentIDs(channels []*discordgo.Channel, exclusions channelExclusions) []string { + channelByID := make(map[string]*discordgo.Channel, len(channels)) + for _, channel := range channels { + if channel != nil { + channelByID[channel.ID] = channel + } + } + parents := make([]string, 0, len(channels)) + for _, channel := range channels { + if isThreadParent(channel) && !exclusions.excludesDiscordChannel(channel, channelByID) { + parents = append(parents, channel.ID) + } + } + return parents +} + func (s *Syncer) appendThreadCatalog(ctx context.Context, allChannels map[string]*discordgo.Channel, parents []string) error { for _, parentID := range uniqueIDs(parents) { channel := allChannels[parentID] diff --git a/internal/syncer/channel_exclusions.go b/internal/syncer/channel_exclusions.go new file mode 100644 index 00000000..c6866d56 --- /dev/null +++ b/internal/syncer/channel_exclusions.go @@ -0,0 +1,255 @@ +package syncer + +import ( + "context" + "strings" + + "github.com/bwmarrin/discordgo" + + "github.com/openclaw/discrawl/internal/store" +) + +type channelExclusions struct { + ids map[string]struct{} + kinds map[string]struct{} + allowedCategoryIDs map[string]struct{} +} + +func newChannelScope(ids, kinds, allowedCategoryIDs []string) channelExclusions { + return channelExclusions{ + ids: normalizedStringSet(ids, false), + kinds: normalizedStringSet(kinds, true), + allowedCategoryIDs: normalizedStringSet(allowedCategoryIDs, false), + } +} + +func normalizedStringSet(values []string, lower bool) map[string]struct{} { + out := make(map[string]struct{}, len(values)) + for _, value := range values { + value = strings.TrimSpace(value) + if lower { + value = strings.ToLower(value) + } + if value != "" { + out[value] = struct{}{} + } + } + return out +} + +func (e channelExclusions) merged(other channelExclusions) channelExclusions { + merged := channelExclusions{ + ids: make(map[string]struct{}, len(e.ids)+len(other.ids)), + kinds: make(map[string]struct{}, len(e.kinds)+len(other.kinds)), + allowedCategoryIDs: intersectOptionalSets(e.allowedCategoryIDs, other.allowedCategoryIDs), + } + for id := range e.ids { + merged.ids[id] = struct{}{} + } + for id := range other.ids { + merged.ids[id] = struct{}{} + } + for kind := range e.kinds { + merged.kinds[kind] = struct{}{} + } + for kind := range other.kinds { + merged.kinds[kind] = struct{}{} + } + return merged +} + +func intersectOptionalSets(left, right map[string]struct{}) map[string]struct{} { + if len(left) == 0 { + return cloneStringSet(right) + } + if len(right) == 0 { + return cloneStringSet(left) + } + out := make(map[string]struct{}, min(len(left), len(right))) + for value := range left { + if _, ok := right[value]; ok { + out[value] = struct{}{} + } + } + return out +} + +func cloneStringSet(in map[string]struct{}) map[string]struct{} { + if len(in) == 0 { + return nil + } + out := make(map[string]struct{}, len(in)) + for value := range in { + out[value] = struct{}{} + } + return out +} + +func (e channelExclusions) excludesID(channelID string) bool { + _, ok := e.ids[channelID] + return ok +} + +func (e channelExclusions) excludesKind(kind string) bool { + kind = strings.ToLower(strings.TrimSpace(kind)) + if _, ok := e.kinds[kind]; ok { + return true + } + if kind == "thread_announcement" { + _, ok := e.kinds["announcement"] + return ok + } + return false +} + +func (e channelExclusions) excludesDiscordChannel(channel *discordgo.Channel, channelByID map[string]*discordgo.Channel) bool { + if channel == nil { + return false + } + if e.excludesID(channel.ID) || e.excludesKind(channelKind(channel)) { + return true + } + if channel.ParentID == "" { + return !e.allowsUnparentedDiscordChannel(channel) + } + if e.excludesID(channel.ParentID) { + return true + } + parent := channelByID[channel.ParentID] + if parent != nil && e.excludesKind(channelKind(parent)) { + return true + } + return !e.allowsDiscordCategory(channel, channelByID) +} + +func (e channelExclusions) excludesStoredChannel(channel store.ChannelRow, channelByID map[string]store.ChannelRow) bool { + if e.excludesID(channel.ID) || e.excludesKind(channel.Kind) { + return true + } + if channel.ParentID == "" { + return !e.allowsUnparentedStoredChannel(channel) + } + if e.excludesID(channel.ParentID) { + return true + } + parent, ok := channelByID[channel.ParentID] + if ok && e.excludesKind(parent.Kind) { + return true + } + return !e.allowsStoredCategory(channel, channelByID) +} + +func (e channelExclusions) allowsUnparentedDiscordChannel(channel *discordgo.Channel) bool { + if len(e.allowedCategoryIDs) == 0 { + return true + } + if channel == nil { + return false + } + _, ok := e.allowedCategoryIDs[channel.ID] + return ok +} + +func (e channelExclusions) allowsUnparentedStoredChannel(channel store.ChannelRow) bool { + if len(e.allowedCategoryIDs) == 0 { + return true + } + _, ok := e.allowedCategoryIDs[channel.ID] + return ok +} + +func (e channelExclusions) allowsDiscordCategory(channel *discordgo.Channel, channelByID map[string]*discordgo.Channel) bool { + if len(e.allowedCategoryIDs) == 0 { + return true + } + for current, seen := channel, map[string]struct{}{}; current != nil; { + if current.ParentID == "" { + return false + } + if _, ok := e.allowedCategoryIDs[current.ParentID]; ok { + return true + } + if _, ok := seen[current.ParentID]; ok { + return false + } + seen[current.ParentID] = struct{}{} + current = channelByID[current.ParentID] + } + return false +} + +func (e channelExclusions) allowsStoredCategory(channel store.ChannelRow, channelByID map[string]store.ChannelRow) bool { + if len(e.allowedCategoryIDs) == 0 { + return true + } + for current, seen := channel, map[string]struct{}{}; ; { + if current.ParentID == "" { + return false + } + if _, ok := e.allowedCategoryIDs[current.ParentID]; ok { + return true + } + if _, ok := seen[current.ParentID]; ok { + return false + } + seen[current.ParentID] = struct{}{} + parent, ok := channelByID[current.ParentID] + if !ok { + return false + } + current = parent + } +} + +func (s *Syncer) effectiveChannelExclusions(opts SyncOptions) channelExclusions { + if s == nil { + return newChannelScope(opts.ExcludeChannelIDs, opts.ExcludeChannelKinds, opts.IncludeCategoryIDs) + } + return s.channelExclusions.merged(newChannelScope(opts.ExcludeChannelIDs, opts.ExcludeChannelKinds, opts.IncludeCategoryIDs)) +} + +func filterExcludedDiscordChannels(channels []*discordgo.Channel, exclusions channelExclusions) []*discordgo.Channel { + if len(exclusions.ids) == 0 && len(exclusions.kinds) == 0 && len(exclusions.allowedCategoryIDs) == 0 { + return channels + } + channelByID := make(map[string]*discordgo.Channel, len(channels)) + for _, channel := range channels { + if channel != nil { + channelByID[channel.ID] = channel + } + } + out := make([]*discordgo.Channel, 0, len(channels)) + for _, channel := range channels { + if channel != nil && !exclusions.excludesDiscordChannel(channel, channelByID) { + out = append(out, channel) + } + } + return out +} + +func (s *Syncer) filterExcludedStoredChannelIDs(ctx context.Context, guildID string, channelIDs []string, opts SyncOptions) ([]string, error) { + exclusions := s.effectiveChannelExclusions(opts) + if len(exclusions.ids) == 0 && len(exclusions.kinds) == 0 && len(exclusions.allowedCategoryIDs) == 0 { + return channelIDs, nil + } + channels, err := s.store.Channels(ctx, guildID) + if err != nil { + return nil, err + } + channelByID := make(map[string]store.ChannelRow, len(channels)) + for _, channel := range channels { + channelByID[channel.ID] = channel + } + out := make([]string, 0, len(channelIDs)) + for _, channelID := range channelIDs { + channel, ok := channelByID[channelID] + if ok && exclusions.excludesStoredChannel(channel, channelByID) { + continue + } + if !ok && exclusions.excludesID(channelID) { + continue + } + out = append(out, channelID) + } + return out, nil +} diff --git a/internal/syncer/channel_exclusions_test.go b/internal/syncer/channel_exclusions_test.go new file mode 100644 index 00000000..cca93e46 --- /dev/null +++ b/internal/syncer/channel_exclusions_test.go @@ -0,0 +1,77 @@ +package syncer + +import ( + "context" + "path/filepath" + "testing" + + "github.com/bwmarrin/discordgo" + "github.com/stretchr/testify/require" + + "github.com/openclaw/discrawl/internal/store" +) + +func TestCategoryScopeFiltersChannelsAndNestedThreads(t *testing.T) { + t.Parallel() + + allowedCategory := &discordgo.Channel{ID: "allowed-category", Type: discordgo.ChannelTypeGuildCategory} + rejectedCategory := &discordgo.Channel{ID: "rejected-category", Type: discordgo.ChannelTypeGuildCategory} + allowedForum := &discordgo.Channel{ID: "allowed-forum", ParentID: allowedCategory.ID, Type: discordgo.ChannelTypeGuildForum} + allowedThread := &discordgo.Channel{ID: "allowed-thread", ParentID: allowedForum.ID, Type: discordgo.ChannelTypeGuildPublicThread} + allowedText := &discordgo.Channel{ID: "allowed-text", ParentID: allowedCategory.ID, Type: discordgo.ChannelTypeGuildText} + rejectedText := &discordgo.Channel{ID: "rejected-text", ParentID: rejectedCategory.ID, Type: discordgo.ChannelTypeGuildText} + rootText := &discordgo.Channel{ID: "root-text", Type: discordgo.ChannelTypeGuildText} + feed := &discordgo.Channel{ID: "feed", ParentID: allowedCategory.ID, Type: discordgo.ChannelTypeGuildNews} + + scope := newChannelScope(nil, []string{"announcement"}, []string{allowedCategory.ID}) + filtered := filterExcludedDiscordChannels( + []*discordgo.Channel{ + allowedCategory, + rejectedCategory, + allowedForum, + allowedThread, + allowedText, + rejectedText, + rootText, + feed, + }, + scope, + ) + require.Equal(t, []string{"allowed-category", "allowed-forum", "allowed-thread", "allowed-text"}, channelIDs(filtered)) + require.Equal(t, []string{"allowed-forum", "allowed-text"}, scopedThreadParentIDs(filtered, scope)) +} + +func TestCategoryScopeFiltersIncompleteStoredChannels(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + for _, channel := range []store.ChannelRecord{ + {ID: "allowed-category", GuildID: "g1", Kind: "category", RawJSON: `{}`}, + {ID: "rejected-category", GuildID: "g1", Kind: "category", RawJSON: `{}`}, + {ID: "allowed-text", GuildID: "g1", ParentID: "allowed-category", Kind: "text", RawJSON: `{}`}, + {ID: "rejected-text", GuildID: "g1", ParentID: "rejected-category", Kind: "text", RawJSON: `{}`}, + } { + require.NoError(t, s.UpsertChannel(ctx, channel)) + } + + svc := New(&fakeClient{}, s, nil) + svc.SetIncludedCategories([]string{"allowed-category"}) + filtered, err := svc.filterExcludedStoredChannelIDs(ctx, "g1", []string{"allowed-text", "rejected-text"}, SyncOptions{}) + require.NoError(t, err) + require.Equal(t, []string{"allowed-text"}, filtered) +} + +func TestTailRepairUsesIncrementalScope(t *testing.T) { + t.Parallel() + + opts := tailRepairSyncOptions([]string{"g1"}) + require.Equal(t, []string{"g1"}, opts.GuildIDs) + require.False(t, opts.Full) + require.True(t, opts.SkipMembers) + require.True(t, opts.LatestOnly) + require.Equal(t, "tail_repair", opts.RepairReason) +} diff --git a/internal/syncer/message_sync.go b/internal/syncer/message_sync.go index 2571b59f..eaa838be 100644 --- a/internal/syncer/message_sync.go +++ b/internal/syncer/message_sync.go @@ -18,7 +18,7 @@ func (s *Syncer) syncMessageChannels( channels []*discordgo.Channel, opts SyncOptions, ) (int, error) { - messageChannels := filterMessageChannels(channels, opts.ChannelIDs) + messageChannels := filterMessageChannels(channels, opts.ChannelIDs, s.effectiveChannelExclusions(opts)) if len(messageChannels) == 0 { return 0, nil } @@ -38,7 +38,11 @@ func (s *Syncer) syncMessageChannels( return total, err } -func filterMessageChannels(channels []*discordgo.Channel, requested []string) []*discordgo.Channel { +func filterMessageChannels(channels []*discordgo.Channel, requested []string, configured ...channelExclusions) []*discordgo.Channel { + exclusions := channelExclusions{} + if len(configured) > 0 { + exclusions = configured[0] + } requestedSet := makeGuildSet(requested) channelByID := make(map[string]*discordgo.Channel, len(channels)) for _, channel := range channels { @@ -51,6 +55,9 @@ func filterMessageChannels(channels []*discordgo.Channel, requested []string) [] if !isMessageChannel(channel) { continue } + if exclusions.excludesDiscordChannel(channel, channelByID) { + continue + } if len(requestedSet) > 0 && !requestedMessageTarget(channel, channelByID, requestedSet) { continue } diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go index f64cff84..124d1d8b 100644 --- a/internal/syncer/syncer.go +++ b/internal/syncer/syncer.go @@ -46,25 +46,61 @@ type Syncer struct { tailRepair func(context.Context, SyncOptions) (SyncStats, error) tailRepairJoinTimeout time.Duration tailRepairMu sync.Mutex + tailRepairOffsetMu sync.RWMutex + tailRepairOffset time.Duration + channelExclusions channelExclusions } type SyncOptions struct { - Full bool - GuildIDs []string - ChannelIDs []string - Concurrency int - Since time.Time - Embeddings bool - SkipMembers bool - RequireMembers bool - LatestOnly bool - RepairReason string + Full bool + GuildIDs []string + ChannelIDs []string + Concurrency int + Since time.Time + Embeddings bool + SkipMembers bool + RequireMembers bool + LatestOnly bool + ExcludeChannelIDs []string + ExcludeChannelKinds []string + IncludeCategoryIDs []string + RepairReason string } func (s *Syncer) SetTailReadyCallback(fn func(context.Context) error) { s.tailReady = fn } +func (s *Syncer) SetChannelExclusions(channelIDs, channelKinds []string) { + s.channelExclusions.ids = normalizedStringSet(channelIDs, false) + s.channelExclusions.kinds = normalizedStringSet(channelKinds, true) +} + +func (s *Syncer) SetIncludedCategories(categoryIDs []string) { + s.channelExclusions.allowedCategoryIDs = normalizedStringSet(categoryIDs, false) +} + +func (s *Syncer) SetRepairOffset(offset time.Duration) { + if s == nil { + return + } + if offset < 0 { + offset = 0 + } + s.tailRepairOffsetMu.Lock() + s.tailRepairOffset = offset + s.tailRepairOffsetMu.Unlock() +} + +func (s *Syncer) repairOffset() time.Duration { + if s == nil { + return 0 + } + s.tailRepairOffsetMu.RLock() + defer s.tailRepairOffsetMu.RUnlock() + return s.tailRepairOffset +} + type SyncStats struct { Guilds int `json:"guilds"` Channels int `json:"channels"` @@ -159,10 +195,12 @@ func (s *Syncer) syncGuild(ctx context.Context, guildID string, opts SyncOptions catalogMode = channelCatalogIncremental } } - channelList, targeted, err := s.channelList(ctx, guildID, opts.ChannelIDs, catalogMode) + exclusions := s.effectiveChannelExclusions(opts) + channelList, targeted, err := s.channelList(ctx, guildID, opts.ChannelIDs, catalogMode, exclusions) if err != nil { return stats, err } + channelList = filterExcludedDiscordChannels(channelList, exclusions) if err := s.storeChannelList(ctx, channelList, &stats); err != nil { return stats, err } @@ -240,6 +278,10 @@ func (s *Syncer) syncGuildIncompleteBatches(ctx context.Context, guildID string, if err != nil { return SyncStats{}, false, err } + incomplete, err = s.filterExcludedStoredChannelIDs(ctx, guildID, incomplete, opts) + if err != nil { + return SyncStats{}, false, err + } if len(incomplete) == 0 { return SyncStats{}, false, nil } diff --git a/internal/syncer/tail.go b/internal/syncer/tail.go index 17862aa7..637a3df5 100644 --- a/internal/syncer/tail.go +++ b/internal/syncer/tail.go @@ -18,12 +18,18 @@ func (s *Syncer) RunTail(ctx context.Context, guildIDs []string, repairEvery tim return err } handler := &tailHandler{ - guilds: makeGuildSet(guildIDs), - store: s.store, - client: s.client, - attachmentTextEnabled: s.attachmentTextEnabled, - onReady: s.tailReady, - logger: s.logger, + guilds: makeGuildSet(guildIDs), + store: s.store, + client: s.client, + attachmentTextEnabled: s.attachmentTextEnabled, + onReady: s.tailReady, + logger: s.logger, + exclusions: s.channelExclusions, + kindExcludedChannelIDs: map[string]struct{}{}, + knownChannelIDs: map[string]struct{}{}, + } + if err := handler.seedChannelExclusions(ctx); err != nil { + return fmt.Errorf("seed tail channel exclusions: %w", err) } if repairEvery <= 0 { return s.client.Tail(ctx, handler) @@ -40,8 +46,8 @@ func (s *Syncer) RunTail(ctx context.Context, guildIDs []string, repairEvery tim go func() { tailDone <- s.client.Tail(tailCtx, handler) }() - ticker := time.NewTicker(repairEvery) - defer ticker.Stop() + repairTimer := time.NewTimer(nextTailRepairDelay(time.Now(), repairEvery, s.repairOffset())) + defer repairTimer.Stop() var activeRepair *tailRepairRun var repairDone <-chan tailRepairResult for { @@ -73,14 +79,16 @@ func (s *Syncer) RunTail(ctx context.Context, guildIDs []string, repairEvery tim s.logTailRepairResult(result) activeRepair = nil repairDone = nil - case <-ticker.C: + case <-repairTimer.C: if activeRepair != nil { + repairTimer.Reset(nextTailRepairDelay(time.Now(), repairEvery, s.repairOffset())) continue } activeRepair = s.startTailRepair(ctx, guildIDs) if activeRepair != nil { repairDone = activeRepair.done } + repairTimer.Reset(nextTailRepairDelay(time.Now(), repairEvery, s.repairOffset())) } } } @@ -123,16 +131,21 @@ func (s *Syncer) startTailRepair(ctx context.Context, guildIDs []string) *tailRe startedAt := time.Now() go func() { defer s.tailRepairMu.Unlock() - _, err := s.runTailRepair(repairCtx, SyncOptions{ - GuildIDs: guildIDs, - Full: false, - RepairReason: "tail_repair", - }) + _, err := s.runTailRepair(repairCtx, tailRepairSyncOptions(guildIDs)) done <- tailRepairResult{err: err, elapsed: time.Since(startedAt)} }() return &tailRepairRun{cancel: cancel, done: done} } +func tailRepairSyncOptions(guildIDs []string) SyncOptions { + return SyncOptions{ + GuildIDs: append([]string(nil), guildIDs...), + SkipMembers: true, + LatestOnly: true, + RepairReason: "tail_repair", + } +} + func (s *Syncer) runTailRepair(ctx context.Context, opts SyncOptions) (SyncStats, error) { if s.tailRepair != nil { return s.tailRepair(ctx, opts) @@ -190,13 +203,17 @@ func (s *Syncer) logTailRepairResult(result tailRepairResult) { } type tailHandler struct { - guilds map[string]struct{} - store *store.Store - client Client - attachmentTextEnabled bool - failureLedgerTimeout time.Duration - onReady func(context.Context) error - logger *slog.Logger + guilds map[string]struct{} + store *store.Store + client Client + attachmentTextEnabled bool + failureLedgerTimeout time.Duration + onReady func(context.Context) error + logger *slog.Logger + exclusions channelExclusions + exclusionMu sync.RWMutex + kindExcludedChannelIDs map[string]struct{} + knownChannelIDs map[string]struct{} } func (t *tailHandler) OnTailReady(ctx context.Context) error { @@ -231,7 +248,7 @@ func (t *tailHandler) OnTailFailure(failure discordclient.TailFailure) { func (t *tailHandler) OnMessageCreate(ctx context.Context, msg *discordgo.Message) error { discordclient.UpdateTailFailureStage(ctx, discordclient.TailFailureStageHandler) - if !t.allowGuild(msg.GuildID) { + if msg == nil || !t.allowGuild(msg.GuildID) || t.excludeChannel(msg.ChannelID) { return nil } discordclient.UpdateTailFailureStage(ctx, discordclient.TailFailureStageMessageBuild) @@ -263,7 +280,7 @@ func (t *tailHandler) OnMessageUpdate(ctx context.Context, msg *discordgo.Messag if msg == nil { return nil } - if msg.GuildID != "" && !t.allowGuild(msg.GuildID) { + if t.excludeChannel(msg.ChannelID) || (msg.GuildID != "" && !t.allowGuild(msg.GuildID)) { return nil } var err error @@ -271,7 +288,7 @@ func (t *tailHandler) OnMessageUpdate(ctx context.Context, msg *discordgo.Messag if err != nil { return err } - if msg == nil || !t.allowGuild(msg.GuildID) { + if msg == nil || !t.allowGuild(msg.GuildID) || t.excludeChannel(msg.ChannelID) { return nil } discordclient.UpdateTailFailureStage(ctx, discordclient.TailFailureStageMessageBuild) @@ -361,7 +378,7 @@ func isPartialMessageUpdate(msg *discordgo.Message) bool { func (t *tailHandler) OnMessageDelete(ctx context.Context, evt *discordgo.MessageDelete) error { discordclient.UpdateTailFailureStage(ctx, discordclient.TailFailureStageHandler) - if !t.allowGuild(evt.GuildID) { + if evt == nil || !t.allowGuild(evt.GuildID) || t.excludeChannel(evt.ChannelID) { return nil } discordclient.UpdateTailFailureStage(ctx, discordclient.TailFailureStageCanonicalDelete) @@ -376,7 +393,11 @@ func (t *tailHandler) OnMessageDelete(ctx context.Context, evt *discordgo.Messag } func (t *tailHandler) OnChannelUpsert(ctx context.Context, channel *discordgo.Channel) error { - if !t.allowGuild(channel.GuildID) { + if channel == nil || !t.allowGuild(channel.GuildID) { + return nil + } + t.trackChannelExclusion(channel) + if t.excludeChannel(channel.ID) { return nil } return t.store.UpsertChannel(ctx, toChannelRecord(channel, marshalJSONString(channel, "{}"))) @@ -419,6 +440,131 @@ func (t *tailHandler) TailAllowsGuild(guildID string) bool { return t.allowGuild(guildID) } +func (t *tailHandler) seedChannelExclusions(ctx context.Context) error { + if t.store == nil { + return nil + } + channels, err := t.store.Channels(ctx, "") + if err != nil { + return err + } + channelByID := make(map[string]store.ChannelRow, len(channels)) + for _, channel := range channels { + channelByID[channel.ID] = channel + } + t.exclusionMu.Lock() + defer t.exclusionMu.Unlock() + if t.kindExcludedChannelIDs == nil { + t.kindExcludedChannelIDs = map[string]struct{}{} + } + if t.knownChannelIDs == nil { + t.knownChannelIDs = map[string]struct{}{} + } + for _, channel := range channels { + t.knownChannelIDs[channel.ID] = struct{}{} + if t.exclusions.excludesStoredChannel(channel, channelByID) { + t.kindExcludedChannelIDs[channel.ID] = struct{}{} + continue + } + delete(t.kindExcludedChannelIDs, channel.ID) + } + return nil +} + +func (t *tailHandler) excludeChannel(channelID string) bool { + if t.exclusions.excludesID(channelID) { + return true + } + t.exclusionMu.RLock() + defer t.exclusionMu.RUnlock() + if _, ok := t.kindExcludedChannelIDs[channelID]; ok { + return true + } + if len(t.exclusions.allowedCategoryIDs) == 0 { + return false + } + _, known := t.knownChannelIDs[channelID] + return !known +} + +func (t *tailHandler) trackChannelExclusion(channel *discordgo.Channel) { + if channel == nil { + return + } + excluded := t.exclusions.excludesID(channel.ID) || t.exclusions.excludesKind(channelKind(channel)) + if !excluded && channel.ParentID == "" { + excluded = !t.exclusions.allowsUnparentedDiscordChannel(channel) + } + if !excluded && channel.ParentID != "" { + if t.exclusions.excludesID(channel.ParentID) { + excluded = true + } else if len(t.exclusions.allowedCategoryIDs) > 0 { + if _, allowedCategory := t.exclusions.allowedCategoryIDs[channel.ParentID]; allowedCategory { + excluded = false + } else { + excluded = t.excludeChannel(channel.ParentID) + } + } else { + excluded = t.excludeChannel(channel.ParentID) + } + } + t.exclusionMu.Lock() + defer t.exclusionMu.Unlock() + if t.kindExcludedChannelIDs == nil { + t.kindExcludedChannelIDs = map[string]struct{}{} + } + if t.knownChannelIDs == nil { + t.knownChannelIDs = map[string]struct{}{} + } + t.knownChannelIDs[channel.ID] = struct{}{} + if excluded { + t.kindExcludedChannelIDs[channel.ID] = struct{}{} + return + } + delete(t.kindExcludedChannelIDs, channel.ID) +} + +func nextTailRepairDelay(now time.Time, repairEvery, repairOffset time.Duration) time.Duration { + if repairEvery <= 0 || repairOffset <= 0 { + return repairEvery + } + normalizedOffset := repairOffset % repairEvery + civilNow := civilTimelineTime(now) + nextCivil := civilNow.Truncate(repairEvery).Add(normalizedOffset) + if !nextCivil.After(civilNow) { + nextCivil = nextCivil.Add(repairEvery) + } + for { + next := time.Date( + nextCivil.Year(), + nextCivil.Month(), + nextCivil.Day(), + nextCivil.Hour(), + nextCivil.Minute(), + nextCivil.Second(), + nextCivil.Nanosecond(), + now.Location(), + ) + if civilTimelineTime(next).Equal(nextCivil) && next.After(now) { + return next.Sub(now) + } + nextCivil = nextCivil.Add(repairEvery) + } +} + +func civilTimelineTime(value time.Time) time.Time { + return time.Date( + value.Year(), + value.Month(), + value.Day(), + value.Hour(), + value.Minute(), + value.Second(), + value.Nanosecond(), + time.UTC, + ) +} + func (t *tailHandler) allowGuild(guildID string) bool { if len(t.guilds) == 0 { return true From bcb67603164ea8ece5595b2137d02e8c65e7a747 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 25 Jul 2026 11:02:54 -0700 Subject: [PATCH 2/3] fix(syncer): make channel scope fail closed --- internal/cli/cli_test.go | 35 ++++ internal/cli/output.go | 4 + internal/config/config_test.go | 26 +++ internal/syncer/channel_catalog.go | 31 +++- internal/syncer/channel_exclusions.go | 58 +++++-- internal/syncer/channel_exclusions_test.go | 192 +++++++++++++++++++++ internal/syncer/message_sync.go | 11 +- internal/syncer/syncer.go | 5 +- internal/syncer/tail.go | 14 +- 9 files changed, 336 insertions(+), 40 deletions(-) diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index 16fb86e1..0cb540bd 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -3662,6 +3662,10 @@ type fakeSyncService struct { replayStats syncer.TailMessageReplayStats replayErr error attachmentTextEnabled bool + excludeChannelIDs []string + excludeChannelKinds []string + includeCategoryIDs []string + repairOffset time.Duration callTailReady bool tailReadyCalls int tailReady func(context.Context) error @@ -3712,6 +3716,19 @@ func (f *fakeSyncService) SetAttachmentTextEnabled(enabled bool) { f.attachmentTextEnabled = enabled } +func (f *fakeSyncService) SetChannelExclusions(channelIDs, channelKinds []string) { + f.excludeChannelIDs = append([]string(nil), channelIDs...) + f.excludeChannelKinds = append([]string(nil), channelKinds...) +} + +func (f *fakeSyncService) SetIncludedCategories(categoryIDs []string) { + f.includeCategoryIDs = append([]string(nil), categoryIDs...) +} + +func (f *fakeSyncService) SetRepairOffset(offset time.Duration) { + f.repairOffset = offset +} + type hybridSyncService struct { store *store.Store sawGitMessage bool @@ -3797,6 +3814,10 @@ func TestRuntimeInitSyncTailAndDoctor(t *testing.T) { require.Equal(t, "g2", cfg.DefaultGuildID) require.True(t, cfg.Search.Embeddings.Enabled) cfg.Desktop.Path = filepath.Join(dir, "empty-discord") + cfg.Sync.ExcludeChannelIDs = []string{"blocked-channel"} + cfg.Sync.ExcludeChannelKinds = []string{"announcement"} + cfg.Sync.IncludeCategoryIDs = []string{"category-a"} + cfg.Sync.RepairOffset = "45m" require.NoError(t, os.MkdirAll(cfg.Desktop.Path, 0o755)) require.NoError(t, config.Write(cfgPath, cfg)) @@ -3806,6 +3827,10 @@ func TestRuntimeInitSyncTailAndDoctor(t *testing.T) { require.True(t, fakeSync.lastSync.LatestOnly) require.True(t, fakeSync.lastSync.SkipMembers) require.True(t, fakeSync.attachmentTextEnabled) + require.Equal(t, []string{"blocked-channel"}, fakeSync.excludeChannelIDs) + require.Equal(t, []string{"announcement"}, fakeSync.excludeChannelKinds) + require.Equal(t, []string{"category-a"}, fakeSync.includeCategoryIDs) + require.Equal(t, 45*time.Minute, fakeSync.repairOffset) rt = newRuntime() require.NoError(t, rt.withServices(true, func() error { return rt.runSync([]string{"--guilds", "g2", "--with-members"}) })) @@ -4165,10 +4190,20 @@ func TestCommandHelpDoesNotOpenConfigOrStore(t *testing.T) { require.Contains(t, stdout.String(), "--stats") require.Empty(t, stderr.String()) + stdout.Reset() + require.NoError(t, Run(context.Background(), []string{"sync", "--help"}, &stdout, &stderr)) + require.Contains(t, stdout.String(), "sync.include_category_ids") + require.Contains(t, stdout.String(), "sync.exclude_channel_ids") + require.Contains(t, stdout.String(), "sync.exclude_channel_kinds") + require.Empty(t, stderr.String()) + stdout.Reset() require.NoError(t, Run(context.Background(), []string{"tail", "--help"}, &stdout, &stderr)) require.Contains(t, stdout.String(), "--replay-failures-only") require.Contains(t, stdout.String(), "--replay-limit N") + require.Contains(t, stdout.String(), "sync.include_category_ids") + require.Contains(t, stdout.String(), "sync.exclude_channel_ids") + require.Contains(t, stdout.String(), "sync.exclude_channel_kinds") require.Empty(t, stderr.String()) err := Run(context.Background(), []string{"help", "wat"}, &bytes.Buffer{}, &bytes.Buffer{}) diff --git a/internal/cli/output.go b/internal/cli/output.go index 35a1fd8b..8c0e372c 100644 --- a/internal/cli/output.go +++ b/internal/cli/output.go @@ -125,10 +125,14 @@ Discover accessible guilds and initialize configuration. "sync": `Usage: discrawl sync [--full] [--all] [--all-channels] [--since RFC3339] [--channels IDS] [--concurrency N] [--source SOURCE] [--with-embeddings] [--with-media] [--skip-members|--with-members] [--latest-only] [--guild ID|--guilds IDS] [--update MODE|--no-update] Sync Discord or desktop-cache data into the local archive. +Configure Discord collection scope with sync.include_category_ids, +sync.exclude_channel_ids, and sync.exclude_channel_kinds; exclusions win. `, "tail": `Usage: discrawl tail [--repair-every DURATION] [--guild ID|--guilds IDS] [--replay-failures-only [--replay-limit N]] Continuously archive new Discord messages. +The sync.include_category_ids, sync.exclude_channel_ids, and +sync.exclude_channel_kinds settings apply to live events and repair syncs. `, "wiretap": `Usage: discrawl wiretap [flags] diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 5910032c..ec9d6f78 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -325,6 +325,10 @@ func TestWriteAndLoadRoundTrip(t *testing.T) { cfg := Default() cfg.DefaultGuildID = "g1" cfg.GuildIDs = []string{"g1", "g2"} + cfg.Sync.RepairOffset = "45m" + cfg.Sync.ExcludeChannelIDs = []string{"c1", "c2"} + cfg.Sync.ExcludeChannelKinds = []string{"announcement"} + cfg.Sync.IncludeCategoryIDs = []string{"category-a"} require.NoError(t, Write(path, cfg)) loaded, err := Load(path) @@ -334,6 +338,28 @@ func TestWriteAndLoadRoundTrip(t *testing.T) { require.Equal(t, DefaultRemoteTokenEnv, loaded.Remote.TokenEnv) require.NotNil(t, loaded.Sync.AttachmentText) require.True(t, *loaded.Sync.AttachmentText) + require.Equal(t, "45m", loaded.Sync.RepairOffset) + require.Equal(t, []string{"c1", "c2"}, loaded.Sync.ExcludeChannelIDs) + require.Equal(t, []string{"announcement"}, loaded.Sync.ExcludeChannelKinds) + require.Equal(t, []string{"category-a"}, loaded.Sync.IncludeCategoryIDs) +} + +func TestNormalizeSyncCollectionScope(t *testing.T) { + t.Parallel() + + cfg := Default() + cfg.Sync.RepairOffset = " 30m " + cfg.Sync.ExcludeChannelIDs = []string{" c1 ", "", "c1", "c2"} + cfg.Sync.ExcludeChannelKinds = []string{" Announcement ", "TEXT", "announcement"} + cfg.Sync.IncludeCategoryIDs = []string{" category-a ", "category-a", "category-b"} + require.NoError(t, cfg.Normalize()) + require.Equal(t, "30m", cfg.Sync.RepairOffset) + require.Equal(t, []string{"c1", "c2"}, cfg.Sync.ExcludeChannelIDs) + require.Equal(t, []string{"announcement", "text"}, cfg.Sync.ExcludeChannelKinds) + require.Equal(t, []string{"category-a", "category-b"}, cfg.Sync.IncludeCategoryIDs) + + cfg.Sync.RepairOffset = "later" + require.ErrorContains(t, cfg.Normalize(), "parse sync.repair_offset") } func TestWriteRejectsNonPositiveEmbeddingTimeout(t *testing.T) { diff --git a/internal/syncer/channel_catalog.go b/internal/syncer/channel_catalog.go index cfe3ae9d..acc48b8c 100644 --- a/internal/syncer/channel_catalog.go +++ b/internal/syncer/channel_catalog.go @@ -3,6 +3,7 @@ package syncer import ( "context" "fmt" + "maps" "slices" "strings" "time" @@ -24,20 +25,22 @@ func (s *Syncer) channelList(ctx context.Context, guildID string, requested []st if err != nil { return nil, false, err } - return channels, false, nil + return filterExcludedDiscordChannels(channels, exclusions), false, nil } requestedSet := makeGuildSet(requested) storedByID := map[string]*discordgo.Channel{} + storedCatalog := map[string]*discordgo.Channel{} if s.store != nil { rows, err := s.store.Channels(ctx, guildID) if err != nil { return nil, false, err } + storedCatalog = storedChannelCatalog(rows) storedByID = selectStoredChannels(rows, requestedSet) if canUseStoredTargets(storedByID, requestedSet) { selected := selectRequestedChannels(nil, storedByID, requestedSet) - return selected, true, nil + return filterExcludedDiscordChannelsWithCatalog(selected, storedCatalog, exclusions), true, nil } } @@ -65,7 +68,11 @@ func (s *Syncer) channelList(ctx context.Context, guildID string, requested []st } selected = selectRequestedChannels(allChannels, storedByID, requestedSet) } - return selected, true, nil + return filterExcludedDiscordChannelsWithCatalog( + selected, + mergedChannelCatalog(storedCatalog, allChannels), + exclusions, + ), true, nil } func (s *Syncer) liveChannelList(ctx context.Context, guildID string, mode channelCatalogMode, exclusions channelExclusions) ([]*discordgo.Channel, error) { @@ -247,6 +254,24 @@ func selectStoredChannels(rows []store.ChannelRow, requested map[string]struct{} return out } +func storedChannelCatalog(rows []store.ChannelRow) map[string]*discordgo.Channel { + if len(rows) == 0 { + return nil + } + out := make(map[string]*discordgo.Channel, len(rows)) + for _, row := range rows { + out[row.ID] = channelFromRow(row) + } + return out +} + +func mergedChannelCatalog(base, overlay map[string]*discordgo.Channel) map[string]*discordgo.Channel { + out := make(map[string]*discordgo.Channel, len(base)+len(overlay)) + maps.Copy(out, base) + maps.Copy(out, overlay) + return out +} + func channelFromRow(row store.ChannelRow) *discordgo.Channel { channelType := channelTypeFromKind(row.Kind) var threadMeta *discordgo.ThreadMetadata diff --git a/internal/syncer/channel_exclusions.go b/internal/syncer/channel_exclusions.go index c6866d56..c5188ba7 100644 --- a/internal/syncer/channel_exclusions.go +++ b/internal/syncer/channel_exclusions.go @@ -13,13 +13,16 @@ type channelExclusions struct { ids map[string]struct{} kinds map[string]struct{} allowedCategoryIDs map[string]struct{} + categoryScopeSet bool } func newChannelScope(ids, kinds, allowedCategoryIDs []string) channelExclusions { + categories := normalizedStringSet(allowedCategoryIDs, false) return channelExclusions{ ids: normalizedStringSet(ids, false), kinds: normalizedStringSet(kinds, true), - allowedCategoryIDs: normalizedStringSet(allowedCategoryIDs, false), + allowedCategoryIDs: categories, + categoryScopeSet: len(categories) > 0, } } @@ -38,10 +41,17 @@ func normalizedStringSet(values []string, lower bool) map[string]struct{} { } func (e channelExclusions) merged(other channelExclusions) channelExclusions { + allowedCategoryIDs, categoryScopeSet := intersectOptionalSets( + e.allowedCategoryIDs, + e.categoryScopeSet, + other.allowedCategoryIDs, + other.categoryScopeSet, + ) merged := channelExclusions{ ids: make(map[string]struct{}, len(e.ids)+len(other.ids)), kinds: make(map[string]struct{}, len(e.kinds)+len(other.kinds)), - allowedCategoryIDs: intersectOptionalSets(e.allowedCategoryIDs, other.allowedCategoryIDs), + allowedCategoryIDs: allowedCategoryIDs, + categoryScopeSet: categoryScopeSet, } for id := range e.ids { merged.ids[id] = struct{}{} @@ -58,12 +68,20 @@ func (e channelExclusions) merged(other channelExclusions) channelExclusions { return merged } -func intersectOptionalSets(left, right map[string]struct{}) map[string]struct{} { - if len(left) == 0 { - return cloneStringSet(right) +func intersectOptionalSets( + left map[string]struct{}, + leftSet bool, + right map[string]struct{}, + rightSet bool, +) (map[string]struct{}, bool) { + if !leftSet && !rightSet { + return nil, false } - if len(right) == 0 { - return cloneStringSet(left) + if !leftSet { + return cloneStringSet(right), true + } + if !rightSet { + return cloneStringSet(left), true } out := make(map[string]struct{}, min(len(left), len(right))) for value := range left { @@ -71,7 +89,7 @@ func intersectOptionalSets(left, right map[string]struct{}) map[string]struct{} out[value] = struct{}{} } } - return out + return out, true } func cloneStringSet(in map[string]struct{}) map[string]struct{} { @@ -140,7 +158,7 @@ func (e channelExclusions) excludesStoredChannel(channel store.ChannelRow, chann } func (e channelExclusions) allowsUnparentedDiscordChannel(channel *discordgo.Channel) bool { - if len(e.allowedCategoryIDs) == 0 { + if !e.categoryScopeSet { return true } if channel == nil { @@ -151,7 +169,7 @@ func (e channelExclusions) allowsUnparentedDiscordChannel(channel *discordgo.Cha } func (e channelExclusions) allowsUnparentedStoredChannel(channel store.ChannelRow) bool { - if len(e.allowedCategoryIDs) == 0 { + if !e.categoryScopeSet { return true } _, ok := e.allowedCategoryIDs[channel.ID] @@ -159,7 +177,7 @@ func (e channelExclusions) allowsUnparentedStoredChannel(channel store.ChannelRo } func (e channelExclusions) allowsDiscordCategory(channel *discordgo.Channel, channelByID map[string]*discordgo.Channel) bool { - if len(e.allowedCategoryIDs) == 0 { + if !e.categoryScopeSet { return true } for current, seen := channel, map[string]struct{}{}; current != nil; { @@ -179,7 +197,7 @@ func (e channelExclusions) allowsDiscordCategory(channel *discordgo.Channel, cha } func (e channelExclusions) allowsStoredCategory(channel store.ChannelRow, channelByID map[string]store.ChannelRow) bool { - if len(e.allowedCategoryIDs) == 0 { + if !e.categoryScopeSet { return true } for current, seen := channel, map[string]struct{}{}; ; { @@ -209,15 +227,23 @@ func (s *Syncer) effectiveChannelExclusions(opts SyncOptions) channelExclusions } func filterExcludedDiscordChannels(channels []*discordgo.Channel, exclusions channelExclusions) []*discordgo.Channel { - if len(exclusions.ids) == 0 && len(exclusions.kinds) == 0 && len(exclusions.allowedCategoryIDs) == 0 { - return channels - } channelByID := make(map[string]*discordgo.Channel, len(channels)) for _, channel := range channels { if channel != nil { channelByID[channel.ID] = channel } } + return filterExcludedDiscordChannelsWithCatalog(channels, channelByID, exclusions) +} + +func filterExcludedDiscordChannelsWithCatalog( + channels []*discordgo.Channel, + channelByID map[string]*discordgo.Channel, + exclusions channelExclusions, +) []*discordgo.Channel { + if len(exclusions.ids) == 0 && len(exclusions.kinds) == 0 && !exclusions.categoryScopeSet { + return channels + } out := make([]*discordgo.Channel, 0, len(channels)) for _, channel := range channels { if channel != nil && !exclusions.excludesDiscordChannel(channel, channelByID) { @@ -229,7 +255,7 @@ func filterExcludedDiscordChannels(channels []*discordgo.Channel, exclusions cha func (s *Syncer) filterExcludedStoredChannelIDs(ctx context.Context, guildID string, channelIDs []string, opts SyncOptions) ([]string, error) { exclusions := s.effectiveChannelExclusions(opts) - if len(exclusions.ids) == 0 && len(exclusions.kinds) == 0 && len(exclusions.allowedCategoryIDs) == 0 { + if len(exclusions.ids) == 0 && len(exclusions.kinds) == 0 && !exclusions.categoryScopeSet { return channelIDs, nil } channels, err := s.store.Channels(ctx, guildID) diff --git a/internal/syncer/channel_exclusions_test.go b/internal/syncer/channel_exclusions_test.go index cca93e46..3418a4de 100644 --- a/internal/syncer/channel_exclusions_test.go +++ b/internal/syncer/channel_exclusions_test.go @@ -4,6 +4,7 @@ import ( "context" "path/filepath" "testing" + "time" "github.com/bwmarrin/discordgo" "github.com/stretchr/testify/require" @@ -65,6 +66,197 @@ func TestCategoryScopeFiltersIncompleteStoredChannels(t *testing.T) { require.Equal(t, []string{"allowed-text"}, filtered) } +func TestCategoryScopeIntersectionDeniesDisjointScopes(t *testing.T) { + t.Parallel() + + configured := newChannelScope(nil, nil, []string{"category-a"}) + requested := newChannelScope(nil, nil, []string{"category-b"}) + merged := configured.merged(requested) + + require.True(t, merged.categoryScopeSet) + require.Empty(t, merged.allowedCategoryIDs) + channel := &discordgo.Channel{ID: "text-a", ParentID: "category-a", Type: discordgo.ChannelTypeGuildText} + require.True(t, merged.excludesDiscordChannel(channel, map[string]*discordgo.Channel{channel.ID: channel})) +} + +func TestCategoryScopeLeavesEmptyConfigurationUnrestricted(t *testing.T) { + t.Parallel() + + scope := newChannelScope(nil, nil, nil) + require.False(t, scope.categoryScopeSet) + channel := &discordgo.Channel{ID: "root-text", Type: discordgo.ChannelTypeGuildText} + require.False(t, scope.excludesDiscordChannel(channel, map[string]*discordgo.Channel{channel.ID: channel})) +} + +func TestCategoryScopeMergePreservesUnsetAndIntersectionStates(t *testing.T) { + t.Parallel() + + unset := newChannelScope(nil, nil, nil) + categoryA := newChannelScope(nil, nil, []string{"category-a"}) + categoryAB := newChannelScope(nil, nil, []string{"category-a", "category-b"}) + + merged := unset.merged(unset) + require.False(t, merged.categoryScopeSet) + require.Nil(t, merged.allowedCategoryIDs) + + merged = unset.merged(categoryA) + require.True(t, merged.categoryScopeSet) + require.Equal(t, map[string]struct{}{"category-a": {}}, merged.allowedCategoryIDs) + + merged = categoryA.merged(unset) + require.True(t, merged.categoryScopeSet) + require.Equal(t, map[string]struct{}{"category-a": {}}, merged.allowedCategoryIDs) + + merged = categoryA.merged(categoryAB) + require.True(t, merged.categoryScopeSet) + require.Equal(t, map[string]struct{}{"category-a": {}}, merged.allowedCategoryIDs) +} + +func TestTargetedStoredThreadUsesFullCategoryAncestry(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + for _, channel := range []store.ChannelRecord{ + {ID: "category-a", GuildID: "g1", Kind: "category", RawJSON: `{}`}, + {ID: "forum-a", GuildID: "g1", ParentID: "category-a", Kind: "forum", RawJSON: `{}`}, + {ID: "thread-a", GuildID: "g1", ParentID: "forum-a", Kind: "thread_public", RawJSON: `{}`}, + } { + require.NoError(t, s.UpsertChannel(ctx, channel)) + } + + svc := New(&fakeClient{}, s, nil) + svc.SetIncludedCategories([]string{"category-a"}) + channels, targeted, err := svc.channelList( + ctx, + "g1", + []string{"thread-a"}, + channelCatalogFull, + svc.effectiveChannelExclusions(SyncOptions{}), + ) + require.NoError(t, err) + require.True(t, targeted) + require.Equal(t, []string{"thread-a"}, channelIDs(channels)) +} + +func TestSyncAppliesCategoryAndChannelExclusions(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + now := time.Now().UTC() + client := &fakeClient{ + guilds: []*discordgo.UserGuild{{ID: "g1", Name: "Guild"}}, + guildByID: map[string]*discordgo.Guild{"g1": {ID: "g1", Name: "Guild"}}, + channels: map[string][]*discordgo.Channel{"g1": { + {ID: "category-a", GuildID: "g1", Type: discordgo.ChannelTypeGuildCategory}, + {ID: "category-b", GuildID: "g1", Type: discordgo.ChannelTypeGuildCategory}, + {ID: "allowed", GuildID: "g1", ParentID: "category-a", Type: discordgo.ChannelTypeGuildText}, + {ID: "blocked-id", GuildID: "g1", ParentID: "category-a", Type: discordgo.ChannelTypeGuildText}, + {ID: "blocked-kind", GuildID: "g1", ParentID: "category-a", Type: discordgo.ChannelTypeGuildNews}, + {ID: "outside", GuildID: "g1", ParentID: "category-b", Type: discordgo.ChannelTypeGuildText}, + }}, + messages: map[string][]*discordgo.Message{ + "allowed": {{ID: "1", GuildID: "g1", ChannelID: "allowed", Content: "keep", Timestamp: now, Author: &discordgo.User{ID: "u1"}}}, + "blocked-id": {{ID: "2", GuildID: "g1", ChannelID: "blocked-id", Content: "drop", Timestamp: now, Author: &discordgo.User{ID: "u1"}}}, + "blocked-kind": {{ID: "3", GuildID: "g1", ChannelID: "blocked-kind", Content: "drop", Timestamp: now, Author: &discordgo.User{ID: "u1"}}}, + "outside": {{ID: "4", GuildID: "g1", ChannelID: "outside", Content: "drop", Timestamp: now, Author: &discordgo.User{ID: "u1"}}}, + }, + } + + svc := New(client, s, nil) + svc.SetIncludedCategories([]string{"category-a"}) + svc.SetChannelExclusions([]string{"blocked-id"}, []string{"announcement"}) + stats, err := svc.Sync(ctx, SyncOptions{GuildIDs: []string{"g1"}, SkipMembers: true}) + require.NoError(t, err) + require.Equal(t, 1, stats.Messages) + require.Equal(t, 1, client.messageCalls["allowed"]) + require.Zero(t, client.messageCalls["blocked-id"]) + require.Zero(t, client.messageCalls["blocked-kind"]) + require.Zero(t, client.messageCalls["outside"]) +} + +func TestTailAppliesCategoryAndChannelExclusions(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + for _, channel := range []store.ChannelRecord{ + {ID: "category-a", GuildID: "g1", Kind: "category", RawJSON: `{}`}, + {ID: "category-b", GuildID: "g1", Kind: "category", RawJSON: `{}`}, + {ID: "allowed", GuildID: "g1", ParentID: "category-a", Kind: "text", RawJSON: `{}`}, + {ID: "blocked-id", GuildID: "g1", ParentID: "category-a", Kind: "text", RawJSON: `{}`}, + {ID: "blocked-kind", GuildID: "g1", ParentID: "category-a", Kind: "announcement", RawJSON: `{}`}, + {ID: "outside", GuildID: "g1", ParentID: "category-b", Kind: "text", RawJSON: `{}`}, + } { + require.NoError(t, s.UpsertChannel(ctx, channel)) + } + + handler := &tailHandler{ + guilds: makeGuildSet([]string{"g1"}), + store: s, + exclusions: newChannelScope([]string{"blocked-id"}, []string{"announcement"}, []string{"category-a"}), + kindExcludedChannelIDs: map[string]struct{}{}, + knownChannelIDs: map[string]struct{}{}, + } + require.NoError(t, handler.seedChannelExclusions(ctx)) + + for index, channelID := range []string{"allowed", "blocked-id", "blocked-kind", "outside"} { + require.NoError(t, handler.OnMessageCreate(ctx, &discordgo.Message{ + ID: string(rune('1' + index)), + GuildID: "g1", + ChannelID: channelID, + Content: channelID, + Timestamp: time.Now().UTC(), + Author: &discordgo.User{ID: "u1"}, + })) + } + + require.NoError(t, handler.OnChannelUpsert(ctx, &discordgo.Channel{ + ID: "new-thread", + GuildID: "g1", + ParentID: "allowed", + Type: discordgo.ChannelTypeGuildPublicThread, + })) + require.NoError(t, handler.OnMessageCreate(ctx, &discordgo.Message{ + ID: "5", + GuildID: "g1", + ChannelID: "new-thread", + Content: "new thread", + Timestamp: time.Now().UTC(), + Author: &discordgo.User{ID: "u1"}, + })) + + messages, err := s.ListMessages(ctx, store.MessageListOptions{GuildIDs: []string{"g1"}, IncludeEmpty: true}) + require.NoError(t, err) + require.Len(t, messages, 2) + require.Equal(t, []string{"allowed", "new-thread"}, []string{messages[0].ChannelID, messages[1].ChannelID}) +} + +func TestTailRepairOffsetScheduling(t *testing.T) { + t.Parallel() + + now := time.Date(2026, 7, 25, 3, 0, 0, 0, time.UTC) + require.Equal(t, 6*time.Hour, nextTailRepairDelay(now, 6*time.Hour, 0)) + require.Equal(t, 5*time.Hour, nextTailRepairDelay(now, 6*time.Hour, 2*time.Hour)) + require.Equal(t, 5*time.Hour, nextTailRepairDelay(now, 6*time.Hour, 8*time.Hour)) + + svc := &Syncer{} + svc.SetRepairOffset(-time.Hour) + require.Zero(t, svc.repairOffset()) + svc.SetRepairOffset(90 * time.Minute) + require.Equal(t, 90*time.Minute, svc.repairOffset()) +} + func TestTailRepairUsesIncrementalScope(t *testing.T) { t.Parallel() diff --git a/internal/syncer/message_sync.go b/internal/syncer/message_sync.go index eaa838be..2571b59f 100644 --- a/internal/syncer/message_sync.go +++ b/internal/syncer/message_sync.go @@ -18,7 +18,7 @@ func (s *Syncer) syncMessageChannels( channels []*discordgo.Channel, opts SyncOptions, ) (int, error) { - messageChannels := filterMessageChannels(channels, opts.ChannelIDs, s.effectiveChannelExclusions(opts)) + messageChannels := filterMessageChannels(channels, opts.ChannelIDs) if len(messageChannels) == 0 { return 0, nil } @@ -38,11 +38,7 @@ func (s *Syncer) syncMessageChannels( return total, err } -func filterMessageChannels(channels []*discordgo.Channel, requested []string, configured ...channelExclusions) []*discordgo.Channel { - exclusions := channelExclusions{} - if len(configured) > 0 { - exclusions = configured[0] - } +func filterMessageChannels(channels []*discordgo.Channel, requested []string) []*discordgo.Channel { requestedSet := makeGuildSet(requested) channelByID := make(map[string]*discordgo.Channel, len(channels)) for _, channel := range channels { @@ -55,9 +51,6 @@ func filterMessageChannels(channels []*discordgo.Channel, requested []string, co if !isMessageChannel(channel) { continue } - if exclusions.excludesDiscordChannel(channel, channelByID) { - continue - } if len(requestedSet) > 0 && !requestedMessageTarget(channel, channelByID, requestedSet) { continue } diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go index 124d1d8b..5fd9e14b 100644 --- a/internal/syncer/syncer.go +++ b/internal/syncer/syncer.go @@ -77,7 +77,9 @@ func (s *Syncer) SetChannelExclusions(channelIDs, channelKinds []string) { } func (s *Syncer) SetIncludedCategories(categoryIDs []string) { - s.channelExclusions.allowedCategoryIDs = normalizedStringSet(categoryIDs, false) + categories := normalizedStringSet(categoryIDs, false) + s.channelExclusions.allowedCategoryIDs = categories + s.channelExclusions.categoryScopeSet = len(categories) > 0 } func (s *Syncer) SetRepairOffset(offset time.Duration) { @@ -200,7 +202,6 @@ func (s *Syncer) syncGuild(ctx context.Context, guildID string, opts SyncOptions if err != nil { return stats, err } - channelList = filterExcludedDiscordChannels(channelList, exclusions) if err := s.storeChannelList(ctx, channelList, &stats); err != nil { return stats, err } diff --git a/internal/syncer/tail.go b/internal/syncer/tail.go index 637a3df5..6c90e79e 100644 --- a/internal/syncer/tail.go +++ b/internal/syncer/tail.go @@ -480,7 +480,7 @@ func (t *tailHandler) excludeChannel(channelID string) bool { if _, ok := t.kindExcludedChannelIDs[channelID]; ok { return true } - if len(t.exclusions.allowedCategoryIDs) == 0 { + if !t.exclusions.categoryScopeSet { return false } _, known := t.knownChannelIDs[channelID] @@ -496,15 +496,9 @@ func (t *tailHandler) trackChannelExclusion(channel *discordgo.Channel) { excluded = !t.exclusions.allowsUnparentedDiscordChannel(channel) } if !excluded && channel.ParentID != "" { - if t.exclusions.excludesID(channel.ParentID) { - excluded = true - } else if len(t.exclusions.allowedCategoryIDs) > 0 { - if _, allowedCategory := t.exclusions.allowedCategoryIDs[channel.ParentID]; allowedCategory { - excluded = false - } else { - excluded = t.excludeChannel(channel.ParentID) - } - } else { + excluded = t.exclusions.excludesID(channel.ParentID) + _, allowedCategory := t.exclusions.allowedCategoryIDs[channel.ParentID] + if !excluded && (!t.exclusions.categoryScopeSet || !allowedCategory) { excluded = t.excludeChannel(channel.ParentID) } } From 973ee475db28b012ba8753c0fd836228ba43e15d Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 25 Jul 2026 11:03:18 -0700 Subject: [PATCH 3/3] docs: explain Discord collection scope --- CHANGELOG.md | 4 ++++ README.md | 16 ++++++++++++++++ SPEC.md | 4 ++++ docs/configuration.md | 8 ++++++++ 4 files changed, 32 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e47bde6..03e5a499 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## 0.11.9 - Unreleased +### Changes + +- Scope Discord history and live tail collection with category allowlists and channel id/kind exclusions, capture new and updated threads immediately, and support wall-clock repair offsets. Thanks @hannesrudolph. + ## 0.11.8 - 2026-07-20 ### Highlights diff --git a/README.md b/README.md index 5a8ac8d8..1a8c6483 100644 --- a/README.md +++ b/README.md @@ -796,6 +796,10 @@ token_keyring_account = "discord_bot_token" source = "both" # use "discord" for bot-only sync or "wiretap" for desktop-cache-only import concurrency = 16 repair_every = "6h" +repair_offset = "0s" # non-zero values align repairs to local wall-clock boundaries +include_category_ids = [] +exclude_channel_ids = [] +exclude_channel_kinds = [] full_history = true attachment_text = true attachment_media = false @@ -834,6 +838,18 @@ stale_after = "" The value above is an example. `init` writes an auto-sized default based on the host: `min(32, max(8, GOMAXPROCS*2))`. +Discord collection can be restricted with `sync.include_category_ids`, which +keeps the listed categories and their channel/thread descendants. Root-level +channels and channels under other categories are skipped while the allowlist is +set. `sync.exclude_channel_ids` and `sync.exclude_channel_kinds` apply to both +historical sync and live tail events and always win over category inclusion. +Kinds use Discrawl's channel names such as `text`, `announcement`, `forum`, +`thread_public`, `thread_private`, and `thread_announcement`. + +`sync.repair_offset` shifts a non-zero periodic tail repair interval onto local +wall-clock boundaries. For example, `repair_every = "6h"` with +`repair_offset = "2h"` runs around 02:00, 08:00, 14:00, and 20:00 local time. + Config override rules: - `--config` beats everything diff --git a/SPEC.md b/SPEC.md index 3c314183..407181a4 100644 --- a/SPEC.md +++ b/SPEC.md @@ -598,6 +598,10 @@ token_keyring_account = "discord_bot_token" [sync] concurrency = 4 repair_every = "6h" +repair_offset = "0s" +include_category_ids = [] +exclude_channel_ids = [] +exclude_channel_kinds = [] full_history = true [search] diff --git a/docs/configuration.md b/docs/configuration.md index 0212aa04..79f47763 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -60,6 +60,10 @@ token_keyring_account = "discord_bot_token" source = "both" # "discord" for bot-only sync, "wiretap" for desktop-cache-only import concurrency = 16 repair_every = "6h" +repair_offset = "0s" +include_category_ids = [] +exclude_channel_ids = [] +exclude_channel_kinds = [] full_history = true attachment_text = true attachment_media = false @@ -130,6 +134,10 @@ Set `discord.token_source = "keyring"` if you want to require keyring lookup and - `default_guild_id` is the implicit scope for `sync`, `tail`, `digest`, and `analytics` when `--guild` is not passed - `guild_ids` is reserved for explicit multi-guild fan-out; usually you do not set this directly +- `sync.include_category_ids` limits Discord sync and tail collection to the listed categories and all channel/thread descendants; root-level and unrelated channels are skipped +- `sync.exclude_channel_ids` and `sync.exclude_channel_kinds` apply to historical sync, live tail events, and repair syncs; exclusions always win over category inclusion +- `sync.exclude_channel_kinds` accepts Discrawl kinds such as `text`, `announcement`, `forum`, `thread_public`, `thread_private`, and `thread_announcement` +- a non-zero `sync.repair_offset` aligns periodic repairs to local wall-clock boundaries; for example, `repair_every = "6h"` with `repair_offset = "2h"` targets 02:00, 08:00, 14:00, and 20:00 local time - changing `[search.embeddings]` provider/model/input version retargets pending jobs and resets prior attempts; existing vectors for another identity remain in SQLite but are not used for semantic search - `[search.embeddings].vector_backend` accepts `exact` or optional `turbovec`; turbovec requires Python plus the `turbovec` package and embedding dimensions divisible by 8. - changing `db_path` does not migrate existing data; copy the file yourself if you want to keep history