Scope Discord collection by category

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## 0.11.9 - Unreleased
 
+### Changes
+
+- Scope Discord history and live tail collection with category allowlists and channel id/kind exclusions, capture new and updated threads immediately, and support wall-clock repair offsets. Thanks @hannesrudolph.
+
 ## 0.11.8 - 2026-07-20
 
 ### Highlights

README.md

@@ -796,6 +796,10 @@ token_keyring_account = "discord_bot_token"
 source = "both" # use "discord" for bot-only sync or "wiretap" for desktop-cache-only import
 concurrency = 16
 repair_every = "6h"
+repair_offset = "0s" # non-zero values align repairs to local wall-clock boundaries
+include_category_ids = []
+exclude_channel_ids = []
+exclude_channel_kinds = []
 full_history = true
 attachment_text = true
 attachment_media = false
@@ -834,6 +838,18 @@ stale_after = ""
 
 The value above is an example. `init` writes an auto-sized default based on the host: `min(32, max(8, GOMAXPROCS*2))`.
 
+Discord collection can be restricted with `sync.include_category_ids`, which
+keeps the listed categories and their channel/thread descendants. Root-level
+channels and channels under other categories are skipped while the allowlist is
+set. `sync.exclude_channel_ids` and `sync.exclude_channel_kinds` apply to both
+historical sync and live tail events and always win over category inclusion.
+Kinds use Discrawl's channel names such as `text`, `announcement`, `forum`,
+`thread_public`, `thread_private`, and `thread_announcement`.
+
+`sync.repair_offset` shifts a non-zero periodic tail repair interval onto local
+wall-clock boundaries. For example, `repair_every = "6h"` with
+`repair_offset = "2h"` runs around 02:00, 08:00, 14:00, and 20:00 local time.
+
 Config override rules:
 
 - `--config` beats everything

SPEC.md

@@ -598,6 +598,10 @@ token_keyring_account = "discord_bot_token"
 [sync]
 concurrency = 4
 repair_every = "6h"
+repair_offset = "0s"
+include_category_ids = []
+exclude_channel_ids = []
+exclude_channel_kinds = []
 full_history = true
 
 [search]

docs/configuration.md

@@ -60,6 +60,10 @@ token_keyring_account = "discord_bot_token"
 source = "both" # "discord" for bot-only sync, "wiretap" for desktop-cache-only import
 concurrency = 16
 repair_every = "6h"
+repair_offset = "0s"
+include_category_ids = []
+exclude_channel_ids = []
+exclude_channel_kinds = []
 full_history = true
 attachment_text = true
 attachment_media = false
@@ -130,6 +134,10 @@ Set `discord.token_source = "keyring"` if you want to require keyring lookup and
 
 - `default_guild_id` is the implicit scope for `sync`, `tail`, `digest`, and `analytics` when `--guild` is not passed
 - `guild_ids` is reserved for explicit multi-guild fan-out; usually you do not set this directly
+- `sync.include_category_ids` limits Discord sync and tail collection to the listed categories and all channel/thread descendants; root-level and unrelated channels are skipped
+- `sync.exclude_channel_ids` and `sync.exclude_channel_kinds` apply to historical sync, live tail events, and repair syncs; exclusions always win over category inclusion
+- `sync.exclude_channel_kinds` accepts Discrawl kinds such as `text`, `announcement`, `forum`, `thread_public`, `thread_private`, and `thread_announcement`
+- a non-zero `sync.repair_offset` aligns periodic repairs to local wall-clock boundaries; for example, `repair_every = "6h"` with `repair_offset = "2h"` targets 02:00, 08:00, 14:00, and 20:00 local time
 - changing `[search.embeddings]` provider/model/input version retargets pending jobs and resets prior attempts; existing vectors for another identity remain in SQLite but are not used for semantic search
 - `[search.embeddings].vector_backend` accepts `exact` or optional `turbovec`; turbovec requires Python plus the `turbovec` package and embedding dimensions divisible by 8.
 - changing `db_path` does not migrate existing data; copy the file yourself if you want to keep history

internal/cli/cli.go

@@ -308,6 +308,18 @@ type attachmentTextConfigurer interface {
 	SetAttachmentTextEnabled(bool)
 }
 
+type channelExclusionConfigurer interface {
+	SetChannelExclusions([]string, []string)
+}
+
+type categoryInclusionConfigurer interface {
+	SetIncludedCategories([]string)
+}
+
+type repairOffsetConfigurer interface {
+	SetRepairOffset(time.Duration)
+}
+
 func (r *runtime) dispatch(rest []string) error {
 	switch rest[0] {
 	case "metadata":
@@ -766,6 +778,19 @@ func (r *runtime) ensureDiscordServices() error {
 	if configurable, ok := r.syncer.(attachmentTextConfigurer); ok {
 		configurable.SetAttachmentTextEnabled(r.cfg.AttachmentTextEnabled())
 	}
+	if configurable, ok := r.syncer.(channelExclusionConfigurer); ok {
+		configurable.SetChannelExclusions(r.cfg.Sync.ExcludeChannelIDs, r.cfg.Sync.ExcludeChannelKinds)
+	}
+	if configurable, ok := r.syncer.(categoryInclusionConfigurer); ok {
+		configurable.SetIncludedCategories(r.cfg.Sync.IncludeCategoryIDs)
+	}
+	if configurable, ok := r.syncer.(repairOffsetConfigurer); ok {
+		offset, err := time.ParseDuration(r.cfg.Sync.RepairOffset)
+		if err != nil {
+			return configErr(fmt.Errorf("parse sync.repair_offset: %w", err))
+		}
+		configurable.SetRepairOffset(offset)
+	}
 	return nil
 }
 

internal/cli/cli_test.go

@@ -3662,6 +3662,10 @@ type fakeSyncService struct {
 	replayStats           syncer.TailMessageReplayStats
 	replayErr             error
 	attachmentTextEnabled bool
+	excludeChannelIDs     []string
+	excludeChannelKinds   []string
+	includeCategoryIDs    []string
+	repairOffset          time.Duration
 	callTailReady         bool
 	tailReadyCalls        int
 	tailReady             func(context.Context) error
@@ -3712,6 +3716,19 @@ func (f *fakeSyncService) SetAttachmentTextEnabled(enabled bool) {
 	f.attachmentTextEnabled = enabled
 }
 
+func (f *fakeSyncService) SetChannelExclusions(channelIDs, channelKinds []string) {
+	f.excludeChannelIDs = append([]string(nil), channelIDs...)
+	f.excludeChannelKinds = append([]string(nil), channelKinds...)
+}
+
+func (f *fakeSyncService) SetIncludedCategories(categoryIDs []string) {
+	f.includeCategoryIDs = append([]string(nil), categoryIDs...)
+}
+
+func (f *fakeSyncService) SetRepairOffset(offset time.Duration) {
+	f.repairOffset = offset
+}
+
 type hybridSyncService struct {
 	store         *store.Store
 	sawGitMessage bool
@@ -3797,6 +3814,10 @@ func TestRuntimeInitSyncTailAndDoctor(t *testing.T) {
 	require.Equal(t, "g2", cfg.DefaultGuildID)
 	require.True(t, cfg.Search.Embeddings.Enabled)
 	cfg.Desktop.Path = filepath.Join(dir, "empty-discord")
+	cfg.Sync.ExcludeChannelIDs = []string{"blocked-channel"}
+	cfg.Sync.ExcludeChannelKinds = []string{"announcement"}
+	cfg.Sync.IncludeCategoryIDs = []string{"category-a"}
+	cfg.Sync.RepairOffset = "45m"
 	require.NoError(t, os.MkdirAll(cfg.Desktop.Path, 0o755))
 	require.NoError(t, config.Write(cfgPath, cfg))
 
@@ -3806,6 +3827,10 @@ func TestRuntimeInitSyncTailAndDoctor(t *testing.T) {
 	require.True(t, fakeSync.lastSync.LatestOnly)
 	require.True(t, fakeSync.lastSync.SkipMembers)
 	require.True(t, fakeSync.attachmentTextEnabled)
+	require.Equal(t, []string{"blocked-channel"}, fakeSync.excludeChannelIDs)
+	require.Equal(t, []string{"announcement"}, fakeSync.excludeChannelKinds)
+	require.Equal(t, []string{"category-a"}, fakeSync.includeCategoryIDs)
+	require.Equal(t, 45*time.Minute, fakeSync.repairOffset)
 
 	rt = newRuntime()
 	require.NoError(t, rt.withServices(true, func() error { return rt.runSync([]string{"--guilds", "g2", "--with-members"}) }))
@@ -4165,10 +4190,20 @@ func TestCommandHelpDoesNotOpenConfigOrStore(t *testing.T) {
 	require.Contains(t, stdout.String(), "--stats")
 	require.Empty(t, stderr.String())
 
+	stdout.Reset()
+	require.NoError(t, Run(context.Background(), []string{"sync", "--help"}, &stdout, &stderr))
+	require.Contains(t, stdout.String(), "sync.include_category_ids")
+	require.Contains(t, stdout.String(), "sync.exclude_channel_ids")
+	require.Contains(t, stdout.String(), "sync.exclude_channel_kinds")
+	require.Empty(t, stderr.String())
+
 	stdout.Reset()
 	require.NoError(t, Run(context.Background(), []string{"tail", "--help"}, &stdout, &stderr))
 	require.Contains(t, stdout.String(), "--replay-failures-only")
 	require.Contains(t, stdout.String(), "--replay-limit N")
+	require.Contains(t, stdout.String(), "sync.include_category_ids")
+	require.Contains(t, stdout.String(), "sync.exclude_channel_ids")
+	require.Contains(t, stdout.String(), "sync.exclude_channel_kinds")
 	require.Empty(t, stderr.String())
 
 	err := Run(context.Background(), []string{"help", "wat"}, &bytes.Buffer{}, &bytes.Buffer{})

internal/cli/output.go

@@ -125,10 +125,14 @@ Discover accessible guilds and initialize configuration.
 	"sync": `Usage: discrawl sync [--full] [--all] [--all-channels] [--since RFC3339] [--channels IDS] [--concurrency N] [--source SOURCE] [--with-embeddings] [--with-media] [--skip-members|--with-members] [--latest-only] [--guild ID|--guilds IDS] [--update MODE|--no-update]
 
 Sync Discord or desktop-cache data into the local archive.
+Configure Discord collection scope with sync.include_category_ids,
+sync.exclude_channel_ids, and sync.exclude_channel_kinds; exclusions win.
 `,
 	"tail": `Usage: discrawl tail [--repair-every DURATION] [--guild ID|--guilds IDS] [--replay-failures-only [--replay-limit N]]
 
 Continuously archive new Discord messages.
+The sync.include_category_ids, sync.exclude_channel_ids, and
+sync.exclude_channel_kinds settings apply to live events and repair syncs.
 `,
 	"wiretap": `Usage:
   discrawl wiretap [flags]

internal/config/config.go

@@ -51,13 +51,17 @@ type DesktopConfig struct {
 }
 
 type SyncConfig struct {
-	Source             string `toml:"source"`
-	Concurrency        int    `toml:"concurrency"`
-	RepairEvery        string `toml:"repair_every"`
-	FullHistory        bool   `toml:"full_history"`
-	AttachmentText     *bool  `toml:"attachment_text"`
-	AttachmentMedia    *bool  `toml:"attachment_media"`
-	MaxAttachmentBytes int64  `toml:"max_attachment_bytes"`
+	Source              string   `toml:"source"`
+	Concurrency         int      `toml:"concurrency"`
+	RepairEvery         string   `toml:"repair_every"`
+	RepairOffset        string   `toml:"repair_offset"`
+	FullHistory         bool     `toml:"full_history"`
+	ExcludeChannelIDs   []string `toml:"exclude_channel_ids,omitempty"`
+	ExcludeChannelKinds []string `toml:"exclude_channel_kinds,omitempty"`
+	IncludeCategoryIDs  []string `toml:"include_category_ids,omitempty"`
+	AttachmentText      *bool    `toml:"attachment_text"`
+	AttachmentMedia     *bool    `toml:"attachment_media"`
+	MaxAttachmentBytes  int64    `toml:"max_attachment_bytes"`
 }
 
 type SearchConfig struct {
@@ -138,6 +142,7 @@ func Default() Config {
 			Source:             "both",
 			Concurrency:        defaultSyncConcurrency(),
 			RepairEvery:        "6h",
+			RepairOffset:       "0s",
 			FullHistory:        true,
 			AttachmentText:     new(true),
 			AttachmentMedia:    new(false),
@@ -268,6 +273,16 @@ func (c *Config) Normalize() error {
 	if c.Sync.RepairEvery == "" {
 		c.Sync.RepairEvery = "6h"
 	}
+	c.Sync.RepairOffset = strings.TrimSpace(c.Sync.RepairOffset)
+	if c.Sync.RepairOffset == "" {
+		c.Sync.RepairOffset = "0s"
+	}
+	if _, err := time.ParseDuration(c.Sync.RepairOffset); err != nil {
+		return fmt.Errorf("parse sync.repair_offset: %w", err)
+	}
+	c.Sync.ExcludeChannelIDs = uniqueStrings(c.Sync.ExcludeChannelIDs)
+	c.Sync.ExcludeChannelKinds = uniqueLowerStrings(c.Sync.ExcludeChannelKinds)
+	c.Sync.IncludeCategoryIDs = uniqueStrings(c.Sync.IncludeCategoryIDs)
 	if c.Sync.AttachmentText == nil {
 		c.Sync.AttachmentText = new(true)
 	}
@@ -439,3 +454,11 @@ func uniqueStrings(in []string) []string {
 	}
 	return out
 }
+
+func uniqueLowerStrings(in []string) []string {
+	out := uniqueStrings(in)
+	for i := range out {
+		out[i] = strings.ToLower(out[i])
+	}
+	return uniqueStrings(out)
+}

internal/config/config_test.go

@@ -325,6 +325,10 @@ func TestWriteAndLoadRoundTrip(t *testing.T) {
 	cfg := Default()
 	cfg.DefaultGuildID = "g1"
 	cfg.GuildIDs = []string{"g1", "g2"}
+	cfg.Sync.RepairOffset = "45m"
+	cfg.Sync.ExcludeChannelIDs = []string{"c1", "c2"}
+	cfg.Sync.ExcludeChannelKinds = []string{"announcement"}
+	cfg.Sync.IncludeCategoryIDs = []string{"category-a"}
 	require.NoError(t, Write(path, cfg))
 
 	loaded, err := Load(path)
@@ -334,6 +338,28 @@ func TestWriteAndLoadRoundTrip(t *testing.T) {
 	require.Equal(t, DefaultRemoteTokenEnv, loaded.Remote.TokenEnv)
 	require.NotNil(t, loaded.Sync.AttachmentText)
 	require.True(t, *loaded.Sync.AttachmentText)
+	require.Equal(t, "45m", loaded.Sync.RepairOffset)
+	require.Equal(t, []string{"c1", "c2"}, loaded.Sync.ExcludeChannelIDs)
+	require.Equal(t, []string{"announcement"}, loaded.Sync.ExcludeChannelKinds)
+	require.Equal(t, []string{"category-a"}, loaded.Sync.IncludeCategoryIDs)
+}
+
+func TestNormalizeSyncCollectionScope(t *testing.T) {
+	t.Parallel()
+
+	cfg := Default()
+	cfg.Sync.RepairOffset = " 30m "
+	cfg.Sync.ExcludeChannelIDs = []string{" c1 ", "", "c1", "c2"}
+	cfg.Sync.ExcludeChannelKinds = []string{" Announcement ", "TEXT", "announcement"}
+	cfg.Sync.IncludeCategoryIDs = []string{" category-a ", "category-a", "category-b"}
+	require.NoError(t, cfg.Normalize())
+	require.Equal(t, "30m", cfg.Sync.RepairOffset)
+	require.Equal(t, []string{"c1", "c2"}, cfg.Sync.ExcludeChannelIDs)
+	require.Equal(t, []string{"announcement", "text"}, cfg.Sync.ExcludeChannelKinds)
+	require.Equal(t, []string{"category-a", "category-b"}, cfg.Sync.IncludeCategoryIDs)
+
+	cfg.Sync.RepairOffset = "later"
+	require.ErrorContains(t, cfg.Normalize(), "parse sync.repair_offset")
 }
 
 func TestWriteRejectsNonPositiveEmbeddingTimeout(t *testing.T) {

internal/discord/client.go

@@ -608,6 +608,34 @@ func (c *Client) Tail(ctx context.Context, handler EventHandler) error {
 			before,
 		))
 	})
+	addHandler(func(_ *discordgo.Session, evt *discordgo.ThreadCreate) {
+		var channel *discordgo.Channel
+		if evt != nil {
+			channel = evt.Channel
+		}
+		c.enqueueTailTask(tailCtx, orderedWorkCh, fatal, newChannelTailTask(
+			"THREAD_CREATE",
+			func(taskCtx context.Context) error {
+				return handler.OnChannelUpsert(taskCtx, channel)
+			},
+			channel,
+		))
+	})
+	addHandler(func(_ *discordgo.Session, evt *discordgo.ThreadUpdate) {
+		var channel, before *discordgo.Channel
+		if evt != nil {
+			channel = evt.Channel
+			before = evt.BeforeUpdate
+		}
+		c.enqueueTailTask(tailCtx, orderedWorkCh, fatal, newChannelTailTask(
+			"THREAD_UPDATE",
+			func(taskCtx context.Context) error {
+				return handler.OnChannelUpsert(taskCtx, channel)
+			},
+			channel,
+			before,
+		))
+	})
 	addHandler(func(_ *discordgo.Session, evt *discordgo.GuildCreate) {
 		guildHandler, ok := handler.(guildEventHandler)
 		if !ok {

internal/discord/client_test.go

@@ -1099,8 +1099,10 @@ func TestTailReceivesGatewayEvents(t *testing.T) {
 			{"op": 0, "t": "MESSAGE_UPDATE", "s": 3, "d": map[string]any{"id": "m1", "guild_id": "g1", "channel_id": "c1", "content": "hello 2", "timestamp": now, "author": map[string]any{"id": "u1", "username": "user"}}},
 			{"op": 0, "t": "MESSAGE_DELETE", "s": 4, "d": map[string]any{"id": "m1", "guild_id": "g1", "channel_id": "c1"}},
 			{"op": 0, "t": "CHANNEL_CREATE", "s": 5, "d": map[string]any{"id": "c1", "guild_id": "g1", "name": "general", "type": 0}},
-			{"op": 0, "t": "GUILD_MEMBER_ADD", "s": 6, "d": map[string]any{"guild_id": "g1", "user": map[string]any{"id": "u1", "username": "user"}, "roles": []string{}}},
-			{"op": 0, "t": "GUILD_MEMBER_REMOVE", "s": 7, "d": map[string]any{"guild_id": "g1", "user": map[string]any{"id": "u1", "username": "user"}}},
+			{"op": 0, "t": "THREAD_CREATE", "s": 6, "d": map[string]any{"id": "thread-1", "guild_id": "g1", "parent_id": "c1", "name": "support request", "type": 11, "newly_created": true}},
+			{"op": 0, "t": "THREAD_UPDATE", "s": 7, "d": map[string]any{"id": "thread-1", "guild_id": "g1", "parent_id": "c1", "name": "support request", "type": 11}},
+			{"op": 0, "t": "GUILD_MEMBER_ADD", "s": 8, "d": map[string]any{"guild_id": "g1", "user": map[string]any{"id": "u1", "username": "user"}, "roles": []string{}}},
+			{"op": 0, "t": "GUILD_MEMBER_REMOVE", "s": 9, "d": map[string]any{"guild_id": "g1", "user": map[string]any{"id": "u1", "username": "user"}}},
 		}
 		for _, event := range events {
 			if err := conn.WriteJSON(event); err != nil {
@@ -1133,7 +1135,7 @@ func TestTailReceivesGatewayEvents(t *testing.T) {
 	require.Equal(t, 1, handler.creates)
 	require.Equal(t, 1, handler.updates)
 	require.Equal(t, 1, handler.deletes)
-	require.Equal(t, 1, handler.channels)
+	require.Equal(t, 3, handler.channels)
 	require.Equal(t, 1, handler.memberUpserts)
 	require.Equal(t, 1, handler.memberDeletes)
 	require.Equal(t, 1, handler.ready)