feat: add safe hybrid local archive collection

README.md

@@ -797,6 +797,8 @@ source = "both" # use "discord" for bot-only sync or "wiretap" for desktop-cache
 concurrency = 16
 repair_every = "6h"
 full_history = true
+exclude_channel_ids = []
+exclude_channel_kinds = []
 attachment_text = true
 attachment_media = false
 max_attachment_bytes = 104857600

docs/commands/sync.md

@@ -27,6 +27,7 @@ discrawl sync --source discord   # bot API only; aliases: key, bot, api
 discrawl sync --source wiretap   # desktop cache only; aliases: desktop, cache
 discrawl sync --guild 123456789012345678 --all-channels
 discrawl sync --channels 111,222 --since 2026-03-01T00:00:00Z
+discrawl sync --exclude-channels 333,444 --exclude-channel-kinds announcement
 discrawl sync --with-embeddings
 discrawl sync --with-media
 ```
@@ -59,8 +60,11 @@ discrawl sync --with-media
 - `--all` - ignore `default_guild_id` and fan out across every discovered guild
 - `--guild <id>` / `--guilds <id,id>` - target specific guilds
 - `--channels <id,id>` - target specific channels (forum ids expand to threads)
+- `--exclude-channels <id,id>` - omit channel messages even if a target or repair would otherwise select them
+- `--exclude-channel-kinds <kind,kind>` - omit message channels by stored Discord kind, such as `announcement`
 - `--since <RFC3339>` - limit initial history and `--full` backfill to messages at or after this timestamp
 - `--concurrency <n>` - override worker count (default auto-sized: floor 8, cap 32)
+- `--channel-timeout <duration>` - bound each channel crawl; the shorter parent deadline still wins
 - `--skip-members` - refresh guild/channel/message data without crawling members
 - `--with-members` - refresh guild members even during the default latest-only sync; fail if the member crawl cannot complete
 - `--with-embeddings` - also enqueue changed messages into `embedding_jobs`
@@ -69,6 +73,8 @@ discrawl sync --with-media
 ## Notes
 
 - `--latest-only` is the default for untargeted `sync`. Use `--all-channels` to opt out without doing a full historical crawl.
+- Collection exclusions apply to Discord API sync and the wiretap portion of `source=both`; exclusions win over explicit targets.
+- With `--update=auto`, the same exclusions filter message-scoped rows from the safe upstream merge while retaining channel metadata and existing local rows.
 - `--with-media` records expired or removed Discord CDN URLs as failed fetches with the HTTP status, commonly `404`.
 - `--with-media` updates the local cache only; run `publish --push` afterward to include cached non-DM media in the Git backup as gzip-compressed files.
 - `--since` does not mark older history as complete, so a later `sync --full` without `--since` can continue the backfill.

docs/commands/tail.md

@@ -9,6 +9,7 @@ discrawl tail
 discrawl tail --guild 123456789012345678
 discrawl tail --repair-every 30m
 discrawl tail --replay-failures-only
+discrawl tail --repair-every 6h --repair-offset 15m
 ```
 
 ## What it does
@@ -18,13 +19,16 @@ discrawl tail --replay-failures-only
 - periodically runs a repair pass to catch anything the live stream missed
 - can replay a bounded set of unresolved exact-message failures without
   starting the Gateway tail
+- ignores messages, edits, and deletes from channels excluded by
+  `[sync].exclude_channel_ids` or `[sync].exclude_channel_kinds`
 
 ## Flags
 
 - `--guild <id>` / `--guilds <id,id>` - tail a specific guild scope (default: `default_guild_id`, or all discovered guilds if unset)
 - `--repair-every <duration>` - frequency of the repair sweep
 - `--replay-failures-only` - replay unresolved exact-message tail failures and exit
 - `--replay-limit <n>` - maximum failures to inspect in replay-only mode (default and maximum: `25`)
+- `--repair-offset <duration>` - for positive values, align repairs to that wall-clock offset within each repair period
 
 ## Notes
 
@@ -33,6 +37,9 @@ discrawl tail --replay-failures-only
 - terminates cleanly on SIGINT / SIGTERM and treats cancellation as normal exit
 - replay-only mode uses the normal exclusive writer lock and does not create
   Gateway message events or update `tail:last_event`
+- retains excluded channel metadata while omitting their message traffic
+- positive-offset repairs skip missed aligned slots after a long repair rather than running catch-up sweeps
+- separation from another scheduled job is only stable when that job also uses a fixed calendar schedule; a drifting `StartInterval` schedule cannot guarantee the offset
 
 ## See also
 

docs/commands/update.md

@@ -37,6 +37,12 @@ discrawl update --force --ref backup-2026-06-19
 
 Normal updates preserve rows learned from live Discord or the desktop cache, even when those rows are absent from the Git snapshot. Generated event history and local sync cursors are not replayed during routine merges. If a safe merge is impossible, Discrawl keeps the current database, marks the snapshot as needing attention in `status --json`, and asks you to rerun with `--force`.
 
+Routine safe merges also honor `[sync].exclude_channel_ids` and
+`[sync].exclude_channel_kinds`. Excluded channel metadata and existing local
+rows remain in place, while new message-scoped snapshot rows for those channels
+are skipped. `--force` is still an explicit exact-reconciliation path and does
+not provide this preservation guarantee.
+
 ## How `sync` interacts
 
 `discrawl sync` does **not** auto-import the share unless `--update=auto` (safe merge when stale) or `--update=force` (exact replacement before live deltas). Routine live refreshes stay fast; explicit imports happen via `update`.

docs/configuration.md

@@ -60,7 +60,10 @@ token_keyring_account = "discord_bot_token"
 source = "both" # "discord" for bot-only sync, "wiretap" for desktop-cache-only import
 concurrency = 16
 repair_every = "6h"
+repair_offset = "0s"
 full_history = true
+exclude_channel_ids = []
+exclude_channel_kinds = [] # for example, ["announcement"]
 attachment_text = true
 attachment_media = false
 max_attachment_bytes = 104857600
@@ -104,6 +107,13 @@ token_env = "DISCRAWL_REMOTE_TOKEN"
 stale_after = ""
 ```
 
+`sync.exclude_channel_ids` and `sync.exclude_channel_kinds` apply to Discord API
+sync, live tail events, periodic tail repair, Discord Desktop cache import, and
+routine safe Git snapshot merges. Exclusions win over an explicit
+`sync --channels` target. Channel metadata and existing local rows are retained,
+but new message-scoped rows from excluded channels are omitted. Forced exact
+snapshot replacement remains a separate destructive operation.
+
 `concurrency` is auto-sized at `init` to `min(32, max(8, GOMAXPROCS*2))`.
 
 ## Token resolution

docs/guides/git-snapshots.md

@@ -65,6 +65,11 @@ discrawl subscribe --no-auto-update https://github.com/example/discord-archive.g
 
 `discrawl update` runs the same safe pull/merge step manually. `discrawl update --force --ref <tag-or-commit>` reads historical Git objects directly and leaves the share checkout unchanged. Snapshot imports are delta-planned from crawlkit shard fingerprints. Older manifests without those fields fall back to Git blob identity, so the common publish shape only imports changed canonical shards. Routine merges preserve destination-only rows and do not replay generated event history or remote sync cursors.
 
+In hybrid mode, routine safe merges honor the collection exclusions under
+`[sync]`. They keep excluded channel metadata and previously stored local rows,
+but skip new message-scoped rows from excluded channels. Forced replacement is
+an explicit destructive exception.
+
 Discrawl does not silently fall back to a full import. Removed shards and incompatible table changes leave the current database intact and require `discrawl update --force`. Forced updates replace public snapshot tables and rebuild search indexes; local DM rows remain untouched.
 
 `discrawl sync` does **not** auto-import the share unless `--update=auto` or `--update=force` is provided. Auto mode uses the safe merge; force mode performs exact replacement before live deltas.

docs/guides/sync-sources.md

@@ -35,8 +35,16 @@ Run one explicit `--full` pass when you want a complete historical guild archive
 - `--guilds 123,456` runs an explicit set
 - `--all` ignores `default_guild_id` and fans out across every discovered guild
 - `--channels 111,222` targets specific channels (forum ids expand to their threads)
+- `--exclude-channels 333,444` omits specific channel messages
+- `--exclude-channel-kinds announcement` omits message channels by Discord kind
 - `--since <RFC3339>` limits initial history and `--full` backfill to messages at or after the timestamp; older history is not marked complete, so a later `sync --full` without `--since` can continue the backfill
 
+The same exclusions apply to routine sync, historical backfill, live tail,
+periodic tail repair, Discord Desktop cache import, and safe upstream snapshot
+merges. Exclusions win over explicit channel targets. Safe merges retain channel
+metadata and existing local rows while omitting new message-scoped rows for
+excluded channels.
+
 ## Performance and resilience
 
 - Long runs emit periodic progress logs to stderr.

go.mod

@@ -10,14 +10,14 @@ require (
 	github.com/zalando/go-keyring v0.2.8
 	golang.org/x/sys v0.47.0
 	golang.org/x/text v0.40.0
+	modernc.org/sqlite v1.54.0
 )
 
 require (
 	github.com/charmbracelet/bubbles v1.0.0 // indirect
 	github.com/clipperhouse/displaywidth v0.11.0 // indirect
 	github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.4.3 // indirect
-	modernc.org/sqlite v1.54.0 // indirect
 )
 
 require (

internal/cli/admin_commands.go

@@ -107,20 +107,27 @@ func (r *runtime) runInit(args []string) error {
 }
 
 func (r *runtime) runSync(args []string) error {
+	return r.runSyncWithShareUpdate(args, shareUpdateNever)
+}
+
+func (r *runtime) runSyncWithShareUpdate(args []string, shareUpdate shareUpdateMode) error {
 	fs := flag.NewFlagSet("sync", flag.ContinueOnError)
 	fs.SetOutput(io.Discard)
 	full := fs.Bool("full", false, "")
 	all := fs.Bool("all", false, "")
 	allChannels := fs.Bool("all-channels", false, "")
 	since := fs.String("since", "", "")
 	channels := fs.String("channels", "", "")
+	excludeChannels := fs.String("exclude-channels", strings.Join(r.cfg.Sync.ExcludeChannelIDs, ","), "")
+	excludeChannelKinds := fs.String("exclude-channel-kinds", strings.Join(r.cfg.Sync.ExcludeChannelKinds, ","), "")
 	concurrency := fs.Int("concurrency", r.cfg.Sync.Concurrency, "")
 	source := fs.String("source", r.cfg.Sync.Source, "")
 	withEmbeddings := fs.Bool("with-embeddings", false, "")
 	withMedia := fs.Bool("with-media", r.cfg.AttachmentMediaEnabled(), "")
 	skipMembers := fs.Bool("skip-members", false, "")
 	withMembers := fs.Bool("with-members", false, "")
 	latestOnly := fs.Bool("latest-only", false, "")
+	channelTimeout := fs.String("channel-timeout", "", "")
 	guildsFlag := fs.String("guilds", "", "")
 	guildFlag := fs.String("guild", "", "")
 	updateMode := fs.String("update", "", "")
@@ -151,23 +158,42 @@ func (r *runtime) runSync(args []string) error {
 		}
 		sinceTime = parsed
 	}
+	var messageChannelTimeout time.Duration
+	if strings.TrimSpace(*channelTimeout) != "" {
+		parsed, err := time.ParseDuration(strings.TrimSpace(*channelTimeout))
+		if err != nil || parsed <= 0 {
+			return usageErr(fmt.Errorf("invalid --channel-timeout %q", *channelTimeout))
+		}
+		messageChannelTimeout = parsed
+	}
 	guildIDs, err := r.resolveSyncGuildsAll(*guildFlag, *guildsFlag, *all)
 	if err != nil {
 		return usageErr(err)
 	}
 	defaultLatest := defaultLatestSyncMode(*full, *allChannels, *since, *channels)
 	opts := syncer.SyncOptions{
-		Full:           *full,
-		GuildIDs:       guildIDs,
-		ChannelIDs:     csvList(*channels),
-		Concurrency:    *concurrency,
-		Since:          sinceTime,
-		Embeddings:     *withEmbeddings,
-		SkipMembers:    syncSkipsMembers(*skipMembers, *withMembers, defaultLatest),
-		RequireMembers: *withMembers,
-		LatestOnly:     syncLatestOnly(*latestOnly, defaultLatest),
+		Full:                  *full,
+		GuildIDs:              guildIDs,
+		ChannelIDs:            csvList(*channels),
+		Concurrency:           *concurrency,
+		Since:                 sinceTime,
+		Embeddings:            *withEmbeddings,
+		SkipMembers:           syncSkipsMembers(*skipMembers, *withMembers, defaultLatest),
+		RequireMembers:        *withMembers,
+		LatestOnly:            syncLatestOnly(*latestOnly, defaultLatest),
+		MessageChannelTimeout: messageChannelTimeout,
+		ExcludeChannelIDs:     csvList(*excludeChannels),
+		ExcludeChannelKinds:   csvList(*excludeChannelKinds),
 	}
 	return r.withSyncLock(func() error {
+		if shareUpdate != shareUpdateNever && os.Getenv("DISCRAWL_NO_AUTO_UPDATE") != "1" {
+			if err := r.autoUpdateShareWithExclusions(shareUpdate, &shareMergeExclusions{
+				channelIDs:   opts.ExcludeChannelIDs,
+				channelKinds: opts.ExcludeChannelKinds,
+			}); err != nil {
+				return err
+			}
+		}
 		return r.runSyncLocked(sources, opts, *withMedia)
 	})
 }
@@ -193,10 +219,12 @@ func (r *runtime) runSyncLocked(sources syncSources, opts syncer.SyncOptions, wi
 	if sources.wiretap {
 		r.setSyncLockPhase("wiretap import")
 		stats, err := discorddesktop.Import(r.ctx, r.store, discorddesktop.Options{
-			Path:         r.cfg.Desktop.Path,
-			MaxFileBytes: r.cfg.Desktop.MaxFileBytes,
-			FullCache:    r.cfg.Desktop.FullCache,
-			Now:          r.now,
+			Path:                r.cfg.Desktop.Path,
+			MaxFileBytes:        r.cfg.Desktop.MaxFileBytes,
+			FullCache:           r.cfg.Desktop.FullCache,
+			ExcludeChannelIDs:   opts.ExcludeChannelIDs,
+			ExcludeChannelKinds: opts.ExcludeChannelKinds,
+			Now:                 r.now,
 		})
 		if err != nil {
 			return err
@@ -332,6 +360,15 @@ func (r *runtime) runTail(args []string) error {
 	repairEvery := fs.Duration("repair-every", mustDuration(r.cfg.Sync.RepairEvery), "")
 	replayFailuresOnly := fs.Bool("replay-failures-only", false, "")
 	replayLimit := fs.Int("replay-limit", syncer.TailMessageReplayLimit, "")
+	repairOffsetDefault := time.Duration(0)
+	if raw := strings.TrimSpace(r.cfg.Sync.RepairOffset); raw != "" {
+		parsed, err := time.ParseDuration(raw)
+		if err != nil {
+			return configErr(fmt.Errorf("parse sync.repair_offset: %w", err))
+		}
+		repairOffsetDefault = parsed
+	}
+	repairOffset := fs.Duration("repair-offset", repairOffsetDefault, "")
 	guildsFlag := fs.String("guilds", "", "")
 	guildFlag := fs.String("guild", "", "")
 	if err := fs.Parse(args); err != nil {
@@ -368,6 +405,9 @@ func (r *runtime) runTail(args []string) error {
 	}
 	ctx, stop := signal.NotifyContext(r.ctx, os.Interrupt, syscall.SIGTERM)
 	defer stop()
+	if configurable, ok := r.syncer.(repairOffsetConfigurer); ok {
+		configurable.SetRepairOffset(*repairOffset)
+	}
 	if configurable, ok := r.syncer.(tailReadyConfigurer); ok {
 		configurable.SetTailReadyCallback(func(context.Context) error {
 			return r.activateTailSyncLock()
@@ -402,11 +442,13 @@ func (r *runtime) runWiretap(args []string) error {
 	var previousCoverage *store.CoverageReport
 	runOnce := func(ctx context.Context) error {
 		stats, err := discorddesktop.Import(ctx, r.store, discorddesktop.Options{
-			Path:         *path,
-			MaxFileBytes: *maxFileBytes,
-			FullCache:    *fullCache,
-			DryRun:       *dryRun,
-			Now:          r.now,
+			Path:                *path,
+			MaxFileBytes:        *maxFileBytes,
+			FullCache:           *fullCache,
+			DryRun:              *dryRun,
+			ExcludeChannelIDs:   r.cfg.Sync.ExcludeChannelIDs,
+			ExcludeChannelKinds: r.cfg.Sync.ExcludeChannelKinds,
+			Now:                 r.now,
 		})
 		if err != nil {
 			return err