diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index 0cb540b..08167fe 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -3616,6 +3616,10 @@ func (f *fakeDiscordClient) Guild(context.Context, string) (*discordgo.Guild, er return &discordgo.Guild{}, nil } +func (f *fakeDiscordClient) Channel(context.Context, string) (*discordgo.Channel, error) { + return nil, nil +} + func (f *fakeDiscordClient) GuildChannels(context.Context, string) ([]*discordgo.Channel, error) { return nil, nil } diff --git a/internal/discord/client.go b/internal/discord/client.go index 0fef474..4d75a06 100644 --- a/internal/discord/client.go +++ b/internal/discord/client.go @@ -293,6 +293,12 @@ func (c *Client) Guild(ctx context.Context, guildID string) (*discordgo.Guild, e return c.session.Guild(guildID, discordgo.WithContext(reqCtx)) } +func (c *Client) Channel(ctx context.Context, channelID string) (*discordgo.Channel, error) { + reqCtx, cancel := c.requestContext(ctx) + defer cancel() + return c.session.Channel(channelID, discordgo.WithContext(reqCtx)) +} + func (c *Client) GuildChannels(ctx context.Context, guildID string) ([]*discordgo.Channel, error) { reqCtx, cancel := c.requestContext(ctx) defer cancel() diff --git a/internal/syncer/channel_catalog.go b/internal/syncer/channel_catalog.go index acc48b8..f75adc4 100644 --- a/internal/syncer/channel_catalog.go +++ b/internal/syncer/channel_catalog.go @@ -14,12 +14,25 @@ import ( type channelCatalogMode int +type directChannelResult struct { + channel *discordgo.Channel + err error +} + const ( channelCatalogFull channelCatalogMode = iota channelCatalogIncremental ) -func (s *Syncer) channelList(ctx context.Context, guildID string, requested []string, mode channelCatalogMode, exclusions channelExclusions) ([]*discordgo.Channel, bool, error) { +func (s *Syncer) channelList( + ctx context.Context, + guildID string, + requested []string, + mode channelCatalogMode, + exclusions channelExclusions, + selectedGuildIDs map[string]struct{}, + directChannelResults map[string]directChannelResult, +) ([]*discordgo.Channel, bool, error) { if len(requested) == 0 { channels, err := s.liveChannelList(ctx, guildID, mode, exclusions) if err != nil { @@ -62,6 +75,16 @@ func (s *Syncer) channelList(ctx context.Context, guildID string, requested []st selected = selectRequestedChannels(allChannels, storedByID, requestedSet) } + if unresolvedRequestedIDs(selected, requestedSet) > 0 { + if directChannelResults == nil { + directChannelResults = make(map[string]directChannelResult, len(requestedSet)) + } + if err := s.appendDirectRequestedChannels(ctx, guildID, allChannels, requestedSet, selectedGuildIDs, directChannelResults); err != nil { + return nil, false, err + } + selected = selectRequestedChannels(allChannels, storedByID, requestedSet) + } + if unresolvedRequestedIDs(selected, requestedSet) > 0 { if err := s.appendThreadCatalog(ctx, allChannels, threadParentIDs(topLevel)); err != nil { return nil, false, err @@ -75,6 +98,48 @@ func (s *Syncer) channelList(ctx context.Context, guildID string, requested []st ), true, nil } +func (s *Syncer) appendDirectRequestedChannels( + ctx context.Context, + guildID string, + allChannels map[string]*discordgo.Channel, + requested map[string]struct{}, + selectedGuildIDs map[string]struct{}, + directChannelResults map[string]directChannelResult, +) error { + for requestedID := range requested { + if _, ok := allChannels[requestedID]; ok { + continue + } + result, cached := directChannelResults[requestedID] + if !cached { + result.channel, result.err = s.client.Channel(ctx, requestedID) + directChannelResults[requestedID] = result + } + if result.err != nil { + return fmt.Errorf("fetch requested channel %s: %w", requestedID, result.err) + } + channel := result.channel + if channel == nil || channel.ID == "" { + continue + } + if channel.GuildID == "" { + return fmt.Errorf("requested channel %s is not a guild channel", channel.ID) + } + if channel.GuildID != guildID { + if _, selected := selectedGuildIDs[channel.GuildID]; !selected { + return fmt.Errorf("requested channel %s belongs to guild %s, not %s", channel.ID, channel.GuildID, guildID) + } + // requested is rebuilt by channelList for each selected guild. Exclude + // the target only from this guild's fallback discovery; its owning guild + // gets a fresh request set and performs the same direct lookup. + delete(requested, requestedID) + continue + } + allChannels[channel.ID] = channel + } + return nil +} + func (s *Syncer) liveChannelList(ctx context.Context, guildID string, mode channelCatalogMode, exclusions channelExclusions) ([]*discordgo.Channel, error) { channels, err := s.client.GuildChannels(ctx, guildID) if err != nil { diff --git a/internal/syncer/channel_catalog_test.go b/internal/syncer/channel_catalog_test.go index 1272251..2afe012 100644 --- a/internal/syncer/channel_catalog_test.go +++ b/internal/syncer/channel_catalog_test.go @@ -123,6 +123,99 @@ func TestSyncChannelSubsetExpandsRequestedForumThreads(t *testing.T) { require.Equal(t, "t1", results[0].ChannelID) } +func TestSyncChannelSubsetFetchesRequestedArchivedThreadDirectly(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + require.NoError(t, s.UpsertGuild(ctx, store.GuildRecord{ID: "g1", Name: "Guild", RawJSON: `{}`})) + + archiveAt := time.Now().UTC().Add(-time.Hour) + client := &fakeClient{ + guilds: []*discordgo.UserGuild{ + {ID: "g-other", Name: "Other Guild"}, + {ID: "g1", Name: "Guild"}, + }, + guildByID: map[string]*discordgo.Guild{ + "g-other": {ID: "g-other", Name: "Other Guild"}, + "g1": {ID: "g1", Name: "Guild"}, + }, + channels: map[string][]*discordgo.Channel{ + "g-other": { + {ID: "f-other", GuildID: "g-other", Name: "other forum", Type: discordgo.ChannelTypeGuildForum}, + }, + "g1": { + {ID: "f1", GuildID: "g1", Name: "support", Type: discordgo.ChannelTypeGuildForum}, + {ID: "c2", GuildID: "g1", Name: "random", Type: discordgo.ChannelTypeGuildText}, + }, + }, + channelByID: map[string]*discordgo.Channel{ + "t-archived": { + ID: "t-archived", + GuildID: "g1", + ParentID: "f1", + Name: "archived support thread", + Type: discordgo.ChannelTypeGuildPublicThread, + ThreadMetadata: &discordgo.ThreadMetadata{ + Archived: true, + ArchiveTimestamp: archiveAt, + }, + LastMessageID: "10", + }, + }, + messages: map[string][]*discordgo.Message{ + "t-archived": {{ + ID: "10", + GuildID: "g1", + ChannelID: "t-archived", + Content: "archived support body", + Timestamp: archiveAt, + Author: &discordgo.User{ID: "u1", Username: "user"}, + }}, + }, + } + + svc := New(client, s, nil) + stats, err := svc.Sync(ctx, SyncOptions{ + Full: true, + GuildIDs: []string{"g-other", "g1"}, + ChannelIDs: []string{"t-archived"}, + }) + require.NoError(t, err) + require.Equal(t, 2, stats.Guilds) + require.Equal(t, 1, stats.Channels) + require.Equal(t, 1, stats.Threads) + require.Equal(t, 1, stats.Messages) + require.Equal(t, 2, client.guildChanCalls) + require.Equal(t, 1, client.channelCalls["t-archived"]) + require.Zero(t, client.threadCalls) + require.Empty(t, client.archivedCalls) + require.Equal(t, 1, client.messageCalls["t-archived"]) + + results, err := s.SearchMessages(ctx, store.SearchOptions{Query: "archived support"}) + require.NoError(t, err) + require.Len(t, results, 1) + require.Equal(t, "t-archived", results[0].ChannelID) + + _, err = svc.Sync(ctx, SyncOptions{ + Full: true, + GuildIDs: []string{"g-other"}, + ChannelIDs: []string{"t-archived"}, + }) + require.ErrorContains(t, err, "requested channel t-archived belongs to guild g1, not g-other") + + client.channelByID["dm"] = &discordgo.Channel{ID: "dm", Type: discordgo.ChannelTypeDM} + _, err = svc.Sync(ctx, SyncOptions{ + Full: true, + GuildIDs: []string{"g1"}, + ChannelIDs: []string{"dm"}, + }) + require.ErrorContains(t, err, "requested channel dm is not a guild channel") +} + func TestSyncToleratesArchivedThread403(t *testing.T) { t.Parallel() diff --git a/internal/syncer/channel_exclusions_test.go b/internal/syncer/channel_exclusions_test.go index 3418a4d..e3128c2 100644 --- a/internal/syncer/channel_exclusions_test.go +++ b/internal/syncer/channel_exclusions_test.go @@ -136,6 +136,8 @@ func TestTargetedStoredThreadUsesFullCategoryAncestry(t *testing.T) { []string{"thread-a"}, channelCatalogFull, svc.effectiveChannelExclusions(SyncOptions{}), + map[string]struct{}{"g1": {}}, + nil, ) require.NoError(t, err) require.True(t, targeted) diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go index 5fd9e14..853ad0c 100644 --- a/internal/syncer/syncer.go +++ b/internal/syncer/syncer.go @@ -18,6 +18,7 @@ type Client interface { Self(context.Context) (*discordgo.User, error) Guilds(context.Context) ([]*discordgo.UserGuild, error) Guild(context.Context, string) (*discordgo.Guild, error) + Channel(context.Context, string) (*discordgo.Channel, error) GuildChannels(context.Context, string) ([]*discordgo.Channel, error) ThreadsActive(context.Context, string) ([]*discordgo.Channel, error) GuildThreadsActive(context.Context, string) ([]*discordgo.Channel, error) @@ -52,19 +53,21 @@ type Syncer struct { } type SyncOptions struct { - Full bool - GuildIDs []string - ChannelIDs []string - Concurrency int - Since time.Time - Embeddings bool - SkipMembers bool - RequireMembers bool - LatestOnly bool - ExcludeChannelIDs []string - ExcludeChannelKinds []string - IncludeCategoryIDs []string - RepairReason string + Full bool + GuildIDs []string + ChannelIDs []string + Concurrency int + Since time.Time + Embeddings bool + SkipMembers bool + RequireMembers bool + LatestOnly bool + ExcludeChannelIDs []string + ExcludeChannelKinds []string + IncludeCategoryIDs []string + RepairReason string + selectedGuildIDs map[string]struct{} + directChannelResults map[string]directChannelResult } func (s *Syncer) SetTailReadyCallback(fn func(context.Context) error) { @@ -154,6 +157,11 @@ func (s *Syncer) Sync(ctx context.Context, opts SyncOptions) (SyncStats, error) return SyncStats{}, fmt.Errorf("requested guilds not accessible: %s", strings.Join(missing, ", ")) } targets := selectGuilds(guilds, opts.GuildIDs) + opts.selectedGuildIDs = make(map[string]struct{}, len(targets)) + opts.directChannelResults = make(map[string]directChannelResult, len(opts.ChannelIDs)) + for _, guild := range targets { + opts.selectedGuildIDs[guild.ID] = struct{}{} + } stats := SyncStats{} for _, guild := range targets { one, err := s.syncGuild(ctx, guild.ID, opts) @@ -198,7 +206,15 @@ func (s *Syncer) syncGuild(ctx context.Context, guildID string, opts SyncOptions } } exclusions := s.effectiveChannelExclusions(opts) - channelList, targeted, err := s.channelList(ctx, guildID, opts.ChannelIDs, catalogMode, exclusions) + channelList, targeted, err := s.channelList( + ctx, + guildID, + opts.ChannelIDs, + catalogMode, + exclusions, + opts.selectedGuildIDs, + opts.directChannelResults, + ) if err != nil { return stats, err } diff --git a/internal/syncer/syncer_test.go b/internal/syncer/syncer_test.go index ac6e4ce..0520449 100644 --- a/internal/syncer/syncer_test.go +++ b/internal/syncer/syncer_test.go @@ -18,6 +18,7 @@ import ( type fakeClient struct { guilds []*discordgo.UserGuild guildByID map[string]*discordgo.Guild + channelByID map[string]*discordgo.Channel channels map[string][]*discordgo.Channel activeThreads map[string][]*discordgo.Channel guildThreads map[string][]*discordgo.Channel @@ -41,6 +42,7 @@ type fakeClient struct { tailCalls int tailHandled chan struct{} messageDelay time.Duration + channelCalls map[string]int guildChanCalls int threadCalls int guildThreadCalls int @@ -68,6 +70,14 @@ func (f *fakeClient) Guild(_ context.Context, guildID string) (*discordgo.Guild, return f.guildByID[guildID], nil } +func (f *fakeClient) Channel(_ context.Context, channelID string) (*discordgo.Channel, error) { + if f.channelCalls == nil { + f.channelCalls = make(map[string]int) + } + f.channelCalls[channelID]++ + return f.channelByID[channelID], nil +} + func (f *fakeClient) GuildChannels(_ context.Context, guildID string) ([]*discordgo.Channel, error) { f.guildChanCalls++ return f.channels[guildID], nil