Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions internal/cli/cli_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3616,6 +3616,10 @@ func (f *fakeDiscordClient) Guild(context.Context, string) (*discordgo.Guild, er
return &discordgo.Guild{}, nil
}

func (f *fakeDiscordClient) Channel(context.Context, string) (*discordgo.Channel, error) {
return nil, nil
}

func (f *fakeDiscordClient) GuildChannels(context.Context, string) ([]*discordgo.Channel, error) {
return nil, nil
}
Expand Down
6 changes: 6 additions & 0 deletions internal/discord/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,12 @@ func (c *Client) Guild(ctx context.Context, guildID string) (*discordgo.Guild, e
return c.session.Guild(guildID, discordgo.WithContext(reqCtx))
}

func (c *Client) Channel(ctx context.Context, channelID string) (*discordgo.Channel, error) {
reqCtx, cancel := c.requestContext(ctx)
defer cancel()
return c.session.Channel(channelID, discordgo.WithContext(reqCtx))
}

func (c *Client) GuildChannels(ctx context.Context, guildID string) ([]*discordgo.Channel, error) {
reqCtx, cancel := c.requestContext(ctx)
defer cancel()
Expand Down
67 changes: 66 additions & 1 deletion internal/syncer/channel_catalog.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,25 @@ import (

type channelCatalogMode int

type directChannelResult struct {
channel *discordgo.Channel
err error
}

const (
channelCatalogFull channelCatalogMode = iota
channelCatalogIncremental
)

func (s *Syncer) channelList(ctx context.Context, guildID string, requested []string, mode channelCatalogMode, exclusions channelExclusions) ([]*discordgo.Channel, bool, error) {
func (s *Syncer) channelList(
ctx context.Context,
guildID string,
requested []string,
mode channelCatalogMode,
exclusions channelExclusions,
selectedGuildIDs map[string]struct{},
directChannelResults map[string]directChannelResult,
) ([]*discordgo.Channel, bool, error) {
if len(requested) == 0 {
channels, err := s.liveChannelList(ctx, guildID, mode, exclusions)
if err != nil {
Expand Down Expand Up @@ -62,6 +75,16 @@ func (s *Syncer) channelList(ctx context.Context, guildID string, requested []st
selected = selectRequestedChannels(allChannels, storedByID, requestedSet)
}

if unresolvedRequestedIDs(selected, requestedSet) > 0 {
if directChannelResults == nil {
directChannelResults = make(map[string]directChannelResult, len(requestedSet))
}
if err := s.appendDirectRequestedChannels(ctx, guildID, allChannels, requestedSet, selectedGuildIDs, directChannelResults); err != nil {
return nil, false, err
}
selected = selectRequestedChannels(allChannels, storedByID, requestedSet)
}

if unresolvedRequestedIDs(selected, requestedSet) > 0 {
if err := s.appendThreadCatalog(ctx, allChannels, threadParentIDs(topLevel)); err != nil {
return nil, false, err
Expand All @@ -75,6 +98,48 @@ func (s *Syncer) channelList(ctx context.Context, guildID string, requested []st
), true, nil
}

func (s *Syncer) appendDirectRequestedChannels(
ctx context.Context,
guildID string,
allChannels map[string]*discordgo.Channel,
requested map[string]struct{},
selectedGuildIDs map[string]struct{},
directChannelResults map[string]directChannelResult,
) error {
for requestedID := range requested {
if _, ok := allChannels[requestedID]; ok {
continue
}
result, cached := directChannelResults[requestedID]
if !cached {
result.channel, result.err = s.client.Channel(ctx, requestedID)
directChannelResults[requestedID] = result
}
if result.err != nil {
return fmt.Errorf("fetch requested channel %s: %w", requestedID, result.err)
}
channel := result.channel
if channel == nil || channel.ID == "" {
continue
}
if channel.GuildID == "" {
return fmt.Errorf("requested channel %s is not a guild channel", channel.ID)
}
if channel.GuildID != guildID {
if _, selected := selectedGuildIDs[channel.GuildID]; !selected {
return fmt.Errorf("requested channel %s belongs to guild %s, not %s", channel.ID, channel.GuildID, guildID)
}
// requested is rebuilt by channelList for each selected guild. Exclude
// the target only from this guild's fallback discovery; its owning guild
// gets a fresh request set and performs the same direct lookup.
delete(requested, requestedID)
continue
}
allChannels[channel.ID] = channel
}
return nil
}

func (s *Syncer) liveChannelList(ctx context.Context, guildID string, mode channelCatalogMode, exclusions channelExclusions) ([]*discordgo.Channel, error) {
channels, err := s.client.GuildChannels(ctx, guildID)
if err != nil {
Expand Down
93 changes: 93 additions & 0 deletions internal/syncer/channel_catalog_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,99 @@ func TestSyncChannelSubsetExpandsRequestedForumThreads(t *testing.T) {
require.Equal(t, "t1", results[0].ChannelID)
}

func TestSyncChannelSubsetFetchesRequestedArchivedThreadDirectly(t *testing.T) {
t.Parallel()

ctx := context.Background()
s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db"))
require.NoError(t, err)
defer func() { _ = s.Close() }()

require.NoError(t, s.UpsertGuild(ctx, store.GuildRecord{ID: "g1", Name: "Guild", RawJSON: `{}`}))

archiveAt := time.Now().UTC().Add(-time.Hour)
client := &fakeClient{
guilds: []*discordgo.UserGuild{
{ID: "g-other", Name: "Other Guild"},
{ID: "g1", Name: "Guild"},
},
guildByID: map[string]*discordgo.Guild{
"g-other": {ID: "g-other", Name: "Other Guild"},
"g1": {ID: "g1", Name: "Guild"},
},
channels: map[string][]*discordgo.Channel{
"g-other": {
{ID: "f-other", GuildID: "g-other", Name: "other forum", Type: discordgo.ChannelTypeGuildForum},
},
"g1": {
{ID: "f1", GuildID: "g1", Name: "support", Type: discordgo.ChannelTypeGuildForum},
{ID: "c2", GuildID: "g1", Name: "random", Type: discordgo.ChannelTypeGuildText},
},
},
channelByID: map[string]*discordgo.Channel{
"t-archived": {
ID: "t-archived",
GuildID: "g1",
ParentID: "f1",
Name: "archived support thread",
Type: discordgo.ChannelTypeGuildPublicThread,
ThreadMetadata: &discordgo.ThreadMetadata{
Archived: true,
ArchiveTimestamp: archiveAt,
},
LastMessageID: "10",
},
},
messages: map[string][]*discordgo.Message{
"t-archived": {{
ID: "10",
GuildID: "g1",
ChannelID: "t-archived",
Content: "archived support body",
Timestamp: archiveAt,
Author: &discordgo.User{ID: "u1", Username: "user"},
}},
},
}

svc := New(client, s, nil)
stats, err := svc.Sync(ctx, SyncOptions{
Full: true,
GuildIDs: []string{"g-other", "g1"},
ChannelIDs: []string{"t-archived"},
})
require.NoError(t, err)
require.Equal(t, 2, stats.Guilds)
require.Equal(t, 1, stats.Channels)
require.Equal(t, 1, stats.Threads)
require.Equal(t, 1, stats.Messages)
require.Equal(t, 2, client.guildChanCalls)
require.Equal(t, 1, client.channelCalls["t-archived"])
require.Zero(t, client.threadCalls)
require.Empty(t, client.archivedCalls)
require.Equal(t, 1, client.messageCalls["t-archived"])

results, err := s.SearchMessages(ctx, store.SearchOptions{Query: "archived support"})
require.NoError(t, err)
require.Len(t, results, 1)
require.Equal(t, "t-archived", results[0].ChannelID)

_, err = svc.Sync(ctx, SyncOptions{
Full: true,
GuildIDs: []string{"g-other"},
ChannelIDs: []string{"t-archived"},
})
require.ErrorContains(t, err, "requested channel t-archived belongs to guild g1, not g-other")

client.channelByID["dm"] = &discordgo.Channel{ID: "dm", Type: discordgo.ChannelTypeDM}
_, err = svc.Sync(ctx, SyncOptions{
Full: true,
GuildIDs: []string{"g1"},
ChannelIDs: []string{"dm"},
})
require.ErrorContains(t, err, "requested channel dm is not a guild channel")
}

func TestSyncToleratesArchivedThread403(t *testing.T) {
t.Parallel()

Expand Down
2 changes: 2 additions & 0 deletions internal/syncer/channel_exclusions_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,8 @@ func TestTargetedStoredThreadUsesFullCategoryAncestry(t *testing.T) {
[]string{"thread-a"},
channelCatalogFull,
svc.effectiveChannelExclusions(SyncOptions{}),
map[string]struct{}{"g1": {}},
nil,
)
require.NoError(t, err)
require.True(t, targeted)
Expand Down
44 changes: 30 additions & 14 deletions internal/syncer/syncer.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ type Client interface {
Self(context.Context) (*discordgo.User, error)
Guilds(context.Context) ([]*discordgo.UserGuild, error)
Guild(context.Context, string) (*discordgo.Guild, error)
Channel(context.Context, string) (*discordgo.Channel, error)
GuildChannels(context.Context, string) ([]*discordgo.Channel, error)
ThreadsActive(context.Context, string) ([]*discordgo.Channel, error)
GuildThreadsActive(context.Context, string) ([]*discordgo.Channel, error)
Expand Down Expand Up @@ -52,19 +53,21 @@ type Syncer struct {
}

type SyncOptions struct {
Full bool
GuildIDs []string
ChannelIDs []string
Concurrency int
Since time.Time
Embeddings bool
SkipMembers bool
RequireMembers bool
LatestOnly bool
ExcludeChannelIDs []string
ExcludeChannelKinds []string
IncludeCategoryIDs []string
RepairReason string
Full bool
GuildIDs []string
ChannelIDs []string
Concurrency int
Since time.Time
Embeddings bool
SkipMembers bool
RequireMembers bool
LatestOnly bool
ExcludeChannelIDs []string
ExcludeChannelKinds []string
IncludeCategoryIDs []string
RepairReason string
selectedGuildIDs map[string]struct{}
directChannelResults map[string]directChannelResult
}

func (s *Syncer) SetTailReadyCallback(fn func(context.Context) error) {
Expand Down Expand Up @@ -154,6 +157,11 @@ func (s *Syncer) Sync(ctx context.Context, opts SyncOptions) (SyncStats, error)
return SyncStats{}, fmt.Errorf("requested guilds not accessible: %s", strings.Join(missing, ", "))
}
targets := selectGuilds(guilds, opts.GuildIDs)
opts.selectedGuildIDs = make(map[string]struct{}, len(targets))
opts.directChannelResults = make(map[string]directChannelResult, len(opts.ChannelIDs))
for _, guild := range targets {
opts.selectedGuildIDs[guild.ID] = struct{}{}
}
stats := SyncStats{}
for _, guild := range targets {
one, err := s.syncGuild(ctx, guild.ID, opts)
Expand Down Expand Up @@ -198,7 +206,15 @@ func (s *Syncer) syncGuild(ctx context.Context, guildID string, opts SyncOptions
}
}
exclusions := s.effectiveChannelExclusions(opts)
channelList, targeted, err := s.channelList(ctx, guildID, opts.ChannelIDs, catalogMode, exclusions)
channelList, targeted, err := s.channelList(
ctx,
guildID,
opts.ChannelIDs,
catalogMode,
exclusions,
opts.selectedGuildIDs,
opts.directChannelResults,
)
if err != nil {
return stats, err
}
Expand Down
10 changes: 10 additions & 0 deletions internal/syncer/syncer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
type fakeClient struct {
guilds []*discordgo.UserGuild
guildByID map[string]*discordgo.Guild
channelByID map[string]*discordgo.Channel
channels map[string][]*discordgo.Channel
activeThreads map[string][]*discordgo.Channel
guildThreads map[string][]*discordgo.Channel
Expand All @@ -41,6 +42,7 @@ type fakeClient struct {
tailCalls int
tailHandled chan struct{}
messageDelay time.Duration
channelCalls map[string]int
guildChanCalls int
threadCalls int
guildThreadCalls int
Expand Down Expand Up @@ -68,6 +70,14 @@ func (f *fakeClient) Guild(_ context.Context, guildID string) (*discordgo.Guild,
return f.guildByID[guildID], nil
}

func (f *fakeClient) Channel(_ context.Context, channelID string) (*discordgo.Channel, error) {
if f.channelCalls == nil {
f.channelCalls = make(map[string]int)
}
f.channelCalls[channelID]++
return f.channelByID[channelID], nil
}

func (f *fakeClient) GuildChannels(_ context.Context, guildID string) ([]*discordgo.Channel, error) {
f.guildChanCalls++
return f.channels[guildID], nil
Expand Down