diff --git a/Dockerfile b/Dockerfile index 1da906ff3..44a5df652 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,6 +11,7 @@ LABEL version="1.0" \ ## to the image to be as small as possible COPY package*.json /opt/safe-settings/ COPY index.js /opt/safe-settings/ +COPY full-sync.js /opt/safe-settings/ COPY lib /opt/safe-settings/lib ## Install the app and dependencies diff --git a/lib/settings.js b/lib/settings.js index fd7ca2693..63a9bd6c6 100644 --- a/lib/settings.js +++ b/lib/settings.js @@ -538,13 +538,23 @@ ${this.results.reduce((x, y) => { async eachRepositoryRepos (github, log) { log.debug('Fetching repositories') - return github.paginate('GET /installation/repositories').then(repositories => { - return Promise.all(repositories.map(repository => { - const { owner, name } = repository - return this.checkAndProcessRepo(owner.login, name) - }) + const repositories = await github.paginate('GET /installation/repositories') + const CONCURRENCY = 10 + const results = [] + for (let i = 0; i < repositories.length; i += CONCURRENCY) { + const chunk = repositories.slice(i, i + CONCURRENCY) + const chunkResults = await Promise.allSettled( + chunk.map(({ owner, name }) => this.checkAndProcessRepo(owner.login, name)) ) - }) + for (const result of chunkResults) { + if (result.status === 'fulfilled') { + results.push(result.value) + } else { + log.error(`Error processing repository in batch: ${result.reason}`) + } + } + } + return results } async checkAndProcessRepo (owner, name) {