From 1c08532f43c83c862595a451f3a45627ea81f62b Mon Sep 17 00:00:00 2001 From: addshore Date: Tue, 12 Mar 2024 15:21:36 +0000 Subject: [PATCH 1/3] Auto retry on 429 in merge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I was getting lots of rate limit messages while trying to merge a bunch of stuff on my account. I was getting responses such as: x-ratelimit: {"name":"AUTHED_API_POST_LIMIT","period":1,"limit":1, "remaining":0,"until":"2024-03-12T15:10:32Z"} retry-after: 1 I added this retry logic which may be useful, but ultimately it looks like the rate-limit tha tis set by the CLI should probably be used, but is not currently used in the executor for the merge? --- trakt_tools/tasks/history/duplicates/merge/executor.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/trakt_tools/tasks/history/duplicates/merge/executor.py b/trakt_tools/tasks/history/duplicates/merge/executor.py index 59b7a9a..463c3ba 100644 --- a/trakt_tools/tasks/history/duplicates/merge/executor.py +++ b/trakt_tools/tasks/history/duplicates/merge/executor.py @@ -1,4 +1,5 @@ from __future__ import print_function +import time from trakt_tools.core.input import boolean_input from ..core.formatter import Formatter @@ -93,6 +94,14 @@ def _remove_records(ids): elif hasattr(ex, 'message'): message = ex.message + # Rate limit exceeded + if isinstance(ex, ClientError) and ex.status_code == 429: + if 'retry-after' in ex.response.headers: + retry_after = int(ex.response.headers['retry-after']) + print(f'Exceeded rate limit, waiting for {retry_after} seconds...') + time.sleep(retry_after) + continue + # Prompt for request retry print('Unable to remove %d history record(s): %s' % (len(ids), message)) From 8e3a0c118c6948e2baa6bc0c227a31dabe2af265 Mon Sep 17 00:00:00 2001 From: addshore Date: Tue, 12 Mar 2024 15:34:32 +0000 Subject: [PATCH 2/3] Just call profile._rate_limit() before remove_records This seems to basiucally fix the issue, and no longer require the retry code that I wrote in the previous patch --- .../tasks/history/duplicates/merge/executor.py | 13 ++++--------- trakt_tools/tasks/history/duplicates/merge/main.py | 5 ++++- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/trakt_tools/tasks/history/duplicates/merge/executor.py b/trakt_tools/tasks/history/duplicates/merge/executor.py index 463c3ba..0386c18 100644 --- a/trakt_tools/tasks/history/duplicates/merge/executor.py +++ b/trakt_tools/tasks/history/duplicates/merge/executor.py @@ -13,9 +13,10 @@ class Executor(object): - def __init__(self, review=True, batch_size=200): + def __init__(self, review=True, batch_size=200, rate_limit=20): self.review = review self.batch_size = batch_size + self.rate_limit = rate_limit def process_shows(self, profile, shows): log.debug('Executing actions on %d shows...', len(shows)) @@ -39,6 +40,7 @@ def process_shows(self, profile, shows): # Remove history records for x in six.moves.xrange(0, len(ids), self.batch_size): + profile._rate_limit() self._remove_records(ids[x:x + self.batch_size]) print() @@ -66,6 +68,7 @@ def process_movies(self, profile, movies): # Remove history records for x in six.moves.xrange(0, len(ids), self.batch_size): + profile._rate_limit() self._remove_records(ids[x:x + self.batch_size]) print() @@ -94,14 +97,6 @@ def _remove_records(ids): elif hasattr(ex, 'message'): message = ex.message - # Rate limit exceeded - if isinstance(ex, ClientError) and ex.status_code == 429: - if 'retry-after' in ex.response.headers: - retry_after = int(ex.response.headers['retry-after']) - print(f'Exceeded rate limit, waiting for {retry_after} seconds...') - time.sleep(retry_after) - continue - # Prompt for request retry print('Unable to remove %d history record(s): %s' % (len(ids), message)) diff --git a/trakt_tools/tasks/history/duplicates/merge/main.py b/trakt_tools/tasks/history/duplicates/merge/main.py index cf28abd..b2af529 100644 --- a/trakt_tools/tasks/history/duplicates/merge/main.py +++ b/trakt_tools/tasks/history/duplicates/merge/main.py @@ -106,7 +106,10 @@ def execute(self, profile, review=None): review = boolean_input('Review every action?', default=True) print() - executor = Executor(review) + executor = Executor( + review, + rate_limit=self.rate_limit + ) if not executor.process_shows(profile, self.scanner.shows): return False From 9ab4fa5c902ea6f044038461d5c5fe376f0e8177 Mon Sep 17 00:00:00 2001 From: addshore Date: Tue, 12 Mar 2024 15:36:37 +0000 Subject: [PATCH 3/3] Cleanup --- trakt_tools/tasks/history/duplicates/merge/executor.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/trakt_tools/tasks/history/duplicates/merge/executor.py b/trakt_tools/tasks/history/duplicates/merge/executor.py index 0386c18..d478e38 100644 --- a/trakt_tools/tasks/history/duplicates/merge/executor.py +++ b/trakt_tools/tasks/history/duplicates/merge/executor.py @@ -1,5 +1,4 @@ from __future__ import print_function -import time from trakt_tools.core.input import boolean_input from ..core.formatter import Formatter @@ -13,10 +12,9 @@ class Executor(object): - def __init__(self, review=True, batch_size=200, rate_limit=20): + def __init__(self, review=True, batch_size=200): self.review = review self.batch_size = batch_size - self.rate_limit = rate_limit def process_shows(self, profile, shows): log.debug('Executing actions on %d shows...', len(shows))