evanferrao
/
dump-mirrorbot
mirror of https://github.com/Box-boi/dump-mirrorbot.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

recursively try again instead of handling switch with backoff 

- Handling service account switching with exponential backoff implementation limits the number of service accounts to be used. recursively retrying again after switch doesnt effect backoff.

Signed-off-by: lzzy12 <jhashivam2020@gmail.com>
This commit is contained in:
jaskaranSM 2020-03-21 10:31:24 +05:30 committed by lzzy12
parent 6cd59d5a15
commit 8d1bf5aa92
1 changed files with 16 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
bot/helper/mirror_utils/upload_utils/gdriveTools.py
View File

@ -46,6 +46,7 @@ class GoogleDriveHelper:
self.updater = None self.updater = None
self.name = name self.name = name
self.update_interval = 3 self.update_interval = 3
self.service_account_count = len(os.listdir("accounts"))
def cancel(self): def cancel(self):
self.is_cancelled = True self.is_cancelled = True
@ -93,6 +94,14 @@ class GoogleDriveHelper:
return self.__service.files().create(supportsTeamDrives=True, return self.__service.files().create(supportsTeamDrives=True,
body=file_metadata, media_body=media_body).execute() body=file_metadata, media_body=media_body).execute()
def switchServiceAccount(self):
global SERVICE_ACCOUNT_INDEX
if SERVICE_ACCOUNT_INDEX == self.service_account_count - 1:
SERVICE_ACCOUNT_INDEX = 0
SERVICE_ACCOUNT_INDEX += 1
LOGGER.info(f"Switching to {SERVICE_ACCOUNT_INDEX}.json service account")
self.__service = self.authorize()
@retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5),
retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG))
def __set_permission(self, drive_id): def __set_permission(self, drive_id):
@ -146,12 +155,13 @@ class GoogleDriveHelper:
except HttpError as err: except HttpError as err:
if err.resp.get('content-type', '').startswith('application/json'): if err.resp.get('content-type', '').startswith('application/json'):
reason = json.loads(err.content).get('error').get('errors')[0].get('reason') reason = json.loads(err.content).get('error').get('errors')[0].get('reason')
if reason == 'userRateLimitExceeded': if reason == 'userRateLimitExceeded' or reason == 'dailyLimitExceeded':
global SERVICE_ACCOUNT_INDEX if USE_SERVICE_ACCOUNTS:
SERVICE_ACCOUNT_INDEX += 1 self.switchServiceAccount()
LOGGER.info(f"Switching to {SERVICE_ACCOUNT_INDEX}.json service account") LOGGER.info(f"Got: {reason}, Trying Again.")
self.__service = self.authorize() self.upload_file(file_path, file_name, mime_type, parent_id)
raise err else:
raise err
self._file_uploaded_bytes = 0 self._file_uploaded_bytes = 0
# Insert new permissions # Insert new permissions
if not IS_TEAM_DRIVE: if not IS_TEAM_DRIVE: