recursively try again instead of handling switch with backoff

- Handling service account switching with exponential backoff implementation limits the number of service accounts to be used. recursively retrying again after switch doesnt effect backoff.

Signed-off-by: lzzy12 <jhashivam2020@gmail.com>
This commit is contained in:
jaskaranSM 2020-03-21 10:31:24 +05:30 committed by lzzy12
parent 6cd59d5a15
commit 8d1bf5aa92

View File

@ -46,6 +46,7 @@ class GoogleDriveHelper:
self.updater = None
self.name = name
self.update_interval = 3
self.service_account_count = len(os.listdir("accounts"))
def cancel(self):
self.is_cancelled = True
@ -93,6 +94,14 @@ class GoogleDriveHelper:
return self.__service.files().create(supportsTeamDrives=True,
body=file_metadata, media_body=media_body).execute()
def switchServiceAccount(self):
global SERVICE_ACCOUNT_INDEX
if SERVICE_ACCOUNT_INDEX == self.service_account_count - 1:
SERVICE_ACCOUNT_INDEX = 0
SERVICE_ACCOUNT_INDEX += 1
LOGGER.info(f"Switching to {SERVICE_ACCOUNT_INDEX}.json service account")
self.__service = self.authorize()
@retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5),
retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG))
def __set_permission(self, drive_id):
@ -146,12 +155,13 @@ class GoogleDriveHelper:
except HttpError as err:
if err.resp.get('content-type', '').startswith('application/json'):
reason = json.loads(err.content).get('error').get('errors')[0].get('reason')
if reason == 'userRateLimitExceeded':
global SERVICE_ACCOUNT_INDEX
SERVICE_ACCOUNT_INDEX += 1
LOGGER.info(f"Switching to {SERVICE_ACCOUNT_INDEX}.json service account")
self.__service = self.authorize()
raise err
if reason == 'userRateLimitExceeded' or reason == 'dailyLimitExceeded':
if USE_SERVICE_ACCOUNTS:
self.switchServiceAccount()
LOGGER.info(f"Got: {reason}, Trying Again.")
self.upload_file(file_path, file_name, mime_type, parent_id)
else:
raise err
self._file_uploaded_bytes = 0
# Insert new permissions
if not IS_TEAM_DRIVE: