diff --git a/bot/helper/ext_utils/shortenurl.py b/bot/helper/ext_utils/shortenurl.py new file mode 100644 index 0000000..03be2f7 --- /dev/null +++ b/bot/helper/ext_utils/shortenurl.py @@ -0,0 +1,27 @@ +import requests +import random +import base64 +import pyshorteners +from urllib.parse import quote +from urllib3 import disable_warnings +from bot import SHORTENER, SHORTENER_API + + +def short_url(longurl): + if "shorte.st" in SHORTENER: + disable_warnings() + return requests.get(f'http://api.shorte.st/stxt/{SHORTENER_API}/{longurl}', verify=False).text + elif "linkvertise" in SHORTENER: + url = quote(base64.b64encode(longurl.encode("utf-8"))) + linkvertise = [ + f"https://link-to.net/{SHORTENER_API}/{random.random() * 1000}/dynamic?r={url}", + f"https://up-to-down.net/{SHORTENER_API}/{random.random() * 1000}/dynamic?r={url}", + f"https://direct-link.net/{SHORTENER_API}/{random.random() * 1000}/dynamic?r={url}", + f"https://file-link.net/{SHORTENER_API}/{random.random() * 1000}/dynamic?r={url}"] + return random.choice(linkvertise) + elif "bitly.com" in SHORTENER: + s = pyshorteners.Shortener(api_key=SHORTENER_API) + bitly = s.bitly.short(longurl) + return bitly + else: + return requests.get(f'https://{SHORTENER}/api?api={SHORTENER_API}&url={longurl}&format=text').text diff --git a/bot/helper/mirror_utils/download_utils/download_helper.py b/bot/helper/mirror_utils/download_utils/download_helper.py deleted file mode 100644 index ebcca14..0000000 --- a/bot/helper/mirror_utils/download_utils/download_helper.py +++ /dev/null @@ -1,27 +0,0 @@ -# An abstract class which will be inherited by the tool specific classes like aria2_helper or mega_download_helper -import threading - - -class MethodNotImplementedError(NotImplementedError): - def __init__(self): - super(self, 'Not implemented method') - - -class DownloadHelper: - def __init__(self): - self.name = '' # Name of the download; empty string if no download has been started - self.size = 0.0 # Size of the download - self.downloaded_bytes = 0.0 # Bytes downloaded - self.speed = 0.0 # Download speed in bytes per second - self.progress = 0.0 - self.progress_string = '0.00%' - self.eta = 0 # Estimated time of download complete - self.eta_string = '0s' # A listener class which have event callbacks - self._resource_lock = threading.Lock() - - def add_download(self, link: str, path): - raise MethodNotImplementedError - - def cancel_download(self): - # Returns None if successfully cancelled, else error string - raise MethodNotImplementedError diff --git a/bot/helper/mirror_utils/download_utils/mega_downloader.py b/bot/helper/mirror_utils/download_utils/mega_downloader.py index 0d0b474..e0ae427 100644 --- a/bot/helper/mirror_utils/download_utils/mega_downloader.py +++ b/bot/helper/mirror_utils/download_utils/mega_downloader.py @@ -77,10 +77,10 @@ class MegaAppListener(MegaListener): self.continue_event.set() def onRequestTemporaryError(self, api, request, error: MegaError): - LOGGER.info(f'Mega Request error in {error}') + LOGGER.error(f'Mega Request error in {error}') if not self.is_cancelled: - self.listener.onDownloadError("RequestTempError: " + error.toString()) self.is_cancelled = True + self.listener.onDownloadError("RequestTempError: " + error.toString()) self.error = error.toString() self.continue_event.set() @@ -90,12 +90,15 @@ class MegaAppListener(MegaListener): def onTransferUpdate(self, api: MegaApi, transfer: MegaTransfer): if self.is_cancelled: api.cancelTransfer(transfer, None) + return self.__speed = transfer.getSpeed() self.__bytes_transferred = transfer.getTransferredBytes() def onTransferFinish(self, api: MegaApi, transfer: MegaTransfer, error): try: - if transfer.isFolderTransfer() and transfer.isFinished() or transfer.getFileName() == self.name and not self.is_cancelled: + if self.is_cancelled: + self.continue_event.set() + elif transfer.isFinished() and (transfer.isFolderTransfer() or transfer.getFileName() == self.name): self.listener.onDownloadComplete() self.continue_event.set() except Exception as e: @@ -105,7 +108,7 @@ class MegaAppListener(MegaListener): filen = transfer.getFileName() state = transfer.getState() errStr = error.toString() - LOGGER.info(f'Mega download error in file {transfer} {filen}: {error}') + LOGGER.error(f'Mega download error in file {transfer} {filen}: {error}') if state == 1 or state == 4: # Sometimes MEGA (offical client) can't stream a node either and raises a temp failed error. @@ -163,7 +166,7 @@ class MegaDownloadHelper: executor.do(folder_api.loginToFolder, (mega_link,)) node = folder_api.authorizeNode(mega_listener.node) if mega_listener.error is not None: - return listener.onDownloadError(str(mega_listener.error)) + return sendMessage(str(mega_listener.error), listener.bot, listener.update) if STOP_DUPLICATE: LOGGER.info(f'Checking File/Folder if already in Drive') mname = node.getName() diff --git a/bot/helper/mirror_utils/upload_utils/gdriveTools.py b/bot/helper/mirror_utils/upload_utils/gdriveTools.py index 72c9a71..ea7589c 100644 --- a/bot/helper/mirror_utils/upload_utils/gdriveTools.py +++ b/bot/helper/mirror_utils/upload_utils/gdriveTools.py @@ -26,14 +26,20 @@ from bot import parent_id, DOWNLOAD_DIR, IS_TEAM_DRIVE, INDEX_URL, \ USE_SERVICE_ACCOUNTS, telegraph_token, BUTTON_FOUR_NAME, BUTTON_FOUR_URL, BUTTON_FIVE_NAME, BUTTON_FIVE_URL, BUTTON_SIX_NAME, BUTTON_SIX_URL, SHORTENER, SHORTENER_API, VIEW_LINK from bot.helper.ext_utils.bot_utils import get_readable_file_size, setInterval from bot.helper.ext_utils.fs_utils import get_mime_type, get_path_size +from bot.helper.ext_utils.shortenurl import short_url LOGGER = logging.getLogger(__name__) logging.getLogger('googleapiclient.discovery').setLevel(logging.ERROR) + if USE_SERVICE_ACCOUNTS: SERVICE_ACCOUNT_INDEX = randrange(len(os.listdir("accounts"))) + TELEGRAPHLIMIT = 80 + class GoogleDriveHelper: + + def __init__(self, name=None, listener=None): self.__G_DRIVE_TOKEN_FILE = "token.pickle" # Check https://developers.google.com/drive/scopes for all available scopes @@ -69,6 +75,7 @@ class GoogleDriveHelper: self.transferred_size = 0 self.sa_count = 0 + def speed(self): """ It calculates the average upload speed and returns it in bytes/seconds unit @@ -79,18 +86,21 @@ class GoogleDriveHelper: except ZeroDivisionError: return 0 + def dspeed(self): try: return self.downloaded_bytes / self.dtotal_time except ZeroDivisionError: return 0 + def cspeed(self): try: return self.transferred_size / int(time.time() - self.start_time) except ZeroDivisionError: return 0 + @staticmethod def getIdFromUrl(link: str): if "folders" in link or "file" in link: @@ -102,6 +112,7 @@ class GoogleDriveHelper: parsed = urlparse.urlparse(link) return parse_qs(parsed.query)['id'][0] + @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) def _on_upload_progress(self): @@ -112,6 +123,7 @@ class GoogleDriveHelper: self.uploaded_bytes += chunk_size self.total_time += self.update_interval + def __upload_empty_file(self, path, file_name, mime_type, parent_id=None): media_body = MediaFileUpload(path, mimetype=mime_type, @@ -125,6 +137,8 @@ class GoogleDriveHelper: file_metadata['parents'] = [parent_id] return self.__service.files().create(supportsTeamDrives=True, body=file_metadata, media_body=media_body).execute() + + def deletefile(self, link: str): try: file_id = self.getIdFromUrl(link) @@ -144,6 +158,7 @@ class GoogleDriveHelper: finally: return msg + def switchServiceAccount(self): global SERVICE_ACCOUNT_INDEX service_account_count = len(os.listdir("accounts")) @@ -153,7 +168,8 @@ class GoogleDriveHelper: SERVICE_ACCOUNT_INDEX += 1 LOGGER.info(f"Switching to {SERVICE_ACCOUNT_INDEX}.json service account") self.__service = self.authorize() - + + @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) def __set_permission(self, drive_id): @@ -166,6 +182,7 @@ class GoogleDriveHelper: return self.__service.permissions().create(supportsTeamDrives=True, fileId=drive_id, body=permissions).execute() + @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) def upload_file(self, file_path, file_name, mime_type, parent_id): @@ -234,6 +251,7 @@ class GoogleDriveHelper: download_url = self.__G_DRIVE_BASE_DOWNLOAD_URL.format(drive_file.get('id')) return download_url + def upload(self, file_name: str): self.is_downloading = False self.is_uploading = True @@ -299,6 +317,7 @@ class GoogleDriveHelper: self.__listener.onUploadComplete(link, size, files, folders, typ) return link + @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) def copyFile(self, file_id, dest_id): @@ -327,12 +346,14 @@ class GoogleDriveHelper: else: raise err + @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) def getFileMetadata(self,file_id): return self.__service.files().get(supportsAllDrives=True, fileId=file_id, fields="name,id,mimeType,size").execute() + @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) def getFilesByFolderId(self,folder_id): @@ -354,6 +375,7 @@ class GoogleDriveHelper: break return files + def clone(self, link): self.is_cloning = True self.start_time = time.time() @@ -385,7 +407,7 @@ class GoogleDriveHelper: msg += f'\nFiles: {self.total_files}' buttons = button_build.ButtonMaker() if SHORTENER is not None and SHORTENER_API is not None: - surl = requests.get(f'https://{SHORTENER}/api?api={SHORTENER_API}&url={durl}&format=text').text + surl = short_url(durl) buttons.buildbutton("☁️ Drive Link", surl) else: buttons.buildbutton("☁️ Drive Link", durl) @@ -393,7 +415,7 @@ class GoogleDriveHelper: url_path = requests.utils.quote(f'{meta.get("name")}') url = f'{INDEX_URL}/{url_path}/' if SHORTENER is not None and SHORTENER_API is not None: - siurl = requests.get(f'https://{SHORTENER}/api?api={SHORTENER_API}&url={url}&format=text').text + siurl = short_url(url) buttons.buildbutton("⚡ Index Link", siurl) else: buttons.buildbutton("⚡ Index Link", url) @@ -409,17 +431,17 @@ class GoogleDriveHelper: durl = self.__G_DRIVE_BASE_DOWNLOAD_URL.format(file.get("id")) buttons = button_build.ButtonMaker() if SHORTENER is not None and SHORTENER_API is not None: - surl = requests.get(f'https://{SHORTENER}/api?api={SHORTENER_API}&url={durl}&format=text').text + surl = short_url(durl) buttons.buildbutton("☁️ Drive Link", surl) else: buttons.buildbutton("☁️ Drive Link", durl) try: - typeee = file.get('mimeType') + typ = file.get('mimeType') except: - typeee = 'File' + typ = 'File' try: msg += f'\nSize: {get_readable_file_size(int(meta.get("size")))}' - msg += f'\nType: {typeee}' + msg += f'\nType: {typ}' except TypeError: pass if INDEX_URL is not None: @@ -427,10 +449,10 @@ class GoogleDriveHelper: url = f'{INDEX_URL}/{url_path}' urls = f'{INDEX_URL}/{url_path}?a=view' if SHORTENER is not None and SHORTENER_API is not None: - siurl = requests.get(f'https://{SHORTENER}/api?api={SHORTENER_API}&url={url}&format=text').text - siurls = requests.get(f'https://{SHORTENER}/api?api={SHORTENER_API}&url={urls}&format=text').text + siurl = short_url(url) buttons.buildbutton("⚡ Index Link", siurl) if VIEW_LINK: + siurls = short_url(urls) buttons.buildbutton("🌐 View Link", siurls) else: buttons.buildbutton("⚡ Index Link", url) @@ -457,6 +479,7 @@ class GoogleDriveHelper: return msg, "" return msg, InlineKeyboardMarkup(buttons.build_menu(2)) + def cloneFolder(self, name, local_path, folder_id, parent_id): LOGGER.info(f"Syncing: {local_path}") files = self.getFilesByFolderId(folder_id) @@ -480,6 +503,7 @@ class GoogleDriveHelper: if self.is_cancelled: break + @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) def create_directory(self, directory_name, parent_id): @@ -496,6 +520,7 @@ class GoogleDriveHelper: LOGGER.info("Created G-Drive Folder:\nName: {}\nID: {} ".format(file.get("name"), file_id)) return file_id + def upload_dir(self, input_directory, parent_id): list_dirs = os.listdir(input_directory) if len(list_dirs) == 0: @@ -518,6 +543,7 @@ class GoogleDriveHelper: break return new_id + def authorize(self): # Get credentials credentials = None @@ -543,6 +569,8 @@ class GoogleDriveHelper: f'accounts/{SERVICE_ACCOUNT_INDEX}.json', scopes=self.__OAUTH_SCOPE) return build('drive', 'v3', credentials=credentials, cache_discovery=False) + + def edit_telegraph(self): nxt_page = 1 prev_page = 0 @@ -564,12 +592,14 @@ class GoogleDriveHelper: html_content=content) return + def escapes(self, str): chars = ['\\', "'", '"', r'\a', r'\b', r'\f', r'\n', r'\r', r'\t'] for char in chars: str = str.replace(char, '\\'+char) return str + def drive_list(self, fileName): msg = "" fileName = self.escapes(str(fileName)) @@ -590,7 +620,7 @@ class GoogleDriveHelper: furl = f"https://drive.google.com/drive/folders/{file.get('id')}" msg += f"📁 {file.get('name')}
(folder)
" if SHORTENER is not None and SHORTENER_API is not None: - sfurl = requests.get(f'https://{SHORTENER}/api?api={SHORTENER_API}&url={furl}&format=text').text + sfurl = short_url(furl) msg += f"Drive Link" else: msg += f"Drive Link" @@ -598,7 +628,7 @@ class GoogleDriveHelper: url_path = requests.utils.quote(f'{file.get("name")}') url = f'{INDEX_URL}/{url_path}/' if SHORTENER is not None and SHORTENER_API is not None: - siurl = requests.get(f'https://{SHORTENER}/api?api={SHORTENER_API}&url={url}&format=text').text + siurl = short_url(url) msg += f' | Index Link' else: msg += f' | Index Link' @@ -610,7 +640,7 @@ class GoogleDriveHelper: furl = f"https://drive.google.com/uc?id={file.get('id')}&export=download" msg += f"📄 {file.get('name')}
({get_readable_file_size(int(file.get('size')))})
" if SHORTENER is not None and SHORTENER_API is not None: - sfurl = requests.get(f'https://{SHORTENER}/api?api={SHORTENER_API}&url={furl}&format=text').text + sfurl = short_url(furl) msg += f"Drive Link" else: msg += f"Drive Link" @@ -619,10 +649,10 @@ class GoogleDriveHelper: url = f'{INDEX_URL}/{url_path}' urls = f'{INDEX_URL}/{url_path}?a=view' if SHORTENER is not None and SHORTENER_API is not None: - siurl = requests.get(f'https://{SHORTENER}/api?api={SHORTENER_API}&url={url}&format=text').text - siurls = requests.get(f'https://{SHORTENER}/api?api={SHORTENER_API}&url={urls}&format=text').text + siurl = short_url(url) msg += f' | Index Link' if VIEW_LINK: + siurls = short_url(urls) msg += f' | View Link' else: msg += f' | Index Link' @@ -662,6 +692,7 @@ class GoogleDriveHelper: else : return '', '' + def count(self, link): try: file_id = self.getIdFromUrl(link) @@ -706,6 +737,7 @@ class GoogleDriveHelper: return msg return msg + def gDrive_file(self, **kwargs): try: size = int(kwargs['size']) @@ -713,6 +745,7 @@ class GoogleDriveHelper: size = 0 self.total_bytes += size + def gDrive_directory(self, **kwargs) -> None: files = self.getFilesByFolderId(kwargs['id']) if len(files) == 0: @@ -725,6 +758,7 @@ class GoogleDriveHelper: self.total_files += 1 self.gDrive_file(**file_) + def clonehelper(self, link): try: file_id = self.getIdFromUrl(link) @@ -757,6 +791,7 @@ class GoogleDriveHelper: return msg, "", "", "" return "", clonesize, name, files + def download(self, link): self.is_downloading = True file_id = self.getIdFromUrl(link) @@ -784,6 +819,7 @@ class GoogleDriveHelper: return self.__listener.onDownloadComplete() + def download_folder(self, folder_id, path, folder_name): if not os.path.exists(path + folder_name): os.makedirs(path + folder_name) @@ -819,6 +855,7 @@ class GoogleDriveHelper: if self.is_cancelled: break + def download_file(self, file_id, path, filename, mime_type): request = self.__service.files().get_media(fileId=file_id) fh = io.FileIO('{}{}'.format(path, filename), 'wb') @@ -850,13 +887,15 @@ class GoogleDriveHelper: raise err self._file_downloaded_bytes = 0 + def _on_download_progress(self): if self.dstatus is not None: chunk_size = self.dstatus.total_size * self.dstatus.progress() - self._file_downloaded_bytes self._file_downloaded_bytes = self.dstatus.total_size * self.dstatus.progress() self.downloaded_bytes += chunk_size self.dtotal_time += self.update_interval - + + def cancel_download(self): self.is_cancelled = True if self.is_downloading: @@ -865,5 +904,5 @@ class GoogleDriveHelper: elif self.is_cloning: LOGGER.info(f"Cancelling Clone: {self.name}") elif self.is_uploading: - LOGGER.info(f"Cancelling upload: {self.name}") + LOGGER.info(f"Cancelling Upload: {self.name}") self.__listener.onUploadError('your upload has been stopped and uploaded data has been deleted!') diff --git a/bot/modules/mirror.py b/bot/modules/mirror.py index 06f53c4..014a4d6 100644 --- a/bot/modules/mirror.py +++ b/bot/modules/mirror.py @@ -5,7 +5,7 @@ from telegram import InlineKeyboardMarkup from bot import Interval, INDEX_URL, BUTTON_FOUR_NAME, BUTTON_FOUR_URL, BUTTON_FIVE_NAME, BUTTON_FIVE_URL, BUTTON_SIX_NAME, BUTTON_SIX_URL, BLOCK_MEGA_FOLDER, BLOCK_MEGA_LINKS, VIEW_LINK, aria2, get_client from bot import dispatcher, DOWNLOAD_DIR, download_dict, download_dict_lock, SHORTENER, SHORTENER_API, TAR_UNZIP_LIMIT from bot.helper.ext_utils import fs_utils, bot_utils -from bot.helper.ext_utils.bot_utils import get_mega_link_type, check_limit +from bot.helper.ext_utils.shortenurl import short_url from bot.helper.ext_utils.exceptions import DirectDownloadLinkException, NotSupportedExtractionArchive from bot.helper.mirror_utils.download_utils.aria2_download import AriaDownloadHelper from bot.helper.mirror_utils.download_utils.mega_downloader import MegaDownloadHelper @@ -164,7 +164,7 @@ class MirrorListener(listeners.MirrorListeners): msg += f'\nType: {typ}' buttons = button_build.ButtonMaker() if SHORTENER is not None and SHORTENER_API is not None: - surl = requests.get(f'https://{SHORTENER}/api?api={SHORTENER_API}&url={link}&format=text').text + surl = short_url(link) buttons.buildbutton("☁️ Drive Link", surl) else: buttons.buildbutton("☁️ Drive Link", link) @@ -175,17 +175,17 @@ class MirrorListener(listeners.MirrorListeners): if os.path.isdir(f'{DOWNLOAD_DIR}/{self.uid}/{download_dict[self.uid].name()}'): share_url += '/' if SHORTENER is not None and SHORTENER_API is not None: - siurl = requests.get(f'https://{SHORTENER}/api?api={SHORTENER_API}&url={share_url}&format=text').text + siurl = short_url(share_url) buttons.buildbutton("⚡ Index Link", siurl) else: buttons.buildbutton("⚡ Index Link", share_url) else: share_urls = f'{INDEX_URL}/{url_path}?a=view' if SHORTENER is not None and SHORTENER_API is not None: - siurl = requests.get(f'https://{SHORTENER}/api?api={SHORTENER_API}&url={share_url}&format=text').text - siurls = requests.get(f'https://{SHORTENER}/api?api={SHORTENER_API}&url={share_urls}&format=text').text + siurl = short_url(share_url) buttons.buildbutton("⚡ Index Link", siurl) if VIEW_LINK: + siurls = short_url(share_urls) buttons.buildbutton("🌐 View Link", siurls) else: buttons.buildbutton("⚡ Index Link", share_url) @@ -335,7 +335,7 @@ def _mirror(bot, update, isTar=False, extract=False, isZip=False, isQbit=False): sendMessage(res, bot, update) return if TAR_UNZIP_LIMIT is not None: - result = check_limit(size, TAR_UNZIP_LIMIT) + result = bot_utils.check_limit(size, TAR_UNZIP_LIMIT) if result: msg = f'Failed, Tar/Unzip limit is {TAR_UNZIP_LIMIT}.\nYour File/Folder size is {get_readable_file_size(size)}.' sendMessage(msg, listener.bot, listener.update) @@ -350,11 +350,12 @@ def _mirror(bot, update, isTar=False, extract=False, isZip=False, isQbit=False): drive.download(link) elif bot_utils.is_mega_link(link): - link_type = get_mega_link_type(link) + if BLOCK_MEGA_LINKS: + sendMessage("Mega links are blocked!", bot, update) + return + link_type = bot_utils.get_mega_link_type(link) if link_type == "folder" and BLOCK_MEGA_FOLDER: sendMessage("Mega folder are blocked!", bot, update) - elif BLOCK_MEGA_LINKS: - sendMessage("Mega links are blocked!", bot, update) else: mega_dl = MegaDownloadHelper() mega_dl.add_download(link, f'{DOWNLOAD_DIR}{listener.uid}/', listener) diff --git a/requirements.txt b/requirements.txt index 2c78ca0..b30ae44 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,3 +30,4 @@ TgCrypto torrentool urllib3 youtube_dl +pyshorteners