From c4e81d5ab1809bfd32dcddf78d4b131932886e67 Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Tue, 12 May 2020 13:50:16 +0530 Subject: [PATCH 01/39] start Aria2 Websocket listener before adding download in queue. --- .../download_utils/aria2_download.py | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/bot/helper/mirror_utils/download_utils/aria2_download.py b/bot/helper/mirror_utils/download_utils/aria2_download.py index 22c403c..47cd4f2 100644 --- a/bot/helper/mirror_utils/download_utils/aria2_download.py +++ b/bot/helper/mirror_utils/download_utils/aria2_download.py @@ -17,6 +17,7 @@ class AriaDownloadHelper(DownloadHelper): def __onDownloadStarted(self, api, gid): with self._resource_lock: + LOGGER.info(f"onDownloadStart: {gid}") if self.gid == gid: download = api.get_download(gid) self.name = download.name @@ -24,15 +25,16 @@ class AriaDownloadHelper(DownloadHelper): def __onDownloadComplete(self, api: API, gid): with self._resource_lock: + LOGGER.info(f"onDownloadComplete: {gid}") if self.gid == gid: download = api.get_download(gid) if download.followed_by_ids: self.gid = download.followed_by_ids[0] with download_dict_lock: download_dict[self.__listener.uid] = AriaDownloadStatus(self, self.__listener) - if download.is_torrent: - download_dict[self.__listener.uid].is_torrent = True - update_all_messages() + if download.is_torrent: + download_dict[self.__listener.uid].is_torrent = True + update_all_messages() LOGGER.info(f'Changed gid from {gid} to {self.gid}') else: self.__listener.onDownloadComplete() @@ -56,22 +58,20 @@ class AriaDownloadHelper(DownloadHelper): self.__listener.onDownloadError(error) def add_download(self, link: str, path): - if is_magnet(link): - download = aria2.add_magnet(link, {'dir': path}) - else: - download = aria2.add_uris([link], {'dir': path}) - self.gid = download.gid - with download_dict_lock: - download_dict[self.__listener.uid] = AriaDownloadStatus(self, self.__listener) - if download.error_message: - self.__listener.onDownloadError(download.error_message) - return - LOGGER.info(f"Started: {self.gid} DIR:{download.dir} ") aria2.listen_to_notifications(threaded=True, on_download_start=self.__onDownloadStarted, on_download_error=self.__onDownloadError, on_download_pause=self.__onDownloadPause, on_download_stop=self.__onDownloadStopped, on_download_complete=self.__onDownloadComplete) + if is_magnet(link): + download = aria2.add_magnet(link, {'dir': path}) + else: + download = aria2.add_uris([link], {'dir': path}) + with self._resource_lock: + self.gid = download.gid + with download_dict_lock: + download_dict[self._listener.uid] = AriaDownloadStatus(self.gid, self._listener) + LOGGER.info(f"Started: {self.gid} DIR:{download.dir} ") def cancel_download(self): download = aria2.get_download(self.gid) From a196ce5b939c0a20fb656789d4a6427229d48d41 Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Tue, 12 May 2020 13:56:57 +0530 Subject: [PATCH 02/39] Handle TG exceptions --- bot/helper/telegram_helper/message_utils.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/bot/helper/telegram_helper/message_utils.py b/bot/helper/telegram_helper/message_utils.py index 7a1fc7e..6980a23 100644 --- a/bot/helper/telegram_helper/message_utils.py +++ b/bot/helper/telegram_helper/message_utils.py @@ -9,9 +9,12 @@ from bot import bot def sendMessage(text: str, bot, update: Update): - return bot.send_message(update.message.chat_id, + try: + return bot.send_message(update.message.chat_id, reply_to_message_id=update.message.message_id, text=text, parse_mode='HTMl') + except Exception as e: + LOGGER.error(str(e)) def editMessage(text: str, message: Message): @@ -19,9 +22,8 @@ def editMessage(text: str, message: Message): bot.edit_message_text(text=text, message_id=message.message_id, chat_id=message.chat.id, parse_mode='HTMl') - except TimedOut as e: + except Exception as e: LOGGER.error(str(e)) - pass def deleteMessage(bot, message: Message): @@ -56,10 +58,8 @@ def delete_all_messages(): try: deleteMessage(bot, message) del status_reply_dict[message.chat.id] - except BadRequest as e: - LOGGER.info(str(e)) - del status_reply_dict[message.chat.id] - pass + except Exception as e: + LOGGER.error(str(e)) def update_all_messages(): @@ -69,7 +69,7 @@ def update_all_messages(): if msg != status_reply_dict[chat_id].text: try: editMessage(msg, status_reply_dict[chat_id]) - except BadRequest as e: + except Exception as e: LOGGER.error(str(e)) status_reply_dict[chat_id].text = msg From 1670b619aec3a278d0e9836038b272cd862f406f Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Tue, 12 May 2020 14:27:24 +0530 Subject: [PATCH 03/39] Fix deadlock --- bot/helper/mirror_utils/download_utils/aria2_download.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/helper/mirror_utils/download_utils/aria2_download.py b/bot/helper/mirror_utils/download_utils/aria2_download.py index 47cd4f2..bd5c783 100644 --- a/bot/helper/mirror_utils/download_utils/aria2_download.py +++ b/bot/helper/mirror_utils/download_utils/aria2_download.py @@ -34,7 +34,7 @@ class AriaDownloadHelper(DownloadHelper): download_dict[self.__listener.uid] = AriaDownloadStatus(self, self.__listener) if download.is_torrent: download_dict[self.__listener.uid].is_torrent = True - update_all_messages() + update_all_messages() LOGGER.info(f'Changed gid from {gid} to {self.gid}') else: self.__listener.onDownloadComplete() @@ -70,7 +70,7 @@ class AriaDownloadHelper(DownloadHelper): with self._resource_lock: self.gid = download.gid with download_dict_lock: - download_dict[self._listener.uid] = AriaDownloadStatus(self.gid, self._listener) + download_dict[self.__listener.uid] = AriaDownloadStatus(self, self.__listener) LOGGER.info(f"Started: {self.gid} DIR:{download.dir} ") def cancel_download(self): From 3c04dc4a6c0203c5db66fc8cbad989a2b0d32351 Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Tue, 12 May 2020 14:34:07 +0530 Subject: [PATCH 04/39] handle exception in onUploadError --- bot/modules/mirror.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/modules/mirror.py b/bot/modules/mirror.py index b9539c3..0295a94 100644 --- a/bot/modules/mirror.py +++ b/bot/modules/mirror.py @@ -130,7 +130,7 @@ class MirrorListener(listeners.MirrorListeners): update_all_messages() def onUploadError(self, error): - e_str = str(error.last_attempt.exception()).replace('<', '').replace('>', '') + e_str = error.replace('<', '').replace('>', '') with download_dict_lock: try: fs_utils.clean_download(download_dict[self.uid].path()) From f4290210b12b644d07a46f8244e1c7d9bfe7ec8d Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Thu, 14 May 2020 19:02:33 +0530 Subject: [PATCH 05/39] Implemented mirror ownership Signed-off-by: lzzy12 --- bot/__init__.py | 2 +- bot/helper/telegram_helper/filters.py | 22 +++++++++++++++++++++- bot/modules/cancel_mirror.py | 16 ++++++++-------- bot/modules/mirror.py | 1 - 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/bot/__init__.py b/bot/__init__.py index 1f5cf40..3097979 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -54,7 +54,7 @@ status_reply_dict_lock = threading.Lock() # Value: telegram.Message status_reply_dict = {} # Key: update.message.message_id -# Value: An object of DownloadStatus +# Value: An object of Status download_dict = {} # Stores list of users and chats the bot is authorized to use in AUTHORIZED_CHATS = set() diff --git a/bot/helper/telegram_helper/filters.py b/bot/helper/telegram_helper/filters.py index a65445f..c850d6f 100644 --- a/bot/helper/telegram_helper/filters.py +++ b/bot/helper/telegram_helper/filters.py @@ -1,5 +1,6 @@ from telegram.ext import BaseFilter -from bot import AUTHORIZED_CHATS, OWNER_ID +from telegram import Message +from bot import AUTHORIZED_CHATS, OWNER_ID, download_dict, download_dict_lock class CustomFilters: @@ -21,3 +22,22 @@ class CustomFilters: return bool(message.chat.id in AUTHORIZED_CHATS) authorized_chat = _AuthorizedChat() + + class _MirrorOwner(BaseFilter): + def filter(self, message: Message): + user_id = message.from_user.id + if user_id == OWNER_ID: + return True + args = str(message.text).split(' ') + if len(args) > 1: + # Cancelling by gid + with download_dict_lock: + for message_id, status in download_dict.items(): + if status.gid == args[1] and status.message.from_user.id == user_id: + return True + else: + return False + # Cancelling by replying to original mirror message + reply_user = message.reply_to_message.from_user.id + return bool(reply_user == user_id) + mirror_owner_filter = _MirrorOwner() diff --git a/bot/modules/cancel_mirror.py b/bot/modules/cancel_mirror.py index c25c460..a4d2d74 100644 --- a/bot/modules/cancel_mirror.py +++ b/bot/modules/cancel_mirror.py @@ -11,14 +11,14 @@ from bot.helper.ext_utils.bot_utils import getDownloadByGid, MirrorStatus @run_async -def cancel_mirror(update,context): - args = update.message.text.split(" ",maxsplit=1) +def cancel_mirror(update, context): + args = update.message.text.split(" ", maxsplit=1) mirror_message = None if len(args) > 1: gid = args[1] dl = getDownloadByGid(gid) if not dl: - sendMessage(f"GID: {gid} not found.",context.bot,update) + sendMessage(f"GID: {gid} not found.", context.bot, update) return with download_dict_lock: keys = list(download_dict.keys()) @@ -33,11 +33,11 @@ def cancel_mirror(update,context): if BotCommands.MirrorCommand in mirror_message.text or \ BotCommands.TarMirrorCommand in mirror_message.text: msg = "Mirror already have been cancelled" - sendMessage(msg,context.bot,update) + sendMessage(msg, context.bot, update) return else: msg = "Please reply to the /mirror message which was used to start the download or /cancel gid to cancel it!" - sendMessage(msg,context.bot,update) + sendMessage(msg, context.bot, update) return if dl.status() == "Uploading": sendMessage("Upload in Progress, Don't Cancel it.", context.bot, update) @@ -56,16 +56,16 @@ def cancel_all(update, context): with download_dict_lock: count = 0 for dlDetails in list(download_dict.values()): - if dlDetails.status() == MirrorStatus.STATUS_DOWNLOADING\ + if dlDetails.status() == MirrorStatus.STATUS_DOWNLOADING \ or dlDetails.status() == MirrorStatus.STATUS_WAITING: dlDetails.download().cancel_download() count += 1 delete_all_messages() - sendMessage(f'Cancelled {count} downloads!', context.bot,update) + sendMessage(f'Cancelled {count} downloads!', context.bot, update) cancel_mirror_handler = CommandHandler(BotCommands.CancelMirror, cancel_mirror, - filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) + filters=(CustomFilters.authorized_chat | CustomFilters.authorized_user) & CustomFilters.mirror_owner_filter) cancel_all_handler = CommandHandler(BotCommands.CancelAllCommand, cancel_all, filters=CustomFilters.owner_filter) dispatcher.add_handler(cancel_all_handler) diff --git a/bot/modules/mirror.py b/bot/modules/mirror.py index 0295a94..0c613c5 100644 --- a/bot/modules/mirror.py +++ b/bot/modules/mirror.py @@ -16,7 +16,6 @@ from bot.helper.mirror_utils.upload_utils import gdriveTools from bot.helper.telegram_helper.bot_commands import BotCommands from bot.helper.telegram_helper.filters import CustomFilters from bot.helper.telegram_helper.message_utils import * -from bot.helper.mirror_utils.download_utils.youtube_dl_download_helper import YoutubeDLHelper import pathlib import os From cdefe1234800e9a8b36ca5f55a51f73f14696293 Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Fri, 15 May 2020 13:34:09 +0530 Subject: [PATCH 06/39] Implement Centralized Aria Websocket Listener --- bot/helper/ext_utils/bot_utils.py | 12 ++- .../download_utils/aria2_download.py | 97 +++++++++---------- .../status_utils/aria_download_status.py | 33 +++++-- bot/helper/telegram_helper/message_utils.py | 2 +- bot/modules/mirror.py | 9 +- 5 files changed, 89 insertions(+), 64 deletions(-) diff --git a/bot/helper/ext_utils/bot_utils.py b/bot/helper/ext_utils/bot_utils.py index 8d0e6ce..5998e53 100644 --- a/bot/helper/ext_utils/bot_utils.py +++ b/bot/helper/ext_utils/bot_utils.py @@ -61,7 +61,7 @@ def get_readable_file_size(size_in_bytes) -> str: def getDownloadByGid(gid): with download_dict_lock: for dl in download_dict.values(): - if dl.status() == MirrorStatus.STATUS_DOWNLOADING or dl.status() == MirrorStatus.STATUS_WAITING: + if dl.status() != MirrorStatus.STATUS_UPLOADING and dl.status() != MirrorStatus.STATUS_ARCHIVING: if dl.gid() == gid: return dl return None @@ -135,3 +135,13 @@ def is_magnet(url: str): if magnet: return True return False + +def new_thread(fn): + """To use as decorator to make a function call threaded. + Needs import + from threading import Thread""" + def wrapper(*args, **kwargs): + thread = threading.Thread(target=fn, args=args, kwargs=kwargs) + thread.start() + return thread + return wrapper \ No newline at end of file diff --git a/bot/helper/mirror_utils/download_utils/aria2_download.py b/bot/helper/mirror_utils/download_utils/aria2_download.py index bd5c783..e30a0a9 100644 --- a/bot/helper/mirror_utils/download_utils/aria2_download.py +++ b/bot/helper/mirror_utils/download_utils/aria2_download.py @@ -1,85 +1,78 @@ -from bot import aria2 +from bot import aria2, download_dict_lock from bot.helper.ext_utils.bot_utils import * from .download_helper import DownloadHelper from bot.helper.mirror_utils.status_utils.aria_download_status import AriaDownloadStatus from bot.helper.telegram_helper.message_utils import * import threading from aria2p import API +from time import sleep class AriaDownloadHelper(DownloadHelper): - def __init__(self, listener): + def __init__(self): super().__init__() - self.gid = None - self.__listener = listener - self._resource_lock = threading.RLock() + @new_thread def __onDownloadStarted(self, api, gid): - with self._resource_lock: - LOGGER.info(f"onDownloadStart: {gid}") - if self.gid == gid: - download = api.get_download(gid) - self.name = download.name - update_all_messages() + LOGGER.info(f"onDownloadStart: {gid}") + update_all_messages() def __onDownloadComplete(self, api: API, gid): - with self._resource_lock: - LOGGER.info(f"onDownloadComplete: {gid}") - if self.gid == gid: - download = api.get_download(gid) - if download.followed_by_ids: - self.gid = download.followed_by_ids[0] - with download_dict_lock: - download_dict[self.__listener.uid] = AriaDownloadStatus(self, self.__listener) - if download.is_torrent: - download_dict[self.__listener.uid].is_torrent = True - update_all_messages() - LOGGER.info(f'Changed gid from {gid} to {self.gid}') - else: - self.__listener.onDownloadComplete() + LOGGER.info(f"onDownloadComplete: {gid}") + dl = getDownloadByGid(gid) + download = api.get_download(gid) + if download.followed_by_ids: + new_gid = download.followed_by_ids[0] + with download_dict_lock: + download_dict[dl.uid()] = AriaDownloadStatus(new_gid,dl.getListener()) + if download.is_torrent: + download_dict[dl.uid()].is_torrent = True + update_all_messages() + LOGGER.info(f'Changed gid from {gid} to {new_gid}') + else: + if dl: threading.Thread(target=dl.getListener().onDownloadComplete).start() + @new_thread def __onDownloadPause(self, api, gid): - if self.gid == gid: - LOGGER.info("Called onDownloadPause") - self.__listener.onDownloadError('Download stopped by user!') + LOGGER.info(f"onDownloadPause: {gid}") + dl = getDownloadByGid(gid) + dl.getListener().onDownloadError('Download stopped by user!') + @new_thread def __onDownloadStopped(self, api, gid): - if self.gid == gid: - LOGGER.info("Called on_download_stop") - self.__listener.onDownloadError('Download stopped by user!') + LOGGER.info(f"onDownloadStop: {gid}") + dl = getDownloadByGid(gid) + if dl: dl.getListener().onDownloadError('Download stopped by user!') + @new_thread def __onDownloadError(self, api, gid): - with self._resource_lock: - if self.gid == gid: - download = api.get_download(gid) - error = download.error_message - LOGGER.info(f"Download Error: {error}") - self.__listener.onDownloadError(error) + sleep(0.5) #sleep for split second to ensure proper dl gid update from onDownloadComplete + LOGGER.info(f"onDownloadError: {gid}") + dl = getDownloadByGid(gid) + download = api.get_download(gid) + error = download.error_message + LOGGER.info(f"Download Error: {error}") + if dl: dl.getListener().onDownloadError(error) - def add_download(self, link: str, path): + def start_listener(self): aria2.listen_to_notifications(threaded=True, on_download_start=self.__onDownloadStarted, on_download_error=self.__onDownloadError, on_download_pause=self.__onDownloadPause, on_download_stop=self.__onDownloadStopped, on_download_complete=self.__onDownloadComplete) + + + def add_download(self, link: str, path,listener): if is_magnet(link): download = aria2.add_magnet(link, {'dir': path}) else: download = aria2.add_uris([link], {'dir': path}) - with self._resource_lock: - self.gid = download.gid + if download.error_message: #no need to proceed further at this point + listener.onDownloadError(download.error_message) + return with download_dict_lock: - download_dict[self.__listener.uid] = AriaDownloadStatus(self, self.__listener) - LOGGER.info(f"Started: {self.gid} DIR:{download.dir} ") + download_dict[listener.uid] = AriaDownloadStatus(download.gid,listener) + LOGGER.info(f"Started: {download.gid} DIR:{download.dir} ") + - def cancel_download(self): - download = aria2.get_download(self.gid) - if download.is_waiting: - aria2.remove([download]) - self.__listener.onDownloadError("Cancelled by user") - return - if len(download.followed_by_ids) != 0: - downloads = aria2.get_downloads(download.followed_by_ids) - aria2.pause(downloads) - aria2.pause([download]) diff --git a/bot/helper/mirror_utils/status_utils/aria_download_status.py b/bot/helper/mirror_utils/status_utils/aria_download_status.py index 502aaf0..c3199f7 100644 --- a/bot/helper/mirror_utils/status_utils/aria_download_status.py +++ b/bot/helper/mirror_utils/status_utils/aria_download_status.py @@ -1,4 +1,4 @@ -from bot import aria2, DOWNLOAD_DIR +from bot import aria2, DOWNLOAD_DIR, LOGGER from bot.helper.ext_utils.bot_utils import MirrorStatus from .status import Status @@ -9,13 +9,12 @@ def get_download(gid): class AriaDownloadStatus(Status): - def __init__(self, obj, listener): + def __init__(self, gid, listener): super().__init__() self.upload_name = None self.is_archiving = False - self.obj = obj - self.__gid = obj.gid - self.__download = get_download(obj.gid) + self.__gid = gid + self.__download = get_download(self.__gid) self.__uid = listener.uid self.__listener = listener self.message = listener.message @@ -75,7 +74,16 @@ class AriaDownloadStatus(Status): return self.__download def download(self): - return self.obj + return self + + def updateName(self,name): + self.__name = name + + def updateGid(self,gid): + self.__gid = gid + + def getListener(self): + return self.__listener def uid(self): return self.__uid @@ -83,3 +91,16 @@ class AriaDownloadStatus(Status): def gid(self): self.__update() return self.__gid + + def cancel_download(self): + LOGGER.info(f"Cancelling Download: {self.name()}") + download = self.aria_download() + if download.is_waiting: + aria2.remove([download]) + self.__listener.onDownloadError("Cancelled by user") + return + if len(download.followed_by_ids) != 0: + downloads = aria2.get_downloads(download.followed_by_ids) + aria2.pause(downloads) + aria2.pause([download]) + diff --git a/bot/helper/telegram_helper/message_utils.py b/bot/helper/telegram_helper/message_utils.py index 6980a23..9d3eb02 100644 --- a/bot/helper/telegram_helper/message_utils.py +++ b/bot/helper/telegram_helper/message_utils.py @@ -66,7 +66,7 @@ def update_all_messages(): msg = get_readable_message() with status_reply_dict_lock: for chat_id in list(status_reply_dict.keys()): - if msg != status_reply_dict[chat_id].text: + if status_reply_dict[chat_id] and msg != status_reply_dict[chat_id].text: try: editMessage(msg, status_reply_dict[chat_id]) except Exception as e: diff --git a/bot/modules/mirror.py b/bot/modules/mirror.py index 0c613c5..f6831fb 100644 --- a/bot/modules/mirror.py +++ b/bot/modules/mirror.py @@ -6,7 +6,7 @@ from bot import dispatcher, DOWNLOAD_DIR, DOWNLOAD_STATUS_UPDATE_INTERVAL, downl from bot.helper.ext_utils import fs_utils, bot_utils from bot.helper.ext_utils.bot_utils import setInterval from bot.helper.ext_utils.exceptions import DirectDownloadLinkException -from bot.helper.mirror_utils.download_utils import aria2_download +from bot.helper.mirror_utils.download_utils.aria2_download import AriaDownloadHelper from bot.helper.mirror_utils.download_utils.direct_link_generator import direct_link_generator from bot.helper.mirror_utils.download_utils.telegram_downloader import TelegramDownloadHelper from bot.helper.mirror_utils.status_utils import listeners @@ -19,6 +19,9 @@ from bot.helper.telegram_helper.message_utils import * import pathlib import os +ariaDlManager = AriaDownloadHelper() +ariaDlManager.start_listener() + class MirrorListener(listeners.MirrorListeners): @@ -82,7 +85,6 @@ class MirrorListener(listeners.MirrorListeners): del download_dict[self.uid] LOGGER.info(f"Deleting folder: {download.path()}") fs_utils.clean_download(download.path()) - LOGGER.info(f"Deleting {download.name()} from download_dict.") LOGGER.info(str(download_dict)) except Exception as e: LOGGER.error(str(e)) @@ -185,8 +187,7 @@ def _mirror(bot, update, isTar=False): except DirectDownloadLinkException as e: LOGGER.info(f'{link}: {e}') listener = MirrorListener(bot, update, isTar, tag) - aria = aria2_download.AriaDownloadHelper(listener) - aria.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/') + ariaDlManager.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/',listener) sendStatusMessage(update, bot) if len(Interval) == 0: Interval.append(setInterval(DOWNLOAD_STATUS_UPDATE_INTERVAL, update_all_messages)) From 0f7b79f7a4fbc0e830af668509bc8400d7da128b Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Thu, 21 May 2020 12:26:58 +0530 Subject: [PATCH 07/39] Improved upload progress implementation Signed-off-by: lzzy12 --- .../mirror_utils/upload_utils/gdriveTools.py | 48 +++++++------------ 1 file changed, 18 insertions(+), 30 deletions(-) diff --git a/bot/helper/mirror_utils/upload_utils/gdriveTools.py b/bot/helper/mirror_utils/upload_utils/gdriveTools.py index 480ddc3..5f30a5c 100644 --- a/bot/helper/mirror_utils/upload_utils/gdriveTools.py +++ b/bot/helper/mirror_utils/upload_utils/gdriveTools.py @@ -39,21 +39,18 @@ class GoogleDriveHelper: self.__service = self.authorize() self.__listener = listener self._file_uploaded_bytes = 0 - self.uploaded_bytes = 0 - self.UPDATE_INTERVAL = 5 + self.__uploaded_bytes = 0 self.start_time = 0 - self.total_time = 0 - self._should_update = True - self.is_uploading = True self.is_cancelled = False - self.status = None - self.updater = None self.name = name - self.update_interval = 3 + self.transferred_size = 0 def cancel(self): self.is_cancelled = True - self.is_uploading = False + + @property + def uploaded_bytes(self): + return self.__uploaded_bytes def speed(self): """ @@ -61,7 +58,7 @@ class GoogleDriveHelper: :return: Upload speed in bytes/second """ try: - return self.uploaded_bytes / self.total_time + return self.uploaded_bytes / (time.time() - self.start_time) except ZeroDivisionError: return 0 @@ -69,7 +66,7 @@ class GoogleDriveHelper: def getIdFromUrl(link: str): if "folders" in link or "file" in link: regex = r"https://drive\.google\.com/(drive)?/?u?/?\d?/?(mobile)?/?(file)?(folders)?/?d?/([-\w]+)[?+]?/?(w+)?" - res = re.search(regex,link) + res = re.search(regex, link) if res is None: raise IndexError("GDrive ID not found.") return res.group(5) @@ -78,14 +75,6 @@ class GoogleDriveHelper: @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) - def _on_upload_progress(self): - if self.status is not None: - chunk_size = self.status.total_size * self.status.progress() - self._file_uploaded_bytes - self._file_uploaded_bytes = self.status.total_size * self.status.progress() - LOGGER.debug(f'Uploading {self.name}, chunk size: {get_readable_file_size(chunk_size)}') - self.uploaded_bytes += chunk_size - self.total_time += self.update_interval - def __upload_empty_file(self, path, file_name, mime_type, parent_id=None): media_body = MediaFileUpload(path, mimetype=mime_type, @@ -155,12 +144,14 @@ class GoogleDriveHelper: drive_file = self.__service.files().create(supportsTeamDrives=True, body=file_metadata, media_body=media_body) response = None + last_uploaded = 0 while response is None: if self.is_cancelled: return None try: - self.status, response = drive_file.next_chunk() + status, response = drive_file.next_chunk() except HttpError as err: + status = None if err.resp.get('content-type', '').startswith('application/json'): reason = json.loads(err.content).get('error').get('errors')[0].get('reason') if reason == 'userRateLimitExceeded' or reason == 'dailyLimitExceeded': @@ -170,6 +161,10 @@ class GoogleDriveHelper: self.upload_file(file_path, file_name, mime_type, parent_id) else: raise err + if status is not None: + chunk_size = status.total_size * status.progress() - last_uploaded + last_uploaded = status.total_size * status.progress() + self.__uploaded_bytes += chunk_size self._file_uploaded_bytes = 0 # Insert new permissions if not IS_TEAM_DRIVE: @@ -180,14 +175,11 @@ class GoogleDriveHelper: return download_url def upload(self, file_name: str): - if USE_SERVICE_ACCOUNTS: - self.service_account_count = len(os.listdir("accounts")) self.__listener.onUploadStarted() file_dir = f"{DOWNLOAD_DIR}{self.__listener.message.message_id}" file_path = f"{file_dir}/{file_name}" LOGGER.info("Uploading File: " + file_path) self.start_time = time.time() - self.updater = setInterval(self.update_interval, self._on_upload_progress) if os.path.isfile(file_path): try: mime_type = get_mime_type(file_path) @@ -204,8 +196,6 @@ class GoogleDriveHelper: LOGGER.error(err) self.__listener.onUploadError(str(err)) return - finally: - self.updater.cancel() else: try: dir_id = self.create_directory(os.path.basename(os.path.abspath(file_name)), parent_id) @@ -223,8 +213,6 @@ class GoogleDriveHelper: LOGGER.error(err) self.__listener.onUploadError(str(err)) return - finally: - self.updater.cancel() LOGGER.info(download_dict) self.__listener.onUploadComplete(link) LOGGER.info("Deleting downloaded file/folder..") @@ -238,7 +226,7 @@ class GoogleDriveHelper: } try: - res = self.__service.files().copy(supportsAllDrives=True,fileId=file_id,body=body).execute() + res = self.__service.files().copy(supportsAllDrives=True, fileId=file_id, body=body).execute() return res except HttpError as err: if err.resp.get('content-type', '').startswith('application/json'): @@ -247,7 +235,7 @@ class GoogleDriveHelper: if USE_SERVICE_ACCOUNTS: self.switchServiceAccount() LOGGER.info(f"Got: {reason}, Trying Again.") - self.copyFile(file_id,dest_id) + self.copyFile(file_id, dest_id) else: raise err @@ -255,7 +243,7 @@ class GoogleDriveHelper: self.transferred_size = 0 try: file_id = self.getIdFromUrl(link) - except (KeyError,IndexError): + except (KeyError, IndexError): msg = "Google drive ID could not be found in the provided link" return msg msg = "" From f3a3ed05f45f22d469cfe8bf57c50acc476da1d3 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Thu, 21 May 2020 17:25:11 +0530 Subject: [PATCH 08/39] Fixed cancel by gid after f4290210b12b644d07a46f8244e1c7d9bfe7ec8d - This fixes #77 Signed-off-by: lzzy12 --- bot/helper/telegram_helper/filters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/helper/telegram_helper/filters.py b/bot/helper/telegram_helper/filters.py index c850d6f..3cd3c93 100644 --- a/bot/helper/telegram_helper/filters.py +++ b/bot/helper/telegram_helper/filters.py @@ -33,7 +33,7 @@ class CustomFilters: # Cancelling by gid with download_dict_lock: for message_id, status in download_dict.items(): - if status.gid == args[1] and status.message.from_user.id == user_id: + if status.gid() == args[1] and status.message.from_user.id == user_id: return True else: return False From 9f5942deaa9ab2b15b4ef01025a2f468ae2fc842 Mon Sep 17 00:00:00 2001 From: Adnan Ahmad <60770759+viper-adnan@users.noreply.github.com> Date: Sat, 23 May 2020 16:57:02 +0530 Subject: [PATCH 09/39] Fixed Start message after changing HelpCommand (#90) --- bot/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/__main__.py b/bot/__main__.py index 4140a3b..adc0df0 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -32,7 +32,7 @@ def stats(update, context): @run_async def start(update, context): sendMessage("This is a bot which can mirror all your links to Google drive!\n" - "Type /help to get a list of available commands", context.bot, update) + "Type /{BotCommands.HelpCommand} to get a list of available commands", context.bot, update) @run_async From b6b8a416e9e286aa529f5b043853f828eba8c180 Mon Sep 17 00:00:00 2001 From: Nageen Chand <25462010+archie9211@users.noreply.github.com> Date: Thu, 28 May 2020 00:26:22 +0530 Subject: [PATCH 10/39] add ability to unzip files (#93) handle unsupported request use shell utils to extract archives use shell utils to extract archives update mirror calls for unzipmirror fix extract location of files install deps handling if archive is not extracted correctly typo fix implement Extract Status handle filenames with spaces handle special character --- Dockerfile | 4 +- bot/__main__.py | 2 + bot/helper/ext_utils/bot_utils.py | 5 ++- bot/helper/ext_utils/fs_utils.py | 25 +++++++++++ .../status_utils/aria_download_status.py | 1 + .../status_utils/extract_status.py | 36 ++++++++++++++++ bot/helper/telegram_helper/bot_commands.py | 1 + bot/modules/mirror.py | 42 ++++++++++++++++--- extract | 34 +++++++++++++++ 9 files changed, 142 insertions(+), 8 deletions(-) create mode 100644 bot/helper/mirror_utils/status_utils/extract_status.py create mode 100755 extract diff --git a/Dockerfile b/Dockerfile index 5fb7f5a..c447db9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,10 +3,12 @@ FROM ubuntu:18.04 WORKDIR /usr/src/app RUN chmod 777 /usr/src/app RUN apt-get -qq update -RUN apt-get -qq install -y aria2 python3 python3-pip \ +RUN apt-get -qq install -y aria2 python3 python3-pip rar unzip \ locales python3-lxml \ curl pv jq ffmpeg COPY requirements.txt . +COPY extract /usr/local/bin +RUN chmod +x /usr/local/bin/extract RUN pip3 install --no-cache-dir -r requirements.txt RUN locale-gen en_US.UTF-8 ENV LANG en_US.UTF-8 diff --git a/bot/__main__.py b/bot/__main__.py index adc0df0..b4ba553 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -65,6 +65,8 @@ def bot_help(update, context): /{BotCommands.MirrorCommand} [download_url][magnet_link]: Start mirroring the link to google drive +/{BotCommands.UnzipMirrorCommand} [download_url][magnet_link] : starts mirroring and if downloaded file is any archive , extracts it to google drive + /{BotCommands.TarMirrorCommand} [download_url][magnet_link]: start mirroring and upload the archived (.tar) version of the download /{BotCommands.WatchCommand} [youtube-dl supported link]: Mirror through youtube-dl diff --git a/bot/helper/ext_utils/bot_utils.py b/bot/helper/ext_utils/bot_utils.py index 5998e53..a311ead 100644 --- a/bot/helper/ext_utils/bot_utils.py +++ b/bot/helper/ext_utils/bot_utils.py @@ -19,6 +19,7 @@ class MirrorStatus: STATUS_FAILED = "Failed. Cleaning download" STATUS_CANCELLED = "Cancelled" STATUS_ARCHIVING = "Archiving" + STATUS_EXTRACTING = "Extracting" PROGRESS_MAX_SIZE = 100 // 8 @@ -91,7 +92,7 @@ def get_readable_message(): for download in list(download_dict.values()): msg += f"{download.name()} - " msg += download.status() - if download.status() != MirrorStatus.STATUS_ARCHIVING: + if download.status() != MirrorStatus.STATUS_ARCHIVING and download.status() != MirrorStatus.STATUS_EXTRACTING: msg += f"\n{get_progress_bar_string(download)} {download.progress()} of " \ f"{download.size()}" \ f" at {download.speed()}, ETA: {download.eta()} " @@ -144,4 +145,4 @@ def new_thread(fn): thread = threading.Thread(target=fn, args=args, kwargs=kwargs) thread.start() return thread - return wrapper \ No newline at end of file + return wrapper diff --git a/bot/helper/ext_utils/fs_utils.py b/bot/helper/ext_utils/fs_utils.py index 0b92aa0..90951d6 100644 --- a/bot/helper/ext_utils/fs_utils.py +++ b/bot/helper/ext_utils/fs_utils.py @@ -56,6 +56,31 @@ def tar(org_path): return tar_path +def get_base_name(orig_path: str): + if orig_path.endswith(".tar.bz2"): + return orig_path.replace(".tar.bz2", "") + elif orig_path.endswith(".tar.gz"): + return orig_path.replace(".tar.gz","") + elif orig_path.endswith(".bz2"): + return orig_path.replace(".bz2","") + elif orig_path.endswith(".gz"): + return orig_path.replace(".gz","") + elif orig_path.endswith(".tar"): + return orig_path.replace(".tar","") + elif orig_path.endswith(".tbz2"): + return orig_path.replace("tbz2","") + elif orig_path.endswith(".tgz"): + return orig_path.replace(".tgz","") + elif orig_path.endswith(".zip"): + return orig_path.replace(".zip","") + elif orig_path.endswith(".Z"): + return orig_path.replace(".Z","") + elif orig_path.endswith(".rar"): + return orig_path.replace(".rar","") + else: + return "unsupported" + + def get_mime_type(file_path): mime = magic.Magic(mime=True) mime_type = mime.from_file(file_path) diff --git a/bot/helper/mirror_utils/status_utils/aria_download_status.py b/bot/helper/mirror_utils/status_utils/aria_download_status.py index c3199f7..ef04a79 100644 --- a/bot/helper/mirror_utils/status_utils/aria_download_status.py +++ b/bot/helper/mirror_utils/status_utils/aria_download_status.py @@ -20,6 +20,7 @@ class AriaDownloadStatus(Status): self.message = listener.message self.last = None self.is_waiting = False + self.is_extracting = False def __update(self): self.__download = get_download(self.__gid) diff --git a/bot/helper/mirror_utils/status_utils/extract_status.py b/bot/helper/mirror_utils/status_utils/extract_status.py new file mode 100644 index 0000000..f7f5e59 --- /dev/null +++ b/bot/helper/mirror_utils/status_utils/extract_status.py @@ -0,0 +1,36 @@ +from .status import Status +from bot.helper.ext_utils.bot_utils import get_readable_file_size, MirrorStatus + + +class ExtractStatus(Status): + def __init__(self, name, path, size): + self.__name = name + self.__path = path + self.__size = size + + # The progress of Tar function cannot be tracked. So we just return dummy values. + # If this is possible in future,we should implement it + + def progress(self): + return '0' + + def speed(self): + return '0' + + def name(self): + return self.__name + + def path(self): + return self.__path + + def size(self): + return get_readable_file_size(self.__size) + + def eta(self): + return '0s' + + def status(self): + return MirrorStatus.STATUS_EXTRACTING + + def processed_bytes(self): + return 0 diff --git a/bot/helper/telegram_helper/bot_commands.py b/bot/helper/telegram_helper/bot_commands.py index fee3fa2..bd1af0c 100644 --- a/bot/helper/telegram_helper/bot_commands.py +++ b/bot/helper/telegram_helper/bot_commands.py @@ -2,6 +2,7 @@ class _BotCommands: def __init__(self): self.StartCommand = 'start' self.MirrorCommand = 'mirror' + self.UnzipMirrorCommand = 'unzipmirror' self.TarMirrorCommand = 'tarmirror' self.CancelMirror = 'cancel' self.CancelAllCommand = 'cancelall' diff --git a/bot/modules/mirror.py b/bot/modules/mirror.py index f6831fb..b253139 100644 --- a/bot/modules/mirror.py +++ b/bot/modules/mirror.py @@ -10,6 +10,7 @@ from bot.helper.mirror_utils.download_utils.aria2_download import AriaDownloadHe from bot.helper.mirror_utils.download_utils.direct_link_generator import direct_link_generator from bot.helper.mirror_utils.download_utils.telegram_downloader import TelegramDownloadHelper from bot.helper.mirror_utils.status_utils import listeners +from bot.helper.mirror_utils.status_utils.extract_status import ExtractStatus from bot.helper.mirror_utils.status_utils.tar_status import TarStatus from bot.helper.mirror_utils.status_utils.upload_status import UploadStatus from bot.helper.mirror_utils.upload_utils import gdriveTools @@ -24,11 +25,11 @@ ariaDlManager.start_listener() class MirrorListener(listeners.MirrorListeners): - - def __init__(self, bot, update, isTar=False, tag=None): + def __init__(self, bot, update, isTar=False,tag=None, extract=False): super().__init__(bot, update) self.isTar = isTar self.tag = tag + self.extract = extract def onDownloadStarted(self): pass @@ -62,6 +63,30 @@ class MirrorListener(listeners.MirrorListeners): LOGGER.info('File to archive not found!') self.onUploadError('Internal error occurred!!') return + elif self.extract: + download.is_extracting = True + + path = fs_utils.get_base_name(m_path) + if path != "unsupported": + LOGGER.info( + f"Extracting : {download_dict[self.uid].name()} " + ) + download_dict[self.uid] = ExtractStatus(name, m_path, size) + os.system(f"extract '{m_path}'") + if not os.path.exists(path): + self.onUploadError("Cannot extract file, check integrity of the file") + return + LOGGER.info( + f'got path : {path}' + ) + try: + os.remove(m_path) + LOGGER.info(f"Deleting archive : {m_path}") + except Exception as e: + LOGGER.error(str(e)) + else: + LOGGER.info("Not any valid archive, uploading file as it is.") + path = f'{DOWNLOAD_DIR}{self.uid}/{download_dict[self.uid].name()}' else: path = f'{DOWNLOAD_DIR}{self.uid}/{download_dict[self.uid].name()}' up_name = pathlib.PurePath(path).name @@ -145,8 +170,7 @@ class MirrorListener(listeners.MirrorListeners): else: update_all_messages() - -def _mirror(bot, update, isTar=False): +def _mirror(bot, update, isTar=False, extract=False): message_args = update.message.text.split(' ') try: link = message_args[1] @@ -186,7 +210,7 @@ def _mirror(bot, update, isTar=False): link = direct_link_generator(link) except DirectDownloadLinkException as e: LOGGER.info(f'{link}: {e}') - listener = MirrorListener(bot, update, isTar, tag) + listener = MirrorListener(bot, update, isTar, tag, extract) ariaDlManager.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/',listener) sendStatusMessage(update, bot) if len(Interval) == 0: @@ -203,9 +227,17 @@ def tar_mirror(update, context): _mirror(context.bot, update, True) +@run_async +def unzip_mirror(update, context): + _mirror(context.bot,update, extract=True) + + mirror_handler = CommandHandler(BotCommands.MirrorCommand, mirror, filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) tar_mirror_handler = CommandHandler(BotCommands.TarMirrorCommand, tar_mirror, filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) +unzip_mirror_handler = CommandHandler(BotCommands.UnzipMirrorCommand, unzip_mirror, + filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) dispatcher.add_handler(mirror_handler) dispatcher.add_handler(tar_mirror_handler) +dispatcher.add_handler(unzip_mirror_handler) diff --git a/extract b/extract new file mode 100755 index 0000000..db113eb --- /dev/null +++ b/extract @@ -0,0 +1,34 @@ +#!/bin/bash + +if [ $# -lt 1 ];then + echo "Usage: `basename $0` FILES" + exit 1 +fi + +extract () { + arg="$1" + cd $(dirname "$arg") + case "$arg" in + *.tar.bz2) + tar xjf "$arg" --one-top-level ;; + *.tar.gz) + tar xzf "$arg" --one-top-level ;; + *.bz2) bunzip2 "$arg" ;; + *.gz) gunzip "$arg" ;; + *.tar) tar xf "$arg" --one-top-level ;; + *.tbz2) tar xjf "$arg" --one-top-level ;; + *.tgz) tar xzf "$arg" --one-top-level ;; + *.zip) + a_dir=`expr "$arg" : '\(.*\).zip'` + unzip "$arg" -d "$a_dir" ;; + *.Z) uncompress "$arg" ;; + *.rar) + a_dir=`expr "$arg" : '\(.*\).rar'` ; + mkdir "$a_dir" ; + rar x "$arg" "$a_dir" ;; # 'rar' must to be installed + *) echo "'$arg' cannot be extracted via extract()" ;; + esac + cd - +} + +extract "$1" From 3f45ba6a7379129e133d3cb10c541b2ffb47de5b Mon Sep 17 00:00:00 2001 From: Yash Garg Date: Thu, 28 May 2020 08:23:02 +0530 Subject: [PATCH 11/39] Fix start message using f strings (#94) Fixes #90 --- bot/__main__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bot/__main__.py b/bot/__main__.py index b4ba553..34df995 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -31,8 +31,11 @@ def stats(update, context): @run_async def start(update, context): - sendMessage("This is a bot which can mirror all your links to Google drive!\n" - "Type /{BotCommands.HelpCommand} to get a list of available commands", context.bot, update) + start_string = f''' +This is a bot which can mirror all your links to Google drive! +Type /{BotCommands.HelpCommand} to get a list of available commands +''' + sendMessage(start_string, context.bot, update) @run_async From 578eeb4004460e71f332f264f2b50cb43fd1e134 Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Thu, 28 May 2020 15:17:19 +0530 Subject: [PATCH 12/39] Implement Mega.nz mirror support. - Co-authored by - lzzy12 (github.com/lzzy12) --- bot/helper/ext_utils/bot_utils.py | 3 + .../download_utils/mega_downloader.py | 132 ++++++++++++++++++ .../status_utils/mega_download_status.py | 62 ++++++++ bot/modules/mirror.py | 7 +- 4 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 bot/helper/mirror_utils/download_utils/mega_downloader.py create mode 100644 bot/helper/mirror_utils/status_utils/mega_download_status.py diff --git a/bot/helper/ext_utils/bot_utils.py b/bot/helper/ext_utils/bot_utils.py index a311ead..a015fb7 100644 --- a/bot/helper/ext_utils/bot_utils.py +++ b/bot/helper/ext_utils/bot_utils.py @@ -137,6 +137,9 @@ def is_magnet(url: str): return True return False +def is_mega_link(url: str): + return "mega.nz" in url + def new_thread(fn): """To use as decorator to make a function call threaded. Needs import diff --git a/bot/helper/mirror_utils/download_utils/mega_downloader.py b/bot/helper/mirror_utils/download_utils/mega_downloader.py new file mode 100644 index 0000000..4714b11 --- /dev/null +++ b/bot/helper/mirror_utils/download_utils/mega_downloader.py @@ -0,0 +1,132 @@ +from bot import LOGGER, MEGA_API_KEY, download_dict_lock, download_dict +import threading +from mega import (MegaApi, MegaListener, MegaRequest, MegaTransfer, MegaError) +from bot.helper.telegram_helper.message_utils import update_all_messages +import os +from bot.helper.mirror_utils.status_utils.mega_download_status import MegaDownloadStatus + +class MegaAppListener(MegaListener): + + def __init__(self, continue_event: threading.Event, listener): + self.continue_event = continue_event + self.node = None + self.listener = listener + self.uid = listener.uid + self.__bytes_transferred = 0 + self.is_cancelled = False + self.__speed = 0 + self.__name = '' + self.__size = 0 + self.error = None + self.gid = "" + super(MegaAppListener, self).__init__() + + @property + def speed(self): + """Returns speed of the download in bytes/second""" + return self.__speed + + @property + def name(self): + """Returns name of the download""" + return self.__name + + def setValues(self,name,size,gid): + self.__name = name + self.__size = size + self.gid = gid + + @property + def size(self): + """Size of download in bytes""" + return self.__size + + @property + def downloaded_bytes(self): + return self.__bytes_transferred + + def onRequestStart(self, api, request): + LOGGER.info('Request start ({})'.format(request)) + + def onRequestFinish(self, api, request, error): + LOGGER.info('Mega Request finished ({}); Result: {}' + .format(request, error)) + + request_type = request.getType() + if request_type == MegaRequest.TYPE_GET_PUBLIC_NODE: + self.node = request.getPublicMegaNode() + if request_type == MegaRequest.TYPE_LOGIN: + LOGGER.info("Fetching Nodes.") + api.fetchNodes() + if request_type == MegaRequest.TYPE_FETCH_NODES: + LOGGER.info("Fetching Root Node.") + self.node = api.getRootNode() + if request_type != MegaRequest.TYPE_LOGIN: + self.continue_event.set() + + def onRequestTemporaryError(self, api, request, error: MegaError): + self.listener.onDownloadError(error.toString()) + self.error = error.toString() + self.continue_event.set() + + def onTransferStart(self, api: MegaApi, transfer: MegaTransfer): + LOGGER.info(f"Transfer Started: {transfer.getFileName()}") + + def onTransferUpdate(self, api: MegaApi, transfer: MegaTransfer): + if self.is_cancelled: + api.cancelTransfer(transfer,None) + self.__speed = transfer.getSpeed() + self.__bytes_transferred = transfer.getTransferredBytes() + + def onTransferFinish(self, api: MegaApi, transfer : MegaTransfer, error): + try: + LOGGER.info(f'Transfer finished ({transfer}); Result: {transfer.getFileName()}') + if str(error) != "No error" and self.is_cancelled: + self.is_cancelled = False + return self.listener.onDownloadError(error.toString()) + if transfer.isFolderTransfer() and transfer.isFinished() and not self.is_cancelled or transfer.getFileName() == self.name and not self.is_cancelled: + self.listener.onDownloadComplete() + except Exception as e: + LOGGER.error(e) + + def onTransferTemporaryError(self, api, transfer, error): + LOGGER.info(f'Mega download error in file {transfer} {transfer.getFileName()}: {error}') + self.listener.onDownloadError(error.toString()) + self.error = error.toString() + self.continue_event.set() + + def cancel_download(self): + self.is_cancelled = True + +class AsyncExecutor(object): + + def __init__(self): + self.continue_event = threading.Event() + + def do(self, function, args): + self.continue_event.clear() + function(*args) + self.continue_event.wait() + +class MegaDownloadHelper: + def __init__(self): + pass + + def add_download(self, mega_link: str, path: str, listener): + executor = AsyncExecutor() + api = MegaApi(MEGA_API_KEY, None, None, 'telegram-mirror-bot') + mega_listener = MegaAppListener(executor.continue_event, listener) + os.makedirs(path) + api.addListener(mega_listener) + executor.do(api.getPublicNode, (mega_link,)) + node = mega_listener.node + if node is None: + executor.do(api.loginToFolder, (mega_link,)) + node = mega_listener.node + if mega_listener.error is not None: + return listener.onDownloadError(str(mega_listener.error)) + mega_listener.setValues(node.getName(),api.getSize(node),mega_link.split("!",1)[-1].split("!",1)[0]) + with download_dict_lock: + download_dict[listener.uid] = MegaDownloadStatus(mega_listener,listener) + threading.Thread(target=executor.do,args=(api.startDownload,(node,path))).start() + update_all_messages() diff --git a/bot/helper/mirror_utils/status_utils/mega_download_status.py b/bot/helper/mirror_utils/status_utils/mega_download_status.py new file mode 100644 index 0000000..bb5cf73 --- /dev/null +++ b/bot/helper/mirror_utils/status_utils/mega_download_status.py @@ -0,0 +1,62 @@ +from bot.helper.ext_utils.bot_utils import get_readable_file_size,MirrorStatus, get_readable_time +from bot import DOWNLOAD_DIR +from .status import Status + + +class MegaDownloadStatus(Status): + + def __init__(self, obj,listener): + self.uid = obj.uid + self.listener = listener + self.obj = obj + + def name(self) -> str: + return self.obj.name + + def progress_raw(self): + try: + return round(self.processed_bytes() / self.obj.size * 100,2) + except ZeroDivisionError: + return 0.0 + + def progress(self): + """Progress of download in percentage""" + return f"{self.progress_raw()}%" + + + def status(self) -> str: + return MirrorStatus.STATUS_DOWNLOADING + + def processed_bytes(self): + return self.obj.downloaded_bytes + + def eta(self): + try: + seconds = (self.size_raw() - self.processed_bytes()) / self.speed_raw() + return f'{get_readable_time(seconds)}' + except ZeroDivisionError: + return '-' + + def size_raw(self): + return self.obj.size + + def size(self) -> str: + return get_readable_file_size(self.size_raw()) + + def downloaded(self) -> str: + return get_readable_file_size(self.obj.downloadedBytes) + + def speed_raw(self): + return self.obj.speed + + def speed(self) -> str: + return f'{get_readable_file_size(self.speed_raw())}/s' + + def gid(self) -> str: + return self.obj.gid + + def path(self) -> str: + return f"{DOWNLOAD_DIR}{self.uid}" + + def download(self): + return self.obj \ No newline at end of file diff --git a/bot/modules/mirror.py b/bot/modules/mirror.py index b253139..a6d2cd0 100644 --- a/bot/modules/mirror.py +++ b/bot/modules/mirror.py @@ -7,6 +7,7 @@ from bot.helper.ext_utils import fs_utils, bot_utils from bot.helper.ext_utils.bot_utils import setInterval from bot.helper.ext_utils.exceptions import DirectDownloadLinkException from bot.helper.mirror_utils.download_utils.aria2_download import AriaDownloadHelper +from bot.helper.mirror_utils.download_utils.mega_downloader import MegaDownloadHelper from bot.helper.mirror_utils.download_utils.direct_link_generator import direct_link_generator from bot.helper.mirror_utils.download_utils.telegram_downloader import TelegramDownloadHelper from bot.helper.mirror_utils.status_utils import listeners @@ -211,7 +212,11 @@ def _mirror(bot, update, isTar=False, extract=False): except DirectDownloadLinkException as e: LOGGER.info(f'{link}: {e}') listener = MirrorListener(bot, update, isTar, tag, extract) - ariaDlManager.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/',listener) + if bot_utils.is_mega_link(link): + mega_dl = MegaDownloadHelper() + mega_dl.add_download(link,f'{DOWNLOAD_DIR}/{listener.uid}/',listener) + else: + ariaDlManager.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/',listener) sendStatusMessage(update, bot) if len(Interval) == 0: Interval.append(setInterval(DOWNLOAD_STATUS_UPDATE_INTERVAL, update_all_messages)) From 82ffb142e44750290e6c1eacb01d27283b27056e Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Fri, 1 Nov 2019 18:20:39 +0530 Subject: [PATCH 13/39] Add Docker steps to install Mega sdk Signed-off-by: lzzy12 --- Dockerfile | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index c447db9..4c89c26 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,10 +2,25 @@ FROM ubuntu:18.04 WORKDIR /usr/src/app RUN chmod 777 /usr/src/app + RUN apt-get -qq update -RUN apt-get -qq install -y aria2 python3 python3-pip rar unzip \ - locales python3-lxml \ - curl pv jq ffmpeg +RUN apt-get -qq install -y python3 python3-pip rar unzip git aria2 g++ gcc autoconf automake \ + m4 libtool qt4-qmake make libqt4-dev libcurl4-openssl-dev \ + libcrypto++-dev libsqlite3-dev libc-ares-dev \ + libsodium-dev libnautilus-extension-dev \ + libssl-dev libfreeimage-dev swig curl pv jq ffmpeg locales python3-lxml + +# Installing mega sdk python binding +ENV MEGA_SDK_VERSION '3.6.4' +RUN git clone https://github.com/meganz/sdk.git sdk +WORKDIR sdk +RUN git checkout v$MEGA_SDK_VERSION && ./autogen.sh && \ + ./configure --disable-silent-rules --enable-python --disable-examples && \ + make -j$(nproc --all) && cd bindings/python/ && \ + python setup.py bdist_wheel && cd dist/ && \ + pip install --no-cache-dir megasdk-$MEGA_SDK_VERSION-*.whl + + COPY requirements.txt . COPY extract /usr/local/bin RUN chmod +x /usr/local/bin/extract @@ -19,3 +34,5 @@ COPY netrc /root/.netrc RUN chmod +x aria.sh CMD ["bash","start.sh"] + + From 3aea903deb865608d8ef18126862a68730c37ce8 Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Thu, 28 May 2020 15:27:16 +0530 Subject: [PATCH 14/39] update config_sample --- config_sample.env | 1 + 1 file changed, 1 insertion(+) diff --git a/config_sample.env b/config_sample.env index 1ef1dc7..453c888 100644 --- a/config_sample.env +++ b/config_sample.env @@ -14,3 +14,4 @@ USER_SESSION_STRING = "" TELEGRAM_API = TELEGRAM_HASH = "" USE_SERVICE_ACCOUNTS = "" +MEGA_API_KEY = "" From df655a356c43617325b7d4121f60d92760d65648 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Thu, 28 May 2020 21:02:04 +0530 Subject: [PATCH 15/39] Import mega api key from config.env file Signed-off-by: lzzy12 --- README.md | 1 + bot/__init__.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/README.md b/README.md index 8e39196..b402db9 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,7 @@ Fill up rest of the fields. Meaning of each fields are discussed below: - **API_KEY** : This is to authenticate to your telegram account for downloading Telegram files. You can get this from https://my.telegram.org DO NOT put this in quotes. - **API_HASH** : This is to authenticate to your telegram account for downloading Telegram files. You can get this from https://my.telegram.org - **USER_SESSION_STRING** : Session string generated by running: +- **MEGA_API_KEY**: Mega.nz api key to mirror mega.nz links. Get it from [Mega SDK Page](https://mega.nz/sdk) ``` python3 generate_string_session.py ``` diff --git a/bot/__init__.py b/bot/__init__.py index 3097979..c08dee6 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -79,6 +79,12 @@ try: except KeyError as e: LOGGER.error("One or more env variables missing! Exiting now") exit(1) + +try: + MEGA_API_KEY = getConfig('MEGA_API_KEY') +except KeyError: + logging.warning('MEGA Api key not provided!') + MEGA_API_KEY = None try: INDEX_URL = getConfig('INDEX_URL') if len(INDEX_URL) == 0: From 8bd01276bec5c4a50b07531835700a152dacb8a0 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Wed, 3 Jun 2020 11:32:09 +0530 Subject: [PATCH 16/39] Add ability to login to mega account for premium accounts Signed-off-by: lzzy12 --- README.md | 2 ++ bot/__init__.py | 13 +++++++-- .../download_utils/mega_downloader.py | 27 +++++++++++++------ config_sample.env | 2 ++ 4 files changed, 34 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index b402db9..5415f8f 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,8 @@ Fill up rest of the fields. Meaning of each fields are discussed below: - **API_HASH** : This is to authenticate to your telegram account for downloading Telegram files. You can get this from https://my.telegram.org - **USER_SESSION_STRING** : Session string generated by running: - **MEGA_API_KEY**: Mega.nz api key to mirror mega.nz links. Get it from [Mega SDK Page](https://mega.nz/sdk) +- **MEGA_EMAIL_ID**: You email id you used to sign up on mega.nz for using premium accounts (Leave th) +- ** ``` python3 generate_string_session.py ``` diff --git a/bot/__init__.py b/bot/__init__.py index c08dee6..76fca77 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -83,8 +83,17 @@ except KeyError as e: try: MEGA_API_KEY = getConfig('MEGA_API_KEY') except KeyError: - logging.warning('MEGA Api key not provided!') - MEGA_API_KEY = None + logging.warning('MEGA API KEY not provided!') + +try: + MEGA_EMAIL_ID = getConfig('MEGA_EMAIL_ID') + MEGA_PASSWORD = getConfig('MEGA_PASSWORD') + if len(MEGA_EMAIL_ID) == 0 or len(MEGA_PASSWORD) == 0: + raise KeyError +except KeyError: + logging.warning('MEGA Credentials not provided!') + MEGA_EMAIL_ID = None + MEGA_PASSWORD = None try: INDEX_URL = getConfig('INDEX_URL') if len(INDEX_URL) == 0: diff --git a/bot/helper/mirror_utils/download_utils/mega_downloader.py b/bot/helper/mirror_utils/download_utils/mega_downloader.py index 4714b11..27d1efd 100644 --- a/bot/helper/mirror_utils/download_utils/mega_downloader.py +++ b/bot/helper/mirror_utils/download_utils/mega_downloader.py @@ -1,10 +1,15 @@ -from bot import LOGGER, MEGA_API_KEY, download_dict_lock, download_dict +from bot import LOGGER, MEGA_API_KEY, download_dict_lock, download_dict, MEGA_EMAIL_ID, MEGA_PASSWORD import threading from mega import (MegaApi, MegaListener, MegaRequest, MegaTransfer, MegaError) from bot.helper.telegram_helper.message_utils import update_all_messages import os from bot.helper.mirror_utils.status_utils.mega_download_status import MegaDownloadStatus + +class MegaDownloaderException(Exception): + pass + + class MegaAppListener(MegaListener): def __init__(self, continue_event: threading.Event, listener): @@ -31,7 +36,7 @@ class MegaAppListener(MegaListener): """Returns name of the download""" return self.__name - def setValues(self,name,size,gid): + def setValues(self, name, size, gid): self.__name = name self.__size = size self.gid = gid @@ -74,11 +79,11 @@ class MegaAppListener(MegaListener): def onTransferUpdate(self, api: MegaApi, transfer: MegaTransfer): if self.is_cancelled: - api.cancelTransfer(transfer,None) + api.cancelTransfer(transfer, None) self.__speed = transfer.getSpeed() self.__bytes_transferred = transfer.getTransferredBytes() - def onTransferFinish(self, api: MegaApi, transfer : MegaTransfer, error): + def onTransferFinish(self, api: MegaApi, transfer: MegaTransfer, error): try: LOGGER.info(f'Transfer finished ({transfer}); Result: {transfer.getFileName()}') if str(error) != "No error" and self.is_cancelled: @@ -98,7 +103,8 @@ class MegaAppListener(MegaListener): def cancel_download(self): self.is_cancelled = True -class AsyncExecutor(object): + +class AsyncExecutor: def __init__(self): self.continue_event = threading.Event() @@ -108,16 +114,21 @@ class AsyncExecutor(object): function(*args) self.continue_event.wait() + class MegaDownloadHelper: def __init__(self): pass def add_download(self, mega_link: str, path: str, listener): + if MEGA_API_KEY is None: + raise MegaDownloaderException('Mega API KEY not provided! Cannot mirror mega links') executor = AsyncExecutor() api = MegaApi(MEGA_API_KEY, None, None, 'telegram-mirror-bot') mega_listener = MegaAppListener(executor.continue_event, listener) os.makedirs(path) api.addListener(mega_listener) + if MEGA_EMAIL_ID is not None and MEGA_PASSWORD is not None: + executor.do(api.login, (MEGA_EMAIL_ID, MEGA_PASSWORD)) executor.do(api.getPublicNode, (mega_link,)) node = mega_listener.node if node is None: @@ -125,8 +136,8 @@ class MegaDownloadHelper: node = mega_listener.node if mega_listener.error is not None: return listener.onDownloadError(str(mega_listener.error)) - mega_listener.setValues(node.getName(),api.getSize(node),mega_link.split("!",1)[-1].split("!",1)[0]) + mega_listener.setValues(node.getName(), api.getSize(node), mega_link.split("!", 1)[-1].split("!", 1)[0]) with download_dict_lock: - download_dict[listener.uid] = MegaDownloadStatus(mega_listener,listener) - threading.Thread(target=executor.do,args=(api.startDownload,(node,path))).start() + download_dict[listener.uid] = MegaDownloadStatus(mega_listener, listener) + threading.Thread(target=executor.do, args=(api.startDownload, (node, path))).start() update_all_messages() diff --git a/config_sample.env b/config_sample.env index 453c888..67ff99c 100644 --- a/config_sample.env +++ b/config_sample.env @@ -15,3 +15,5 @@ TELEGRAM_API = TELEGRAM_HASH = "" USE_SERVICE_ACCOUNTS = "" MEGA_API_KEY = "" +MEGA_EMAIL_ID = "" +MEGA_PASSWORD = "" \ No newline at end of file From 4754fb7a231828e8e6c303b62e8b3981717f2676 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Wed, 3 Jun 2020 11:35:02 +0530 Subject: [PATCH 17/39] Fixed docs Signed-off-by: lzzy12 --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5415f8f..31f3b52 100644 --- a/README.md +++ b/README.md @@ -61,8 +61,8 @@ Fill up rest of the fields. Meaning of each fields are discussed below: - **API_HASH** : This is to authenticate to your telegram account for downloading Telegram files. You can get this from https://my.telegram.org - **USER_SESSION_STRING** : Session string generated by running: - **MEGA_API_KEY**: Mega.nz api key to mirror mega.nz links. Get it from [Mega SDK Page](https://mega.nz/sdk) -- **MEGA_EMAIL_ID**: You email id you used to sign up on mega.nz for using premium accounts (Leave th) -- ** +- **MEGA_EMAIL_ID**: Your email id you used to sign up on mega.nz for using premium accounts (Leave th) +- **MEGA_PASSWORD**: Your password for your mega.nz account ``` python3 generate_string_session.py ``` @@ -73,7 +73,7 @@ Note: You can limit maximum concurrent downloads by changing the value of MAX_CO - Visit the [Google Cloud Console](https://console.developers.google.com/apis/credentials) - Go to the OAuth Consent tab, fill it, and save. - Go to the Credentials tab and click Create Credentials -> OAuth Client ID -- Choose Other and Create. +- Choose Desktop and Create. - Use the download button to download your credentials. - Move that file to the root of mirror-bot, and rename it to credentials.json - Visit [Google API page](https://console.developers.google.com/apis/library) From 8b3805c55a76b5376bf855660751a70263e3fa0a Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Fri, 5 Jun 2020 12:53:17 +0530 Subject: [PATCH 18/39] Use prebuilt Docker image for mega-sdk Signed-off-by: lzzy12 --- Dockerfile | 19 ++----------------- extract | 10 +++++----- 2 files changed, 7 insertions(+), 22 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4c89c26..8cc39e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,25 +1,10 @@ -FROM ubuntu:18.04 +FROM lzzy12/mega-sdk-python:latest WORKDIR /usr/src/app RUN chmod 777 /usr/src/app RUN apt-get -qq update -RUN apt-get -qq install -y python3 python3-pip rar unzip git aria2 g++ gcc autoconf automake \ - m4 libtool qt4-qmake make libqt4-dev libcurl4-openssl-dev \ - libcrypto++-dev libsqlite3-dev libc-ares-dev \ - libsodium-dev libnautilus-extension-dev \ - libssl-dev libfreeimage-dev swig curl pv jq ffmpeg locales python3-lxml - -# Installing mega sdk python binding -ENV MEGA_SDK_VERSION '3.6.4' -RUN git clone https://github.com/meganz/sdk.git sdk -WORKDIR sdk -RUN git checkout v$MEGA_SDK_VERSION && ./autogen.sh && \ - ./configure --disable-silent-rules --enable-python --disable-examples && \ - make -j$(nproc --all) && cd bindings/python/ && \ - python setup.py bdist_wheel && cd dist/ && \ - pip install --no-cache-dir megasdk-$MEGA_SDK_VERSION-*.whl - +RUN apt-get -qq install -y p7zip-full aria2 curl pv jq ffmpeg locales python3-lxml COPY requirements.txt . COPY extract /usr/local/bin diff --git a/extract b/extract index db113eb..8a31a9e 100755 --- a/extract +++ b/extract @@ -7,7 +7,7 @@ fi extract () { arg="$1" - cd $(dirname "$arg") + cd "$(dirname "$arg")" || exit case "$arg" in *.tar.bz2) tar xjf "$arg" --one-top-level ;; @@ -20,15 +20,15 @@ extract () { *.tgz) tar xzf "$arg" --one-top-level ;; *.zip) a_dir=`expr "$arg" : '\(.*\).zip'` - unzip "$arg" -d "$a_dir" ;; + 7z "$arg" -o"$a_dir" ;; *.Z) uncompress "$arg" ;; - *.rar) + *.rar) a_dir=`expr "$arg" : '\(.*\).rar'` ; mkdir "$a_dir" ; - rar x "$arg" "$a_dir" ;; # 'rar' must to be installed + 7z x "$arg" -o"$a_dir" ;; *) echo "'$arg' cannot be extracted via extract()" ;; esac - cd - + cd - || exit } extract "$1" From 3217b67df03022655899ac821e90e05fecd4bdbd Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Fri, 5 Jun 2020 13:15:00 +0530 Subject: [PATCH 19/39] Fixup for meganz login Signed-off-by: lzzy12 --- .../mirror_utils/download_utils/mega_downloader.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/bot/helper/mirror_utils/download_utils/mega_downloader.py b/bot/helper/mirror_utils/download_utils/mega_downloader.py index 27d1efd..a7a52e1 100644 --- a/bot/helper/mirror_utils/download_utils/mega_downloader.py +++ b/bot/helper/mirror_utils/download_utils/mega_downloader.py @@ -11,6 +11,8 @@ class MegaDownloaderException(Exception): class MegaAppListener(MegaListener): + _NO_EVENT_ON = (MegaRequest.TYPE_LOGIN, + MegaRequest.TYPE_FETCH_NODES) def __init__(self, continue_event: threading.Event, listener): self.continue_event = continue_event @@ -58,15 +60,14 @@ class MegaAppListener(MegaListener): .format(request, error)) request_type = request.getType() - if request_type == MegaRequest.TYPE_GET_PUBLIC_NODE: - self.node = request.getPublicMegaNode() if request_type == MegaRequest.TYPE_LOGIN: - LOGGER.info("Fetching Nodes.") api.fetchNodes() - if request_type == MegaRequest.TYPE_FETCH_NODES: + elif request_type == MegaRequest.TYPE_GET_PUBLIC_NODE: + self.node = request.getPublicMegaNode() + elif request_type == MegaRequest.TYPE_FETCH_NODES: LOGGER.info("Fetching Root Node.") self.node = api.getRootNode() - if request_type != MegaRequest.TYPE_LOGIN: + if request_type not in self._NO_EVENT_ON: self.continue_event.set() def onRequestTemporaryError(self, api, request, error: MegaError): From c791b39785b222783ebeefee5338e7ee86957682 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Fri, 5 Jun 2020 13:20:40 +0530 Subject: [PATCH 20/39] Revert "Improved upload progress implementation" This reverts commit 0f7b79f7a4fbc0e830af668509bc8400d7da128b. --- .../mirror_utils/upload_utils/gdriveTools.py | 48 ++++++++++++------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/bot/helper/mirror_utils/upload_utils/gdriveTools.py b/bot/helper/mirror_utils/upload_utils/gdriveTools.py index 5f30a5c..480ddc3 100644 --- a/bot/helper/mirror_utils/upload_utils/gdriveTools.py +++ b/bot/helper/mirror_utils/upload_utils/gdriveTools.py @@ -39,18 +39,21 @@ class GoogleDriveHelper: self.__service = self.authorize() self.__listener = listener self._file_uploaded_bytes = 0 - self.__uploaded_bytes = 0 + self.uploaded_bytes = 0 + self.UPDATE_INTERVAL = 5 self.start_time = 0 + self.total_time = 0 + self._should_update = True + self.is_uploading = True self.is_cancelled = False + self.status = None + self.updater = None self.name = name - self.transferred_size = 0 + self.update_interval = 3 def cancel(self): self.is_cancelled = True - - @property - def uploaded_bytes(self): - return self.__uploaded_bytes + self.is_uploading = False def speed(self): """ @@ -58,7 +61,7 @@ class GoogleDriveHelper: :return: Upload speed in bytes/second """ try: - return self.uploaded_bytes / (time.time() - self.start_time) + return self.uploaded_bytes / self.total_time except ZeroDivisionError: return 0 @@ -66,7 +69,7 @@ class GoogleDriveHelper: def getIdFromUrl(link: str): if "folders" in link or "file" in link: regex = r"https://drive\.google\.com/(drive)?/?u?/?\d?/?(mobile)?/?(file)?(folders)?/?d?/([-\w]+)[?+]?/?(w+)?" - res = re.search(regex, link) + res = re.search(regex,link) if res is None: raise IndexError("GDrive ID not found.") return res.group(5) @@ -75,6 +78,14 @@ class GoogleDriveHelper: @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) + def _on_upload_progress(self): + if self.status is not None: + chunk_size = self.status.total_size * self.status.progress() - self._file_uploaded_bytes + self._file_uploaded_bytes = self.status.total_size * self.status.progress() + LOGGER.debug(f'Uploading {self.name}, chunk size: {get_readable_file_size(chunk_size)}') + self.uploaded_bytes += chunk_size + self.total_time += self.update_interval + def __upload_empty_file(self, path, file_name, mime_type, parent_id=None): media_body = MediaFileUpload(path, mimetype=mime_type, @@ -144,14 +155,12 @@ class GoogleDriveHelper: drive_file = self.__service.files().create(supportsTeamDrives=True, body=file_metadata, media_body=media_body) response = None - last_uploaded = 0 while response is None: if self.is_cancelled: return None try: - status, response = drive_file.next_chunk() + self.status, response = drive_file.next_chunk() except HttpError as err: - status = None if err.resp.get('content-type', '').startswith('application/json'): reason = json.loads(err.content).get('error').get('errors')[0].get('reason') if reason == 'userRateLimitExceeded' or reason == 'dailyLimitExceeded': @@ -161,10 +170,6 @@ class GoogleDriveHelper: self.upload_file(file_path, file_name, mime_type, parent_id) else: raise err - if status is not None: - chunk_size = status.total_size * status.progress() - last_uploaded - last_uploaded = status.total_size * status.progress() - self.__uploaded_bytes += chunk_size self._file_uploaded_bytes = 0 # Insert new permissions if not IS_TEAM_DRIVE: @@ -175,11 +180,14 @@ class GoogleDriveHelper: return download_url def upload(self, file_name: str): + if USE_SERVICE_ACCOUNTS: + self.service_account_count = len(os.listdir("accounts")) self.__listener.onUploadStarted() file_dir = f"{DOWNLOAD_DIR}{self.__listener.message.message_id}" file_path = f"{file_dir}/{file_name}" LOGGER.info("Uploading File: " + file_path) self.start_time = time.time() + self.updater = setInterval(self.update_interval, self._on_upload_progress) if os.path.isfile(file_path): try: mime_type = get_mime_type(file_path) @@ -196,6 +204,8 @@ class GoogleDriveHelper: LOGGER.error(err) self.__listener.onUploadError(str(err)) return + finally: + self.updater.cancel() else: try: dir_id = self.create_directory(os.path.basename(os.path.abspath(file_name)), parent_id) @@ -213,6 +223,8 @@ class GoogleDriveHelper: LOGGER.error(err) self.__listener.onUploadError(str(err)) return + finally: + self.updater.cancel() LOGGER.info(download_dict) self.__listener.onUploadComplete(link) LOGGER.info("Deleting downloaded file/folder..") @@ -226,7 +238,7 @@ class GoogleDriveHelper: } try: - res = self.__service.files().copy(supportsAllDrives=True, fileId=file_id, body=body).execute() + res = self.__service.files().copy(supportsAllDrives=True,fileId=file_id,body=body).execute() return res except HttpError as err: if err.resp.get('content-type', '').startswith('application/json'): @@ -235,7 +247,7 @@ class GoogleDriveHelper: if USE_SERVICE_ACCOUNTS: self.switchServiceAccount() LOGGER.info(f"Got: {reason}, Trying Again.") - self.copyFile(file_id, dest_id) + self.copyFile(file_id,dest_id) else: raise err @@ -243,7 +255,7 @@ class GoogleDriveHelper: self.transferred_size = 0 try: file_id = self.getIdFromUrl(link) - except (KeyError, IndexError): + except (KeyError,IndexError): msg = "Google drive ID could not be found in the provided link" return msg msg = "" From c4c7575000814a408951f9e048fc64fa6db140ed Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Fri, 5 Jun 2020 14:01:36 +0530 Subject: [PATCH 21/39] Generate our own gid for mega mirrors Signed-off-by: lzzy12 --- .../mirror_utils/download_utils/mega_downloader.py | 6 ++++-- .../mirror_utils/status_utils/mega_download_status.py | 3 +-- bot/modules/mirror.py | 11 ++++++----- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/bot/helper/mirror_utils/download_utils/mega_downloader.py b/bot/helper/mirror_utils/download_utils/mega_downloader.py index a7a52e1..1c8c987 100644 --- a/bot/helper/mirror_utils/download_utils/mega_downloader.py +++ b/bot/helper/mirror_utils/download_utils/mega_downloader.py @@ -4,7 +4,8 @@ from mega import (MegaApi, MegaListener, MegaRequest, MegaTransfer, MegaError) from bot.helper.telegram_helper.message_utils import update_all_messages import os from bot.helper.mirror_utils.status_utils.mega_download_status import MegaDownloadStatus - +import random +import string class MegaDownloaderException(Exception): pass @@ -137,7 +138,8 @@ class MegaDownloadHelper: node = mega_listener.node if mega_listener.error is not None: return listener.onDownloadError(str(mega_listener.error)) - mega_listener.setValues(node.getName(), api.getSize(node), mega_link.split("!", 1)[-1].split("!", 1)[0]) + gid = ''.join(random.SystemRandom().choices(string.ascii_letters + string.digits, k=8)) + mega_listener.setValues(node.getName(), api.getSize(node), gid) with download_dict_lock: download_dict[listener.uid] = MegaDownloadStatus(mega_listener, listener) threading.Thread(target=executor.do, args=(api.startDownload, (node, path))).start() diff --git a/bot/helper/mirror_utils/status_utils/mega_download_status.py b/bot/helper/mirror_utils/status_utils/mega_download_status.py index bb5cf73..cc7f7b4 100644 --- a/bot/helper/mirror_utils/status_utils/mega_download_status.py +++ b/bot/helper/mirror_utils/status_utils/mega_download_status.py @@ -5,7 +5,7 @@ from .status import Status class MegaDownloadStatus(Status): - def __init__(self, obj,listener): + def __init__(self, obj, listener): self.uid = obj.uid self.listener = listener self.obj = obj @@ -22,7 +22,6 @@ class MegaDownloadStatus(Status): def progress(self): """Progress of download in percentage""" return f"{self.progress_raw()}%" - def status(self) -> str: return MirrorStatus.STATUS_DOWNLOADING diff --git a/bot/modules/mirror.py b/bot/modules/mirror.py index a6d2cd0..dc1c3fa 100644 --- a/bot/modules/mirror.py +++ b/bot/modules/mirror.py @@ -26,7 +26,7 @@ ariaDlManager.start_listener() class MirrorListener(listeners.MirrorListeners): - def __init__(self, bot, update, isTar=False,tag=None, extract=False): + def __init__(self, bot, update, isTar=False, tag=None, extract=False): super().__init__(bot, update) self.isTar = isTar self.tag = tag @@ -66,7 +66,7 @@ class MirrorListener(listeners.MirrorListeners): return elif self.extract: download.is_extracting = True - + path = fs_utils.get_base_name(m_path) if path != "unsupported": LOGGER.info( @@ -171,6 +171,7 @@ class MirrorListener(listeners.MirrorListeners): else: update_all_messages() + def _mirror(bot, update, isTar=False, extract=False): message_args = update.message.text.split(' ') try: @@ -214,9 +215,9 @@ def _mirror(bot, update, isTar=False, extract=False): listener = MirrorListener(bot, update, isTar, tag, extract) if bot_utils.is_mega_link(link): mega_dl = MegaDownloadHelper() - mega_dl.add_download(link,f'{DOWNLOAD_DIR}/{listener.uid}/',listener) + mega_dl.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/', listener) else: - ariaDlManager.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/',listener) + ariaDlManager.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/', listener) sendStatusMessage(update, bot) if len(Interval) == 0: Interval.append(setInterval(DOWNLOAD_STATUS_UPDATE_INTERVAL, update_all_messages)) @@ -234,7 +235,7 @@ def tar_mirror(update, context): @run_async def unzip_mirror(update, context): - _mirror(context.bot,update, extract=True) + _mirror(context.bot, update, extract=True) mirror_handler = CommandHandler(BotCommands.MirrorCommand, mirror, From cfe636f5ceb35f476a62cef4881a9406a8daefaa Mon Sep 17 00:00:00 2001 From: Shivam Jha Date: Mon, 8 Jun 2020 20:24:26 +0530 Subject: [PATCH 22/39] Add missing x argument --- extract | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extract b/extract index 8a31a9e..f124d7c 100755 --- a/extract +++ b/extract @@ -20,7 +20,7 @@ extract () { *.tgz) tar xzf "$arg" --one-top-level ;; *.zip) a_dir=`expr "$arg" : '\(.*\).zip'` - 7z "$arg" -o"$a_dir" ;; + 7z x "$arg" -o"$a_dir" ;; *.Z) uncompress "$arg" ;; *.rar) a_dir=`expr "$arg" : '\(.*\).rar'` ; From 87cc9206fa17aa9b8092b20e29a284fc8952a3d7 Mon Sep 17 00:00:00 2001 From: Shivam Jha Date: Tue, 9 Jun 2020 19:42:06 +0530 Subject: [PATCH 23/39] Use local fork of youtube-dl for hotstar fix --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1699c9b..078a15e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,4 @@ python-magic beautifulsoup4>=4.8.2,<4.8.10 Pyrogram>=0.16.0,<0.16.10 TgCrypto>=1.1.1,<1.1.10 -youtube-dl +git+git://github.com/lzzy12/youtube-dl@d7c2b43#youtube_dl From 3720eacf135a7bb64b8bd541726ef77ebffca58b Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Thu, 18 Jun 2020 12:57:31 +0530 Subject: [PATCH 24/39] Optimize Clone implementation --- .../mirror_utils/upload_utils/gdriveTools.py | 50 ++++++++++++------- bot/modules/clone.py | 7 +-- 2 files changed, 36 insertions(+), 21 deletions(-) diff --git a/bot/helper/mirror_utils/upload_utils/gdriveTools.py b/bot/helper/mirror_utils/upload_utils/gdriveTools.py index 480ddc3..0fe2f19 100644 --- a/bot/helper/mirror_utils/upload_utils/gdriveTools.py +++ b/bot/helper/mirror_utils/upload_utils/gdriveTools.py @@ -6,6 +6,7 @@ from urllib.parse import parse_qs import re import json import requests +import logging from google.auth.transport.requests import Request from google.oauth2 import service_account @@ -16,7 +17,7 @@ from googleapiclient.http import MediaFileUpload from tenacity import * from bot import parent_id, DOWNLOAD_DIR, IS_TEAM_DRIVE, INDEX_URL, \ - USE_SERVICE_ACCOUNTS + USE_SERVICE_ACCOUNTS, download_dict from bot.helper.ext_utils.bot_utils import * from bot.helper.ext_utils.fs_utils import get_mime_type @@ -167,7 +168,7 @@ class GoogleDriveHelper: if USE_SERVICE_ACCOUNTS: self.switchServiceAccount() LOGGER.info(f"Got: {reason}, Trying Again.") - self.upload_file(file_path, file_name, mime_type, parent_id) + return self.upload_file(file_path, file_name, mime_type, parent_id) else: raise err self._file_uploaded_bytes = 0 @@ -247,10 +248,37 @@ class GoogleDriveHelper: if USE_SERVICE_ACCOUNTS: self.switchServiceAccount() LOGGER.info(f"Got: {reason}, Trying Again.") - self.copyFile(file_id,dest_id) + return self.copyFile(file_id,dest_id) else: raise err + @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), + retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) + def getFileMetadata(self,file_id): + return self.__service.files().get(supportsAllDrives=True, fileId=file_id, + fields="name,id,mimeType,size").execute() + + @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), + retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) + def getFilesByFolderId(self,folder_id): + page_token = None + q = f"'{folder_id}' in parents" + files = [] + while True: + response = self.__service.files().list(supportsTeamDrives=True, + includeTeamDriveItems=True, + q=q, + spaces='drive', + pageSize=200, + fields='nextPageToken, files(id, name, mimeType,size)', + pageToken=page_token).execute() + for file in response.get('files', []): + files.append(file) + page_token = response.get('nextPageToken', None) + if page_token is None: + break + return files + def clone(self, link): self.transferred_size = 0 try: @@ -304,23 +332,9 @@ class GoogleDriveHelper: return msg def cloneFolder(self, name, local_path, folder_id, parent_id): - page_token = None - q = f"'{folder_id}' in parents" - files = [] LOGGER.info(f"Syncing: {local_path}") + files = self.getFilesByFolderId(folder_id) new_id = None - while True: - response = self.__service.files().list(supportsTeamDrives=True, - includeTeamDriveItems=True, - q=q, - spaces='drive', - fields='nextPageToken, files(id, name, mimeType,size)', - pageToken=page_token).execute() - for file in response.get('files', []): - files.append(file) - page_token = response.get('nextPageToken', None) - if page_token is None: - break if len(files) == 0: return parent_id for file in files: diff --git a/bot/modules/clone.py b/bot/modules/clone.py index 407e353..47202ff 100644 --- a/bot/modules/clone.py +++ b/bot/modules/clone.py @@ -1,12 +1,13 @@ -from telegram.ext import CommandHandler, run_async +from telegram.ext import CommandHandler from bot.helper.mirror_utils.upload_utils.gdriveTools import GoogleDriveHelper from bot.helper.telegram_helper.message_utils import * from bot.helper.telegram_helper.filters import CustomFilters from bot.helper.telegram_helper.bot_commands import BotCommands +from bot.helper.ext_utils.bot_utils import new_thread from bot import dispatcher -@run_async +@new_thread def cloneNode(update,context): args = update.message.text.split(" ",maxsplit=1) if len(args) > 1: @@ -17,7 +18,7 @@ def cloneNode(update,context): deleteMessage(context.bot,msg) sendMessage(result,context.bot,update) else: - sendMessage("Provide G-Drive Shareable Link to Clone.",bot,update) + sendMessage("Provide G-Drive Shareable Link to Clone.",context.bot,update) clone_handler = CommandHandler(BotCommands.CloneCommand,cloneNode,filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) dispatcher.add_handler(clone_handler) \ No newline at end of file From c6a430a44d60683cca0c72b46975fb2cbf9a0127 Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Thu, 18 Jun 2020 12:59:52 +0530 Subject: [PATCH 25/39] Handle more exceptions --- bot/helper/mirror_utils/upload_utils/gdriveTools.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bot/helper/mirror_utils/upload_utils/gdriveTools.py b/bot/helper/mirror_utils/upload_utils/gdriveTools.py index 0fe2f19..61e611a 100644 --- a/bot/helper/mirror_utils/upload_utils/gdriveTools.py +++ b/bot/helper/mirror_utils/upload_utils/gdriveTools.py @@ -289,8 +289,7 @@ class GoogleDriveHelper: msg = "" LOGGER.info(f"File ID: {file_id}") try: - meta = self.__service.files().get(supportsAllDrives=True, fileId=file_id, - fields="name,id,mimeType,size").execute() + meta = self.getFileMetadata(file_id) except Exception as e: return f"{str(e).replace('>', '').replace('<', '')}" if meta.get("mimeType") == self.__G_DRIVE_DIR_MIME_TYPE: From 45f75fb7a6ddec95f91296129c34744174add5aa Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Fri, 19 Jun 2020 15:33:00 +0530 Subject: [PATCH 26/39] Fix crash if Mega API key is not provided Signed-off-by: lzzy12 --- bot/__init__.py | 2 +- bot/helper/mirror_utils/download_utils/mega_downloader.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bot/__init__.py b/bot/__init__.py index 76fca77..a42a0ae 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -84,7 +84,7 @@ try: MEGA_API_KEY = getConfig('MEGA_API_KEY') except KeyError: logging.warning('MEGA API KEY not provided!') - + MEGA_API_KEY = None try: MEGA_EMAIL_ID = getConfig('MEGA_EMAIL_ID') MEGA_PASSWORD = getConfig('MEGA_PASSWORD') diff --git a/bot/helper/mirror_utils/download_utils/mega_downloader.py b/bot/helper/mirror_utils/download_utils/mega_downloader.py index 1c8c987..081356a 100644 --- a/bot/helper/mirror_utils/download_utils/mega_downloader.py +++ b/bot/helper/mirror_utils/download_utils/mega_downloader.py @@ -121,7 +121,8 @@ class MegaDownloadHelper: def __init__(self): pass - def add_download(self, mega_link: str, path: str, listener): + @staticmethod + def add_download(mega_link: str, path: str, listener): if MEGA_API_KEY is None: raise MegaDownloaderException('Mega API KEY not provided! Cannot mirror mega links') executor = AsyncExecutor() From 1b79bbda6e4918dbf62b0f94e4206f6e4165c1bc Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Sun, 21 Jun 2020 13:39:15 +0530 Subject: [PATCH 27/39] Fix error extraction --- .../mirror_utils/upload_utils/gdriveTools.py | 57 +++++++------------ 1 file changed, 22 insertions(+), 35 deletions(-) diff --git a/bot/helper/mirror_utils/upload_utils/gdriveTools.py b/bot/helper/mirror_utils/upload_utils/gdriveTools.py index 61e611a..e918506 100644 --- a/bot/helper/mirror_utils/upload_utils/gdriveTools.py +++ b/bot/helper/mirror_utils/upload_utils/gdriveTools.py @@ -290,44 +290,31 @@ class GoogleDriveHelper: LOGGER.info(f"File ID: {file_id}") try: meta = self.getFileMetadata(file_id) - except Exception as e: - return f"{str(e).replace('>', '').replace('<', '')}" - if meta.get("mimeType") == self.__G_DRIVE_DIR_MIME_TYPE: - dir_id = self.create_directory(meta.get('name'), parent_id) - try: + if meta.get("mimeType") == self.__G_DRIVE_DIR_MIME_TYPE: + dir_id = self.create_directory(meta.get('name'), parent_id) result = self.cloneFolder(meta.get('name'), meta.get('name'), meta.get('id'), dir_id) - except Exception as e: - if isinstance(e, RetryError): - LOGGER.info(f"Total Attempts: {e.last_attempt.attempt_number}") - err = e.last_attempt.exception() - else: - err = str(e).replace('>', '').replace('<', '') - LOGGER.error(err) - return err - msg += f'{meta.get("name")}' \ - f' ({get_readable_file_size(self.transferred_size)})' - if INDEX_URL is not None: - url = requests.utils.requote_uri(f'{INDEX_URL}/{meta.get("name")}/') - msg += f' | Index URL' - else: - try: - file = self.copyFile(meta.get('id'), parent_id) - except Exception as e: - if isinstance(e, RetryError): - LOGGER.info(f"Total Attempts: {e.last_attempt.attempt_number}") - err = e.last_attempt.exception() - else: - err = str(e).replace('>', '').replace('<', '') - LOGGER.error(err) - return err - msg += f'{file.get("name")}' - try: - msg += f' ({get_readable_file_size(int(meta.get("size")))}) ' + msg += f'{meta.get("name")}' \ + f' ({get_readable_file_size(self.transferred_size)})' if INDEX_URL is not None: - url = requests.utils.requote_uri(f'{INDEX_URL}/{file.get("name")}') + url = requests.utils.requote_uri(f'{INDEX_URL}/{meta.get("name")}/') msg += f' | Index URL' - except TypeError: - pass + else: + file = self.copyFile(meta.get('id'), parent_id) + msg += f'{file.get("name")}' + try: + msg += f' ({get_readable_file_size(int(meta.get("size")))}) ' + except TypeError: + pass + if INDEX_URL is not None: + url = requests.utils.requote_uri(f'{INDEX_URL}/{file.get("name")}') + msg += f' | Index URL' + except Exception as err: + if isinstance(err, RetryError): + LOGGER.info(f"Total Attempts: {err.last_attempt.attempt_number}") + err = err.last_attempt.exception() + err = str(err).replace('>', '').replace('<', '') + LOGGER.error(err) + return err return msg def cloneFolder(self, name, local_path, folder_id, parent_id): From 53c34db027d5a52af63d9d775dad32a180c94a98 Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Sun, 21 Jun 2020 14:02:24 +0530 Subject: [PATCH 28/39] Added cpu and ram usage in stats --- bot/__main__.py | 9 +++++++-- requirements.txt | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/bot/__main__.py b/bot/__main__.py index 34df995..bc432bb 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -1,9 +1,10 @@ -import shutil +import shutil, psutil import signal import pickle from os import execl, path, remove from sys import executable +import time from telegram.ext import CommandHandler, run_async from bot import dispatcher, updater, botStartTime @@ -22,10 +23,14 @@ def stats(update, context): total = get_readable_file_size(total) used = get_readable_file_size(used) free = get_readable_file_size(free) + cpuUsage = psutil.cpu_percent(interval=0.5) + memory = psutil.virtual_memory().percent stats = f'Bot Uptime: {currentTime}\n' \ f'Total disk space: {total}\n' \ f'Used: {used}\n' \ - f'Free: {free}' + f'Free: {free}\n' \ + f'CPU: {cpuUsage}%\n' \ + f'RAM: {memory}%' sendMessage(stats, context.bot, update) diff --git a/requirements.txt b/requirements.txt index 078a15e..9a915ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ requests +psutil python-telegram-bot==12.6.1 google-api-python-client>=1.7.11,<1.7.20 google-auth-httplib2>=0.0.3,<0.1.0 From e77290528e1275a5e9fc00f738821785e971cb19 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Sat, 27 Jun 2020 20:18:08 +0530 Subject: [PATCH 29/39] Fixed downloads from torrent files won't show seeders and peers Signed-off-by: lzzy12 # Conflicts: # bot/helper/mirror_utils/download_utils/aria2_download.py --- bot/helper/mirror_utils/download_utils/aria2_download.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bot/helper/mirror_utils/download_utils/aria2_download.py b/bot/helper/mirror_utils/download_utils/aria2_download.py index e30a0a9..e85a07b 100644 --- a/bot/helper/mirror_utils/download_utils/aria2_download.py +++ b/bot/helper/mirror_utils/download_utils/aria2_download.py @@ -1,4 +1,4 @@ -from bot import aria2, download_dict_lock +from bot import aria2 from bot.helper.ext_utils.bot_utils import * from .download_helper import DownloadHelper from bot.helper.mirror_utils.status_utils.aria_download_status import AriaDownloadStatus @@ -24,9 +24,10 @@ class AriaDownloadHelper(DownloadHelper): download = api.get_download(gid) if download.followed_by_ids: new_gid = download.followed_by_ids[0] + new_download = api.get_download(new_gid) with download_dict_lock: - download_dict[dl.uid()] = AriaDownloadStatus(new_gid,dl.getListener()) - if download.is_torrent: + download_dict[dl.uid()] = AriaDownloadStatus(new_gid, dl.getListener()) + if new_download.is_torrent: download_dict[dl.uid()].is_torrent = True update_all_messages() LOGGER.info(f'Changed gid from {gid} to {new_gid}') From b5a162f15d1715faa5529fa998cfcdb29f0cccbc Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Sat, 27 Jun 2020 23:40:15 +0530 Subject: [PATCH 30/39] Use subprocess.run to call extract shell script Signed-off-by: lzzy12 --- bot/helper/ext_utils/bot_utils.py | 2 ++ bot/helper/ext_utils/exceptions.py | 6 +++++ bot/helper/ext_utils/fs_utils.py | 21 +++++++++-------- bot/modules/mirror.py | 37 +++++++++++++++--------------- extract | 6 +++-- 5 files changed, 42 insertions(+), 30 deletions(-) diff --git a/bot/helper/ext_utils/bot_utils.py b/bot/helper/ext_utils/bot_utils.py index a015fb7..15a7feb 100644 --- a/bot/helper/ext_utils/bot_utils.py +++ b/bot/helper/ext_utils/bot_utils.py @@ -137,9 +137,11 @@ def is_magnet(url: str): return True return False + def is_mega_link(url: str): return "mega.nz" in url + def new_thread(fn): """To use as decorator to make a function call threaded. Needs import diff --git a/bot/helper/ext_utils/exceptions.py b/bot/helper/ext_utils/exceptions.py index 25ff87f..a2f600c 100644 --- a/bot/helper/ext_utils/exceptions.py +++ b/bot/helper/ext_utils/exceptions.py @@ -1,2 +1,8 @@ class DirectDownloadLinkException(Exception): + """Not method found for extracting direct download link from the http link""" + pass + + +class NotSupportedExtractionArchive(Exception): + """The archive format use is trying to extract is not supported""" pass diff --git a/bot/helper/ext_utils/fs_utils.py b/bot/helper/ext_utils/fs_utils.py index 90951d6..8044c76 100644 --- a/bot/helper/ext_utils/fs_utils.py +++ b/bot/helper/ext_utils/fs_utils.py @@ -5,6 +5,7 @@ import os import pathlib import magic import tarfile +from .exceptions import NotSupportedExtractionArchive def clean_download(path: str): @@ -60,25 +61,25 @@ def get_base_name(orig_path: str): if orig_path.endswith(".tar.bz2"): return orig_path.replace(".tar.bz2", "") elif orig_path.endswith(".tar.gz"): - return orig_path.replace(".tar.gz","") + return orig_path.replace(".tar.gz", "") elif orig_path.endswith(".bz2"): - return orig_path.replace(".bz2","") + return orig_path.replace(".bz2", "") elif orig_path.endswith(".gz"): - return orig_path.replace(".gz","") + return orig_path.replace(".gz", "") elif orig_path.endswith(".tar"): - return orig_path.replace(".tar","") + return orig_path.replace(".tar", "") elif orig_path.endswith(".tbz2"): - return orig_path.replace("tbz2","") + return orig_path.replace("tbz2", "") elif orig_path.endswith(".tgz"): - return orig_path.replace(".tgz","") + return orig_path.replace(".tgz", "") elif orig_path.endswith(".zip"): - return orig_path.replace(".zip","") + return orig_path.replace(".zip", "") elif orig_path.endswith(".Z"): - return orig_path.replace(".Z","") + return orig_path.replace(".Z", "") elif orig_path.endswith(".rar"): - return orig_path.replace(".rar","") + return orig_path.replace(".rar", "") else: - return "unsupported" + raise NotSupportedExtractionArchive('File format not supported for extraction') def get_mime_type(file_path): diff --git a/bot/modules/mirror.py b/bot/modules/mirror.py index dc1c3fa..e86d0b3 100644 --- a/bot/modules/mirror.py +++ b/bot/modules/mirror.py @@ -5,7 +5,7 @@ from bot import Interval, INDEX_URL from bot import dispatcher, DOWNLOAD_DIR, DOWNLOAD_STATUS_UPDATE_INTERVAL, download_dict, download_dict_lock from bot.helper.ext_utils import fs_utils, bot_utils from bot.helper.ext_utils.bot_utils import setInterval -from bot.helper.ext_utils.exceptions import DirectDownloadLinkException +from bot.helper.ext_utils.exceptions import DirectDownloadLinkException, NotSupportedExtractionArchive from bot.helper.mirror_utils.download_utils.aria2_download import AriaDownloadHelper from bot.helper.mirror_utils.download_utils.mega_downloader import MegaDownloadHelper from bot.helper.mirror_utils.download_utils.direct_link_generator import direct_link_generator @@ -20,6 +20,8 @@ from bot.helper.telegram_helper.filters import CustomFilters from bot.helper.telegram_helper.message_utils import * import pathlib import os +import subprocess +import threading ariaDlManager = AriaDownloadHelper() ariaDlManager.start_listener() @@ -66,30 +68,29 @@ class MirrorListener(listeners.MirrorListeners): return elif self.extract: download.is_extracting = True - - path = fs_utils.get_base_name(m_path) - if path != "unsupported": + try: + path = fs_utils.get_base_name(m_path) LOGGER.info( - f"Extracting : {download_dict[self.uid].name()} " + f"Extracting : {name} " ) - download_dict[self.uid] = ExtractStatus(name, m_path, size) - os.system(f"extract '{m_path}'") - if not os.path.exists(path): - self.onUploadError("Cannot extract file, check integrity of the file") - return + with download_dict_lock: + download_dict[self.uid] = ExtractStatus(name, m_path, size) + archive_result = subprocess.run(["extract", m_path]) + if archive_result.returncode == 0: + threading.Thread(target=os.remove, args=(m_path,)).start() + LOGGER.info(f"Deleting archive : {m_path}") + else: + LOGGER.warning('Unable to extract archive! Uploading anyway') + path = f'{DOWNLOAD_DIR}{self.uid}/{name}' LOGGER.info( f'got path : {path}' ) - try: - os.remove(m_path) - LOGGER.info(f"Deleting archive : {m_path}") - except Exception as e: - LOGGER.error(str(e)) - else: + + except NotSupportedExtractionArchive: LOGGER.info("Not any valid archive, uploading file as it is.") - path = f'{DOWNLOAD_DIR}{self.uid}/{download_dict[self.uid].name()}' + path = f'{DOWNLOAD_DIR}{self.uid}/{name}' else: - path = f'{DOWNLOAD_DIR}{self.uid}/{download_dict[self.uid].name()}' + path = f'{DOWNLOAD_DIR}{self.uid}/{name}' up_name = pathlib.PurePath(path).name LOGGER.info(f"Upload Name : {up_name}") drive = gdriveTools.GoogleDriveHelper(up_name, self) diff --git a/extract b/extract index f124d7c..2a5aa6c 100755 --- a/extract +++ b/extract @@ -25,8 +25,10 @@ extract () { *.rar) a_dir=`expr "$arg" : '\(.*\).rar'` ; mkdir "$a_dir" ; - 7z x "$arg" -o"$a_dir" ;; - *) echo "'$arg' cannot be extracted via extract()" ;; + 7z x "$arg" -o"$a_dir" ;; + *) echo "'$arg' cannot be extracted via extract()" 1>&2 + exit 1 ;; + esac cd - || exit } From fed3be2411e98d8d59060b0e3ec249488cb5a6cc Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Sun, 28 Jun 2020 14:26:41 +0530 Subject: [PATCH 31/39] Return correct exit codes from extract script Signed-off-by: lzzy12 --- extract | 84 ++++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 56 insertions(+), 28 deletions(-) diff --git a/extract b/extract index 2a5aa6c..8e47862 100755 --- a/extract +++ b/extract @@ -1,36 +1,64 @@ #!/bin/bash -if [ $# -lt 1 ];then - echo "Usage: `basename $0` FILES" +if [ $# -lt 1 ]; then + echo "Usage: $(basename $0) FILES" exit 1 fi -extract () { - arg="$1" - cd "$(dirname "$arg")" || exit - case "$arg" in - *.tar.bz2) - tar xjf "$arg" --one-top-level ;; - *.tar.gz) - tar xzf "$arg" --one-top-level ;; - *.bz2) bunzip2 "$arg" ;; - *.gz) gunzip "$arg" ;; - *.tar) tar xf "$arg" --one-top-level ;; - *.tbz2) tar xjf "$arg" --one-top-level ;; - *.tgz) tar xzf "$arg" --one-top-level ;; - *.zip) - a_dir=`expr "$arg" : '\(.*\).zip'` - 7z x "$arg" -o"$a_dir" ;; - *.Z) uncompress "$arg" ;; - *.rar) - a_dir=`expr "$arg" : '\(.*\).rar'` ; - mkdir "$a_dir" ; - 7z x "$arg" -o"$a_dir" ;; - *) echo "'$arg' cannot be extracted via extract()" 1>&2 - exit 1 ;; - - esac - cd - || exit +extract() { + arg="$1" + cd "$(dirname "$arg")" || exit + case "$arg" in + *.tar.bz2) + tar xjf "$arg" --one-top-level + local code=$? + ;; + *.tar.gz) + tar xzf "$arg" --one-top-level + local code=$? + ;; + *.bz2) + bunzip2 "$arg" + local code=$? + ;; + *.gz) + gunzip "$arg" + local code=$? + ;; + *.tar) + tar xf "$arg" --one-top-level + local code=$? + ;; + *.tbz2) + (tar xjf "$arg" --one-top-level) + local code=$? + ;; + *.tgz) + tar xzf "$arg" --one-top-level + local code=$? + ;; + *.zip) + a_dir=$(expr "$arg" : '\(.*\).zip') + 7z x "$arg" -o"$a_dir" + local code=$? + ;; + *.Z) + uncompress "$arg" + local code=$? + ;; + *.rar) + a_dir=$(expr "$arg" : '\(.*\).rar') + mkdir "$a_dir" + 7z x "$arg" -o"$a_dir" + local code=$? + ;; + *) + echo "'$arg' cannot be extracted via extract()" 1>&2 + exit 1 + ;; + esac + cd - || exit $? + exit $code } extract "$1" From 1aed50a5f40b6f5acf44fff02d4dca76e168f612 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Sun, 28 Jun 2020 14:51:55 +0530 Subject: [PATCH 32/39] Fixed multiple /unzipmirror commands gets all the mirrors stuck Signed-off-by: lzzy12 --- bot/helper/ext_utils/bot_utils.py | 6 +++++- bot/helper/mirror_utils/status_utils/extract_status.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/bot/helper/ext_utils/bot_utils.py b/bot/helper/ext_utils/bot_utils.py index 15a7feb..35452f2 100644 --- a/bot/helper/ext_utils/bot_utils.py +++ b/bot/helper/ext_utils/bot_utils.py @@ -62,7 +62,9 @@ def get_readable_file_size(size_in_bytes) -> str: def getDownloadByGid(gid): with download_dict_lock: for dl in download_dict.values(): - if dl.status() != MirrorStatus.STATUS_UPLOADING and dl.status() != MirrorStatus.STATUS_ARCHIVING: + status = dl.status() + if status != MirrorStatus.STATUS_UPLOADING and status != MirrorStatus.STATUS_ARCHIVING\ + and status != MirrorStatus.STATUS_EXTRACTING: if dl.gid() == gid: return dl return None @@ -146,8 +148,10 @@ def new_thread(fn): """To use as decorator to make a function call threaded. Needs import from threading import Thread""" + def wrapper(*args, **kwargs): thread = threading.Thread(target=fn, args=args, kwargs=kwargs) thread.start() return thread + return wrapper diff --git a/bot/helper/mirror_utils/status_utils/extract_status.py b/bot/helper/mirror_utils/status_utils/extract_status.py index f7f5e59..41b6ff5 100644 --- a/bot/helper/mirror_utils/status_utils/extract_status.py +++ b/bot/helper/mirror_utils/status_utils/extract_status.py @@ -8,7 +8,7 @@ class ExtractStatus(Status): self.__path = path self.__size = size - # The progress of Tar function cannot be tracked. So we just return dummy values. + # The progress of extract function cannot be tracked. So we just return dummy values. # If this is possible in future,we should implement it def progress(self): From aa8428fdfae2cee3557b3351a67b49e242372a61 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Wed, 1 Jul 2020 12:43:33 +0530 Subject: [PATCH 33/39] Dockerfile: Install rar codecs for 7z Signed-off-by: lzzy12 --- Dockerfile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8cc39e3..4d85264 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,8 +3,13 @@ FROM lzzy12/mega-sdk-python:latest WORKDIR /usr/src/app RUN chmod 777 /usr/src/app -RUN apt-get -qq update -RUN apt-get -qq install -y p7zip-full aria2 curl pv jq ffmpeg locales python3-lxml +RUN apt-get -qq update && \ + apt-get install -y software-properties-common && \ + rm -rf /var/lib/apt/lists/* && \ + apt-add-repository non-free && \ + apt-get -qq update && \ + apt-get -qq install -y p7zip-full p7zip-rar aria2 curl pv jq ffmpeg locales python3-lxml && \ + apt-get purge -y software-properties-common COPY requirements.txt . COPY extract /usr/local/bin From 0096b6328c1b19bffe8e9a4928f824f693cb4960 Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Fri, 3 Jul 2020 08:00:40 +0530 Subject: [PATCH 34/39] Revert "Implement mega.nz support" This reverts commit 45f75fb7a6ddec95f91296129c34744174add5aa. --- Dockerfile | 4 +- README.md | 7 +- bot/__init__.py | 15 -- bot/helper/ext_utils/bot_utils.py | 5 - .../download_utils/mega_downloader.py | 147 ------------------ .../status_utils/mega_download_status.py | 61 -------- bot/modules/mirror.py | 14 +- config_sample.env | 3 - extract | 2 +- 9 files changed, 8 insertions(+), 250 deletions(-) delete mode 100644 bot/helper/mirror_utils/download_utils/mega_downloader.py delete mode 100644 bot/helper/mirror_utils/status_utils/mega_download_status.py diff --git a/Dockerfile b/Dockerfile index 4d85264..9af3671 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM lzzy12/mega-sdk-python:latest +FROM ubuntu:18.04 WORKDIR /usr/src/app RUN chmod 777 /usr/src/app @@ -24,5 +24,3 @@ COPY netrc /root/.netrc RUN chmod +x aria.sh CMD ["bash","start.sh"] - - diff --git a/README.md b/README.md index 31f3b52..2daa623 100644 --- a/README.md +++ b/README.md @@ -60,10 +60,7 @@ Fill up rest of the fields. Meaning of each fields are discussed below: - **API_KEY** : This is to authenticate to your telegram account for downloading Telegram files. You can get this from https://my.telegram.org DO NOT put this in quotes. - **API_HASH** : This is to authenticate to your telegram account for downloading Telegram files. You can get this from https://my.telegram.org - **USER_SESSION_STRING** : Session string generated by running: -- **MEGA_API_KEY**: Mega.nz api key to mirror mega.nz links. Get it from [Mega SDK Page](https://mega.nz/sdk) -- **MEGA_EMAIL_ID**: Your email id you used to sign up on mega.nz for using premium accounts (Leave th) -- **MEGA_PASSWORD**: Your password for your mega.nz account -``` + python3 generate_string_session.py ``` Note: You can limit maximum concurrent downloads by changing the value of MAX_CONCURRENT_DOWNLOADS in aria.sh. By default, it's set to 2 @@ -73,7 +70,7 @@ Note: You can limit maximum concurrent downloads by changing the value of MAX_CO - Visit the [Google Cloud Console](https://console.developers.google.com/apis/credentials) - Go to the OAuth Consent tab, fill it, and save. - Go to the Credentials tab and click Create Credentials -> OAuth Client ID -- Choose Desktop and Create. +- Choose Other and Create. - Use the download button to download your credentials. - Move that file to the root of mirror-bot, and rename it to credentials.json - Visit [Google API page](https://console.developers.google.com/apis/library) diff --git a/bot/__init__.py b/bot/__init__.py index a42a0ae..3097979 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -79,21 +79,6 @@ try: except KeyError as e: LOGGER.error("One or more env variables missing! Exiting now") exit(1) - -try: - MEGA_API_KEY = getConfig('MEGA_API_KEY') -except KeyError: - logging.warning('MEGA API KEY not provided!') - MEGA_API_KEY = None -try: - MEGA_EMAIL_ID = getConfig('MEGA_EMAIL_ID') - MEGA_PASSWORD = getConfig('MEGA_PASSWORD') - if len(MEGA_EMAIL_ID) == 0 or len(MEGA_PASSWORD) == 0: - raise KeyError -except KeyError: - logging.warning('MEGA Credentials not provided!') - MEGA_EMAIL_ID = None - MEGA_PASSWORD = None try: INDEX_URL = getConfig('INDEX_URL') if len(INDEX_URL) == 0: diff --git a/bot/helper/ext_utils/bot_utils.py b/bot/helper/ext_utils/bot_utils.py index 35452f2..56bc930 100644 --- a/bot/helper/ext_utils/bot_utils.py +++ b/bot/helper/ext_utils/bot_utils.py @@ -139,11 +139,6 @@ def is_magnet(url: str): return True return False - -def is_mega_link(url: str): - return "mega.nz" in url - - def new_thread(fn): """To use as decorator to make a function call threaded. Needs import diff --git a/bot/helper/mirror_utils/download_utils/mega_downloader.py b/bot/helper/mirror_utils/download_utils/mega_downloader.py deleted file mode 100644 index 081356a..0000000 --- a/bot/helper/mirror_utils/download_utils/mega_downloader.py +++ /dev/null @@ -1,147 +0,0 @@ -from bot import LOGGER, MEGA_API_KEY, download_dict_lock, download_dict, MEGA_EMAIL_ID, MEGA_PASSWORD -import threading -from mega import (MegaApi, MegaListener, MegaRequest, MegaTransfer, MegaError) -from bot.helper.telegram_helper.message_utils import update_all_messages -import os -from bot.helper.mirror_utils.status_utils.mega_download_status import MegaDownloadStatus -import random -import string - -class MegaDownloaderException(Exception): - pass - - -class MegaAppListener(MegaListener): - _NO_EVENT_ON = (MegaRequest.TYPE_LOGIN, - MegaRequest.TYPE_FETCH_NODES) - - def __init__(self, continue_event: threading.Event, listener): - self.continue_event = continue_event - self.node = None - self.listener = listener - self.uid = listener.uid - self.__bytes_transferred = 0 - self.is_cancelled = False - self.__speed = 0 - self.__name = '' - self.__size = 0 - self.error = None - self.gid = "" - super(MegaAppListener, self).__init__() - - @property - def speed(self): - """Returns speed of the download in bytes/second""" - return self.__speed - - @property - def name(self): - """Returns name of the download""" - return self.__name - - def setValues(self, name, size, gid): - self.__name = name - self.__size = size - self.gid = gid - - @property - def size(self): - """Size of download in bytes""" - return self.__size - - @property - def downloaded_bytes(self): - return self.__bytes_transferred - - def onRequestStart(self, api, request): - LOGGER.info('Request start ({})'.format(request)) - - def onRequestFinish(self, api, request, error): - LOGGER.info('Mega Request finished ({}); Result: {}' - .format(request, error)) - - request_type = request.getType() - if request_type == MegaRequest.TYPE_LOGIN: - api.fetchNodes() - elif request_type == MegaRequest.TYPE_GET_PUBLIC_NODE: - self.node = request.getPublicMegaNode() - elif request_type == MegaRequest.TYPE_FETCH_NODES: - LOGGER.info("Fetching Root Node.") - self.node = api.getRootNode() - if request_type not in self._NO_EVENT_ON: - self.continue_event.set() - - def onRequestTemporaryError(self, api, request, error: MegaError): - self.listener.onDownloadError(error.toString()) - self.error = error.toString() - self.continue_event.set() - - def onTransferStart(self, api: MegaApi, transfer: MegaTransfer): - LOGGER.info(f"Transfer Started: {transfer.getFileName()}") - - def onTransferUpdate(self, api: MegaApi, transfer: MegaTransfer): - if self.is_cancelled: - api.cancelTransfer(transfer, None) - self.__speed = transfer.getSpeed() - self.__bytes_transferred = transfer.getTransferredBytes() - - def onTransferFinish(self, api: MegaApi, transfer: MegaTransfer, error): - try: - LOGGER.info(f'Transfer finished ({transfer}); Result: {transfer.getFileName()}') - if str(error) != "No error" and self.is_cancelled: - self.is_cancelled = False - return self.listener.onDownloadError(error.toString()) - if transfer.isFolderTransfer() and transfer.isFinished() and not self.is_cancelled or transfer.getFileName() == self.name and not self.is_cancelled: - self.listener.onDownloadComplete() - except Exception as e: - LOGGER.error(e) - - def onTransferTemporaryError(self, api, transfer, error): - LOGGER.info(f'Mega download error in file {transfer} {transfer.getFileName()}: {error}') - self.listener.onDownloadError(error.toString()) - self.error = error.toString() - self.continue_event.set() - - def cancel_download(self): - self.is_cancelled = True - - -class AsyncExecutor: - - def __init__(self): - self.continue_event = threading.Event() - - def do(self, function, args): - self.continue_event.clear() - function(*args) - self.continue_event.wait() - - -class MegaDownloadHelper: - def __init__(self): - pass - - @staticmethod - def add_download(mega_link: str, path: str, listener): - if MEGA_API_KEY is None: - raise MegaDownloaderException('Mega API KEY not provided! Cannot mirror mega links') - executor = AsyncExecutor() - api = MegaApi(MEGA_API_KEY, None, None, 'telegram-mirror-bot') - mega_listener = MegaAppListener(executor.continue_event, listener) - os.makedirs(path) - api.addListener(mega_listener) - if MEGA_EMAIL_ID is not None and MEGA_PASSWORD is not None: - executor.do(api.login, (MEGA_EMAIL_ID, MEGA_PASSWORD)) - executor.do(api.getPublicNode, (mega_link,)) - node = mega_listener.node - if node is None: - executor.do(api.loginToFolder, (mega_link,)) - node = mega_listener.node - if mega_listener.error is not None: - return listener.onDownloadError(str(mega_listener.error)) - gid = ''.join(random.SystemRandom().choices(string.ascii_letters + string.digits, k=8)) - mega_listener.setValues(node.getName(), api.getSize(node), gid) - with download_dict_lock: - download_dict[listener.uid] = MegaDownloadStatus(mega_listener, listener) - threading.Thread(target=executor.do, args=(api.startDownload, (node, path))).start() - update_all_messages() diff --git a/bot/helper/mirror_utils/status_utils/mega_download_status.py b/bot/helper/mirror_utils/status_utils/mega_download_status.py deleted file mode 100644 index cc7f7b4..0000000 --- a/bot/helper/mirror_utils/status_utils/mega_download_status.py +++ /dev/null @@ -1,61 +0,0 @@ -from bot.helper.ext_utils.bot_utils import get_readable_file_size,MirrorStatus, get_readable_time -from bot import DOWNLOAD_DIR -from .status import Status - - -class MegaDownloadStatus(Status): - - def __init__(self, obj, listener): - self.uid = obj.uid - self.listener = listener - self.obj = obj - - def name(self) -> str: - return self.obj.name - - def progress_raw(self): - try: - return round(self.processed_bytes() / self.obj.size * 100,2) - except ZeroDivisionError: - return 0.0 - - def progress(self): - """Progress of download in percentage""" - return f"{self.progress_raw()}%" - - def status(self) -> str: - return MirrorStatus.STATUS_DOWNLOADING - - def processed_bytes(self): - return self.obj.downloaded_bytes - - def eta(self): - try: - seconds = (self.size_raw() - self.processed_bytes()) / self.speed_raw() - return f'{get_readable_time(seconds)}' - except ZeroDivisionError: - return '-' - - def size_raw(self): - return self.obj.size - - def size(self) -> str: - return get_readable_file_size(self.size_raw()) - - def downloaded(self) -> str: - return get_readable_file_size(self.obj.downloadedBytes) - - def speed_raw(self): - return self.obj.speed - - def speed(self) -> str: - return f'{get_readable_file_size(self.speed_raw())}/s' - - def gid(self) -> str: - return self.obj.gid - - def path(self) -> str: - return f"{DOWNLOAD_DIR}{self.uid}" - - def download(self): - return self.obj \ No newline at end of file diff --git a/bot/modules/mirror.py b/bot/modules/mirror.py index e86d0b3..dbbff29 100644 --- a/bot/modules/mirror.py +++ b/bot/modules/mirror.py @@ -1,13 +1,12 @@ import requests from telegram.ext import CommandHandler, run_async -from bot import Interval, INDEX_URL +from bot import Interval, INDEX_URL,LOGGER from bot import dispatcher, DOWNLOAD_DIR, DOWNLOAD_STATUS_UPDATE_INTERVAL, download_dict, download_dict_lock from bot.helper.ext_utils import fs_utils, bot_utils from bot.helper.ext_utils.bot_utils import setInterval from bot.helper.ext_utils.exceptions import DirectDownloadLinkException, NotSupportedExtractionArchive from bot.helper.mirror_utils.download_utils.aria2_download import AriaDownloadHelper -from bot.helper.mirror_utils.download_utils.mega_downloader import MegaDownloadHelper from bot.helper.mirror_utils.download_utils.direct_link_generator import direct_link_generator from bot.helper.mirror_utils.download_utils.telegram_downloader import TelegramDownloadHelper from bot.helper.mirror_utils.status_utils import listeners @@ -28,7 +27,7 @@ ariaDlManager.start_listener() class MirrorListener(listeners.MirrorListeners): - def __init__(self, bot, update, isTar=False, tag=None, extract=False): + def __init__(self, bot, update, isTar=False,tag=None, extract=False): super().__init__(bot, update) self.isTar = isTar self.tag = tag @@ -172,7 +171,6 @@ class MirrorListener(listeners.MirrorListeners): else: update_all_messages() - def _mirror(bot, update, isTar=False, extract=False): message_args = update.message.text.split(' ') try: @@ -214,11 +212,7 @@ def _mirror(bot, update, isTar=False, extract=False): except DirectDownloadLinkException as e: LOGGER.info(f'{link}: {e}') listener = MirrorListener(bot, update, isTar, tag, extract) - if bot_utils.is_mega_link(link): - mega_dl = MegaDownloadHelper() - mega_dl.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/', listener) - else: - ariaDlManager.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/', listener) + ariaDlManager.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/',listener) sendStatusMessage(update, bot) if len(Interval) == 0: Interval.append(setInterval(DOWNLOAD_STATUS_UPDATE_INTERVAL, update_all_messages)) @@ -236,7 +230,7 @@ def tar_mirror(update, context): @run_async def unzip_mirror(update, context): - _mirror(context.bot, update, extract=True) + _mirror(context.bot,update, extract=True) mirror_handler = CommandHandler(BotCommands.MirrorCommand, mirror, diff --git a/config_sample.env b/config_sample.env index 67ff99c..1ef1dc7 100644 --- a/config_sample.env +++ b/config_sample.env @@ -14,6 +14,3 @@ USER_SESSION_STRING = "" TELEGRAM_API = TELEGRAM_HASH = "" USE_SERVICE_ACCOUNTS = "" -MEGA_API_KEY = "" -MEGA_EMAIL_ID = "" -MEGA_PASSWORD = "" \ No newline at end of file diff --git a/extract b/extract index 8e47862..0d17ae0 100755 --- a/extract +++ b/extract @@ -61,4 +61,4 @@ extract() { exit $code } -extract "$1" +extract "$1" \ No newline at end of file From 121fb52e0372cacf4bf7d746c6228dc5862e438d Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Fri, 3 Jul 2020 19:43:08 +0530 Subject: [PATCH 35/39] Fixed Dockerfile --- Dockerfile | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9af3671..db631e1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,12 +4,7 @@ WORKDIR /usr/src/app RUN chmod 777 /usr/src/app RUN apt-get -qq update && \ - apt-get install -y software-properties-common && \ - rm -rf /var/lib/apt/lists/* && \ - apt-add-repository non-free && \ - apt-get -qq update && \ - apt-get -qq install -y p7zip-full p7zip-rar aria2 curl pv jq ffmpeg locales python3-lxml && \ - apt-get purge -y software-properties-common + apt-get -qq install -y python3 python3-pip p7zip-full p7zip-rar aria2 curl pv jq ffmpeg gcc git locales python3-lxml COPY requirements.txt . COPY extract /usr/local/bin From 10593d8a66a614e5452207e7151c0b39993df24f Mon Sep 17 00:00:00 2001 From: Shubham dubey Date: Wed, 29 Jul 2020 20:53:34 +0530 Subject: [PATCH 36/39] Fix /list if fileName string contains multiple quotes (#141) --- bot/helper/mirror_utils/upload_utils/gdriveTools.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bot/helper/mirror_utils/upload_utils/gdriveTools.py b/bot/helper/mirror_utils/upload_utils/gdriveTools.py index e918506..554c789 100644 --- a/bot/helper/mirror_utils/upload_utils/gdriveTools.py +++ b/bot/helper/mirror_utils/upload_utils/gdriveTools.py @@ -407,8 +407,15 @@ class GoogleDriveHelper: scopes=self.__OAUTH_SCOPE) return build('drive', 'v3', credentials=credentials, cache_discovery=False) + def escapes(self, str): + chars = ['\\', "'", '"', r'\a', r'\b', r'\f', r'\n', r'\r', r'\t'] + for char in chars: + str = str.replace(char, '\\'+char) + return str + def drive_list(self, fileName): msg = "" + fileName = self.escapes(str(fileName)) # Create Search Query for API request. query = f"'{parent_id}' in parents and (name contains '{fileName}')" response = self.__service.files().list(supportsTeamDrives=True, From 940237a2161fd6355f4587425d37120252c8f431 Mon Sep 17 00:00:00 2001 From: Adnan Ahmad <60770759+viper-adnan@users.noreply.github.com> Date: Wed, 29 Jul 2020 20:54:31 +0530 Subject: [PATCH 37/39] Fixed Typo (#128) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2daa623..611df26 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ Fill up rest of the fields. Meaning of each fields are discussed below: - **API_KEY** : This is to authenticate to your telegram account for downloading Telegram files. You can get this from https://my.telegram.org DO NOT put this in quotes. - **API_HASH** : This is to authenticate to your telegram account for downloading Telegram files. You can get this from https://my.telegram.org - **USER_SESSION_STRING** : Session string generated by running: - +``` python3 generate_string_session.py ``` Note: You can limit maximum concurrent downloads by changing the value of MAX_CONCURRENT_DOWNLOADS in aria.sh. By default, it's set to 2 @@ -128,4 +128,4 @@ For using your premium accounts in youtube-dl, edit the netrc file (in the root ``` machine host login username password my_youtube_password ``` -where host is the name of extractor (eg. youtube, twitch). Multiple accounts of different hosts can be added each separated by a new line \ No newline at end of file +where host is the name of extractor (eg. youtube, twitch). Multiple accounts of different hosts can be added each separated by a new line From 0288770ec484d40b4c0d7af18ecae871128d134b Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Fri, 31 Jul 2020 20:06:52 +0530 Subject: [PATCH 38/39] Merge upstream youtube-dl Signed-off-by: lzzy12 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9a915ff..bd80f10 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,4 +11,4 @@ python-magic beautifulsoup4>=4.8.2,<4.8.10 Pyrogram>=0.16.0,<0.16.10 TgCrypto>=1.1.1,<1.1.10 -git+git://github.com/lzzy12/youtube-dl@d7c2b43#youtube_dl +git+git://github.com/lzzy12/youtube-dl@efa40fc#youtube_dl From 009746aa010cb92496a2732bb4aa4b9e470b0253 Mon Sep 17 00:00:00 2001 From: Dev Singh Rajput Date: Wed, 29 Jul 2020 20:52:39 +0530 Subject: [PATCH 39/39] added ability of extracting .7z files (#144) --- bot/helper/ext_utils/fs_utils.py | 2 ++ extract | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/bot/helper/ext_utils/fs_utils.py b/bot/helper/ext_utils/fs_utils.py index 8044c76..cbf50b2 100644 --- a/bot/helper/ext_utils/fs_utils.py +++ b/bot/helper/ext_utils/fs_utils.py @@ -74,6 +74,8 @@ def get_base_name(orig_path: str): return orig_path.replace(".tgz", "") elif orig_path.endswith(".zip"): return orig_path.replace(".zip", "") + elif orig_path.endswith(".7z"): + return orig_path.replace(".7z", "") elif orig_path.endswith(".Z"): return orig_path.replace(".Z", "") elif orig_path.endswith(".rar"): diff --git a/extract b/extract index 0d17ae0..da9847b 100755 --- a/extract +++ b/extract @@ -42,6 +42,11 @@ extract() { 7z x "$arg" -o"$a_dir" local code=$? ;; + *.7z) + a_dir=$(expr "$arg" : '\(.*\).7z') + 7z x "$arg" -o"$a_dir" + local code=$? + ;; *.Z) uncompress "$arg" local code=$?