From 77904d77f057aea0df90490b1b59c25248e2b7c0 Mon Sep 17 00:00:00 2001 From: Hafitz Setya <71178188+breakdowns@users.noreply.github.com> Date: Mon, 1 Mar 2021 07:44:54 +0700 Subject: [PATCH] v2.5 (#9) * Added mega.nz support * Added option to disable mega.nz links * Some fixes and optimization --- .netrc | 1 + Dockerfile | 21 ++- README.md | 1 + app.json | 18 +- bot/__init__.py | 25 +++ bot/helper/ext_utils/bot_utils.py | 12 ++ .../download_utils/mega_downloader.py | 172 ++++++++++++++++++ .../status_utils/mega_download_status.py | 62 +++++++ bot/helper/telegram_helper/message_utils.py | 5 + bot/modules/mirror.py | 17 +- config_sample.env | 9 +- netrc | 0 12 files changed, 328 insertions(+), 15 deletions(-) create mode 100644 .netrc create mode 100644 bot/helper/mirror_utils/download_utils/mega_downloader.py create mode 100644 bot/helper/mirror_utils/status_utils/mega_download_status.py delete mode 100644 netrc diff --git a/.netrc b/.netrc new file mode 100644 index 0000000..d3f5a12 --- /dev/null +++ b/.netrc @@ -0,0 +1 @@ + diff --git a/Dockerfile b/Dockerfile index 12a382d..626df92 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,26 @@ -FROM ubuntu:18.04 +FROM lzzy12/mega-sdk-python:latest WORKDIR /usr/src/app RUN chmod 777 /usr/src/app -RUN apt -qq update -RUN apt -qq install -y aria2 python3 python3-pip \ - locales python3-lxml \ - curl pv jq ffmpeg \ - p7zip-full p7zip-rar + +RUN apt-get -qq update && \ + apt-get install -y software-properties-common && \ + rm -rf /var/lib/apt/lists/* && \ + apt-add-repository non-free && \ + apt-get -qq update && \ + apt-get -qq install -y p7zip-full p7zip-rar aria2 curl pv jq ffmpeg locales python3-lxml && \ + apt-get purge -y software-properties-common + COPY requirements.txt . COPY extract /usr/local/bin RUN chmod +x /usr/local/bin/extract RUN pip3 install --no-cache-dir -r requirements.txt -COPY . . -RUN chmod +x aria.sh RUN locale-gen en_US.UTF-8 ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 +COPY . . +COPY .netrc /root/.netrc +RUN chmod +x aria.sh CMD ["bash","start.sh"] diff --git a/README.md b/README.md index f01b47c..c759d38 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ Fork this repo, than upload credentials.json and token.pickle to your forks ## Features supported: - Mirroring direct download links to Google Drive +- Mirroring Mega.nz links to google drive (In development stage) - Download progress - Upload progress - Download/upload speeds and ETAs diff --git a/app.json b/app.json index 7b442e7..a714784 100644 --- a/app.json +++ b/app.json @@ -7,7 +7,7 @@ "python" ], "repository": "https://github.com/breakdowns/slam-mirrorbot", - "website": "https://github.com/breakdowns/slam-mirrorbot", + "website": "https://t.me/SlamMirrorSupport", "success_url": "https://github.com/breakdowns/slam-mirrorbot/blob/master/README.md", "stack": "container", "env": { @@ -65,6 +65,22 @@ "TELEGRAM_HASH": { "description": "This is to authenticate to your telegram account for downloading Telegram files. You can get this from https://my.telegram.org.", "required": true + }, + "MEGA_API_KEY": { + "description": "Mega.nz api key to mirror mega.nz links. Get it from https://mega.nz/sdk.", + "required": false + }, + "MEGA_EMAIL_ID": { + "description": "Your email id you used to sign up on mega.nz.", + "required": false + }, + "MEGA_PASSWORD": { + "description": "Your password for your mega.nz account.", + "required": false + }, + "BLOCK_MEGA_LINKS": { + "description": "If you want to remove mega.nz mirror support, set it to True.", + "required": false } } } diff --git a/bot/__init__.py b/bot/__init__.py index 7cc450e..e858f36 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -87,6 +87,21 @@ except KeyError as e: LOGGER.error("One or more env variables missing! Exiting now") exit(1) +try: + MEGA_API_KEY = getConfig('MEGA_API_KEY') +except KeyError: + LOGGER.warning('MEGA API KEY not provided!') + MEGA_API_KEY = None +try: + MEGA_EMAIL_ID = getConfig('MEGA_EMAIL_ID') + MEGA_PASSWORD = getConfig('MEGA_PASSWORD') + if len(MEGA_EMAIL_ID) == 0 or len(MEGA_PASSWORD) == 0: + raise KeyError +except KeyError: + LOGGER.warning('MEGA Credentials not provided!') + MEGA_EMAIL_ID = None + MEGA_PASSWORD = None + LOGGER.info("Generating USER_SESSION_STRING") with Client(':memory:', api_id=int(TELEGRAM_API), api_hash=TELEGRAM_HASH, bot_token=BOT_TOKEN) as app: USER_SESSION_STRING = app.export_session_string() @@ -113,6 +128,16 @@ try: USE_SERVICE_ACCOUNTS = False except KeyError: USE_SERVICE_ACCOUNTS = False + +try: + BLOCK_MEGA_LINKS = getConfig('BLOCK_MEGA_LINKS') + if BLOCK_MEGA_LINKS.lower() == 'true': + BLOCK_MEGA_LINKS = True + else: + BLOCK_MEGA_LINKS = False +except KeyError: + BLOCK_MEGA_LINKS = False + app = Client('slam', api_id=TELEGRAM_API, api_hash=TELEGRAM_HASH, bot_token=BOT_TOKEN) updater = tg.Updater(token=BOT_TOKEN,use_context=True) bot = updater.bot diff --git a/bot/helper/ext_utils/bot_utils.py b/bot/helper/ext_utils/bot_utils.py index b6aec41..ab36f0b 100644 --- a/bot/helper/ext_utils/bot_utils.py +++ b/bot/helper/ext_utils/bot_utils.py @@ -134,6 +134,18 @@ def is_url(url: str): return False +def is_mega_link(url: str): + return "mega.nz" in url + +def get_mega_link_type(url: str): + if "folder" in url: + return "folder" + elif "file" in url: + return "file" + elif "/#F!" in url: + return "folder" + return "file" + def is_magnet(url: str): magnet = re.findall(MAGNET_REGEX, url) if magnet: diff --git a/bot/helper/mirror_utils/download_utils/mega_downloader.py b/bot/helper/mirror_utils/download_utils/mega_downloader.py new file mode 100644 index 0000000..e1f43a5 --- /dev/null +++ b/bot/helper/mirror_utils/download_utils/mega_downloader.py @@ -0,0 +1,172 @@ +from bot import LOGGER, MEGA_API_KEY, download_dict_lock, download_dict, MEGA_EMAIL_ID, MEGA_PASSWORD +import threading +from mega import (MegaApi, MegaListener, MegaRequest, MegaTransfer, MegaError) +from bot.helper.telegram_helper.message_utils import update_all_messages +import os +from bot.helper.ext_utils.bot_utils import new_thread, get_mega_link_type +from bot.helper.mirror_utils.status_utils.mega_download_status import MegaDownloadStatus +import random +import string + +class MegaDownloaderException(Exception): + pass + + +class MegaAppListener(MegaListener): + _NO_EVENT_ON = (MegaRequest.TYPE_LOGIN,MegaRequest.TYPE_FETCH_NODES) + NO_ERROR = "no error" + + def __init__(self, continue_event: threading.Event, listener): + self.continue_event = continue_event + self.node = None + self.public_node = None + self.listener = listener + self.uid = listener.uid + self.__bytes_transferred = 0 + self.is_cancelled = False + self.__speed = 0 + self.__name = '' + self.__size = 0 + self.error = None + self.gid = "" + super(MegaAppListener, self).__init__() + + @property + def speed(self): + """Returns speed of the download in bytes/second""" + return self.__speed + + @property + def name(self): + """Returns name of the download""" + return self.__name + + def setValues(self, name, size, gid): + self.__name = name + self.__size = size + self.gid = gid + + @property + def size(self): + """Size of download in bytes""" + return self.__size + + @property + def downloaded_bytes(self): + return self.__bytes_transferred + + def onRequestStart(self, api, request): + LOGGER.info('Request start ({})'.format(request)) + + def onRequestFinish(self, api, request, error): + LOGGER.info('Mega Request finished ({}); Result: {}' + .format(request, error)) + if str(error).lower() != "no error": + self.error = error.copy() + return + request_type = request.getType() + if request_type == MegaRequest.TYPE_LOGIN: + api.fetchNodes() + elif request_type == MegaRequest.TYPE_GET_PUBLIC_NODE: + self.public_node = request.getPublicMegaNode() + elif request_type == MegaRequest.TYPE_FETCH_NODES: + LOGGER.info("Fetching Root Node.") + self.node = api.getRootNode() + LOGGER.info(f"Node Name: {self.node.getName()}") + if request_type not in self._NO_EVENT_ON or self.node and "cloud drive" not in self.node.getName().lower(): + self.continue_event.set() + + def onRequestTemporaryError(self, api, request, error: MegaError): + LOGGER.info(f'Mega Request error in {error}') + if not self.is_cancelled: + self.listener.onDownloadError("RequestTempError: " + error.toString()) + self.is_cancelled = True + self.error = error.toString() + self.continue_event.set() + + def onTransferStart(self, api: MegaApi, transfer: MegaTransfer): + LOGGER.info(f"Transfer Started: {transfer.getFileName()}") + + def onTransferUpdate(self, api: MegaApi, transfer: MegaTransfer): + if self.is_cancelled: + api.cancelTransfer(transfer, None) + self.__speed = transfer.getSpeed() + self.__bytes_transferred = transfer.getTransferredBytes() + + def onTransferFinish(self, api: MegaApi, transfer: MegaTransfer, error): + try: + LOGGER.info(f'Transfer finished ({transfer}); Result: {transfer.getFileName()}') + if transfer.isFolderTransfer() and transfer.isFinished() or transfer.getFileName() == self.name and not self.is_cancelled: + self.listener.onDownloadComplete() + self.continue_event.set() + except Exception as e: + LOGGER.error(e) + + def onTransferTemporaryError(self, api, transfer, error): + filen = transfer.getFileName() + state = transfer.getState() + errStr = error.toString() + LOGGER.info(f'Mega download error in file {transfer} {filen}: {error}') + + if state == 1 or state == 4: + # Sometimes MEGA (offical client) can't stream a node either and raises a temp failed error. + # Don't break the transfer queue if transfer's in queued (1) or retrying (4) state [causes seg fault] + return + + self.error = errStr + if not self.is_cancelled: + self.is_cancelled = True + self.listener.onDownloadError(f"TransferTempError: {errStr} ({filen})") + + def cancel_download(self): + self.is_cancelled = True + self.listener.onDownloadError("Download Canceled by user") + + +class AsyncExecutor: + + def __init__(self): + self.continue_event = threading.Event() + + def do(self, function, args): + self.continue_event.clear() + function(*args) + self.continue_event.wait() + +listeners = [] + +class MegaDownloadHelper: + def __init__(self): + pass + + @staticmethod + @new_thread + def add_download(mega_link: str, path: str, listener): + if MEGA_API_KEY is None: + raise MegaDownloaderException('Mega API KEY not provided! Cannot mirror mega links') + executor = AsyncExecutor() + api = MegaApi(MEGA_API_KEY, None, None, 'telegram-mirror-bot') + global listeners + mega_listener = MegaAppListener(executor.continue_event, listener) + listeners.append(mega_listener) + with download_dict_lock: + download_dict[listener.uid] = MegaDownloadStatus(mega_listener, listener) + os.makedirs(path) + api.addListener(mega_listener) + if MEGA_EMAIL_ID is not None and MEGA_PASSWORD is not None: + executor.do(api.login, (MEGA_EMAIL_ID, MEGA_PASSWORD)) + link_type = get_mega_link_type(mega_link) + if link_type == "file": + executor.do(api.getPublicNode, (mega_link,)) + node = mega_listener.public_node + else: + LOGGER.info("Logging into mega folder") + folder_api = MegaApi(MEGA_API_KEY,None,None,'TgBot') + folder_api.addListener(mega_listener) + executor.do(folder_api.loginToFolder, (mega_link,)) + node = folder_api.authorizeNode(mega_listener.node) + if mega_listener.error is not None: + return listener.onDownloadError(str(mega_listener.error)) + gid = ''.join(random.SystemRandom().choices(string.ascii_letters + string.digits, k=8)) + mega_listener.setValues(node.getName(), api.getSize(node), gid) + executor.do(api.startDownload,(node,path)) diff --git a/bot/helper/mirror_utils/status_utils/mega_download_status.py b/bot/helper/mirror_utils/status_utils/mega_download_status.py new file mode 100644 index 0000000..e4f5656 --- /dev/null +++ b/bot/helper/mirror_utils/status_utils/mega_download_status.py @@ -0,0 +1,62 @@ +from bot.helper.ext_utils.bot_utils import get_readable_file_size,MirrorStatus, get_readable_time +from bot import DOWNLOAD_DIR +from .status import Status + + +class MegaDownloadStatus(Status): + + def __init__(self, obj, listener): + self.uid = obj.uid + self.listener = listener + self.obj = obj + self.message = listener.message + + def name(self) -> str: + return self.obj.name + + def progress_raw(self): + try: + return round(self.processed_bytes() / self.obj.size * 100,2) + except ZeroDivisionError: + return 0.0 + + def progress(self): + """Progress of download in percentage""" + return f"{self.progress_raw()}%" + + def status(self) -> str: + return MirrorStatus.STATUS_DOWNLOADING + + def processed_bytes(self): + return self.obj.downloaded_bytes + + def eta(self): + try: + seconds = (self.size_raw() - self.processed_bytes()) / self.speed_raw() + return f'{get_readable_time(seconds)}' + except ZeroDivisionError: + return '-' + + def size_raw(self): + return self.obj.size + + def size(self) -> str: + return get_readable_file_size(self.size_raw()) + + def downloaded(self) -> str: + return get_readable_file_size(self.obj.downloadedBytes) + + def speed_raw(self): + return self.obj.speed + + def speed(self) -> str: + return f'{get_readable_file_size(self.speed_raw())}/s' + + def gid(self) -> str: + return self.obj.gid + + def path(self) -> str: + return f"{DOWNLOAD_DIR}{self.uid}" + + def download(self): + return self.obj diff --git a/bot/helper/telegram_helper/message_utils.py b/bot/helper/telegram_helper/message_utils.py index 9d3eb02..285b655 100644 --- a/bot/helper/telegram_helper/message_utils.py +++ b/bot/helper/telegram_helper/message_utils.py @@ -67,6 +67,8 @@ def update_all_messages(): with status_reply_dict_lock: for chat_id in list(status_reply_dict.keys()): if status_reply_dict[chat_id] and msg != status_reply_dict[chat_id].text: + if len(msg) == 0: + msg = "Starting DL" try: editMessage(msg, status_reply_dict[chat_id]) except Exception as e: @@ -74,6 +76,7 @@ def update_all_messages(): status_reply_dict[chat_id].text = msg + def sendStatusMessage(msg, bot): progress = get_readable_message() with status_reply_dict_lock: @@ -86,5 +89,7 @@ def sendStatusMessage(msg, bot): LOGGER.error(str(e)) del status_reply_dict[msg.message.chat.id] pass + if len(progress) == 0: + progress = "Starting DL" message = sendMessage(progress, bot, msg) status_reply_dict[msg.message.chat.id] = message diff --git a/bot/modules/mirror.py b/bot/modules/mirror.py index 0201822..c9f0041 100644 --- a/bot/modules/mirror.py +++ b/bot/modules/mirror.py @@ -1,12 +1,13 @@ import requests from telegram.ext import CommandHandler, run_async -from bot import Interval, INDEX_URL, LOGGER +from bot import Interval, INDEX_URL, BLOCK_MEGA_LINKS from bot import dispatcher, DOWNLOAD_DIR, DOWNLOAD_STATUS_UPDATE_INTERVAL, download_dict, download_dict_lock from bot.helper.ext_utils import fs_utils, bot_utils from bot.helper.ext_utils.bot_utils import setInterval from bot.helper.ext_utils.exceptions import DirectDownloadLinkException, NotSupportedExtractionArchive from bot.helper.mirror_utils.download_utils.aria2_download import AriaDownloadHelper +from bot.helper.mirror_utils.download_utils.mega_downloader import MegaDownloadHelper from bot.helper.mirror_utils.download_utils.direct_link_generator import direct_link_generator from bot.helper.mirror_utils.download_utils.telegram_downloader import TelegramDownloadHelper from bot.helper.mirror_utils.status_utils import listeners @@ -26,7 +27,7 @@ ariaDlManager = AriaDownloadHelper() ariaDlManager.start_listener() class MirrorListener(listeners.MirrorListeners): - def __init__(self, bot, update, isTar=False,tag=None, extract=False): + def __init__(self, bot, update, isTar=False, tag=None, extract=False): super().__init__(bot, update) self.isTar = isTar self.tag = tag @@ -218,8 +219,16 @@ def _mirror(bot, update, isTar=False, extract=False): LOGGER.info(f'{link}: {e}') listener = MirrorListener(bot, update, isTar, tag, extract) - ariaDlManager.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/',listener) - sendStatusMessage(update, bot) + if bot_utils.is_mega_link(link): + if BLOCK_MEGA_LINKS: + sendMessage("Mega links are blocked bcoz mega downloading is too much unstable and buggy. mega support will be added back after fix", bot, update) + else: + mega_dl = MegaDownloadHelper() + mega_dl.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/', listener) + sendStatusMessage(update, bot) + else: + ariaDlManager.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/', listener) + sendStatusMessage(update, bot) if len(Interval) == 0: Interval.append(setInterval(DOWNLOAD_STATUS_UPDATE_INTERVAL, update_all_messages)) diff --git a/config_sample.env b/config_sample.env index cc2e43e..8dc3d8b 100644 --- a/config_sample.env +++ b/config_sample.env @@ -9,8 +9,13 @@ DOWNLOAD_DIR = "/home/username/mirror-bot/downloads" DOWNLOAD_STATUS_UPDATE_INTERVAL = 5 AUTO_DELETE_MESSAGE_DURATION = 20 IS_TEAM_DRIVE = "" -INDEX_URL = "" TELEGRAM_API = TELEGRAM_HASH = "" USE_SERVICE_ACCOUNTS = "" -AUTHORIZED_CHATS = "" +# Optional config +AUTHORIZED_CHATS = "" +INDEX_URL = "" +MEGA_API_KEY = "" +MEGA_EMAIL_ID = "" +MEGA_PASSWORD = "" +BLOCK_MEGA_LINKS = "" diff --git a/netrc b/netrc deleted file mode 100644 index e69de29..0000000