v2.5 (#9)

* Added mega.nz support
* Added option to disable mega.nz links
* Some fixes and optimization

.netrc

@@ -0,0 +1 @@
+

Dockerfile

@@ -1,21 +1,26 @@
-FROM ubuntu:18.04
+FROM lzzy12/mega-sdk-python:latest
 
 WORKDIR /usr/src/app
 RUN chmod 777 /usr/src/app
-RUN apt -qq update
-RUN apt -qq install -y aria2 python3 python3-pip \
-    locales python3-lxml \
-    curl pv jq ffmpeg \
-    p7zip-full p7zip-rar
+
+RUN apt-get -qq update && \
+    apt-get install -y software-properties-common && \
+    rm -rf /var/lib/apt/lists/* && \
+    apt-add-repository non-free && \
+    apt-get -qq update && \
+    apt-get -qq install -y p7zip-full p7zip-rar aria2 curl pv jq ffmpeg locales python3-lxml && \
+    apt-get purge -y software-properties-common
+
 COPY requirements.txt .
 COPY extract /usr/local/bin
 RUN chmod +x /usr/local/bin/extract
 RUN pip3 install --no-cache-dir -r requirements.txt
-COPY . .
-RUN chmod +x aria.sh
 RUN locale-gen en_US.UTF-8
 ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8
+COPY . .
+COPY .netrc /root/.netrc
+RUN chmod +x aria.sh
 
 CMD ["bash","start.sh"]

README.md

@@ -32,6 +32,7 @@ Fork this repo, than upload credentials.json and token.pickle to your forks
 
 ## Features supported:
 - Mirroring direct download links to Google Drive
+- Mirroring Mega.nz links to google drive (In development stage)
 - Download progress
 - Upload progress
 - Download/upload speeds and ETAs

app.json

@@ -7,7 +7,7 @@
       "python"
     ],
     "repository": "https://github.com/breakdowns/slam-mirrorbot",
-    "website": "https://github.com/breakdowns/slam-mirrorbot",
+    "website": "https://t.me/SlamMirrorSupport",
     "success_url": "https://github.com/breakdowns/slam-mirrorbot/blob/master/README.md",
     "stack": "container",
     "env": {
@@ -65,6 +65,22 @@
       "TELEGRAM_HASH": {
         "description": "This is to authenticate to your telegram account for downloading Telegram files. You can get this from https://my.telegram.org.",
         "required": true
+      },
+      "MEGA_API_KEY": {
+        "description": "Mega.nz api key to mirror mega.nz links. Get it from https://mega.nz/sdk.",
+        "required": false
+      },
+      "MEGA_EMAIL_ID": {
+        "description": "Your email id you used to sign up on mega.nz.",
+        "required": false
+      },
+      "MEGA_PASSWORD": {
+        "description": "Your password for your mega.nz account.",
+        "required": false
+      },
+      "BLOCK_MEGA_LINKS": {
+        "description": "If you want to remove mega.nz mirror support, set it to True.",
+        "required": false
       }
     }
 }

bot/__init__.py

@@ -87,6 +87,21 @@ except KeyError as e:
     LOGGER.error("One or more env variables missing! Exiting now")
     exit(1)
 
+try:
+    MEGA_API_KEY = getConfig('MEGA_API_KEY')
+except KeyError:
+    LOGGER.warning('MEGA API KEY not provided!')
+    MEGA_API_KEY = None
+try:
+    MEGA_EMAIL_ID = getConfig('MEGA_EMAIL_ID')
+    MEGA_PASSWORD = getConfig('MEGA_PASSWORD')
+    if len(MEGA_EMAIL_ID) == 0 or len(MEGA_PASSWORD) == 0:
+        raise KeyError
+except KeyError:
+    LOGGER.warning('MEGA Credentials not provided!')
+    MEGA_EMAIL_ID = None
+    MEGA_PASSWORD = None
+
 LOGGER.info("Generating USER_SESSION_STRING")
 with Client(':memory:', api_id=int(TELEGRAM_API), api_hash=TELEGRAM_HASH, bot_token=BOT_TOKEN) as app:
     USER_SESSION_STRING = app.export_session_string()
@@ -113,6 +128,16 @@ try:
         USE_SERVICE_ACCOUNTS = False
 except KeyError:
     USE_SERVICE_ACCOUNTS = False
+
+try:
+    BLOCK_MEGA_LINKS = getConfig('BLOCK_MEGA_LINKS')
+    if BLOCK_MEGA_LINKS.lower() == 'true':
+        BLOCK_MEGA_LINKS = True
+    else:
+        BLOCK_MEGA_LINKS = False
+except KeyError:
+    BLOCK_MEGA_LINKS = False
+
 app = Client('slam', api_id=TELEGRAM_API, api_hash=TELEGRAM_HASH, bot_token=BOT_TOKEN)
 updater = tg.Updater(token=BOT_TOKEN,use_context=True)
 bot = updater.bot

bot/helper/ext_utils/bot_utils.py

@@ -134,6 +134,18 @@ def is_url(url: str):
     return False
 
 
+def is_mega_link(url: str):
+    return "mega.nz" in url
+
+def get_mega_link_type(url: str):
+    if "folder" in url:
+        return "folder"
+    elif "file" in url:
+        return "file"
+    elif "/#F!" in url:
+        return "folder"
+    return "file"
+
 def is_magnet(url: str):
     magnet = re.findall(MAGNET_REGEX, url)
     if magnet:

bot/helper/mirror_utils/download_utils/mega_downloader.py

@@ -0,0 +1,172 @@
+from bot import LOGGER, MEGA_API_KEY, download_dict_lock, download_dict, MEGA_EMAIL_ID, MEGA_PASSWORD
+import threading
+from mega import (MegaApi, MegaListener, MegaRequest, MegaTransfer, MegaError)
+from bot.helper.telegram_helper.message_utils import update_all_messages
+import os
+from bot.helper.ext_utils.bot_utils import new_thread, get_mega_link_type
+from bot.helper.mirror_utils.status_utils.mega_download_status import MegaDownloadStatus
+import random
+import string
+
+class MegaDownloaderException(Exception):
+    pass
+
+
+class MegaAppListener(MegaListener):
+    _NO_EVENT_ON = (MegaRequest.TYPE_LOGIN,MegaRequest.TYPE_FETCH_NODES)
+    NO_ERROR = "no error"
+
+    def __init__(self, continue_event: threading.Event, listener):
+        self.continue_event = continue_event
+        self.node = None
+        self.public_node = None
+        self.listener = listener
+        self.uid = listener.uid
+        self.__bytes_transferred = 0
+        self.is_cancelled = False
+        self.__speed = 0
+        self.__name = ''
+        self.__size = 0
+        self.error = None
+        self.gid = ""
+        super(MegaAppListener, self).__init__()
+
+    @property
+    def speed(self):
+        """Returns speed of the download in bytes/second"""
+        return self.__speed
+
+    @property
+    def name(self):
+        """Returns name of the download"""
+        return self.__name
+
+    def setValues(self, name, size, gid):
+        self.__name = name
+        self.__size = size
+        self.gid = gid
+
+    @property
+    def size(self):
+        """Size of download in bytes"""
+        return self.__size
+
+    @property
+    def downloaded_bytes(self):
+        return self.__bytes_transferred
+
+    def onRequestStart(self, api, request):
+        LOGGER.info('Request start ({})'.format(request))
+
+    def onRequestFinish(self, api, request, error):
+        LOGGER.info('Mega Request finished ({}); Result: {}'
+                    .format(request, error))
+        if str(error).lower() != "no error":
+            self.error = error.copy()
+            return
+        request_type = request.getType()
+        if request_type == MegaRequest.TYPE_LOGIN:
+            api.fetchNodes()
+        elif request_type == MegaRequest.TYPE_GET_PUBLIC_NODE:
+            self.public_node = request.getPublicMegaNode()
+        elif request_type == MegaRequest.TYPE_FETCH_NODES:
+            LOGGER.info("Fetching Root Node.")
+            self.node = api.getRootNode()
+            LOGGER.info(f"Node Name: {self.node.getName()}")
+        if request_type not in self._NO_EVENT_ON or self.node and "cloud drive" not in self.node.getName().lower():
+            self.continue_event.set()
+
+    def onRequestTemporaryError(self, api, request, error: MegaError):
+        LOGGER.info(f'Mega Request error in {error}')
+        if not self.is_cancelled:
+            self.listener.onDownloadError("RequestTempError: " + error.toString())
+            self.is_cancelled = True
+        self.error = error.toString()
+        self.continue_event.set()
+
+    def onTransferStart(self, api: MegaApi, transfer: MegaTransfer):
+        LOGGER.info(f"Transfer Started: {transfer.getFileName()}")
+
+    def onTransferUpdate(self, api: MegaApi, transfer: MegaTransfer):
+        if self.is_cancelled:
+            api.cancelTransfer(transfer, None)
+        self.__speed = transfer.getSpeed()
+        self.__bytes_transferred = transfer.getTransferredBytes()
+
+    def onTransferFinish(self, api: MegaApi, transfer: MegaTransfer, error):
+        try:
+            LOGGER.info(f'Transfer finished ({transfer}); Result: {transfer.getFileName()}')
+            if transfer.isFolderTransfer() and transfer.isFinished() or transfer.getFileName() == self.name and not self.is_cancelled:
+                self.listener.onDownloadComplete()
+                self.continue_event.set()
+        except Exception as e:
+            LOGGER.error(e)
+
+    def onTransferTemporaryError(self, api, transfer, error):
+        filen = transfer.getFileName()
+        state = transfer.getState()
+        errStr = error.toString()
+        LOGGER.info(f'Mega download error in file {transfer} {filen}: {error}')
+
+        if state == 1 or state == 4:
+            # Sometimes MEGA (offical client) can't stream a node either and raises a temp failed error.
+            # Don't break the transfer queue if transfer's in queued (1) or retrying (4) state [causes seg fault]
+            return
+
+        self.error = errStr
+        if not self.is_cancelled:
+            self.is_cancelled = True
+            self.listener.onDownloadError(f"TransferTempError: {errStr} ({filen})")
+
+    def cancel_download(self):
+        self.is_cancelled = True
+        self.listener.onDownloadError("Download Canceled by user")
+
+
+class AsyncExecutor:
+
+    def __init__(self):
+        self.continue_event = threading.Event()
+
+    def do(self, function, args):
+        self.continue_event.clear()
+        function(*args)
+        self.continue_event.wait()
+
+listeners = []
+
+class MegaDownloadHelper:
+    def __init__(self):
+        pass
+
+    @staticmethod
+    @new_thread
+    def add_download(mega_link: str, path: str, listener):
+        if MEGA_API_KEY is None:
+            raise MegaDownloaderException('Mega API KEY not provided! Cannot mirror mega links')
+        executor = AsyncExecutor()
+        api = MegaApi(MEGA_API_KEY, None, None, 'telegram-mirror-bot')
+        global listeners
+        mega_listener = MegaAppListener(executor.continue_event, listener)
+        listeners.append(mega_listener)
+        with download_dict_lock:
+            download_dict[listener.uid] = MegaDownloadStatus(mega_listener, listener)
+        os.makedirs(path)
+        api.addListener(mega_listener)
+        if MEGA_EMAIL_ID is not None and MEGA_PASSWORD is not None:
+            executor.do(api.login, (MEGA_EMAIL_ID, MEGA_PASSWORD))
+        link_type = get_mega_link_type(mega_link)
+        if link_type == "file":
+            executor.do(api.getPublicNode, (mega_link,))
+            node = mega_listener.public_node
+        else:
+            LOGGER.info("Logging into mega folder")
+            folder_api = MegaApi(MEGA_API_KEY,None,None,'TgBot')
+            folder_api.addListener(mega_listener)
+            executor.do(folder_api.loginToFolder, (mega_link,))
+            node = folder_api.authorizeNode(mega_listener.node)
+        if mega_listener.error is not None:
+            return listener.onDownloadError(str(mega_listener.error))
+        gid = ''.join(random.SystemRandom().choices(string.ascii_letters + string.digits, k=8))
+        mega_listener.setValues(node.getName(), api.getSize(node), gid)
+        executor.do(api.startDownload,(node,path))

bot/helper/mirror_utils/status_utils/mega_download_status.py

@@ -0,0 +1,62 @@
+from bot.helper.ext_utils.bot_utils import get_readable_file_size,MirrorStatus, get_readable_time
+from bot import DOWNLOAD_DIR
+from .status import Status
+
+
+class MegaDownloadStatus(Status):
+
+    def __init__(self, obj, listener):
+        self.uid = obj.uid
+        self.listener = listener
+        self.obj = obj
+        self.message = listener.message
+
+    def name(self) -> str:
+        return self.obj.name
+
+    def progress_raw(self):
+        try:
+            return round(self.processed_bytes() / self.obj.size * 100,2)
+        except ZeroDivisionError:
+            return 0.0
+
+    def progress(self):
+        """Progress of download in percentage"""
+        return f"{self.progress_raw()}%"
+
+    def status(self) -> str:
+        return MirrorStatus.STATUS_DOWNLOADING
+
+    def processed_bytes(self):
+        return self.obj.downloaded_bytes
+
+    def eta(self):
+        try:
+            seconds = (self.size_raw() - self.processed_bytes()) / self.speed_raw()
+            return f'{get_readable_time(seconds)}'
+        except ZeroDivisionError:
+            return '-'
+
+    def size_raw(self):
+        return self.obj.size
+
+    def size(self) -> str:
+        return get_readable_file_size(self.size_raw())
+
+    def downloaded(self) -> str:
+        return get_readable_file_size(self.obj.downloadedBytes)
+
+    def speed_raw(self):
+        return self.obj.speed
+
+    def speed(self) -> str:
+        return f'{get_readable_file_size(self.speed_raw())}/s' 
+
+    def gid(self) -> str:
+        return self.obj.gid
+
+    def path(self) -> str:
+        return f"{DOWNLOAD_DIR}{self.uid}"
+
+    def download(self):
+        return self.obj

bot/helper/telegram_helper/message_utils.py

@@ -67,6 +67,8 @@ def update_all_messages():
     with status_reply_dict_lock:
         for chat_id in list(status_reply_dict.keys()):
             if status_reply_dict[chat_id] and msg != status_reply_dict[chat_id].text:
+                if len(msg) == 0:
+                    msg = "Starting DL"
                 try:
                     editMessage(msg, status_reply_dict[chat_id])
                 except Exception as e:
@@ -74,6 +76,7 @@ def update_all_messages():
                 status_reply_dict[chat_id].text = msg
 
 
+
 def sendStatusMessage(msg, bot):
     progress = get_readable_message()
     with status_reply_dict_lock:
@@ -86,5 +89,7 @@ def sendStatusMessage(msg, bot):
                 LOGGER.error(str(e))
                 del status_reply_dict[msg.message.chat.id]
                 pass
+        if len(progress) == 0:
+            progress = "Starting DL"
         message = sendMessage(progress, bot, msg)
         status_reply_dict[msg.message.chat.id] = message

bot/modules/mirror.py

@@ -1,12 +1,13 @@
 import requests
 from telegram.ext import CommandHandler, run_async
 
-from bot import Interval, INDEX_URL, LOGGER
+from bot import Interval, INDEX_URL, BLOCK_MEGA_LINKS
 from bot import dispatcher, DOWNLOAD_DIR, DOWNLOAD_STATUS_UPDATE_INTERVAL, download_dict, download_dict_lock
 from bot.helper.ext_utils import fs_utils, bot_utils
 from bot.helper.ext_utils.bot_utils import setInterval
 from bot.helper.ext_utils.exceptions import DirectDownloadLinkException, NotSupportedExtractionArchive
 from bot.helper.mirror_utils.download_utils.aria2_download import AriaDownloadHelper
+from bot.helper.mirror_utils.download_utils.mega_downloader import MegaDownloadHelper
 from bot.helper.mirror_utils.download_utils.direct_link_generator import direct_link_generator
 from bot.helper.mirror_utils.download_utils.telegram_downloader import TelegramDownloadHelper
 from bot.helper.mirror_utils.status_utils import listeners
@@ -26,7 +27,7 @@ ariaDlManager = AriaDownloadHelper()
 ariaDlManager.start_listener()
 
 class MirrorListener(listeners.MirrorListeners):
-    def __init__(self, bot, update, isTar=False,tag=None, extract=False):
+    def __init__(self, bot, update, isTar=False, tag=None, extract=False):
         super().__init__(bot, update)
         self.isTar = isTar
         self.tag = tag
@@ -218,8 +219,16 @@ def _mirror(bot, update, isTar=False, extract=False):
         LOGGER.info(f'{link}: {e}')
 
     listener = MirrorListener(bot, update, isTar, tag, extract)
-    ariaDlManager.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/',listener)
-    sendStatusMessage(update, bot)
+    if bot_utils.is_mega_link(link):
+        if BLOCK_MEGA_LINKS:
+            sendMessage("Mega links are blocked bcoz mega downloading is too much unstable and buggy. mega support will be added back after fix", bot, update)
+        else:
+            mega_dl = MegaDownloadHelper()
+            mega_dl.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/', listener)
+            sendStatusMessage(update, bot)
+    else:
+        ariaDlManager.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/', listener)
+        sendStatusMessage(update, bot)
     if len(Interval) == 0:
         Interval.append(setInterval(DOWNLOAD_STATUS_UPDATE_INTERVAL, update_all_messages))
 

config_sample.env

@@ -9,8 +9,13 @@ DOWNLOAD_DIR = "/home/username/mirror-bot/downloads"
 DOWNLOAD_STATUS_UPDATE_INTERVAL = 5
 AUTO_DELETE_MESSAGE_DURATION = 20
 IS_TEAM_DRIVE = ""
-INDEX_URL = ""
 TELEGRAM_API = 
 TELEGRAM_HASH = ""
 USE_SERVICE_ACCOUNTS = ""
-AUTHORIZED_CHATS = "" 
+# Optional config
+AUTHORIZED_CHATS = ""
+INDEX_URL = "" 
+MEGA_API_KEY = ""
+MEGA_EMAIL_ID = ""
+MEGA_PASSWORD = ""  
+BLOCK_MEGA_LINKS = ""