Added Youtube-dl support

Signed-off-by: lzzy12 <jhashivam2020@gmail.com>

bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py

@@ -0,0 +1,104 @@
+from .download_helper import DownloadHelper
+import time
+from youtube_dl import YoutubeDL
+import threading
+from bot import LOGGER,download_dict_lock,download_dict,DOWNLOAD_DIR
+from ..status_utils.youtube_dl_download_status import YoutubeDLDownloadStatus
+
+class YoutubeDLHelper(DownloadHelper):
+    def __init__(self, listener):
+        super().__init__()
+        self.__name = ""
+        self.__start_time = time.time()
+        self.__listener = listener
+        self.__gid = ""
+        self.opts = {
+        'format': 'bestaudio/best',
+        'progress_hooks':[self.__onDownloadProgress],
+        'outtmpl': f"{DOWNLOAD_DIR}{self.__listener.uid}/%(title)s.%(ext)s"
+        }
+        self.ydl = YoutubeDL(self.opts)
+        self.__download_speed = 0
+        self.download_speed_readable = ""
+        self.downloaded_bytes = 0
+        self.size = 0
+        self.is_playlist = False
+        self.last_downloaded = 0
+        self.is_cancelled = False
+        self.__resource_lock = threading.RLock()
+
+    @property
+    def download_speed(self):
+        with self.__resource_lock:
+            return self.__download_speed
+
+    @property
+    def gid(self):
+        with self.__resource_lock:
+            return self.__gid
+
+    def __onDownloadProgress(self,d):
+        if self.is_cancelled:
+            raise ValueError("Cancelling Download..")
+        if d['status'] == "finished":
+            if self.is_playlist:
+                self.last_downloaded = 0
+                if self.downloaded_bytes == self.size:
+                    self.__onDownloadComplete()
+            else: 
+                self.__onDownloadComplete()
+        elif d['status'] == "downloading":
+            with self.__resource_lock:
+                self.progress = self.downloaded_bytes / self.size * 100
+                self.__download_speed = d['speed']
+                if self.is_playlist:
+                    chunk_size = self.size * self.progress - self.last_downloaded
+                    self.last_downloaded = self.size * self.progress
+                    self.downloaded_bytes += chunk_size
+                else:
+                    self.download_speed_readable = d['_speed_str']
+                    self.downloaded_bytes = d['downloaded_bytes']
+        
+    def __onDownloadStart(self):
+        with download_dict_lock:
+            download_dict[self.__listener.uid] = YoutubeDLDownloadStatus(self,self.__listener.uid)
+
+    def __onDownloadComplete(self):
+        self.__listener.onDownloadComplete()
+
+    def __onDownloadError(self,error):
+        self.__listener.onDownloadError(error)
+
+    def extractMetaData(self,link):
+        result = self.ydl.extract_info(link,download=False)
+        if 'entries' in result:
+            video = result['entries'][0]
+            for v in result['entries']:
+                self.size += int(v['filesize'])
+            self.name = result.get('title')
+            self.vid_id = video.get('id')
+            self.is_playlist = True
+            self.opts['outtmpl'] = f"{DOWNLOAD_DIR}{self.__listener.uid}/%(playlist)s/%(title)s.%(ext)s"
+            self.ydl = YoutubeDL(self.opts)
+        else:
+            video = result
+            self.size = int(video.get('filesize'))
+            self.name = video.get('title')
+            self.vid_id = video.get('id')
+        return video
+
+    def __download(self,link):
+        try:
+            self.ydl.download([link],)
+        except ValueError:
+            LOGGER.info("Download Cancelled by User!")
+            self.__onDownloadError("Download Cancelled by User!")
+
+    def add_download(self,link):
+        LOGGER.info(f"Downloading with YT-DL: {link}")
+        self.__gid = f"{self.vid_id}{self.__listener.uid}"
+        threading.Thread(target=self.__download,args=(link,)).start()
+        self.__onDownloadStart()
+
+    def cancel_download(self):
+        self.is_cancelled = True

bot/helper/mirror_utils/status_utils/youtube_dl_download_status.py

@@ -0,0 +1,54 @@
+from bot import DOWNLOAD_DIR
+from bot.helper.ext_utils.bot_utils import MirrorStatus, get_readable_file_size, get_readable_time
+from .status import Status
+
+class YoutubeDLDownloadStatus(Status):
+    def __init__(self, obj, uid):
+        self.obj = obj
+        self.uid = uid
+
+    def gid(self):
+        return self.obj.gid
+
+    def path(self):
+        return f"{DOWNLOAD_DIR}{self.uid}"
+
+    def processed_bytes(self):
+        return self.obj.downloaded_bytes
+
+    def size_raw(self):
+        return self.obj.size
+
+    def size(self):
+        return get_readable_file_size(self.size_raw())
+
+    def status(self):
+        return MirrorStatus.STATUS_DOWNLOADING
+
+    def name(self):
+        return self.obj.name
+
+    def progress_raw(self):
+        return self.obj.progress
+
+    def progress(self):
+        return f'{round(self.progress_raw(), 2)}%'
+
+    def speed_raw(self):
+        """
+        :return: Download speed in Bytes/Seconds
+        """
+        return self.obj.download_speed
+
+    def speed(self):
+        return f'{get_readable_file_size(self.speed_raw())}/s'
+
+    def eta(self):
+        try:
+            seconds = (self.size_raw() - self.processed_bytes()) / self.speed_raw()
+            return f'{get_readable_time(seconds)}'
+        except ZeroDivisionError:
+            return '-'
+
+    def download(self):
+        return self.obj

bot/modules/cancel_mirror.py

@@ -3,13 +3,16 @@ from time import sleep
 from telegram.ext import CommandHandler, run_async
 
 from bot import download_dict, aria2, dispatcher, download_dict_lock, DOWNLOAD_DIR
-from bot.helper.ext_utils.bot_utils import getDownloadByGid
 from bot.helper.ext_utils.fs_utils import clean_download
 from bot.helper.telegram_helper.bot_commands import BotCommands
 from bot.helper.telegram_helper.filters import CustomFilters
 from bot.helper.telegram_helper.message_utils import *
 from ..helper.mirror_utils.download_utils.telegram_downloader import TelegramDownloadHelper
 
+from time import sleep
+from bot.helper.ext_utils.bot_utils import getDownloadByGid
+from ..helper.mirror_utils.download_utils.youtube_dl_download_helper import YoutubeDLHelper
+
 
 @run_async
 def cancel_mirror(bot, update):
@@ -52,6 +55,8 @@ def cancel_mirror(bot, update):
         download = dl.download()
         if isinstance(download, TelegramDownloadHelper):
             download.cancel_download()
+        if isinstance(download, YoutubeDLHelper):
+            download.cancel_download()
         else:
             if len(download.followed_by_ids) != 0:
                 downloads = aria2.get_downloads(download.followed_by_ids)

bot/modules/mirror.py

@@ -1,6 +1,3 @@
-import os
-import pathlib
-
 import requests
 from telegram.ext import CommandHandler, run_async
 
@@ -19,6 +16,9 @@ from bot.helper.mirror_utils.upload_utils import gdriveTools
 from bot.helper.telegram_helper.bot_commands import BotCommands
 from bot.helper.telegram_helper.filters import CustomFilters
 from bot.helper.telegram_helper.message_utils import *
+from bot.helper.mirror_utils.download_utils.youtube_dl_download_helper import YoutubeDLHelper
+import pathlib
+import os
 
 
 class MirrorListener(listeners.MirrorListeners):
@@ -175,8 +175,16 @@ def _mirror(bot, update, isTar=False):
     except DirectDownloadLinkException as e:
         LOGGER.info(f'{link}: {e}')
     listener = MirrorListener(bot, update, isTar, tag)
-    aria = aria2_download.AriaDownloadHelper(listener)
-    aria.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/')
+    try:
+        ydl = YoutubeDLHelper(listener)
+        sup_link = ydl.extractMetaData(link)
+    except Exception as e:
+        sup_link = None
+    if sup_link:
+        ydl.add_download(link)
+    else:
+        aria = aria2_download.AriaDownloadHelper(listener)
+        aria.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/')
     sendStatusMessage(update, bot)
     if len(Interval) == 0:
         Interval.append(setInterval(DOWNLOAD_STATUS_UPDATE_INTERVAL, update_all_messages))

requirements.txt

@@ -5,8 +5,10 @@ google-auth-httplib2>=0.0.3,<0.1.0
 google-auth-oauthlib>=0.4.1,<0.10.0
 aria2p>=0.3.0,<0.10.0
 python-dotenv>=0.10
-tenacity>=6.0.0 
+tenacity>=6.0.0
 python-magic
 beautifulsoup4>=4.8.2,<4.8.10
 Pyrogram>=0.16.0,<0.16.10
-TgCrypto>=1.1.1,<1.1.10
+TgCrypto>=1.1.1,<1.1.10
+tenacity>=6.0.0
+youtube-dl