From df192e41452fa99941e1759e09cf4e9047fdd952 Mon Sep 17 00:00:00 2001 From: Hafitz Setya <71178188+breakdowns@users.noreply.github.com> Date: Thu, 4 Mar 2021 06:30:32 +0700 Subject: [PATCH] v3.5 (#12) - Added quality option in /watch command - Added url Shorteners support (Optional) - Random name while generating Telegraph Token --- README.md | 2 +- app.json | 12 +++- bot/__init__.py | 31 ++++++++- bot/__main__.py | 4 +- .../youtube_dl_download_helper.py | 27 +++++--- .../youtube_dl_download_status.py | 9 ++- .../mirror_utils/upload_utils/gdriveTools.py | 63 +++++++++++++++---- bot/modules/mirror.py | 22 ++++--- bot/modules/speedtest.py | 20 +++--- bot/modules/watch.py | 12 +++- config_sample.env | 2 + 11 files changed, 154 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 79428e4..1f9550b 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ Fork this repo, than upload credentials.json and token.pickle to your forks - Add stickers to your pack - Nyaa.si and Sukebei Torrent search - Shell and Executor -- Inline buttons support +- Shortener support - Speedtest with picture results - Extract these filetypes and uploads to google drive > ZIP, RAR, TAR, 7z, ISO, WIM, CAB, GZIP, BZIP2, diff --git a/app.json b/app.json index a714784..a8b5fc0 100644 --- a/app.json +++ b/app.json @@ -3,6 +3,7 @@ "description": "Telegram mirror bot with python", "logo": "https://telegra.ph/file/6507910fd06d18dfaba82.jpg", "keywords": [ + "bot", "telegram", "python" ], @@ -25,7 +26,7 @@ }, "DOWNLOAD_DIR": { "description": "The path to the local folder where the downloads should be downloaded to.", - "value": "/home/username/mirror-bot/downloads", + "value": "/home/username/mirror-bot/downloads", "required": false }, "DOWNLOAD_STATUS_UPDATE_INTERVAL": { @@ -47,7 +48,6 @@ }, "USE_SERVICE_ACCOUNTS": { "description": "Whether to use service accounts or not. For this to work see 'Using service accounts' section below.", - "value": "False", "required": false }, "AUTHORIZED_CHATS": { @@ -81,6 +81,14 @@ "BLOCK_MEGA_LINKS": { "description": "If you want to remove mega.nz mirror support, set it to True.", "required": false + }, + "SHORTENER": { + "description": "if you want to use shortener in Gdrive and index link.", + "required": false + }, + "SHORTENER_API": { + "description": "Fill your shortener api key if you are using shortener.", + "required": false } } } diff --git a/bot/__init__.py b/bot/__init__.py index ab6eed4..4d453aa 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -2,6 +2,8 @@ import logging import os import threading import time +import random +import string import aria2p import telegram.ext as tg from dotenv import load_dotenv @@ -107,11 +109,27 @@ LOGGER.info("Generating USER_SESSION_STRING") with Client(':memory:', api_id=int(TELEGRAM_API), api_hash=TELEGRAM_HASH, bot_token=BOT_TOKEN) as app: USER_SESSION_STRING = app.export_session_string() #Generate Telegraph Token -LOGGER.info("Generating Telegraph Token") +sname = ''.join(random.SystemRandom().choices(string.ascii_letters, k=8)) +LOGGER.info("Generating Telegraph Token using '" + sname + "' name") telegraph = Telegraph() -telegraph.create_account(short_name="mirror_bot") +telegraph.create_account(short_name=sname) telegraph_token = telegraph.get_access_token() +LOGGER.info("Telegraph Token Generated: '" + telegraph_token + "'") +try: + MEGA_API_KEY = getConfig('MEGA_API_KEY') +except KeyError: + logging.warning('MEGA API KEY not provided!') + MEGA_API_KEY = None +try: + MEGA_EMAIL_ID = getConfig('MEGA_EMAIL_ID') + MEGA_PASSWORD = getConfig('MEGA_PASSWORD') + if len(MEGA_EMAIL_ID) == 0 or len(MEGA_PASSWORD) == 0: + raise KeyError +except KeyError: + logging.warning('MEGA Credentials not provided!') + MEGA_EMAIL_ID = None + MEGA_PASSWORD = None try: INDEX_URL = getConfig('INDEX_URL') if len(INDEX_URL) == 0: @@ -145,6 +163,15 @@ try: except KeyError: BLOCK_MEGA_LINKS = False +try: + SHORTENER = getConfig('SHORTENER') + SHORTENER_API = getConfig('SHORTENER_API') + if len(SHORTENER) == 0 or len(SHORTENER_API) == 0: + raise KeyError +except KeyError: + SHORTENER = None + SHORTENER_API = None + app = Client('slam', api_id=TELEGRAM_API, api_hash=TELEGRAM_HASH, bot_token=BOT_TOKEN) updater = tg.Updater(token=BOT_TOKEN,use_context=True) bot = updater.bot diff --git a/bot/__main__.py b/bot/__main__.py index 8ab2397..49519bb 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -33,7 +33,7 @@ def stats(update, context): f'Total disk space: {total}\n' \ f'Used: {used} ' \ f'Free: {free}\n\n' \ - f'📊Data Usage📊\nUpload: {sent}\n' \ + f'Data Usage\nUpload: {sent}\n' \ f'Down: {recv}\n\n' \ f'CPU: {cpuUsage}%\n' \ f'RAM: {memory}%\n' \ @@ -86,7 +86,7 @@ def bot_help(update, context): /{BotCommands.TarMirrorCommand} [download_url][magnet_link]: Start mirroring and upload the archived (.tar) version of the download -/{BotCommands.WatchCommand} [youtube-dl supported link]: Mirror through youtube-dl +/{BotCommands.WatchCommand} [youtube-dl supported link]: Mirror through youtube-dl. Click /{BotCommands.WatchCommand} for more help. /{BotCommands.TarWatchCommand} [youtube-dl supported link]: Mirror through youtube-dl and tar before uploading diff --git a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py index e4e4776..cf517b9 100644 --- a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py +++ b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py @@ -40,8 +40,7 @@ class YoutubeDLHelper(DownloadHelper): self.opts = { 'progress_hooks': [self.__onDownloadProgress], 'logger': MyLogger(self), - 'usenetrc': True, - 'format': "best/bestvideo+bestaudio" + 'usenetrc': True } self.__download_speed = 0 self.download_speed_readable = '' @@ -72,10 +71,14 @@ class YoutubeDLHelper(DownloadHelper): elif d['status'] == "downloading": with self.__resource_lock: self.__download_speed = d['speed'] + try: + tbyte = d['total_bytes'] + except KeyError: + tbyte = d['total_bytes_estimate'] if self.is_playlist: - progress = d['downloaded_bytes'] / d['total_bytes'] + progress = d['downloaded_bytes'] / tbyte chunk_size = d['downloaded_bytes'] - self.last_downloaded - self.last_downloaded = d['total_bytes'] * progress + self.last_downloaded = tbyte * progress self.downloaded_bytes += chunk_size try: self.progress = (self.downloaded_bytes / self.size) * 100 @@ -95,14 +98,17 @@ class YoutubeDLHelper(DownloadHelper): def onDownloadError(self, error): self.__listener.onDownloadError(error) - def extractMetaData(self, link): - if 'hotstar' in link: + def extractMetaData(self, link, qual): + if 'hotstar' or 'sonyliv' in link: self.opts['geo_bypass_country'] = 'IN' with YoutubeDL(self.opts) as ydl: try: result = ydl.extract_info(link, download=False) name = ydl.prepare_filename(result) + # noobway hack for changing extension after converting to mp3 + if qual == "audio": + name = name.replace(".mp4", ".mp3").replace(".webm", ".mp3") except DownloadError as e: self.onDownloadError(str(e)) return @@ -138,14 +144,19 @@ class YoutubeDLHelper(DownloadHelper): LOGGER.info("Download Cancelled by User!") self.onDownloadError("Download Cancelled by User!") - def add_download(self, link, path): + def add_download(self, link, path, qual): pattern = '^.*(youtu\.be\/|youtube.com\/)(playlist?)' if re.match(pattern, link): self.opts['ignoreerrors'] = True self.__onDownloadStart() - self.extractMetaData(link) + self.extractMetaData(link, qual) LOGGER.info(f"Downloading with YT-DL: {link}") self.__gid = f"{self.vid_id}{self.__listener.uid}" + if qual == "audio": + self.opts['format'] = 'bestaudio/best' + self.opts['postprocessors'] = [{'key': 'FFmpegExtractAudio','preferredcodec': 'mp3','preferredquality': '192',}] + else: + self.opts['format'] = qual if not self.is_playlist: self.opts['outtmpl'] = f"{path}/{self.name}" else: diff --git a/bot/helper/mirror_utils/status_utils/youtube_dl_download_status.py b/bot/helper/mirror_utils/status_utils/youtube_dl_download_status.py index a29013d..3950bad 100644 --- a/bot/helper/mirror_utils/status_utils/youtube_dl_download_status.py +++ b/bot/helper/mirror_utils/status_utils/youtube_dl_download_status.py @@ -1,7 +1,7 @@ from bot import DOWNLOAD_DIR from bot.helper.ext_utils.bot_utils import MirrorStatus, get_readable_file_size, get_readable_time from .status import Status - +from bot.helper.ext_utils.fs_utils import get_path_size class YoutubeDLDownloadStatus(Status): def __init__(self, obj, listener): @@ -16,7 +16,10 @@ class YoutubeDLDownloadStatus(Status): return f"{DOWNLOAD_DIR}{self.uid}" def processed_bytes(self): - return self.obj.downloaded_bytes + if self.obj.downloaded_bytes != 0: + return self.obj.downloaded_bytes + else: + return get_path_size(f"{DOWNLOAD_DIR}{self.uid}") def size_raw(self): return self.obj.size @@ -49,7 +52,7 @@ class YoutubeDLDownloadStatus(Status): try: seconds = (self.size_raw() - self.processed_bytes()) / self.speed_raw() return f'{get_readable_time(seconds)}' - except ZeroDivisionError: + except: return '-' def download(self): diff --git a/bot/helper/mirror_utils/upload_utils/gdriveTools.py b/bot/helper/mirror_utils/upload_utils/gdriveTools.py index f00de50..28b9c40 100644 --- a/bot/helper/mirror_utils/upload_utils/gdriveTools.py +++ b/bot/helper/mirror_utils/upload_utils/gdriveTools.py @@ -20,9 +20,9 @@ from telegram import InlineKeyboardMarkup from bot.helper.telegram_helper import button_build from telegraph import Telegraph from bot import parent_id, DOWNLOAD_DIR, IS_TEAM_DRIVE, INDEX_URL, \ - USE_SERVICE_ACCOUNTS, download_dict, telegraph_token + USE_SERVICE_ACCOUNTS, download_dict, telegraph_token, SHORTENER, SHORTENER_API from bot.helper.ext_utils.bot_utils import * -from bot.helper.ext_utils.fs_utils import get_mime_type +from bot.helper.ext_utils.fs_utils import get_mime_type, get_path_size LOGGER = logging.getLogger(__name__) logging.getLogger('googleapiclient.discovery').setLevel(logging.ERROR) @@ -209,6 +209,7 @@ class GoogleDriveHelper: self.__listener.onUploadStarted() file_dir = f"{DOWNLOAD_DIR}{self.__listener.message.message_id}" file_path = f"{file_dir}/{file_name}" + size = get_readable_file_size(get_path_size(file_path)) LOGGER.info("Uploading File: " + file_path) self.start_time = time.time() self.updater = setInterval(self.update_interval, self._on_upload_progress) @@ -250,7 +251,7 @@ class GoogleDriveHelper: finally: self.updater.cancel() LOGGER.info(download_dict) - self.__listener.onUploadComplete(link) + self.__listener.onUploadComplete(link, size) LOGGER.info("Deleting downloaded file/folder..") return link @@ -317,22 +318,40 @@ class GoogleDriveHelper: dir_id = self.create_directory(meta.get('name'), parent_id) result = self.cloneFolder(meta.get('name'), meta.get('name'), meta.get('id'), dir_id) msg += f'Filename : {meta.get("name")}\nSize : {get_readable_file_size(self.transferred_size)}' + durl = self.__G_DRIVE_DIR_BASE_DOWNLOAD_URL.format(dir_id) buttons = button_build.ButtonMaker() - buttons.buildbutton("✨Drive Link✨", self.__G_DRIVE_DIR_BASE_DOWNLOAD_URL.format(dir_id)) + if SHORTENER is not None and SHORTENER_API is not None: + surl = requests.get('https://{}/api?api={}&url={}&format=text'.format(SHORTENER, SHORTENER_API, durl)).text + buttons.buildbutton("✨Drive Link✨", surl) + else: + buttons.buildbutton("✨Drive Link✨", durl) if INDEX_URL is not None: url = requests.utils.requote_uri(f'{INDEX_URL}/{meta.get("name")}/') - buttons.buildbutton("⚡Index Link⚡", url) + if SHORTENER is not None and SHORTENER_API is not None: + siurl = requests.get('https://{}/api?api={}&url={}&format=text'.format(SHORTENER, SHORTENER_API, url)).text + buttons.buildbutton("⚡Index Link⚡", siurl) + else: + buttons.buildbutton("⚡Index Link⚡", url) else: file = self.copyFile(meta.get('id'), parent_id) msg += f'Filename : {file.get("name")}' + durl = self.__G_DRIVE_BASE_DOWNLOAD_URL.format(file.get("id")) buttons = button_build.ButtonMaker() - buttons.buildbutton("✨Drive Link✨", self.__G_DRIVE_BASE_DOWNLOAD_URL.format(file.get("id"))) + if SHORTENER is not None and SHORTENER_API is not None: + surl = requests.get('https://{}/api?api={}&url={}&format=text'.format(SHORTENER, SHORTENER_API, durl)).text + buttons.buildbutton("✨Drive Link ✨", surl) + else: + buttons.buildbutton("✨Drive Link✨", durl) try: msg += f'\nSize : {get_readable_file_size(int(meta.get("size")))}' except TypeError: pass if INDEX_URL is not None: - url = requests.utils.requote_uri(f'{INDEX_URL}/{file.get("name")}') + url = requests.utils.requote_uri(f'{INDEX_URL}/{file.get("name")}') + if SHORTENER is not None and SHORTENER_API is not None: + siurl = requests.get('https://{}/api?api={}&url={}&format=text'.format(SHORTENER, SHORTENER_API, url)).text + buttons.buildbutton("⚡Index Link⚡", siurl) + else: buttons.buildbutton("⚡Index Link⚡", url) except Exception as err: if isinstance(err, RetryError): @@ -476,17 +495,35 @@ class GoogleDriveHelper: msg += f'

Results : {fileName}



' for file in response.get('files', []): if file.get('mimeType') == "application/vnd.google-apps.folder": # Detect Whether Current Entity is a Folder or File. - msg += f"⁍{file.get('name')}
(folder📁)
" \ - f"Drive Link" + furl = f"https://drive.google.com/drive/folders/{file.get('id')}" + msg += f"⁍{file.get('name')}
(folder📁)
" + if SHORTENER is not None and SHORTENER_API is not None: + sfurl = requests.get('https://{}/api?api={}&url={}&format=text'.format(SHORTENER, SHORTENER_API, furl)).text + msg += f"Drive Link" + else: + msg += f"Drive Link" if INDEX_URL is not None: url = requests.utils.requote_uri(f'{INDEX_URL}/{file.get("name")}/') - msg += f' | Index Link' + if SHORTENER is not None and SHORTENER_API is not None: + siurl = requests.get('https://{}/api?api={}&url={}&format=text'.format(SHORTENER, SHORTENER_API, url)).text + msg += f' | Index Link' + else: + msg += f' | Index Link' else: - msg += f"⁍{file.get('name')}
({get_readable_file_size(int(file.get('size')))})📄
" \ - f"Drive Link" + furl = f"https://drive.google.com/uc?id={file.get('id')}&export=download" + msg += f"⁍{file.get('name')}
({get_readable_file_size(int(file.get('size')))})📄
" + if SHORTENER is not None and SHORTENER_API is not None: + sfurl = requests.get('https://{}/api?api={}&url={}&format=text'.format(SHORTENER, SHORTENER_API, furl)).text + msg += f"Drive Link" + else: + msg += f"Drive Link" if INDEX_URL is not None: url = requests.utils.requote_uri(f'{INDEX_URL}/{file.get("name")}') - msg += f' | Index Link' + if SHORTENER is not None and SHORTENER_API is not None: + siurl = requests.get('https://{}/api?api={}&url={}&format=text'.format(SHORTENER, SHORTENER_API, url)).text + msg += f' | Index Link' + else: + msg += f' | Index Link' msg += '

' content_count += 1 if content_count == TELEGRAPHLIMIT : diff --git a/bot/modules/mirror.py b/bot/modules/mirror.py index 4a7876c..376021a 100644 --- a/bot/modules/mirror.py +++ b/bot/modules/mirror.py @@ -3,7 +3,7 @@ from telegram.ext import CommandHandler, run_async from telegram import InlineKeyboardMarkup from bot import Interval, INDEX_URL, BLOCK_MEGA_LINKS -from bot import dispatcher, DOWNLOAD_DIR, DOWNLOAD_STATUS_UPDATE_INTERVAL, download_dict, download_dict_lock +from bot import dispatcher, DOWNLOAD_DIR, DOWNLOAD_STATUS_UPDATE_INTERVAL, download_dict, download_dict_lock, SHORTENER, SHORTENER_API from bot.helper.ext_utils import fs_utils, bot_utils from bot.helper.ext_utils.bot_utils import setInterval from bot.helper.ext_utils.exceptions import DirectDownloadLinkException, NotSupportedExtractionArchive @@ -93,12 +93,12 @@ class MirrorListener(listeners.MirrorListeners): else: path = f'{DOWNLOAD_DIR}{self.uid}/{name}' up_name = pathlib.PurePath(path).name + up_path = f'{DOWNLOAD_DIR}{self.uid}/{up_name}' if up_name == "None": up_name = "".join(os.listdir(f'{DOWNLOAD_DIR}{self.uid}/')) LOGGER.info(f"Upload Name : {up_name}") drive = gdriveTools.GoogleDriveHelper(up_name, self) - if size == 0: - size = fs_utils.get_path_size(m_path) + size = fs_utils.get_path_size(up_path) upload_status = UploadStatus(drive, size, self) with download_dict_lock: download_dict[self.uid] = upload_status @@ -137,17 +137,25 @@ class MirrorListener(listeners.MirrorListeners): def onUploadProgress(self): pass - def onUploadComplete(self, link: str): + def onUploadComplete(self, link: str, size): with download_dict_lock: - msg = f'Filename : {download_dict[self.uid].name()}\nSize : {download_dict[self.uid].size()}' + msg = f'Filename : {download_dict[self.uid].name()}\nSize : {size}' buttons = button_build.ButtonMaker() - buttons.buildbutton("✨Drive Link✨", link) + if SHORTENER is not None and SHORTENER_API is not None: + surl = requests.get('https://{}/api?api={}&url={}&format=text'.format(SHORTENER, SHORTENER_API, link)).text + buttons.buildbutton("✨Drive Link✨", surl) + else: + buttons.buildbutton("✨Drive Link✨", link) LOGGER.info(f'Done Uploading {download_dict[self.uid].name()}') if INDEX_URL is not None: share_url = requests.utils.requote_uri(f'{INDEX_URL}/{download_dict[self.uid].name()}') if os.path.isdir(f'{DOWNLOAD_DIR}/{self.uid}/{download_dict[self.uid].name()}'): share_url += '/' - buttons.buildbutton("⚡Index Link⚡", share_url) + if SHORTENER is not None and SHORTENER_API is not None: + siurl = requests.get('https://{}/api?api={}&url={}&format=text'.format(SHORTENER, SHORTENER_API, share_url)).text + buttons.buildbutton("⚡Index Link⚡", siurl) + else: + buttons.buildbutton("⚡Index Link⚡", share_url) if self.message.from_user.username: uname = f"@{self.message.from_user.username}" else: diff --git a/bot/modules/speedtest.py b/bot/modules/speedtest.py index 7ccf864..7ab1515 100644 --- a/bot/modules/speedtest.py +++ b/bot/modules/speedtest.py @@ -7,7 +7,7 @@ from telegram.ext import CallbackContext, Filters, run_async, CommandHandler @run_async -def speedtst(update, context): +def speedtest(update, context): message = update.effective_message ed_msg = message.reply_text("Running Speed Test . . . ") test = Speedtest() @@ -18,16 +18,16 @@ def speedtst(update, context): result = test.results.dict() path = (result['share']) string_speed = f''' -📬 Server -🌀 Name: {result['server']['name']} -🏁 Country: {result['server']['country']}, {result['server']['cc']} -🌐 Sponsor: {result['server']['sponsor']} +Server +Name: {result['server']['name']} +Country: {result['server']['country']}, {result['server']['cc']} +Sponsor: {result['server']['sponsor']} SpeedTest Results -🔼 Upload: {speed_convert(result['upload'] / 8)} -🔽 Download: {speed_convert(result['download'] / 8)} -📶 Ping: {result['ping']} ms -🖥️ ISP: {result['client']['isp']} +Upload: {speed_convert(result['upload'] / 8)} +Download: {speed_convert(result['download'] / 8)} +Ping: {result['ping']} ms +ISP: {result['client']['isp']} ''' ed_msg.delete() try: @@ -46,7 +46,7 @@ def speed_convert(size): return f"{round(size, 2)} {units[zero]}" -SPEED_HANDLER = CommandHandler(BotCommands.SpeedCommand, speedtst, +SPEED_HANDLER = CommandHandler(BotCommands.SpeedCommand, speedtest, filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) dispatcher.add_handler(SPEED_HANDLER) diff --git a/bot/modules/watch.py b/bot/modules/watch.py index 42bdb1d..a2ec2d7 100644 --- a/bot/modules/watch.py +++ b/bot/modules/watch.py @@ -14,8 +14,16 @@ def _watch(bot: Bot, update: Update, args: list, isTar=False): try: link = args[0] except IndexError: - sendMessage(f'/{BotCommands.WatchCommand} [yt_dl supported link] to mirror with youtube_dl', bot, update) + msg = f"/{BotCommands.WatchCommand} [yt_dl supported link] [quality] to mirror with youtube_dl.\n\n" + msg += "Example of quality :- audio, 144, 360, 720, 1080.\nNote :- Quality is optional" + sendMessage(msg, bot, update) return + try: + qual = args[1] + if qual != "audio": + qual = f'bestvideo[height<={qual}]+bestaudio/best[height<={qual}]' + except IndexError: + qual = "bestvideo+bestaudio/best" reply_to = update.message.reply_to_message if reply_to is not None: tag = reply_to.from_user.username @@ -24,7 +32,7 @@ def _watch(bot: Bot, update: Update, args: list, isTar=False): listener = MirrorListener(bot, update, isTar, tag) ydl = YoutubeDLHelper(listener) - threading.Thread(target=ydl.add_download,args=(link, f'{DOWNLOAD_DIR}{listener.uid}')).start() + threading.Thread(target=ydl.add_download,args=(link, f'{DOWNLOAD_DIR}{listener.uid}', qual)).start() sendStatusMessage(update, bot) if len(Interval) == 0: Interval.append(setInterval(DOWNLOAD_STATUS_UPDATE_INTERVAL, update_all_messages)) diff --git a/config_sample.env b/config_sample.env index 8dc3d8b..d006b44 100644 --- a/config_sample.env +++ b/config_sample.env @@ -19,3 +19,5 @@ MEGA_API_KEY = "" MEGA_EMAIL_ID = "" MEGA_PASSWORD = "" BLOCK_MEGA_LINKS = "" +SHORTENER = "" +SHORTENER_API = ""