From b6956f6f6ceb652dad4d311924ec63cbfb1bede1 Mon Sep 17 00:00:00 2001 From: RemixDev Date: Fri, 10 Apr 2020 16:12:21 +0200 Subject: [PATCH] Implemented socket.io Started work on darkmode Implemented queue system Started work on download tab --- deemix/api/deezer.py | 6 +- deemix/app/cli.py | 26 +---- deemix/app/downloader.py | 103 ++++++-------------- deemix/app/main.py | 28 +----- deemix/app/queuemanager.py | 189 +++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + 6 files changed, 228 insertions(+), 125 deletions(-) create mode 100644 deemix/app/queuemanager.py diff --git a/deemix/api/deezer.py b/deemix/api/deezer.py index a1ce412..2a2f148 100755 --- a/deemix/api/deezer.py +++ b/deemix/api/deezer.py @@ -268,7 +268,11 @@ class Deezer: i += 1 def stream_track(self, track_id, url, stream): - request = requests.get(url, headers=self.http_headers, stream=True) + try: + request = requests.get(url, headers=self.http_headers, stream=True, timeout=30) + except: + time.sleep(2) + return self.stream_track(track_id, url, stream) request.raise_for_status() blowfish_key = str.encode(self._get_blowfish_key(str(track_id))) i = 0 diff --git a/deemix/app/cli.py b/deemix/app/cli.py index 167f33a..08d6e35 100644 --- a/deemix/app/cli.py +++ b/deemix/app/cli.py @@ -2,7 +2,7 @@ from deemix.api.deezer import Deezer import deemix.utils.localpaths as localpaths from deemix.utils.misc import getIDFromLink, getTypeFromLink, getBitrateInt -from deemix.app.downloader import download_track, download_album, download_playlist, download_artist, download_spotifytrack, download_spotifyalbum +from deemix.app.queuemanager import addToQueue from os import system as execute import os.path as path from os import mkdir @@ -31,26 +31,4 @@ def login(): f.write(arl) def downloadLink(url, settings, bitrate=None): - forcedBitrate = getBitrateInt(bitrate) - type = getTypeFromLink(url) - id = getIDFromLink(url, type) - folder = settings['downloadLocation'] - if type == None or id == None: - print("URL not recognized") - if type == "track": - folder = download_track(dz, id, settings, forcedBitrate) - elif type == "album": - folder = download_album(dz, id, settings, forcedBitrate) - elif type == "playlist": - folder = download_playlist(dz, id, settings, forcedBitrate) - elif type == "artist": - download_artist(dz, id, settings, forcedBitrate) - elif type == "spotifytrack": - folder = download_spotifytrack(dz, id, settings, forcedBitrate) - elif type == "spotifyalbum": - folder = download_spotifyalbum(dz, id, settings, forcedBitrate) - else: - print("URL not supported yet") - return None - if settings['executeCommand'] != "": - execute(settings['executeCommand'].replace("%folder%", folder)) + addToQueue(dz, url, settings, bitrate) diff --git a/deemix/app/downloader.py b/deemix/app/downloader.py index 74097be..e2b2592 100644 --- a/deemix/app/downloader.py +++ b/deemix/app/downloader.py @@ -329,7 +329,7 @@ def getTrackData(dz, trackAPI_gw, trackAPI = None, albumAPI_gw = None, albumAPI return track -def downloadTrackObj(dz, trackAPI, settings, overwriteBitrate=False, extraTrack=None): +def downloadTrackObj(dz, trackAPI, settings, bitrate, uuid, extraTrack=None, socket=None): result = {} # Get the metadata if extraTrack: @@ -348,7 +348,7 @@ def downloadTrackObj(dz, trackAPI, settings, overwriteBitrate=False, extraTrack= if not 'MD5_ORIGIN' in trackNew: trackNew['MD5_ORIGIN'] = dz.get_track_md5(trackNew['SNG_ID']) track = parseEssentialTrackData(track, trackNew) - return downloadTrackObj(dz, trackAPI, settings, extraTrack=track) + return downloadTrackObj(dz, trackAPI, settings, bitrate, uuid, extraTrack=track, socket=socket) elif not 'searched' in track and settings['fallbackSearch']: print("Track not yet encoded, searching for alternative") searchedId = dz.get_track_from_metadata(track['mainArtist']['name'], track['title'], track['album']['title']) @@ -358,7 +358,7 @@ def downloadTrackObj(dz, trackAPI, settings, overwriteBitrate=False, extraTrack= trackNew['MD5_ORIGIN'] = dz.get_track_md5(trackNew['SNG_ID']) track = parseEssentialTrackData(track, trackNew) track['searched'] = True - return downloadTrackObj(dz, trackAPI, settings, extraTrack=track) + return downloadTrackObj(dz, trackAPI, settings, bitrate, uuid, extraTrack=track, socket=socket) else: print("ERROR: Track not yet encoded and no alternative found!") result['error'] = { @@ -375,7 +375,6 @@ def downloadTrackObj(dz, trackAPI, settings, overwriteBitrate=False, extraTrack= return result # Get the selected bitrate - bitrate = overwriteBitrate if overwriteBitrate else settings['maxBitrate'] (format, filesize) = getPreferredBitrate(track['filesize'], bitrate, settings['fallbackBitrate']) if format == -100: print("ERROR: Track not found at desired bitrate. Enable fallback to lower bitrates to fix this issue.") @@ -471,14 +470,14 @@ def downloadTrackObj(dz, trackAPI, settings, overwriteBitrate=False, extraTrack= if track['selectedFormat'] == 9 and settings['fallbackBitrate']: print("Track not available in flac, trying mp3") track['filesize']['flac'] = 0 - return downloadTrackObj(dz, trackAPI, settings, extraTrack=track) + return downloadTrackObj(dz, trackAPI, settings, bitrate, uuid, extraTrack=track, socket=socket) elif track['fallbackId'] != 0: print("Track not available, using fallback id") trackNew = dz.get_track_gw(track['fallbackId']) if not 'MD5_ORIGIN' in trackNew: trackNew['MD5_ORIGIN'] = dz.get_track_md5(trackNew['SNG_ID']) track = parseEssentialTrackData(track, trackNew) - return downloadTrackObj(dz, trackAPI, settings, extraTrack=track) + return downloadTrackObj(dz, trackAPI, settings, bitrate, uuid, extraTrack=track, socket=socket) elif not 'searched' in track and settings['fallbackSearch']: print("Track not available, searching for alternative") searchedId = dz.get_track_from_metadata(track['mainArtist']['name'], track['title'], track['album']['title']) @@ -488,7 +487,7 @@ def downloadTrackObj(dz, trackAPI, settings, overwriteBitrate=False, extraTrack= trackNew['MD5_ORIGIN'] = dz.get_track_md5(trackNew['SNG_ID']) track = parseEssentialTrackData(track, trackNew) track['searched'] = True - return downloadTrackObj(dz, trackAPI, settings, extraTrack=track) + return downloadTrackObj(dz, trackAPI, settings, bitrate, uuid, extraTrack=track, socket=socket) else: print("ERROR: Track not available on deezer's servers and no alternative found!") result['error'] = { @@ -510,8 +509,29 @@ def downloadTrackObj(dz, trackAPI, settings, overwriteBitrate=False, extraTrack= if 'searched' in track: result['searched'] = f'{track["mainArtist"]["name"]} - {track["title"]}' print("Done!") + if socket: + socket.emit("updateQueue", {'uuid': uuid, 'downloaded': True}) return result +def download(dz, queueItem, socket=None): + settings = queueItem['settings'] + bitrate = queueItem['bitrate'] + if 'single' in queueItem: + result = downloadTrackObj(dz, queueItem['single'], settings, bitrate, queueItem['uuid'], socket=socket) + download_path = after_download_single(result, settings) + elif 'collection' in queueItem: + print("Downloading collection") + playlist = [None] * len(queueItem['collection']) + with ThreadPoolExecutor(settings['queueConcurrency']) as executor: + for pos, track in enumerate(queueItem['collection'], start=0): + playlist[pos] = executor.submit(downloadTrackObj, dz, track, settings, bitrate, queueItem['uuid'], socket=socket) + download_path = after_download(playlist, settings) + return { + 'dz': dz, + 'socket': socket, + 'download_path': download_path + } + def after_download(tracks, settings): extrasPath = None playlist = [None] * len(tracks) @@ -558,72 +578,3 @@ def after_download_single(track, settings): return track['extrasPath'] else: return None - -def download_track(dz, id, settings, overwriteBitrate=False): - trackAPI = dz.get_track_gw(id) - trackAPI['FILENAME_TEMPLATE'] = settings['tracknameTemplate'] - trackAPI['SINGLE_TRACK'] = True - result = downloadTrackObj(dz, trackAPI, settings, overwriteBitrate) - return after_download_single(result, settings) - -def download_spotifytrack(dz, id, settings, overwriteBitrate=False): - track_id = get_trackid_spotify(dz, id, settings['fallbackSearch']) - if track_id == "Not Enabled": - print("Spotify Features is not setted up correctly.") - if track_id != 0: - return download_track(dz, track_id, settings, overwriteBitrate) - else: - print("Track not found on deezer!") - return None - -def download_album(dz, id, settings, overwriteBitrate=False): - albumAPI = dz.get_album(id) - albumAPI_gw = dz.get_album_gw(id) - albumAPI['nb_disk'] = albumAPI_gw['NUMBER_DISK'] - albumAPI['copyright'] = albumAPI_gw['COPYRIGHT'] - if albumAPI['nb_tracks'] == 1: - trackAPI = dz.get_track_gw(albumAPI['tracks']['data'][0]['id']) - trackAPI['_EXTRA_ALBUM'] = albumAPI - trackAPI['FILENAME_TEMPLATE'] = settings['tracknameTemplate'] - trackAPI['SINGLE_TRACK'] = True - result = downloadTrackObj(dz, trackAPI, settings, overwriteBitrate) - return after_download_single(result, settings) - else: - tracksArray = dz.get_album_tracks_gw(id) - playlist = [None] * len(tracksArray) - with ThreadPoolExecutor(settings['queueConcurrency']) as executor: - for pos, trackAPI in enumerate(tracksArray, start=1): - trackAPI['_EXTRA_ALBUM'] = albumAPI - trackAPI['POSITION'] = pos - trackAPI['FILENAME_TEMPLATE'] = settings['albumTracknameTemplate'] - playlist[pos-1] = executor.submit(downloadTrackObj, dz, trackAPI, settings, overwriteBitrate) - - return after_download(playlist, settings) - -def download_spotifyalbum(dz, id, settings, overwriteBitrate=False): - album_id = get_albumid_spotify(dz, id) - if album_id == "Not Enabled": - print("Spotify Features is not setted up correctly.") - if album_id != 0: - return download_album(dz, album_id, settings, overwriteBitrate) - else: - print("Album not found on deezer!") - return None - -def download_artist(dz, id, settings, overwriteBitrate=False): - artistAPI = dz.get_artist_albums(id) - for album in artistAPI['data']: - print(f"Album: {album['title']}") - download_album(dz, album['id'], settings, overwriteBitrate) - -def download_playlist(dz, id, settings, overwriteBitrate=False): - playlistAPI = dz.get_playlist(id) - playlistTracksAPI = dz.get_playlist_tracks_gw(id) - playlist = [None] * len(playlistTracksAPI) - with ThreadPoolExecutor(settings['queueConcurrency']) as executor: - for pos, trackAPI in enumerate(playlistTracksAPI, start=1): - trackAPI['_EXTRA_PLAYLIST'] = playlistAPI - trackAPI['POSITION'] = pos - trackAPI['FILENAME_TEMPLATE'] = settings['playlistTracknameTemplate'] - playlist[pos-1] = executor.submit(downloadTrackObj, dz, trackAPI, settings, overwriteBitrate) - return after_download(playlist, settings) diff --git a/deemix/app/main.py b/deemix/app/main.py index 0b942f0..98e2fa7 100644 --- a/deemix/app/main.py +++ b/deemix/app/main.py @@ -1,7 +1,6 @@ from deemix.api.deezer import Deezer import deemix.utils.localpaths as localpaths -from deemix.utils.misc import getIDFromLink, getTypeFromLink, getBitrateInt, isValidLink -from deemix.app.downloader import download_track, download_album, download_playlist, download_artist, download_spotifytrack, download_spotifyalbum +from deemix.app.queuemanager import addToQueue from deemix.app.settings import initSettings from os import system as execute import os.path as path @@ -43,28 +42,9 @@ def mainSearch(term): def search(term, type, start, nb): return dz.search_gw(term, type, start, nb) +def addToQueue_link(url, bitrate=None, socket=None): + addToQueue(dz, url, settings, bitrate, socket) + def downloadLink(url, bitrate=None): - global settings - forcedBitrate = getBitrateInt(bitrate) - type = getTypeFromLink(url) - id = getIDFromLink(url, type) - folder = settings['downloadLocation'] - if type == None or id == None: - print("URL not recognized") - if type == "track": - folder = download_track(dz, id, settings, forcedBitrate) - elif type == "album": - folder = download_album(dz, id, settings, forcedBitrate) - elif type == "playlist": - folder = download_playlist(dz, id, settings, forcedBitrate) - elif type == "artist": - download_artist(dz, id, settings, forcedBitrate) - elif type == "spotifytrack": - folder = download_spotifytrack(dz, id, settings, forcedBitrate) - elif type == "spotifyalbum": - folder = download_spotifyalbum(dz, id, settings, forcedBitrate) - else: - print("URL not supported yet") - return None if settings['executeCommand'] != "": execute(settings['executeCommand'].replace("%folder%", folder)) diff --git a/deemix/app/queuemanager.py b/deemix/app/queuemanager.py new file mode 100644 index 0000000..5b86a49 --- /dev/null +++ b/deemix/app/queuemanager.py @@ -0,0 +1,189 @@ +from deemix.utils.misc import getIDFromLink, getTypeFromLink, getBitrateInt +from deemix.utils.spotifyHelper import get_trackid_spotify, get_albumid_spotify +from concurrent.futures import ProcessPoolExecutor +from deemix.app.downloader import download + +queue = [] +queueList = {} +currentItem = "" +currentJob = None + +""" +queueItem base structure + title + artist + cover + size + downloaded + failed + progress + type + id + bitrate + uuid: type+id+bitrate +if its a single track + single +if its an album/playlist + collection +""" + +def generateQueueItem(dz, url, settings, bitrate=None, albumAPI=None): + forcedBitrate = getBitrateInt(bitrate) + bitrate = forcedBitrate if forcedBitrate else settings['maxBitrate'] + type = getTypeFromLink(url) + id = getIDFromLink(url, type) + result = {} + if type == None or id == None: + print("URL not recognized") + result['error'] = "URL not recognized" + elif type == "track": + trackAPI = dz.get_track_gw(id) + if albumAPI: + trackAPI['_EXTRA_ALBUM'] = albumAPI + trackAPI['FILENAME_TEMPLATE'] = settings['tracknameTemplate'] + trackAPI['SINGLE_TRACK'] = True + + result['title'] = trackAPI['SNG_TITLE'] + if 'VERSION' in trackAPI and trackAPI['VERSION']: + result['title'] += " " + trackAPI['VERSION'] + result['artist'] = trackAPI['ART_NAME'] + result['cover'] = f"https://e-cdns-images.dzcdn.net/images/cover/{trackAPI['ART_PICTURE']}/128x128-000000-80-0-0.jpg" + result['size'] = 1 + result['downloaded'] = 0 + result['failed'] = 0 + result['progress'] = 0 + result['type'] = 'track' + result['id'] = id + result['bitrate'] = bitrate + result['uuid'] = f"{result['type']}:{id}:{bitrate}" + result['settings'] = settings or {} + result['single'] = trackAPI + + elif type == "album": + albumAPI = dz.get_album(id) + albumAPI_gw = dz.get_album_gw(id) + albumAPI['nb_disk'] = albumAPI_gw['NUMBER_DISK'] + albumAPI['copyright'] = albumAPI_gw['COPYRIGHT'] + if albumAPI['nb_tracks'] == 1: + return generateQueueItem(dz, f"https://www.deezer.com/track/{albumAPI['tracks']['data'][0]['id']}", settings, bitrate, albumAPI) + tracksArray = dz.get_album_tracks_gw(id) + + result['title'] = albumAPI['title'] + result['artist'] = albumAPI['artist']['name'] + result['cover'] = albumAPI['cover_small'][:-24]+'/128x128-000000-80-0-0.jpg' + result['size'] = albumAPI['nb_tracks'] + result['downloaded'] = 0 + result['failed'] = 0 + result['progress'] = 0 + result['type'] = 'album' + result['id'] = id + result['bitrate'] = bitrate + result['uuid'] = f"{result['type']}:{id}:{bitrate}" + result['settings'] = settings or {} + result['collection'] = [] + for pos, trackAPI in enumerate(tracksArray, start=1): + trackAPI['_EXTRA_ALBUM'] = albumAPI + trackAPI['POSITION'] = pos + trackAPI['FILENAME_TEMPLATE'] = settings['albumTracknameTemplate'] + result['collection'].append(trackAPI) + + elif type == "playlist": + playlistAPI = dz.get_playlist(id) + playlistTracksAPI = dz.get_playlist_tracks_gw(id) + + result['title'] = playlistAPI['title'] + result['artist'] = playlistAPI['creator']['name'] + result['cover'] = playlistAPI['picture_small'][:-24]+'/128x128-000000-80-0-0.jpg' + result['size'] = playlistAPI['nb_tracks'] + result['downloaded'] = 0 + result['failed'] = 0 + result['progress'] = 0 + result['type'] = 'playlist' + result['id'] = id + result['bitrate'] = bitrate + result['uuid'] = f"{result['type']}:{id}:{bitrate}" + result['settings'] = settings or {} + result['collection'] = [] + for pos, trackAPI in enumerate(playlistTracksAPI, start=1): + trackAPI['_EXTRA_PLAYLIST'] = playlistAPI + trackAPI['POSITION'] = pos + trackAPI['FILENAME_TEMPLATE'] = settings['playlistTracknameTemplate'] + result['collection'].append(trackAPI) + + elif type == "artist": + artistAPI = dz.get_artist_albums(id) + albumList = [] + for album in artistAPI['data']: + albumList.append(generateQueueItem(dz, album['link'], settings, bitrate)) + return albumList + elif type == "spotifytrack": + track_id = get_trackid_spotify(dz, id, settings['fallbackSearch']) + result = {} + if track_id == "Not Enabled": + print("Spotify Features is not setted up correctly.") + result['error'] = "Spotify Features is not setted up correctly." + elif track_id != 0: + return generateQueueItem(dz, f'https://www.deezer.com/track/{track_id}', settings, bitrate) + else: + print("Track not found on deezer!") + result['error'] = "Track not found on deezer!" + elif type == "spotifyalbum": + album_id = get_albumid_spotify(dz, id) + if album_id == "Not Enabled": + print("Spotify Features is not setted up correctly.") + result['error'] = "Spotify Features is not setted up correctly." + elif album_id != 0: + return generateQueueItem(dz, f'https://www.deezer.com/album/{track_id}', settings, bitrate) + else: + print("Album not found on deezer!") + result['error'] = "Album not found on deezer!" + else: + print("URL not supported yet") + result['error'] = "URL not supported yet" + return result + +def addToQueue(dz, url, settings, bitrate=None, socket=None): + global currentItem, currentJob, queueList, queue + queueItem = generateQueueItem(dz, url, settings, bitrate) + if 'error' in queueItem: + if socket: + socket.emit("message", queueItem['error']) + return None + if queueItem['uuid'] in list(queueList.keys()): + print("Already in queue!") + if socket: + socket.emit("message", "Already in queue!") + return None + if type(queueItem) is list: + for x in queueItem: + if socket: + socket.emit("addedToQueue", x) + queue.append(x['uuid']) + queueList[x['uuid']] = x + else: + if socket: + socket.emit("addedToQueue", queueItem) + queue.append(queueItem['uuid']) + queueList[queueItem['uuid']] = queueItem + nextItem(dz, socket) + +def nextItem(dz, socket=None): + global currentItem, currentJob, queueList, queue + if currentItem != "": + return None + else: + if len(queue)>0: + currentItem = queue.pop(0) + else: + return None + if socket: + socket.emit("message", f"Started downloading {currentItem}") + result = download(dz, queueList[currentItem], socket) + callbackQueueDone(result) + +def callbackQueueDone(result): + global currentItem, currentJob, queueList, queue + result['socket'] + del queueList[currentItem] + currentItem = "" + nextItem(result['dz'], result['socket']) diff --git a/requirements.txt b/requirements.txt index 3f7e282..da5029d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ requests spotipy pywebview flask +flask-socketio