From dc6adc7887297cb4d454ea249a37f9d80f8341c8 Mon Sep 17 00:00:00 2001 From: RemixDev Date: Fri, 19 Mar 2021 15:44:21 +0100 Subject: [PATCH] More work on the library (WIP) --- deemix/__init__.py | 8 +- deemix/decryption.py | 136 ++++++++++++++++++++++++++++++++ deemix/downloader.py | 96 +++------------------- deemix/itemgen.py | 6 +- deemix/types/DownloadObjects.py | 22 ++++++ deemix/types/Playlist.py | 2 +- deemix/types/Track.py | 11 +-- deemix/utils/pathtemplates.py | 6 +- 8 files changed, 187 insertions(+), 100 deletions(-) diff --git a/deemix/__init__.py b/deemix/__init__.py index ea1b8ae..8f10dc1 100644 --- a/deemix/__init__.py +++ b/deemix/__init__.py @@ -26,19 +26,19 @@ def parseLink(link): id = link[link.rfind("/") + 1:] elif '/playlist' in link: type = 'playlist' - id = re.search("\/playlist\/(\d+)", link)[0] + id = re.search("\/playlist\/(\d+)", link).group(1) elif '/album' in link: type = 'album' id = link[link.rfind("/") + 1:] elif re.search("\/artist\/(\d+)\/top_track", link): type = 'artist_top' - id = re.search("\/artist\/(\d+)\/top_track", link)[0] + id = re.search("\/artist\/(\d+)\/top_track", link).group(1) elif re.search("\/artist\/(\d+)\/discography", link): type = 'artist_discography' - id = re.search("\/artist\/(\d+)\/discography", link)[0] + id = re.search("\/artist\/(\d+)\/discography", link).group(1) elif '/artist' in link: type = 'artist' - id = re.search("\/artist\/(\d+)", link)[0] + id = re.search("\/artist\/(\d+)", link).group(1) return (link, type, id) diff --git a/deemix/decryption.py b/deemix/decryption.py index 0dec77f..867cb52 100644 --- a/deemix/decryption.py +++ b/deemix/decryption.py @@ -2,6 +2,18 @@ import binascii from Cryptodome.Cipher import Blowfish, AES from Cryptodome.Hash import MD5 +from deemix import USER_AGENT_HEADER +from deemix.types.DownloadObjects import Single, Collection + +from requests import get + +from requests.exceptions import ConnectionError, ReadTimeout +from ssl import SSLError +from urllib3.exceptions import SSLError as u3SSLError + +import logging +logger = logging.getLogger('deemix') + def _md5(data): h = MD5.new() h.update(str.encode(data) if isinstance(data, str) else data) @@ -40,3 +52,127 @@ def generateUnencryptedStreamURL(sng_id, md5, media_version, format): def reverseStreamURL(url): urlPart = url[url.find("/1/")+3:] return generateStreamPath(urlPart) + +def streamUnencryptedTrack(outputStream, track, start=0, downloadObject=None, interface=None): + headers= {'User-Agent': USER_AGENT_HEADER} + chunkLength = start + percentage = 0 + + itemName = f"[{track.mainArtist.name} - {track.title}]" + + try: + with get(track.downloadUrl, headers=headers, stream=True, timeout=10) as request: + request.raise_for_status() + + complete = int(request.headers["Content-Length"]) + if complete == 0: raise DownloadEmpty + if start != 0: + responseRange = request.headers["Content-Range"] + logger.info(f'{itemName} downloading range {responseRange}') + else: + logger.info(f'{itemName} downloading {complete} bytes') + + for chunk in request.iter_content(2048 * 3): + outputStream.write(chunk) + chunkLength += len(chunk) + + if downloadObject: + if isinstance(downloadObject, Single): + percentage = (chunkLength / (complete + start)) * 100 + downloadObject.progressNext = percentage + else: + chunkProgres = (len(chunk) / (complete + start)) / downloadObject.size * 100 + downloadObject.progressNext += chunkProgres + downloadObject.updateProgress(interface) + + except (SSLError, u3SSLError) as e: + logger.info(f'{itemName} retrying from byte {chunkLength}') + return streamUnencryptedTrack(outputStream, track, chunkLength, downloadObject, interface) + except (ConnectionError, ReadTimeout): + sleep(2) + return streamUnencryptedTrack(outputStream, track, start, downloadObject, interface) + +def streamUnencryptedTrack(outputStream, track, start=0, downloadObject=None, interface=None): + headers= {'User-Agent': USER_AGENT_HEADER} + chunkLength = start + percentage = 0 + + itemName = f"[{track.mainArtist.name} - {track.title}]" + + try: + with get(track.downloadUrl, headers=headers, stream=True, timeout=10) as request: + request.raise_for_status() + + complete = int(request.headers["Content-Length"]) + if complete == 0: raise DownloadEmpty + if start != 0: + responseRange = request.headers["Content-Range"] + logger.info(f'{itemName} downloading range {responseRange}') + else: + logger.info(f'{itemName} downloading {complete} bytes') + + for chunk in request.iter_content(2048 * 3): + outputStream.write(chunk) + chunkLength += len(chunk) + + if downloadObject: + if isinstance(downloadObject, Single): + percentage = (chunkLength / (complete + start)) * 100 + downloadObject.progressNext = percentage + else: + chunkProgres = (len(chunk) / (complete + start)) / downloadObject.size * 100 + downloadObject.progressNext += chunkProgres + downloadObject.updateProgress(interface) + + except (SSLError, u3SSLError) as e: + logger.info(f'{itemName} retrying from byte {chunkLength}') + return streamUnencryptedTrack(outputStream, track, chunkLength, downloadObject, interface) + except (ConnectionError, ReadTimeout): + sleep(2) + return streamUnencryptedTrack(outputStream, track, start, downloadObject, interface) + +def streamTrack(outputStream, track, start=0, downloadObject=None, interface=None): + headers= {'User-Agent': USER_AGENT_HEADER} + chunkLength = start + percentage = 0 + + itemName = f"[{track.mainArtist.name} - {track.title}]" + + try: + with get(track.downloadUrl, headers=headers, stream=True, timeout=10) as request: + request.raise_for_status() + blowfish_key = str.encode(generateBlowfishKey(str(track.id))) + + complete = int(request.headers["Content-Length"]) + if complete == 0: raise DownloadEmpty + if start != 0: + responseRange = request.headers["Content-Range"] + logger.info(f'{itemName} downloading range {responseRange}') + else: + logger.info(f'{itemName} downloading {complete} bytes') + + for chunk in request.iter_content(2048 * 3): + if len(chunk) >= 2048: + chunk = Blowfish.new(blowfish_key, Blowfish.MODE_CBC, b"\x00\x01\x02\x03\x04\x05\x06\x07").decrypt(chunk[0:2048]) + chunk[2048:] + + outputStream.write(chunk) + chunkLength += len(chunk) + + if downloadObject: + if isinstance(downloadObject, Single): + percentage = (chunkLength / (complete + start)) * 100 + downloadObject.progressNext = percentage + else: + chunkProgres = (len(chunk) / (complete + start)) / downloadObject.size * 100 + downloadObject.progressNext += chunkProgres + downloadObject.updateProgress(interface) + + except (SSLError, u3SSLError) as e: + logger.info(f'{itemName} retrying from byte {chunkLength}') + return streamTrack(outputStream, track, chunkLength, downloadObject, interface) + except (ConnectionError, ReadTimeout): + sleep(2) + return streamTrack(outputStream, track, start, downloadObject, interface) + +class DownloadEmpty(Exception): + pass diff --git a/deemix/downloader.py b/deemix/downloader.py index ff8da30..f2a985f 100644 --- a/deemix/downloader.py +++ b/deemix/downloader.py @@ -11,24 +11,21 @@ import re import errno from ssl import SSLError -from os import makedirs from urllib3.exceptions import SSLError as u3SSLError +from os import makedirs from deemix.types.DownloadObjects import Single, Collection from deemix.types.Track import Track, AlbumDoesntExists -from deemix.utils import changeCase from deemix.utils.pathtemplates import generateFilename, generateFilepath, settingsRegexAlbum, settingsRegexArtist, settingsRegexPlaylistFile from deezer import TrackFormats from deemix import USER_AGENT_HEADER from deemix.taggers import tagID3, tagFLAC -from deemix.decryption import generateStreamURL, generateBlowfishKey +from deemix.decryption import generateUnencryptedStreamURL, streamUnencryptedTrack from deemix.settings import OverwriteOption -from Cryptodome.Cipher import Blowfish from mutagen.flac import FLACNoHeaderError, error as FLACError -import logging -logging.basicConfig(level=logging.INFO) +import logging logger = logging.getLogger('deemix') from tempfile import gettempdir @@ -124,7 +121,7 @@ def getPreferredBitrate(track, preferredBitrate, shouldFallback, downloadObjectU if int(track.filesizes[f"FILESIZE_{formatName}"]) != 0: return formatNumber if not track.filesizes[f"FILESIZE_{formatName}_TESTED"]: request = requests.head( - generateStreamURL(track.id, track.MD5, track.mediaVersion, formatNumber), + generateUnencryptedStreamURL(track.id, track.MD5, track.mediaVersion, formatNumber), headers={'User-Agent': USER_AGENT_HEADER}, timeout=30 ) @@ -159,8 +156,6 @@ class Downloader: self.settings = settings self.bitrate = downloadObject.bitrate self.interface = interface - self.downloadPercentage = 0 - self.lastPercentage = 0 self.extrasPath = None self.playlistCoverName = None self.playlistURLs = [] @@ -184,7 +179,6 @@ class Downloader: if trackAPI_gw['SNG_ID'] == "0": raise DownloadFailed("notOnDeezer") # Create Track object - print(track) if not track: logger.info(f"[{trackAPI_gw['ART_NAME']} - {trackAPI_gw['SNG_TITLE']}] Getting the tags") try: @@ -252,7 +246,7 @@ class Downloader: url = track.album.pic.generatePictureURL(self.settings['localArtworkSize'], extendedFormat) if self.settings['tags']['savePlaylistAsCompilation'] \ and track.playlist \ - and track.playlist.pic.url \ + and track.playlist.pic.staticUrl \ and not format.startswith("jpg"): continue result['albumURLs'].append({'url': url, 'ext': format}) @@ -280,7 +274,7 @@ class Downloader: extendedFormat = format if extendedFormat == "jpg": extendedFormat += f"-{self.settings['jpegImageQuality']}" url = track.playlist.pic.generatePictureURL(self.settings['localArtworkSize'], extendedFormat) - if track.playlist.pic.url and not format.startswith("jpg"): continue + if track.playlist.pic.staticUrl and not format.startswith("jpg"): continue self.playlistURLs.append({'url': url, 'ext': format}) if not self.playlistCoverName: track.playlist.bitrate = selectedFormat @@ -316,12 +310,12 @@ class Downloader: if not trackAlreadyDownloaded or self.settings['overwriteFile'] == OverwriteOption.OVERWRITE: logger.info(f"[{track.mainArtist.name} - {track.title}] Downloading the track") - track.downloadUrl = generateStreamURL(track.id, track.MD5, track.mediaVersion, track.selectedFormat) + track.downloadUrl = generateUnencryptedStreamURL(track.id, track.MD5, track.mediaVersion, track.selectedFormat) def downloadMusic(track, trackAPI_gw): try: with open(writepath, 'wb') as stream: - self.streamTrack(stream, track) + streamUnencryptedTrack(stream, track, downloadObject=self.downloadObject, interface=self.interface) except DownloadCancelled: if writepath.is_file(): writepath.unlink() raise DownloadCancelled @@ -382,7 +376,7 @@ class Downloader: if not trackDownloaded: return self.download(trackAPI_gw, track=track) else: logger.info(f"[{track.mainArtist.name} - {track.title}] Skipping track as it's already downloaded") - self.completeTrackPercentage() + self.downloadObject.completeTrackProgress(self.interface) # Adding tags if (not trackAlreadyDownloaded or self.settings['overwriteFile'] in [OverwriteOption.ONLY_TAGS, OverwriteOption.OVERWRITE]) and not track.localTrack: @@ -395,7 +389,7 @@ class Downloader: except (FLACNoHeaderError, FLACError): if writepath.is_file(): writepath.unlink() logger.warn(f"[{track.mainArtist.name} - {track.title}] Track not available in FLAC, falling back if necessary") - self.removeTrackPercentage() + self.downloadObject.removeTrackProgress(self.interface) track.filesizes['FILESIZE_FLAC'] = "0" track.filesizes['FILESIZE_FLAC_TESTED'] = True return self.download(trackAPI_gw, track=track) @@ -409,71 +403,6 @@ class Downloader: self.interface.send("updateQueue", {'uuid': self.downloadObject.uuid, 'downloaded': True, 'downloadPath': str(writepath), 'extrasPath': str(self.extrasPath)}) return result - def streamTrack(self, stream, track, start=0): - - headers=dict(self.dz.http_headers) - if range != 0: headers['Range'] = f'bytes={start}-' - chunkLength = start - percentage = 0 - - itemName = f"[{track.mainArtist.name} - {track.title}]" - - try: - with self.dz.session.get(track.downloadUrl, headers=headers, stream=True, timeout=10) as request: - request.raise_for_status() - blowfish_key = str.encode(generateBlowfishKey(str(track.id))) - - complete = int(request.headers["Content-Length"]) - if complete == 0: raise DownloadEmpty - if start != 0: - responseRange = request.headers["Content-Range"] - logger.info(f'{itemName} downloading range {responseRange}') - else: - logger.info(f'{itemName} downloading {complete} bytes') - - for chunk in request.iter_content(2048 * 3): - - if len(chunk) >= 2048: - chunk = Blowfish.new(blowfish_key, Blowfish.MODE_CBC, b"\x00\x01\x02\x03\x04\x05\x06\x07").decrypt(chunk[0:2048]) + chunk[2048:] - - stream.write(chunk) - chunkLength += len(chunk) - - if isinstance(self.downloadObject, Single): - percentage = (chunkLength / (complete + start)) * 100 - self.downloadPercentage = percentage - else: - chunkProgres = (len(chunk) / (complete + start)) / self.downloadObject.size * 100 - self.downloadPercentage += chunkProgres - self.updatePercentage() - - except (SSLError, u3SSLError) as e: - logger.info(f'{itemName} retrying from byte {chunkLength}') - return self.streamTrack(stream, track, chunkLength) - except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout): - sleep(2) - return self.streamTrack(stream, track, start) - - def updatePercentage(self): - if round(self.downloadPercentage) != self.lastPercentage and round(self.downloadPercentage) % 2 == 0: - self.lastPercentage = round(self.downloadPercentage) - self.downloadObject.progress = self.lastPercentage - if self.interface: self.interface.send("updateQueue", {'uuid': self.downloadObject.uuid, 'progress': self.lastPercentage}) - - def completeTrackPercentage(self): - if isinstance(self.downloadObject, Single): - self.downloadPercentage = 100 - else: - self.downloadPercentage += (1 / self.downloadObject.size) * 100 - self.updatePercentage() - - def removeTrackPercentage(self): - if isinstance(self.downloadObject, Single): - self.downloadPercentage = 0 - else: - self.downloadPercentage -= (1 / self.downloadObject.size) * 100 - self.updatePercentage() - def downloadWrapper(self, trackAPI_gw, trackAPI=None, albumAPI=None, playlistAPI=None, track=None): # Temp metadata to generate logs tempTrack = { @@ -531,7 +460,7 @@ class Downloader: }} if 'error' in result: - self.completeTrackPercentage() + self.downloadObject.completeTrackProgress(self.interface) self.downloadObject.failed += 1 self.downloadObject.errors.append(result['error']) if self.interface: @@ -640,9 +569,6 @@ class DownloadFailed(DownloadError): class DownloadCancelled(DownloadError): pass -class DownloadEmpty(DownloadError): - pass - class PreferredBitrateNotFound(DownloadError): pass diff --git a/deemix/itemgen.py b/deemix/itemgen.py index 9629044..15d6eb0 100644 --- a/deemix/itemgen.py +++ b/deemix/itemgen.py @@ -1,4 +1,6 @@ from deemix.types.DownloadObjects import Single, Collection +from deezer.api import APIError +from deezer.gw import GWAPIError, LyricsStatus class GenerationError(Exception): def __init__(self, link, message, errid=None): @@ -29,7 +31,7 @@ def generateTrackItem(dz, id, bitrate, trackAPI=None, albumAPI=None): # Get essential track info try: trackAPI_gw = dz.gw.get_track_with_fallback(id) - except gwAPIError as e: + except GWAPIError as e: e = str(e) message = "Wrong URL" if "DATA_ERROR" in e: message += f": {e['DATA_ERROR']}" @@ -116,7 +118,7 @@ def generatePlaylistItem(dz, id, bitrate, playlistAPI=None, playlistTracksAPI=No try: userPlaylist = dz.gw.get_playlist_page(id) playlistAPI = map_user_playlist(userPlaylist['DATA']) - except gwAPIError as e: + except GWAPIError as e: e = str(e) message = "Wrong URL" if "DATA_ERROR" in e: diff --git a/deemix/types/DownloadObjects.py b/deemix/types/DownloadObjects.py index e7b43b5..0fd2664 100644 --- a/deemix/types/DownloadObjects.py +++ b/deemix/types/DownloadObjects.py @@ -28,6 +28,7 @@ class IDownloadObject: self.progress = 0 self.errors = [] self.files = [] + self.progressNext = 0 self.uuid = f"{self.type}_{self.id}_{self.bitrate}" self.ack = None self.__type__ = None @@ -69,6 +70,11 @@ class IDownloadObject: del light[property] return light + def updateProgress(self, interface=None): + if round(self.progressNext) != self.progress and round(self.progressNext) % 2 == 0: + self.progress = round(self.progressNext) + if interface: interface.send("updateQueue", {'uuid': self.uuid, 'progress': self.progress}) + class Single(IDownloadObject): def __init__(self, type=None, id=None, bitrate=None, title=None, artist=None, cover=None, explicit=False, trackAPI_gw=None, trackAPI=None, albumAPI=None, dictItem=None): if dictItem: @@ -88,6 +94,14 @@ class Single(IDownloadObject): item['single'] = self.single return item + def completeTrackProgress(self, interface=None): + self.progressNext = 100 + self.updateProgress(interface) + + def removeTrackProgress(self, interface=None): + self.progressNext = 0 + self.updateProgress(interface) + class Collection(IDownloadObject): def __init__(self, type=None, id=None, bitrate=None, title=None, artist=None, cover=None, explicit=False, size=None, tracks_gw=None, albumAPI=None, playlistAPI=None, dictItem=None): if dictItem: @@ -107,6 +121,14 @@ class Collection(IDownloadObject): item['collection'] = self.collection return item + def completeTrackProgress(self, interface=None): + self.progressNext += (1 / self.size) * 100 + self.updateProgress(interface) + + def removeTrackProgress(self, interface=None): + self.progressNext -= (1 / self.size) * 100 + self.updateProgress(interface) + class Convertable(Collection): def __init__(self, type=None, id=None, bitrate=None, title=None, artist=None, cover=None, explicit=False, size=None, plugin=None, conversion_data=None, dictItem=None): if dictItem: diff --git a/deemix/types/Playlist.py b/deemix/types/Playlist.py index e358674..61412ee 100644 --- a/deemix/types/Playlist.py +++ b/deemix/types/Playlist.py @@ -39,7 +39,7 @@ class Playlist: if 'various_artist' in playlistAPI: pic_md5 = playlistAPI['various_artist']['picture_small'] - pic_md5 = pic_md5[pic_md5.indexOf('artist/') + 7:-24] + pic_md5 = pic_md5[pic_md5.find('artist/') + 7:-24] self.variousArtists = Artist( id = playlistAPI['various_artist']['id'], name = playlistAPI['various_artist']['name'], diff --git a/deemix/types/Track.py b/deemix/types/Track.py index b75db67..e130578 100644 --- a/deemix/types/Track.py +++ b/deemix/types/Track.py @@ -5,9 +5,10 @@ import logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger('deemix') -from deezer.gw import APIError as gwAPIError +from deezer.gw import GWAPIError from deezer.api import APIError -from deemix.utils import removeFeatures, andCommaConcat, removeDuplicateArtists, generateReplayGainString +from deemix.utils import removeFeatures, andCommaConcat, removeDuplicateArtists, generateReplayGainString, changeCase + from deemix.types.Album import Album from deemix.types.Artist import Artist from deemix.types.Date import Date @@ -114,7 +115,7 @@ class Track: # Get Lyrics data if not "LYRICS" in trackAPI_gw and self.lyrics.id != "0": try: trackAPI_gw["LYRICS"] = dz.gw.get_track_lyrics(self.id) - except gwAPIError: self.lyrics.id = "0" + except GWAPIError: self.lyrics.id = "0" if self.lyrics.id != "0": self.lyrics.parseLyrics(trackAPI_gw["LYRICS"]) # Parse Album data @@ -132,7 +133,7 @@ class Track: # Get album_gw Data if not albumAPI_gw: try: albumAPI_gw = dz.gw.get_album(self.album.id) - except gwAPIError: albumAPI_gw = None + except GWAPIError: albumAPI_gw = None if albumAPI: self.album.parseAlbum(albumAPI) @@ -261,7 +262,7 @@ class Track: def applySettings(self, settings, TEMPDIR, embeddedImageFormat): from deemix.settings import FeaturesOption - + # Check if should save the playlist as a compilation if self.playlist and settings['tags']['savePlaylistAsCompilation']: self.trackNumber = self.position diff --git a/deemix/utils/pathtemplates.py b/deemix/utils/pathtemplates.py index 3d04dce..3d5dafe 100644 --- a/deemix/utils/pathtemplates.py +++ b/deemix/utils/pathtemplates.py @@ -148,7 +148,7 @@ def settingsRegex(filename, track, settings): filename = filename.replace("%album_id%", str(track.album.id)) filename = filename.replace("%artist_id%", str(track.mainArtist.id)) if track.playlist: - filename = filename.replace("%playlist_id%", str(track.playlist.playlistId)) + filename = filename.replace("%playlist_id%", str(track.playlist.playlistID)) filename = filename.replace("%position%", pad(track.position, track.playlist.trackTotal, settings)) else: filename = filename.replace("%playlist_id%", '') @@ -159,7 +159,7 @@ def settingsRegex(filename, track, settings): def settingsRegexAlbum(foldername, album, settings, playlist=None): if playlist and settings['tags']['savePlaylistAsCompilation']: - foldername = foldername.replace("%album_id%", "pl_" + str(playlist.playlistId)) + foldername = foldername.replace("%album_id%", "pl_" + str(playlist.playlistID)) foldername = foldername.replace("%genre%", "Compile") else: foldername = foldername.replace("%album_id%", str(album.id)) @@ -205,7 +205,7 @@ def settingsRegexArtist(foldername, artist, settings, rootArtist=None): def settingsRegexPlaylist(foldername, playlist, settings): foldername = foldername.replace("%playlist%", fixName(playlist.title, settings['illegalCharacterReplacer'])) - foldername = foldername.replace("%playlist_id%", fixName(playlist.playlistId, settings['illegalCharacterReplacer'])) + foldername = foldername.replace("%playlist_id%", fixName(playlist.playlistID, settings['illegalCharacterReplacer'])) foldername = foldername.replace("%owner%", fixName(playlist.owner['name'], settings['illegalCharacterReplacer'])) foldername = foldername.replace("%owner_id%", str(playlist.owner['id'])) foldername = foldername.replace("%year%", str(playlist.date.year))