From 6b41e6cb0ad1625deaa9103908004570ad79ae3a Mon Sep 17 00:00:00 2001 From: RemixDev Date: Thu, 23 Dec 2021 19:02:33 +0100 Subject: [PATCH] version parity --- deemix/downloader.py | 95 +++++++++------- deemix/errors.py | 3 +- deemix/itemgen.py | 90 ++++++++------- deemix/plugins/spotify.py | 38 +++---- deemix/settings.py | 2 + deemix/types/Album.py | 60 ++++------ deemix/types/Track.py | 224 +++++++++++++++++--------------------- setup.py | 2 +- 8 files changed, 248 insertions(+), 266 deletions(-) diff --git a/deemix/downloader.py b/deemix/downloader.py index 4f5ac74..5286c48 100644 --- a/deemix/downloader.py +++ b/deemix/downloader.py @@ -20,6 +20,7 @@ from mutagen.flac import FLACNoHeaderError, error as FLACError from deezer import TrackFormats from deezer.errors import WrongLicense, WrongGeolocation +from deezer.utils import map_track from deemix.types.DownloadObjects import Single, Collection from deemix.types.Track import Track from deemix.types.Picture import StaticPicture @@ -74,7 +75,7 @@ def downloadImage(url, path, overwrite=OverwriteOption.DONT_OVERWRITE): pictureSize = int(pictureUrl[:pictureUrl.find("x")]) if pictureSize > 1200: return downloadImage(urlBase+pictureUrl.replace(f"{pictureSize}x{pictureSize}", '1200x1200'), path, overwrite) - except (requests.exceptions.ConnectionError, requests.exceptions.ChunkedEncodingError, u3SSLError) as e: + except (requests.exceptions.ConnectionError, requests.exceptions.ChunkedEncodingError, u3SSLError): if path.is_file(): path.unlink() sleep(5) return downloadImage(url, path, overwrite) @@ -84,7 +85,7 @@ def downloadImage(url, path, overwrite=OverwriteOption.DONT_OVERWRITE): logger.exception("Error while downloading an image, you should report this to the developers: %s", e) return None -def getPreferredBitrate(dz, track, preferredBitrate, shouldFallback, uuid=None, listener=None): +def getPreferredBitrate(dz, track, preferredBitrate, shouldFallback, feelingLucky, uuid=None, listener=None): preferredBitrate = int(preferredBitrate) falledBack = False @@ -101,32 +102,31 @@ def getPreferredBitrate(dz, track, preferredBitrate, shouldFallback, uuid=None, ) try: request.raise_for_status() - track.filesizes[f"FILESIZE_{formatName}"] = int(request.headers["Content-Length"]) - track.filesizes[f"FILESIZE_{formatName}_TESTED"] = True - return track.filesizes[f"FILESIZE_{formatName}"] != 0 + track.filesizes[f"{formatName.lower()}"] = int(request.headers["Content-Length"]) + track.filesizes[f"{formatName.lower()}_TESTED"] = True + return track.filesizes[f"{formatName.lower()}"] != 0 except requests.exceptions.HTTPError: # if the format is not available, Deezer returns a 403 error return False - def getCorrectURL(track, formatName, formatNumber): + def getCorrectURL(track, formatName, formatNumber, feelingLucky): nonlocal wrongLicense, isGeolocked url = None # Check the track with the legit method - try: - url = dz.get_track_url(track.trackToken, formatName) - if testURL(track, url, formatName): return url - url = None - except (WrongLicense, WrongGeolocation) as e: - wrongLicense = isinstance(e, WrongLicense) - isGeolocked = isinstance(e, WrongGeolocation) + if formatName.lower() in track.filesizes and track.filesizes[formatName.lower()] != "0": + try: + url = dz.get_track_url(track.trackToken, formatName) + except (WrongLicense, WrongGeolocation) as e: + wrongLicense = isinstance(e, WrongLicense) + isGeolocked = isinstance(e, WrongGeolocation) # Fallback to old method - if not url: + if not url and feelingLucky: url = generateCryptedStreamURL(track.id, track.MD5, track.mediaVersion, formatNumber) if testURL(track, url, formatName): return url url = None return url if track.local: - url = getCorrectURL(track, "MP3_MISC", TrackFormats.LOCAL) + url = getCorrectURL(track, "MP3_MISC", TrackFormats.LOCAL, feelingLucky) track.urls["MP3_MISC"] = url return TrackFormats.LOCAL @@ -150,20 +150,23 @@ def getPreferredBitrate(dz, track, preferredBitrate, shouldFallback, uuid=None, else: formats = formats_non_360 + # check and renew trackToken before starting the check + track.checkAndRenewTrackToken(dz) for formatNumber, formatName in formats.items(): # Current bitrate is higher than preferred bitrate; skip if formatNumber > preferredBitrate: continue currentTrack = track - url = getCorrectURL(currentTrack, formatName, formatNumber) + url = getCorrectURL(currentTrack, formatName, formatNumber, feelingLucky) newTrack = None while True: if not url and hasAlternative: newTrack = dz.gw.get_track_with_fallback(currentTrack.fallbackID) + newTrack = map_track(newTrack) currentTrack = Track() currentTrack.parseEssentialData(newTrack) hasAlternative = currentTrack.fallbackID != "0" - if not url: getCorrectURL(currentTrack, formatName, formatNumber) + if not url: getCorrectURL(currentTrack, formatName, formatNumber, feelingLucky) if (url or not hasAlternative): break if url: @@ -189,7 +192,7 @@ def getPreferredBitrate(dz, track, preferredBitrate, shouldFallback, uuid=None, }, }) if is360format: raise TrackNot360 - url = getCorrectURL(track, "MP3_MISC", TrackFormats.DEFAULT) + url = getCorrectURL(track, "MP3_MISC", TrackFormats.DEFAULT, feelingLucky) track.urls["MP3_MISC"] = url return TrackFormats.DEFAULT @@ -208,17 +211,16 @@ class Downloader: if not self.downloadObject.isCanceled: if isinstance(self.downloadObject, Single): track = self.downloadWrapper({ - 'trackAPI_gw': self.downloadObject.single['trackAPI_gw'], 'trackAPI': self.downloadObject.single.get('trackAPI'), 'albumAPI': self.downloadObject.single.get('albumAPI') }) if track: self.afterDownloadSingle(track) elif isinstance(self.downloadObject, Collection): - tracks = [None] * len(self.downloadObject.collection['tracks_gw']) + tracks = [None] * len(self.downloadObject.collection['tracks']) with ThreadPoolExecutor(self.settings['queueConcurrency']) as executor: - for pos, track in enumerate(self.downloadObject.collection['tracks_gw'], start=0): + for pos, track in enumerate(self.downloadObject.collection['tracks'], start=0): tracks[pos] = executor.submit(self.downloadWrapper, { - 'trackAPI_gw': track, + 'trackAPI': track, 'albumAPI': self.downloadObject.collection.get('albumAPI'), 'playlistAPI': self.downloadObject.collection.get('playlistAPI') }) @@ -241,17 +243,17 @@ class Downloader: def download(self, extraData, track=None): returnData = {} - trackAPI_gw = extraData['trackAPI_gw'] trackAPI = extraData.get('trackAPI') albumAPI = extraData.get('albumAPI') playlistAPI = extraData.get('playlistAPI') + trackAPI['size'] = self.downloadObject.size if self.downloadObject.isCanceled: raise DownloadCanceled - if trackAPI_gw['SNG_ID'] == "0": raise DownloadFailed("notOnDeezer") + if int(trackAPI['id']) == 0: raise DownloadFailed("notOnDeezer") itemData = { - 'id': trackAPI_gw['SNG_ID'], - 'title': trackAPI_gw['SNG_TITLE'].strip(), - 'artist': trackAPI_gw['ART_NAME'] + 'id': trackAPI['id'], + 'title': trackAPI['title'], + 'artist': trackAPI['artist']['name'] } # Create Track object @@ -260,7 +262,7 @@ class Downloader: try: track = Track().parseData( dz=self.dz, - trackAPI_gw=trackAPI_gw, + track_id=trackAPI['id'], trackAPI=trackAPI, albumAPI=albumAPI, playlistAPI=playlistAPI @@ -287,13 +289,13 @@ class Downloader: self.dz, track, self.bitrate, - self.settings['fallbackBitrate'], + self.settings['fallbackBitrate'], self.settings['feelingLucky'], self.downloadObject.uuid, self.listener ) except WrongLicense as e: raise DownloadFailed("wrongLicense") from e except WrongGeolocation as e: - raise DownloadFailed("wrongGeolocation") from e + raise DownloadFailed("wrongGeolocation", track) from e except PreferredBitrateNotFound as e: raise DownloadFailed("wrongBitrate", track) from e except TrackNot360 as e: @@ -432,7 +434,7 @@ class Downloader: self.downloadObject.removeTrackProgress(self.listener) track.filesizes['FILESIZE_FLAC'] = "0" track.filesizes['FILESIZE_FLAC_TESTED'] = True - return self.download(trackAPI_gw, track=track) + return self.download(extraData, track=track) self.log(itemData, "tagged") if track.searched: returnData['searched'] = True @@ -450,18 +452,13 @@ class Downloader: return returnData def downloadWrapper(self, extraData, track=None): - trackAPI_gw = extraData['trackAPI_gw'] - if trackAPI_gw.get('_EXTRA_TRACK'): - extraData['trackAPI'] = trackAPI_gw['_EXTRA_TRACK'].copy() - del extraData['trackAPI_gw']['_EXTRA_TRACK'] + trackAPI = extraData['trackAPI'] # Temp metadata to generate logs itemData = { - 'id': trackAPI_gw['SNG_ID'], - 'title': trackAPI_gw['SNG_TITLE'].strip(), - 'artist': trackAPI_gw['ART_NAME'] + 'id': trackAPI['id'], + 'title': trackAPI['title'], + 'artist': trackAPI['artist']['name'] } - if trackAPI_gw.get('VERSION') and trackAPI_gw['VERSION'] not in trackAPI_gw['SNG_TITLE']: - itemData['title'] += f" {trackAPI_gw['VERSION']}".strip() try: result = self.download(extraData, track) @@ -471,16 +468,30 @@ class Downloader: if track.fallbackID != "0": self.warn(itemData, error.errid, 'fallback') newTrack = self.dz.gw.get_track_with_fallback(track.fallbackID) + newTrack = map_track(newTrack) track.parseEssentialData(newTrack) - track.retriveFilesizes(self.dz) return self.downloadWrapper(extraData, track) + if len(track.albumsFallback) != 0 and self.settings.fallbackISRC: + newAlbumID = track.albumsFallback.pop() + newAlbum = self.dz.gw.get_album_page(newAlbumID) + fallbackID = 0 + for newTrack in newAlbum['SONGS']['data']: + if newTrack['ISRC'] == track.ISRC: + fallbackID = newTrack['SNG_ID'] + break + if fallbackID != 0: + self.warn(itemData, error.errid, 'fallback') + newTrack = self.dz.gw.get_track_with_fallback(fallbackID) + newTrack = map_track(newTrack) + track.parseEssentialData(newTrack) + return self.downloadWrapper(extraData, track) if not track.searched and self.settings['fallbackSearch']: self.warn(itemData, error.errid, 'search') searchedId = self.dz.api.get_track_id_from_metadata(track.mainArtist.name, track.title, track.album.title) if searchedId != "0": newTrack = self.dz.gw.get_track_with_fallback(searchedId) + newTrack = map_track(newTrack) track.parseEssentialData(newTrack) - track.retriveFilesizes(self.dz) track.searched = True self.log(itemData, "searchFallback") return self.downloadWrapper(extraData, track) diff --git a/deemix/errors.py b/deemix/errors.py index 62001ce..c4b6f6d 100644 --- a/deemix/errors.py +++ b/deemix/errors.py @@ -60,7 +60,8 @@ ErrorMessages = { 'noSpaceLeft': "No space left on target drive, clean up some space for the tracks", 'albumDoesntExists': "Track's album does not exsist, failed to gather info.", 'notLoggedIn': "You need to login to download tracks.", - 'wrongGeolocation': "Your account can't stream the track from your current country." + 'wrongGeolocation': "Your account can't stream the track from your current country.", + 'wrongGeolocationNoAlternative': "Your account can't stream the track from your current country and no alternative found." } class DownloadFailed(DownloadError): diff --git a/deemix/itemgen.py b/deemix/itemgen.py index 8366d76..ec0881c 100644 --- a/deemix/itemgen.py +++ b/deemix/itemgen.py @@ -1,8 +1,7 @@ import logging -from deezer.gw import LyricsStatus from deezer.errors import GWAPIError, APIError -from deezer.utils import map_user_playlist +from deezer.utils import map_user_playlist, map_track, map_album from deemix.types.DownloadObjects import Single, Collection from deemix.errors import GenerationError, ISRCnotOnDeezer, InvalidID, NotYourPrivatePlaylist @@ -10,40 +9,44 @@ from deemix.errors import GenerationError, ISRCnotOnDeezer, InvalidID, NotYourPr logger = logging.getLogger('deemix') def generateTrackItem(dz, link_id, bitrate, trackAPI=None, albumAPI=None): - # Check if is an isrc: url - if str(link_id).startswith("isrc"): - try: - trackAPI = dz.api.get_track(link_id) - except APIError as e: - raise GenerationError(f"https://deezer.com/track/{link_id}", str(e)) from e + # Get essential track info + if not trackAPI: + if str(link_id).startswith("isrc") or int(link_id) > 0: + try: + trackAPI = dz.api.get_track(link_id) + except APIError as e: + raise GenerationError(f"https://deezer.com/track/{link_id}", str(e)) from e - if 'id' in trackAPI and 'title' in trackAPI: - link_id = trackAPI['id'] + # Check if is an isrc: url + if str(link_id).startswith("isrc"): + if 'id' in trackAPI and 'title' in trackAPI: + link_id = trackAPI['id'] + else: + raise ISRCnotOnDeezer(f"https://deezer.com/track/{link_id}") else: - raise ISRCnotOnDeezer(f"https://deezer.com/track/{link_id}") + trackAPI_gw = dz.gw.get_track(link_id) + trackAPI = map_track(trackAPI_gw) + else: + link_id = trackAPI['id'] if not str(link_id).strip('-').isdecimal(): raise InvalidID(f"https://deezer.com/track/{link_id}") - # Get essential track info - try: - trackAPI_gw = dz.gw.get_track_with_fallback(link_id) - except GWAPIError as e: - raise GenerationError(f"https://deezer.com/track/{link_id}", str(e)) from e + cover = None + if trackAPI['album']['cover_small']: + cover = trackAPI['album']['cover_small'][:-24] + '/75x75-000000-80-0-0.jpg' + else: + cover = f"https://e-cdns-images.dzcdn.net/images/cover/{trackAPI['md5_image']}/75x75-000000-80-0-0.jpg" - title = trackAPI_gw['SNG_TITLE'].strip() - if trackAPI_gw.get('VERSION') and trackAPI_gw['VERSION'] not in trackAPI_gw['SNG_TITLE']: - title += f" {trackAPI_gw['VERSION']}".strip() - explicit = bool(int(trackAPI_gw.get('EXPLICIT_LYRICS', 0))) + del trackAPI['track_token'] return Single({ 'type': 'track', 'id': link_id, 'bitrate': bitrate, - 'title': title, - 'artist': trackAPI_gw['ART_NAME'], - 'cover': f"https://e-cdns-images.dzcdn.net/images/cover/{trackAPI_gw['ALB_PICTURE']}/75x75-000000-80-0-0.jpg", - 'explicit': explicit, + 'title': trackAPI['title'], + 'artist': trackAPI['artist']['name'], + 'cover': cover, + 'explicit': trackAPI.explicit_lyrics, 'single': { - 'trackAPI_gw': trackAPI_gw, 'trackAPI': trackAPI, 'albumAPI': albumAPI } @@ -66,7 +69,14 @@ def generateAlbumItem(dz, link_id, bitrate, rootArtist=None): link_id = albumAPI['id'] else: try: - albumAPI = dz.api.get_album(link_id) + albumAPI_gw_page = dz.gw.get_album_page(link_id) + if 'DATA' in albumAPI_gw_page: + albumAPI = map_album(albumAPI_gw_page['DATA']) + link_id = albumAPI_gw_page['DATA']['ALB_ID'] + albumAPI_new = dz.api.get_album(link_id) + albumAPI.update(albumAPI_new) + else: + raise GenerationError(f"https://deezer.com/album/{link_id}", "Can't find the album") except APIError as e: raise GenerationError(f"https://deezer.com/album/{link_id}", str(e)) from e @@ -75,9 +85,9 @@ def generateAlbumItem(dz, link_id, bitrate, rootArtist=None): # Get extra info about album # This saves extra api calls when downloading albumAPI_gw = dz.gw.get_album(link_id) - albumAPI['nb_disk'] = albumAPI_gw['NUMBER_DISK'] - albumAPI['copyright'] = albumAPI_gw['COPYRIGHT'] - albumAPI['release_date'] = albumAPI_gw['PHYSICAL_RELEASE_DATE'] + albumAPI_gw = map_album(albumAPI_gw) + albumAPI_gw.update(albumAPI) + albumAPI = albumAPI_gw albumAPI['root_artist'] = rootArtist # If the album is a single download as a track @@ -91,18 +101,17 @@ def generateAlbumItem(dz, link_id, bitrate, rootArtist=None): if albumAPI['cover_small'] is not None: cover = albumAPI['cover_small'][:-24] + '/75x75-000000-80-0-0.jpg' else: - cover = f"https://e-cdns-images.dzcdn.net/images/cover/{albumAPI_gw['ALB_PICTURE']}/75x75-000000-80-0-0.jpg" + cover = f"https://e-cdns-images.dzcdn.net/images/cover/{albumAPI['md5_image']}/75x75-000000-80-0-0.jpg" totalSize = len(tracksArray) albumAPI['nb_tracks'] = totalSize collection = [] for pos, trackAPI in enumerate(tracksArray, start=1): - trackAPI['POSITION'] = pos - trackAPI['SIZE'] = totalSize + trackAPI = map_track(trackAPI) + del trackAPI['track_token'] + trackAPI['position'] = pos collection.append(trackAPI) - explicit = albumAPI_gw.get('EXPLICIT_ALBUM_CONTENT', {}).get('EXPLICIT_LYRICS_STATUS', LyricsStatus.UNKNOWN) in [LyricsStatus.EXPLICIT, LyricsStatus.PARTIALLY_EXPLICIT] - return Collection({ 'type': 'album', 'id': link_id, @@ -110,10 +119,10 @@ def generateAlbumItem(dz, link_id, bitrate, rootArtist=None): 'title': albumAPI['title'], 'artist': albumAPI['artist']['name'], 'cover': cover, - 'explicit': explicit, + 'explicit': albumAPI['explicit_lyrics'], 'size': totalSize, 'collection': { - 'tracks_gw': collection, + 'tracks': collection, 'albumAPI': albumAPI } }) @@ -147,10 +156,11 @@ def generatePlaylistItem(dz, link_id, bitrate, playlistAPI=None, playlistTracksA playlistAPI['nb_tracks'] = totalSize collection = [] for pos, trackAPI in enumerate(playlistTracksAPI, start=1): - if trackAPI.get('EXPLICIT_TRACK_CONTENT', {}).get('EXPLICIT_LYRICS_STATUS', LyricsStatus.UNKNOWN) in [LyricsStatus.EXPLICIT, LyricsStatus.PARTIALLY_EXPLICIT]: + trackAPI = map_track(trackAPI) + if trackAPI['explicit_lyrics']: playlistAPI['explicit'] = True - trackAPI['POSITION'] = pos - trackAPI['SIZE'] = totalSize + del trackAPI['track_token'] + trackAPI['position'] = pos collection.append(trackAPI) if 'explicit' not in playlistAPI: playlistAPI['explicit'] = False @@ -165,7 +175,7 @@ def generatePlaylistItem(dz, link_id, bitrate, playlistAPI=None, playlistTracksA 'explicit': playlistAPI['explicit'], 'size': totalSize, 'collection': { - 'tracks_gw': collection, + 'tracks': collection, 'playlistAPI': playlistAPI } }) diff --git a/deemix/plugins/spotify.py b/deemix/plugins/spotify.py index f553927..63bfde7 100644 --- a/deemix/plugins/spotify.py +++ b/deemix/plugins/spotify.py @@ -143,7 +143,7 @@ class Spotify(Plugin): 'explicit': playlistAPI['explicit'], 'size': len(tracklist), 'collection': { - 'tracks_gw': [], + 'tracks': [], 'playlistAPI': playlistAPI }, 'plugin': 'spotify', @@ -217,31 +217,31 @@ class Spotify(Plugin): if cachedTrack.get('id', "0") != "0": trackAPI = dz.api.get_track(cachedTrack['id']) - deezerTrack = None if not trackAPI: - deezerTrack = { - 'SNG_ID': "0", - 'SNG_TITLE': track['name'], - 'DURATION': 0, - 'MD5_ORIGIN': 0, - 'MEDIA_VERSION': 0, - 'FILESIZE': 0, - 'ALB_TITLE': track['album']['name'], - 'ALB_PICTURE': "", - 'ART_ID': 0, - 'ART_NAME': track['artists'][0]['name'] + trackAPI = { + 'id': "0", + 'title': track['name'], + 'duration': 0, + 'md5_origin': 0, + 'media_version': 0, + 'filesizes': {}, + 'album': { + 'title': track['album']['name'], + 'md5_image': "" + }, + 'artist': { + 'id': 0, + 'name': track['artists'][0]['name'] + } } - else: - deezerTrack = dz.gw.get_track_with_fallback(trackAPI['id']) - deezerTrack['_EXTRA_TRACK'] = trackAPI - deezerTrack['POSITION'] = pos+1 + trackAPI['position'] = pos+1 conversion['next'] += (1 / downloadObject.size) * 100 if round(conversion['next']) != conversion['now'] and round(conversion['next']) % 2 == 0: conversion['now'] = round(conversion['next']) if listener: listener.send("updateQueue", {'uuid': downloadObject.uuid, 'conversion': conversion['now']}) - return deezerTrack + return trackAPI def convert(self, dz, downloadObject, settings, listener=None): cache = self.loadCache() @@ -259,7 +259,7 @@ class Spotify(Plugin): cache, listener ).result() - downloadObject.collection['tracks_gw'] = collection + downloadObject.collection['tracks'] = collection downloadObject.size = len(collection) downloadObject = Collection(downloadObject.toDict()) if listener: listener.send("finishConversion", downloadObject.getSlimmedDict()) diff --git a/deemix/settings.py b/deemix/settings.py index 1f0656c..9cb0e7c 100644 --- a/deemix/settings.py +++ b/deemix/settings.py @@ -39,8 +39,10 @@ DEFAULTS = { "illegalCharacterReplacer": "_", "queueConcurrency": 3, "maxBitrate": str(TrackFormats.MP3_320), + "feelingLucky": False, "fallbackBitrate": False, "fallbackSearch": False, + "fallbackISRC": False, "logErrors": True, "logSearched": False, "overwriteFile": OverwriteOption.DONT_OVERWRITE, diff --git a/deemix/types/Album.py b/deemix/types/Album.py index 7522e27..3edf40f 100644 --- a/deemix/types/Album.py +++ b/deemix/types/Album.py @@ -1,5 +1,3 @@ -from deezer.gw import LyricsStatus - from deemix.utils import removeDuplicateArtists, removeFeatures from deemix.types.Artist import Artist from deemix.types.Date import Date @@ -30,7 +28,7 @@ class Album: self.rootArtist = None self.variousArtists = None - self.playlistId = None + self.playlistID = None self.owner = None self.isPlaylist = False @@ -39,8 +37,9 @@ class Album: # Getting artist image ID # ex: https://e-cdns-images.dzcdn.net/images/artist/f2bc007e9133c946ac3c3907ddc5d2ea/56x56-000000-80-0-0.jpg - art_pic = albumAPI['artist']['picture_small'] - art_pic = art_pic[art_pic.find('artist/') + 7:-24] + art_pic = albumAPI['artist'].get('picture_small') + if art_pic: art_pic = art_pic[art_pic.find('artist/') + 7:-24] + else: art_pic = "" self.mainArtist = Artist( albumAPI['artist']['id'], albumAPI['artist']['name'], @@ -83,52 +82,31 @@ class Album: self.barcode = albumAPI.get('upc', self.barcode) self.label = albumAPI.get('label', self.label) self.explicit = bool(albumAPI.get('explicit_lyrics', False)) - if 'release_date' in albumAPI: - self.date.day = albumAPI["release_date"][8:10] - self.date.month = albumAPI["release_date"][5:7] - self.date.year = albumAPI["release_date"][0:4] + release_date = albumAPI.get('release_date') + if 'physical_release_date' in albumAPI: + release_date = albumAPI['physical_release_date'] + if release_date: + self.date.day = release_date[8:10] + self.date.month = release_date[5:7] + self.date.year = release_date[0:4] self.date.fixDayMonth() self.discTotal = albumAPI.get('nb_disk') self.copyright = albumAPI.get('copyright') - if self.pic.md5 == "" and albumAPI.get('cover_small'): - # Getting album cover MD5 - # ex: https://e-cdns-images.dzcdn.net/images/cover/2e018122cb56986277102d2041a592c8/56x56-000000-80-0-0.jpg - alb_pic = albumAPI['cover_small'] - self.pic.md5 = alb_pic[alb_pic.find('cover/') + 6:-24] + if self.pic.md5 == "": + if albumAPI.get('md5_image'): + self.pic.md5 = albumAPI['md5_image'] + elif albumAPI.get('cover_small'): + # Getting album cover MD5 + # ex: https://e-cdns-images.dzcdn.net/images/cover/2e018122cb56986277102d2041a592c8/56x56-000000-80-0-0.jpg + alb_pic = albumAPI['cover_small'] + self.pic.md5 = alb_pic[alb_pic.find('cover/') + 6:-24] if albumAPI.get('genres') and len(albumAPI['genres'].get('data', [])) > 0: for genre in albumAPI['genres']['data']: self.genre.append(genre['name']) - def parseAlbumGW(self, albumAPI_gw): - self.title = albumAPI_gw['ALB_TITLE'] - self.mainArtist = Artist( - art_id = albumAPI_gw['ART_ID'], - name = albumAPI_gw['ART_NAME'], - role = "Main" - ) - - self.artists = [albumAPI_gw['ART_NAME']] - self.trackTotal = albumAPI_gw['NUMBER_TRACK'] - self.discTotal = albumAPI_gw['NUMBER_DISK'] - self.label = albumAPI_gw.get('LABEL_NAME', self.label) - - explicitLyricsStatus = albumAPI_gw.get('EXPLICIT_ALBUM_CONTENT', {}).get('EXPLICIT_LYRICS_STATUS', LyricsStatus.UNKNOWN) - self.explicit = explicitLyricsStatus in [LyricsStatus.EXPLICIT, LyricsStatus.PARTIALLY_EXPLICIT] - - self.addExtraAlbumGWData(albumAPI_gw) - - def addExtraAlbumGWData(self, albumAPI_gw): - if self.pic.md5 == "" and albumAPI_gw.get('ALB_PICTURE'): - self.pic.md5 = albumAPI_gw['ALB_PICTURE'] - if 'PHYSICAL_RELEASE_DATE' in albumAPI_gw: - self.date.day = albumAPI_gw["PHYSICAL_RELEASE_DATE"][8:10] - self.date.month = albumAPI_gw["PHYSICAL_RELEASE_DATE"][5:7] - self.date.year = albumAPI_gw["PHYSICAL_RELEASE_DATE"][0:4] - self.date.fixDayMonth() - def makePlaylistCompilation(self, playlist): self.variousArtists = playlist.variousArtists self.mainArtist = playlist.mainArtist diff --git a/deemix/types/Track.py b/deemix/types/Track.py index 7716a78..40bf065 100644 --- a/deemix/types/Track.py +++ b/deemix/types/Track.py @@ -1,9 +1,9 @@ -from time import sleep import re -import requests +from datetime import datetime +from deezer.utils import map_track, map_album from deezer.errors import APIError, GWAPIError -from deemix.errors import TrackError, NoDataToParse, AlbumDoesntExists +from deemix.errors import NoDataToParse, AlbumDoesntExists from deemix.utils import removeFeatures, andCommaConcat, removeDuplicateArtists, generateReplayGainString, changeCase @@ -24,8 +24,10 @@ class Track: self.MD5 = "" self.mediaVersion = "" self.trackToken = "" + self.trackTokenExpiration = 0 self.duration = 0 self.fallbackID = "0" + self.albumsFallback = [] self.filesizes = {} self.local = False self.mainArtist = None @@ -42,6 +44,7 @@ class Track: self.explicit = False self.ISRC = "" self.replayGain = "" + self.rank = 0 self.playlist = None self.position = None self.searched = False @@ -53,79 +56,54 @@ class Track: self.featArtistsString = "" self.urls = {} - def parseEssentialData(self, trackAPI_gw, trackAPI=None): - self.id = str(trackAPI_gw['SNG_ID']) - self.duration = trackAPI_gw['DURATION'] - self.trackToken = trackAPI_gw['TRACK_TOKEN'] - self.MD5 = trackAPI_gw.get('MD5_ORIGIN') - if not self.MD5: - if trackAPI and trackAPI.get('md5_origin'): - self.MD5 = trackAPI['md5_origin'] - #else: - # raise MD5NotFound - self.mediaVersion = trackAPI_gw['MEDIA_VERSION'] + def parseEssentialData(self, trackAPI): + self.id = str(trackAPI['id']) + self.duration = trackAPI['duration'] + self.trackToken = trackAPI['track_token'] + self.trackTokenExpiration = trackAPI['track_token_expire'] + self.MD5 = trackAPI.get('md5_origin') + self.mediaVersion = trackAPI['media_version'] self.fallbackID = "0" - if 'FALLBACK' in trackAPI_gw: - self.fallbackID = trackAPI_gw['FALLBACK']['SNG_ID'] + if 'fallback_id' in trackAPI: + self.fallbackID = trackAPI['fallback_id'] self.local = int(self.id) < 0 self.urls = {} - def retriveFilesizes(self, dz): - guest_sid = dz.session.cookies.get('sid') - try: - site = requests.post( - "https://api.deezer.com/1.0/gateway.php", - params={ - 'api_key': "4VCYIJUCDLOUELGD1V8WBVYBNVDYOXEWSLLZDONGBBDFVXTZJRXPR29JRLQFO6ZE", - 'sid': guest_sid, - 'input': '3', - 'output': '3', - 'method': 'song_getData' - }, - timeout=30, - json={'sng_id': self.id}, - headers=dz.http_headers - ) - result_json = site.json() - except: - sleep(2) - self.retriveFilesizes(dz) - if len(result_json['error']): - raise TrackError(result_json.dumps(result_json['error'])) - response = result_json.get("results", {}) - filesizes = {} - for key, value in response.items(): - if key.startswith("FILESIZE_"): - filesizes[key] = int(value) - filesizes[key+"_TESTED"] = False - self.filesizes = filesizes + def parseData(self, dz, track_id=None, trackAPI=None, albumAPI=None, playlistAPI=None): + if track_id and (not trackAPI or trackAPI and not trackAPI.get('track_token')): + trackAPI_new = dz.gw.get_track_with_fallback(track_id) + trackAPI_new = map_track(trackAPI_new) + if not trackAPI: trackAPI = {} + trackAPI_new.update(trackAPI) + trackAPI = trackAPI_new + elif not trackAPI: raise NoDataToParse - def parseData(self, dz, track_id=None, trackAPI_gw=None, trackAPI=None, albumAPI_gw=None, albumAPI=None, playlistAPI=None): - if track_id and not trackAPI_gw: trackAPI_gw = dz.gw.get_track_with_fallback(track_id) - elif not trackAPI_gw: raise NoDataToParse - if not trackAPI: - try: trackAPI = dz.api.get_track(trackAPI_gw['SNG_ID']) - except APIError: trackAPI = None + self.parseEssentialData(trackAPI) - self.parseEssentialData(trackAPI_gw, trackAPI) + # only public api has bpm + if not trackAPI.get('bpm') and not self.local: + try: + trackAPI_new = dz.api.get_track(trackAPI['id']) + trackAPI_new['release_date'] = trackAPI['release_date'] + trackAPI.update(trackAPI_new) + except APIError: pass if self.local: - self.parseLocalTrackData(trackAPI_gw) + self.parseLocalTrackData(trackAPI) else: - self.retriveFilesizes(dz) - self.parseTrackGW(trackAPI_gw) + self.parseTrack(trackAPI) # Get Lyrics data - if not "LYRICS" in trackAPI_gw and self.lyrics.id != "0": - try: trackAPI_gw["LYRICS"] = dz.gw.get_track_lyrics(self.id) + if not trackAPI.get("lyrics") and self.lyrics.id != "0": + try: trackAPI["lyrics"] = dz.gw.get_track_lyrics(self.id) except GWAPIError: self.lyrics.id = "0" - if self.lyrics.id != "0": self.lyrics.parseLyrics(trackAPI_gw["LYRICS"]) + if self.lyrics.id != "0": self.lyrics.parseLyrics(trackAPI["lyrics"]) # Parse Album Data self.album = Album( - alb_id = trackAPI_gw['ALB_ID'], - title = trackAPI_gw['ALB_TITLE'], - pic_md5 = trackAPI_gw.get('ALB_PICTURE') + alb_id = trackAPI['album']['id'], + title = trackAPI['album']['title'], + pic_md5 = trackAPI['album'].get('md5_origin') ) # Get album Data @@ -134,31 +112,31 @@ class Track: except APIError: albumAPI = None # Get album_gw Data - if not albumAPI_gw: - try: albumAPI_gw = dz.gw.get_album(self.album.id) - except GWAPIError: albumAPI_gw = None + # Only gw has disk number + if not albumAPI or albumAPI and not albumAPI.get('nb_disk'): + try: + albumAPI_gw = dz.gw.get_album(self.album.id) + albumAPI_gw = map_album(albumAPI_gw) + except GWAPIError: albumAPI_gw = {} + if not albumAPI: albumAPI = {} + albumAPI_gw.update(albumAPI) + albumAPI = albumAPI_gw - if albumAPI: - self.album.parseAlbum(albumAPI) - elif albumAPI_gw: - self.album.parseAlbumGW(albumAPI_gw) - # albumAPI_gw doesn't contain the artist cover - # Getting artist image ID - # ex: https://e-cdns-images.dzcdn.net/images/artist/f2bc007e9133c946ac3c3907ddc5d2ea/56x56-000000-80-0-0.jpg + if not albumAPI: raise AlbumDoesntExists + + self.album.parseAlbum(albumAPI) + # albumAPI_gw doesn't contain the artist cover + # Getting artist image ID + # ex: https://e-cdns-images.dzcdn.net/images/artist/f2bc007e9133c946ac3c3907ddc5d2ea/56x56-000000-80-0-0.jpg + if not self.album.mainArtist.pic.md5 or self.album.mainArtist.pic.md5 == "": artistAPI = dz.api.get_artist(self.album.mainArtist.id) self.album.mainArtist.pic.md5 = artistAPI['picture_small'][artistAPI['picture_small'].find('artist/') + 7:-24] - else: - raise AlbumDoesntExists # Fill missing data - if albumAPI_gw: self.album.addExtraAlbumGWData(albumAPI_gw) if self.album.date and not self.date: self.date = self.album.date - if not self.album.discTotal: self.album.discTotal = albumAPI_gw.get('NUMBER_DISK', "1") - if not self.copyright: self.copyright = albumAPI_gw['COPYRIGHT'] - if 'GENRES' in trackAPI_gw: - for genre in trackAPI_gw['GENRES']: + if 'genres' in trackAPI: + for genre in trackAPI['genres']: if genre not in self.album.genre: self.album.genre.append(genre) - self.parseTrack(trackAPI) # Remove unwanted charaters in track name # Example: track/127793 @@ -168,7 +146,7 @@ class Track: if len(self.artist['Main']) == 0: self.artist['Main'] = [self.mainArtist.name] - self.position = trackAPI_gw.get('POSITION') + self.position = trackAPI.get('position') # Add playlist data if track is in a playlist if playlistAPI: self.playlist = Playlist(playlistAPI) @@ -176,63 +154,52 @@ class Track: self.generateMainFeatStrings() return self - def parseLocalTrackData(self, trackAPI_gw): + def parseLocalTrackData(self, trackAPI): # Local tracks has only the trackAPI_gw page and # contains only the tags provided by the file - self.title = trackAPI_gw['SNG_TITLE'] - self.album = Album(title=trackAPI_gw['ALB_TITLE']) + self.title = trackAPI['title'] + self.album = Album(title=trackAPI['album']['title']) self.album.pic = Picture( - md5 = trackAPI_gw.get('ALB_PICTURE', ""), + md5 = trackAPI.get('md5_image', ""), pic_type = "cover" ) - self.mainArtist = Artist(name=trackAPI_gw['ART_NAME'], role="Main") - self.artists = [trackAPI_gw['ART_NAME']] + self.mainArtist = Artist(name=trackAPI['artist']['name'], role="Main") + self.artists = [trackAPI['artist']['name']] self.artist = { - 'Main': [trackAPI_gw['ART_NAME']] + 'Main': [trackAPI['artist']['name']] } self.album.artist = self.artist self.album.artists = self.artists self.album.date = self.date self.album.mainArtist = self.mainArtist - def parseTrackGW(self, trackAPI_gw): - self.title = trackAPI_gw['SNG_TITLE'].strip() - if trackAPI_gw.get('VERSION') and not trackAPI_gw['VERSION'].strip() in self.title: - self.title += f" {trackAPI_gw['VERSION'].strip()}" - - self.discNumber = trackAPI_gw.get('DISK_NUMBER') - self.explicit = bool(int(trackAPI_gw.get('EXPLICIT_LYRICS', "0"))) - self.copyright = trackAPI_gw.get('COPYRIGHT') - if 'GAIN' in trackAPI_gw: self.replayGain = generateReplayGainString(trackAPI_gw['GAIN']) - self.ISRC = trackAPI_gw.get('ISRC') - self.trackNumber = trackAPI_gw['TRACK_NUMBER'] - self.contributors = trackAPI_gw['SNG_CONTRIBUTORS'] - self.rank = trackAPI_gw['RANK_SNG'] - - self.lyrics = Lyrics(trackAPI_gw.get('LYRICS_ID', "0")) - - self.mainArtist = Artist( - art_id = trackAPI_gw['ART_ID'], - name = trackAPI_gw['ART_NAME'], - role = "Main", - pic_md5 = trackAPI_gw.get('ART_PICTURE') - ) - - if 'PHYSICAL_RELEASE_DATE' in trackAPI_gw: - self.date.day = trackAPI_gw["PHYSICAL_RELEASE_DATE"][8:10] - self.date.month = trackAPI_gw["PHYSICAL_RELEASE_DATE"][5:7] - self.date.year = trackAPI_gw["PHYSICAL_RELEASE_DATE"][0:4] - self.date.fixDayMonth() - def parseTrack(self, trackAPI): + self.title = trackAPI['title'] + + self.discNumber = trackAPI.get('disk_number') + self.explicit = trackAPI.get('explicit_lyrics', False) + self.copyright = trackAPI.get('copyright') + if 'gain' in trackAPI: self.replayGain = generateReplayGainString(trackAPI['gain']) + self.ISRC = trackAPI.get('isrc') + self.trackNumber = trackAPI['track_position'] + self.contributors = trackAPI.get('song_contributors') + self.rank = trackAPI['rank'] self.bpm = trackAPI['bpm'] - if not self.replayGain and 'gain' in trackAPI: - self.replayGain = generateReplayGainString(trackAPI['gain']) - if not self.explicit: - self.explicit = trackAPI['explicit_lyrics'] - if not self.discNumber: - self.discNumber = trackAPI['disk_number'] + self.lyrics = Lyrics(trackAPI.get('lyrics_id', "0")) + + self.mainArtist = Artist( + art_id = trackAPI['artist']['id'], + name = trackAPI['artist']['name'], + role = "Main", + pic_md5 = trackAPI['artist'].get('md5_image') + ) + + if 'physical_release_date' in trackAPI: + self.date.day = trackAPI["physical_release_date"][8:10] + self.date.month = trackAPI["physical_release_date"][5:7] + self.date.year = trackAPI["physical_release_date"][0:4] + self.date.fixDayMonth() for artist in trackAPI['contributors']: isVariousArtists = str(artist['id']) == VARIOUS_ARTISTS @@ -249,6 +216,11 @@ class Track: self.artist[artist['role']] = [] self.artist[artist['role']].append(artist['name']) + if 'alternative_albums' in trackAPI and trackAPI['alternative_albums']: + for album in trackAPI['alternative_albums']['data']: + if 'RIGHTS' in album and album['RIGHTS'].get('STREAM_ADS_AVAILABLE') or album['RIGHTS'].get('STREAM_SUB_AVAILABLE'): + self.albumsFallback.append(album['ALB_ID']) + def removeDuplicateArtists(self): (self.artist, self.artists) = removeDuplicateArtists(self.artist, self.artists) @@ -267,6 +239,14 @@ class Track: if 'Featured' in self.artist: self.featArtistsString = "feat. "+andCommaConcat(self.artist['Featured']) + def checkAndRenewTrackToken(self, dz): + now = datetime.now() + expiration = datetime.fromtimestamp(self.trackTokenExpiration) + if now > expiration: + newTrack = dz.gw.get_track_with_fallback(self.id) + self.trackToken = newTrack['TRACK_TOKEN'] + self.trackTokenExpiration = newTrack['TRACK_TOKEN_EXPIRE'] + def applySettings(self, settings): # Check if should save the playlist as a compilation diff --git a/setup.py b/setup.py index d647330..8ac77e9 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ setup( python_requires='>=3.7', packages=find_packages(exclude=("tests",)), include_package_data=True, - install_requires=["click", "pycryptodomex", "mutagen", "requests", "deezer-py>=1.2.8"], + install_requires=["click", "pycryptodomex", "mutagen", "requests", "deezer-py>=1.3.0"], extras_require={ "spotify": ["spotipy>=2.11.0"] },