danolo
/
deemix-py
mirror of https://gitlab.com/RemixDev/deemix-py.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

More work on the library (WIP)

This commit is contained in:
RemixDev 2021-03-19 15:44:21 +01:00
parent 5ee81ced44
commit dc6adc7887
No known key found for this signature in database
GPG Key ID: B33962B465BDB51C
8 changed files with 187 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
deemix/__init__.py
View File

@ -26,19 +26,19 @@ def parseLink(link):
id = link[link.rfind("/") + 1:]
elif '/playlist' in link:
type = 'playlist'
id = re.search("\/playlist\/(\d+)", link)[0]
id = re.search("\/playlist\/(\d+)", link).group(1)
elif '/album' in link:
type = 'album'
id = link[link.rfind("/") + 1:]
elif re.search("\/artist\/(\d+)\/top_track", link):
type = 'artist_top'
id = re.search("\/artist\/(\d+)\/top_track", link)[0]
id = re.search("\/artist\/(\d+)\/top_track", link).group(1)
elif re.search("\/artist\/(\d+)\/discography", link):
type = 'artist_discography'
id = re.search("\/artist\/(\d+)\/discography", link)[0]
id = re.search("\/artist\/(\d+)\/discography", link).group(1)
elif '/artist' in link:
type = 'artist'
id = re.search("\/artist\/(\d+)", link)[0]
id = re.search("\/artist\/(\d+)", link).group(1)
return (link, type, id)

136
deemix/decryption.py
View File

@ -2,6 +2,18 @@ import binascii
from Cryptodome.Cipher import Blowfish, AES
from Cryptodome.Hash import MD5
from deemix import USER_AGENT_HEADER
from deemix.types.DownloadObjects import Single, Collection
from requests import get
from requests.exceptions import ConnectionError, ReadTimeout
from ssl import SSLError
from urllib3.exceptions import SSLError as u3SSLError
import logging
logger = logging.getLogger('deemix')
def _md5(data):
h = MD5.new()
h.update(str.encode(data) if isinstance(data, str) else data)
@ -40,3 +52,127 @@ def generateUnencryptedStreamURL(sng_id, md5, media_version, format):
def reverseStreamURL(url):
urlPart = url[url.find("/1/")+3:]
return generateStreamPath(urlPart)
def streamUnencryptedTrack(outputStream, track, start=0, downloadObject=None, interface=None):
headers= {'User-Agent': USER_AGENT_HEADER}
chunkLength = start
percentage = 0
itemName = f"[{track.mainArtist.name} - {track.title}]"
try:
with get(track.downloadUrl, headers=headers, stream=True, timeout=10) as request:
request.raise_for_status()
complete = int(request.headers["Content-Length"])
if complete == 0: raise DownloadEmpty
if start != 0:
responseRange = request.headers["Content-Range"]
logger.info(f'{itemName} downloading range {responseRange}')
else:
logger.info(f'{itemName} downloading {complete} bytes')
for chunk in request.iter_content(2048 * 3):
outputStream.write(chunk)
chunkLength += len(chunk)
if downloadObject:
if isinstance(downloadObject, Single):
percentage = (chunkLength / (complete + start)) * 100
downloadObject.progressNext = percentage
else:
chunkProgres = (len(chunk) / (complete + start)) / downloadObject.size * 100
downloadObject.progressNext += chunkProgres
downloadObject.updateProgress(interface)
except (SSLError, u3SSLError) as e:
logger.info(f'{itemName} retrying from byte {chunkLength}')
return streamUnencryptedTrack(outputStream, track, chunkLength, downloadObject, interface)
except (ConnectionError, ReadTimeout):
sleep(2)
return streamUnencryptedTrack(outputStream, track, start, downloadObject, interface)
def streamUnencryptedTrack(outputStream, track, start=0, downloadObject=None, interface=None):
headers= {'User-Agent': USER_AGENT_HEADER}
chunkLength = start
percentage = 0
itemName = f"[{track.mainArtist.name} - {track.title}]"
try:
with get(track.downloadUrl, headers=headers, stream=True, timeout=10) as request:
request.raise_for_status()
complete = int(request.headers["Content-Length"])
if complete == 0: raise DownloadEmpty
if start != 0:
responseRange = request.headers["Content-Range"]
logger.info(f'{itemName} downloading range {responseRange}')
else:
logger.info(f'{itemName} downloading {complete} bytes')
for chunk in request.iter_content(2048 * 3):
outputStream.write(chunk)
chunkLength += len(chunk)
if downloadObject:
if isinstance(downloadObject, Single):
percentage = (chunkLength / (complete + start)) * 100
downloadObject.progressNext = percentage
else:
chunkProgres = (len(chunk) / (complete + start)) / downloadObject.size * 100
downloadObject.progressNext += chunkProgres
downloadObject.updateProgress(interface)
except (SSLError, u3SSLError) as e:
logger.info(f'{itemName} retrying from byte {chunkLength}')
return streamUnencryptedTrack(outputStream, track, chunkLength, downloadObject, interface)
except (ConnectionError, ReadTimeout):
sleep(2)
return streamUnencryptedTrack(outputStream, track, start, downloadObject, interface)
def streamTrack(outputStream, track, start=0, downloadObject=None, interface=None):
headers= {'User-Agent': USER_AGENT_HEADER}
chunkLength = start
percentage = 0
itemName = f"[{track.mainArtist.name} - {track.title}]"
try:
with get(track.downloadUrl, headers=headers, stream=True, timeout=10) as request:
request.raise_for_status()
blowfish_key = str.encode(generateBlowfishKey(str(track.id)))
complete = int(request.headers["Content-Length"])
if complete == 0: raise DownloadEmpty
if start != 0:
responseRange = request.headers["Content-Range"]
logger.info(f'{itemName} downloading range {responseRange}')
else:
logger.info(f'{itemName} downloading {complete} bytes')
for chunk in request.iter_content(2048 * 3):
if len(chunk) >= 2048:
chunk = Blowfish.new(blowfish_key, Blowfish.MODE_CBC, b"\x00\x01\x02\x03\x04\x05\x06\x07").decrypt(chunk[0:2048]) + chunk[2048:]
outputStream.write(chunk)
chunkLength += len(chunk)
if downloadObject:
if isinstance(downloadObject, Single):
percentage = (chunkLength / (complete + start)) * 100
downloadObject.progressNext = percentage
else:
chunkProgres = (len(chunk) / (complete + start)) / downloadObject.size * 100
downloadObject.progressNext += chunkProgres
downloadObject.updateProgress(interface)
except (SSLError, u3SSLError) as e:
logger.info(f'{itemName} retrying from byte {chunkLength}')
return streamTrack(outputStream, track, chunkLength, downloadObject, interface)
except (ConnectionError, ReadTimeout):
sleep(2)
return streamTrack(outputStream, track, start, downloadObject, interface)
class DownloadEmpty(Exception):
pass

96
deemix/downloader.py
View File

@ -11,24 +11,21 @@ import re
import errno
from ssl import SSLError
from os import makedirs
from urllib3.exceptions import SSLError as u3SSLError
from os import makedirs
from deemix.types.DownloadObjects import Single, Collection
from deemix.types.Track import Track, AlbumDoesntExists
from deemix.utils import changeCase
from deemix.utils.pathtemplates import generateFilename, generateFilepath, settingsRegexAlbum, settingsRegexArtist, settingsRegexPlaylistFile
from deezer import TrackFormats
from deemix import USER_AGENT_HEADER
from deemix.taggers import tagID3, tagFLAC
from deemix.decryption import generateStreamURL, generateBlowfishKey
from deemix.decryption import generateUnencryptedStreamURL, streamUnencryptedTrack
from deemix.settings import OverwriteOption
from Cryptodome.Cipher import Blowfish
from mutagen.flac import FLACNoHeaderError, error as FLACError
import logging
logging.basicConfig(level=logging.INFO)
import logging
logger = logging.getLogger('deemix')
from tempfile import gettempdir
@ -124,7 +121,7 @@ def getPreferredBitrate(track, preferredBitrate, shouldFallback, downloadObjectU
if int(track.filesizes[f"FILESIZE_{formatName}"]) != 0: return formatNumber
if not track.filesizes[f"FILESIZE_{formatName}_TESTED"]:
request = requests.head(
generateStreamURL(track.id, track.MD5, track.mediaVersion, formatNumber),
generateUnencryptedStreamURL(track.id, track.MD5, track.mediaVersion, formatNumber),
headers={'User-Agent': USER_AGENT_HEADER},
timeout=30
)
@ -159,8 +156,6 @@ class Downloader:
self.settings = settings
self.bitrate = downloadObject.bitrate
self.interface = interface
self.downloadPercentage = 0
self.lastPercentage = 0
self.extrasPath = None
self.playlistCoverName = None
self.playlistURLs = []
@ -184,7 +179,6 @@ class Downloader:
if trackAPI_gw['SNG_ID'] == "0": raise DownloadFailed("notOnDeezer")
# Create Track object
print(track)
if not track:
logger.info(f"[{trackAPI_gw['ART_NAME']} - {trackAPI_gw['SNG_TITLE']}] Getting the tags")
try:
@ -252,7 +246,7 @@ class Downloader:
url = track.album.pic.generatePictureURL(self.settings['localArtworkSize'], extendedFormat)
if self.settings['tags']['savePlaylistAsCompilation'] \
and track.playlist \
and track.playlist.pic.url \
and track.playlist.pic.staticUrl \
and not format.startswith("jpg"):
continue
result['albumURLs'].append({'url': url, 'ext': format})
@ -280,7 +274,7 @@ class Downloader:
extendedFormat = format
if extendedFormat == "jpg": extendedFormat += f"-{self.settings['jpegImageQuality']}"
url = track.playlist.pic.generatePictureURL(self.settings['localArtworkSize'], extendedFormat)
if track.playlist.pic.url and not format.startswith("jpg"): continue
if track.playlist.pic.staticUrl and not format.startswith("jpg"): continue
self.playlistURLs.append({'url': url, 'ext': format})
if not self.playlistCoverName:
track.playlist.bitrate = selectedFormat
@ -316,12 +310,12 @@ class Downloader:
if not trackAlreadyDownloaded or self.settings['overwriteFile'] == OverwriteOption.OVERWRITE:
logger.info(f"[{track.mainArtist.name} - {track.title}] Downloading the track")
track.downloadUrl = generateStreamURL(track.id, track.MD5, track.mediaVersion, track.selectedFormat)
track.downloadUrl = generateUnencryptedStreamURL(track.id, track.MD5, track.mediaVersion, track.selectedFormat)
def downloadMusic(track, trackAPI_gw):
try:
with open(writepath, 'wb') as stream:
self.streamTrack(stream, track)
streamUnencryptedTrack(stream, track, downloadObject=self.downloadObject, interface=self.interface)
except DownloadCancelled:
if writepath.is_file(): writepath.unlink()
raise DownloadCancelled
@ -382,7 +376,7 @@ class Downloader:
if not trackDownloaded: return self.download(trackAPI_gw, track=track)
else:
logger.info(f"[{track.mainArtist.name} - {track.title}] Skipping track as it's already downloaded")
self.completeTrackPercentage()
self.downloadObject.completeTrackProgress(self.interface)
# Adding tags
if (not trackAlreadyDownloaded or self.settings['overwriteFile'] in [OverwriteOption.ONLY_TAGS, OverwriteOption.OVERWRITE]) and not track.localTrack:
@ -395,7 +389,7 @@ class Downloader:
except (FLACNoHeaderError, FLACError):
if writepath.is_file(): writepath.unlink()
logger.warn(f"[{track.mainArtist.name} - {track.title}] Track not available in FLAC, falling back if necessary")
self.removeTrackPercentage()
self.downloadObject.removeTrackProgress(self.interface)
track.filesizes['FILESIZE_FLAC'] = "0"
track.filesizes['FILESIZE_FLAC_TESTED'] = True
return self.download(trackAPI_gw, track=track)
@ -409,71 +403,6 @@ class Downloader:
self.interface.send("updateQueue", {'uuid': self.downloadObject.uuid, 'downloaded': True, 'downloadPath': str(writepath), 'extrasPath': str(self.extrasPath)})
return result
def streamTrack(self, stream, track, start=0):
headers=dict(self.dz.http_headers)
if range != 0: headers['Range'] = f'bytes={start}-'
chunkLength = start
percentage = 0
itemName = f"[{track.mainArtist.name} - {track.title}]"
try:
with self.dz.session.get(track.downloadUrl, headers=headers, stream=True, timeout=10) as request:
request.raise_for_status()
blowfish_key = str.encode(generateBlowfishKey(str(track.id)))
complete = int(request.headers["Content-Length"])
if complete == 0: raise DownloadEmpty
if start != 0:
responseRange = request.headers["Content-Range"]
logger.info(f'{itemName} downloading range {responseRange}')
else:
logger.info(f'{itemName} downloading {complete} bytes')
for chunk in request.iter_content(2048 * 3):
if len(chunk) >= 2048:
chunk = Blowfish.new(blowfish_key, Blowfish.MODE_CBC, b"\x00\x01\x02\x03\x04\x05\x06\x07").decrypt(chunk[0:2048]) + chunk[2048:]
stream.write(chunk)
chunkLength += len(chunk)
if isinstance(self.downloadObject, Single):
percentage = (chunkLength / (complete + start)) * 100
self.downloadPercentage = percentage
else:
chunkProgres = (len(chunk) / (complete + start)) / self.downloadObject.size * 100
self.downloadPercentage += chunkProgres
self.updatePercentage()
except (SSLError, u3SSLError) as e:
logger.info(f'{itemName} retrying from byte {chunkLength}')
return self.streamTrack(stream, track, chunkLength)
except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout):
sleep(2)
return self.streamTrack(stream, track, start)
def updatePercentage(self):
if round(self.downloadPercentage) != self.lastPercentage and round(self.downloadPercentage) % 2 == 0:
self.lastPercentage = round(self.downloadPercentage)
self.downloadObject.progress = self.lastPercentage
if self.interface: self.interface.send("updateQueue", {'uuid': self.downloadObject.uuid, 'progress': self.lastPercentage})
def completeTrackPercentage(self):
if isinstance(self.downloadObject, Single):
self.downloadPercentage = 100
else:
self.downloadPercentage += (1 / self.downloadObject.size) * 100
self.updatePercentage()
def removeTrackPercentage(self):
if isinstance(self.downloadObject, Single):
self.downloadPercentage = 0
else:
self.downloadPercentage -= (1 / self.downloadObject.size) * 100
self.updatePercentage()
def downloadWrapper(self, trackAPI_gw, trackAPI=None, albumAPI=None, playlistAPI=None, track=None):
# Temp metadata to generate logs
tempTrack = {
@ -531,7 +460,7 @@ class Downloader:
}}
if 'error' in result:
self.completeTrackPercentage()
self.downloadObject.completeTrackProgress(self.interface)
self.downloadObject.failed += 1
self.downloadObject.errors.append(result['error'])
if self.interface:
@ -640,9 +569,6 @@ class DownloadFailed(DownloadError):
class DownloadCancelled(DownloadError):
pass
class DownloadEmpty(DownloadError):
pass
class PreferredBitrateNotFound(DownloadError):
pass

6
deemix/itemgen.py
View File

@ -1,4 +1,6 @@
from deemix.types.DownloadObjects import Single, Collection
from deezer.api import APIError
from deezer.gw import GWAPIError, LyricsStatus
class GenerationError(Exception):
def __init__(self, link, message, errid=None):
@ -29,7 +31,7 @@ def generateTrackItem(dz, id, bitrate, trackAPI=None, albumAPI=None):
# Get essential track info
try:
trackAPI_gw = dz.gw.get_track_with_fallback(id)
except gwAPIError as e:
except GWAPIError as e:
e = str(e)
message = "Wrong URL"
if "DATA_ERROR" in e: message += f": {e['DATA_ERROR']}"
@ -116,7 +118,7 @@ def generatePlaylistItem(dz, id, bitrate, playlistAPI=None, playlistTracksAPI=No
try:
userPlaylist = dz.gw.get_playlist_page(id)
playlistAPI = map_user_playlist(userPlaylist['DATA'])
except gwAPIError as e:
except GWAPIError as e:
e = str(e)
message = "Wrong URL"
if "DATA_ERROR" in e:

22
deemix/types/DownloadObjects.py
View File

@ -28,6 +28,7 @@ class IDownloadObject:
self.progress = 0
self.errors = []
self.files = []
self.progressNext = 0
self.uuid = f"{self.type}_{self.id}_{self.bitrate}"
self.ack = None
self.__type__ = None
@ -69,6 +70,11 @@ class IDownloadObject:
del light[property]
return light
def updateProgress(self, interface=None):
if round(self.progressNext) != self.progress and round(self.progressNext) % 2 == 0:
self.progress = round(self.progressNext)
if interface: interface.send("updateQueue", {'uuid': self.uuid, 'progress': self.progress})
class Single(IDownloadObject):
def __init__(self, type=None, id=None, bitrate=None, title=None, artist=None, cover=None, explicit=False, trackAPI_gw=None, trackAPI=None, albumAPI=None, dictItem=None):
if dictItem:
@ -88,6 +94,14 @@ class Single(IDownloadObject):
item['single'] = self.single
return item
def completeTrackProgress(self, interface=None):
self.progressNext = 100
self.updateProgress(interface)
def removeTrackProgress(self, interface=None):
self.progressNext = 0
self.updateProgress(interface)
class Collection(IDownloadObject):
def __init__(self, type=None, id=None, bitrate=None, title=None, artist=None, cover=None, explicit=False, size=None, tracks_gw=None, albumAPI=None, playlistAPI=None, dictItem=None):
if dictItem:
@ -107,6 +121,14 @@ class Collection(IDownloadObject):
item['collection'] = self.collection
return item
def completeTrackProgress(self, interface=None):
self.progressNext += (1 / self.size) * 100
self.updateProgress(interface)
def removeTrackProgress(self, interface=None):
self.progressNext -= (1 / self.size) * 100
self.updateProgress(interface)
class Convertable(Collection):
def __init__(self, type=None, id=None, bitrate=None, title=None, artist=None, cover=None, explicit=False, size=None, plugin=None, conversion_data=None, dictItem=None):
if dictItem:

2
deemix/types/Playlist.py
View File

@ -39,7 +39,7 @@ class Playlist:
if 'various_artist' in playlistAPI:
pic_md5 = playlistAPI['various_artist']['picture_small']
pic_md5 = pic_md5[pic_md5.indexOf('artist/') + 7:-24]
pic_md5 = pic_md5[pic_md5.find('artist/') + 7:-24]
self.variousArtists = Artist(
id = playlistAPI['various_artist']['id'],
name = playlistAPI['various_artist']['name'],

9
deemix/types/Track.py
View File

@ -5,9 +5,10 @@ import logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger('deemix')
from deezer.gw import APIError as gwAPIError
from deezer.gw import GWAPIError
from deezer.api import APIError
from deemix.utils import removeFeatures, andCommaConcat, removeDuplicateArtists, generateReplayGainString
from deemix.utils import removeFeatures, andCommaConcat, removeDuplicateArtists, generateReplayGainString, changeCase
from deemix.types.Album import Album
from deemix.types.Artist import Artist
from deemix.types.Date import Date
@ -114,7 +115,7 @@ class Track:
# Get Lyrics data
if not "LYRICS" in trackAPI_gw and self.lyrics.id != "0":
try: trackAPI_gw["LYRICS"] = dz.gw.get_track_lyrics(self.id)
except gwAPIError: self.lyrics.id = "0"
except GWAPIError: self.lyrics.id = "0"
if self.lyrics.id != "0": self.lyrics.parseLyrics(trackAPI_gw["LYRICS"])
# Parse Album data
@ -132,7 +133,7 @@ class Track:
# Get album_gw Data
if not albumAPI_gw:
try: albumAPI_gw = dz.gw.get_album(self.album.id)
except gwAPIError: albumAPI_gw = None
except GWAPIError: albumAPI_gw = None
if albumAPI:
self.album.parseAlbum(albumAPI)

6
deemix/utils/pathtemplates.py
View File

@ -148,7 +148,7 @@ def settingsRegex(filename, track, settings):
filename = filename.replace("%album_id%", str(track.album.id))
filename = filename.replace("%artist_id%", str(track.mainArtist.id))
if track.playlist:
filename = filename.replace("%playlist_id%", str(track.playlist.playlistId))
filename = filename.replace("%playlist_id%", str(track.playlist.playlistID))
filename = filename.replace("%position%", pad(track.position, track.playlist.trackTotal, settings))
else:
filename = filename.replace("%playlist_id%", '')
@ -159,7 +159,7 @@ def settingsRegex(filename, track, settings):
def settingsRegexAlbum(foldername, album, settings, playlist=None):
if playlist and settings['tags']['savePlaylistAsCompilation']:
foldername = foldername.replace("%album_id%", "pl_" + str(playlist.playlistId))
foldername = foldername.replace("%album_id%", "pl_" + str(playlist.playlistID))
foldername = foldername.replace("%genre%", "Compile")
else:
foldername = foldername.replace("%album_id%", str(album.id))
@ -205,7 +205,7 @@ def settingsRegexArtist(foldername, artist, settings, rootArtist=None):
def settingsRegexPlaylist(foldername, playlist, settings):
foldername = foldername.replace("%playlist%", fixName(playlist.title, settings['illegalCharacterReplacer']))
foldername = foldername.replace("%playlist_id%", fixName(playlist.playlistId, settings['illegalCharacterReplacer']))
foldername = foldername.replace("%playlist_id%", fixName(playlist.playlistID, settings['illegalCharacterReplacer']))
foldername = foldername.replace("%owner%", fixName(playlist.owner['name'], settings['illegalCharacterReplacer']))
foldername = foldername.replace("%owner_id%", str(playlist.owner['id']))
foldername = foldername.replace("%year%", str(playlist.date.year))